From 091e2422fd6ca8c1deb556b042d3db9cec94f45c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Cea=20Fontenla?= Date: Wed, 2 Oct 2024 13:37:34 +0200 Subject: [PATCH 001/194] Fix MemoryTracking...ArraysTests bulkSet errors (#113934) Fixes https://github.com/elastic/elasticsearch/issues/113914 Fixes https://github.com/elastic/elasticsearch/issues/113744 Some random numbers in the tests were leading to negative indexes in some cases. Just refactored them a bit so no impossible values are generated. And it's now a bit more _readable_... --- .../MemoryTrackingTDigestArraysTests.java | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArraysTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArraysTests.java index e57f39becef7c..c5fb078cae66b 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArraysTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArraysTests.java @@ -84,11 +84,11 @@ public void testIntResize() { } public void testIntBulkSet() { - int initialSize = randomIntBetween(10, 1000); - int sourceArraySize = randomIntBetween(0, initialSize); + int targetArraySize = randomIntBetween(10, 1000); + int sourceArraySize = randomIntBetween(0, 1000); - try (TDigestIntArray array = intArray(initialSize); TDigestIntArray source = intArray(sourceArraySize)) { - assertThat(array.size(), equalTo(initialSize)); + try (TDigestIntArray target = intArray(targetArraySize); TDigestIntArray source = intArray(sourceArraySize)) { + assertThat(target.size(), equalTo(targetArraySize)); assertThat(source.size(), equalTo(sourceArraySize)); int value = randomInt(); @@ -96,17 +96,17 @@ public void testIntBulkSet() { source.set(i, value); } - int initialOffset = randomIntBetween(0, initialSize - sourceArraySize); - int sourceOffset = randomIntBetween(0, sourceArraySize - 1); - int elementsToCopy = randomIntBetween(1, sourceArraySize - sourceOffset); + int targetOffset = randomIntBetween(0, targetArraySize); + int sourceOffset = randomIntBetween(0, sourceArraySize); + int elementsToCopy = randomIntBetween(0, Math.min(sourceArraySize - sourceOffset, targetArraySize - targetOffset)); - array.set(initialOffset, source, sourceOffset, elementsToCopy); + target.set(targetOffset, source, sourceOffset, elementsToCopy); - for (int i = 0; i < initialSize; i++) { - if (i < initialOffset || i >= initialOffset + elementsToCopy) { - assertThat(array.get(i), equalTo(0)); + for (int i = 0; i < targetArraySize; i++) { + if (i < targetOffset || i >= targetOffset + elementsToCopy) { + assertThat(target.get(i), equalTo(0)); } else { - assertThat(array.get(i), equalTo(value)); + assertThat(target.get(i), equalTo(value)); } } } @@ -289,11 +289,11 @@ public void testDoubleAdd() { } public void testDoubleBulkSet() { - int initialSize = randomIntBetween(10, 1000); - int sourceArraySize = randomIntBetween(0, initialSize); + int targetArraySize = randomIntBetween(10, 1000); + int sourceArraySize = randomIntBetween(0, 1000); - try (TDigestDoubleArray array = doubleArray(initialSize); TDigestDoubleArray source = doubleArray(sourceArraySize)) { - assertThat(array.size(), equalTo(initialSize)); + try (TDigestDoubleArray target = doubleArray(targetArraySize); TDigestDoubleArray source = doubleArray(sourceArraySize)) { + assertThat(target.size(), equalTo(targetArraySize)); assertThat(source.size(), equalTo(sourceArraySize)); double value = randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true); @@ -301,17 +301,17 @@ public void testDoubleBulkSet() { source.set(i, value); } - int initialOffset = randomIntBetween(0, initialSize - sourceArraySize); - int sourceOffset = randomIntBetween(0, sourceArraySize - 1); - int elementsToCopy = randomIntBetween(1, sourceArraySize - sourceOffset); + int targetOffset = randomIntBetween(0, targetArraySize); + int sourceOffset = randomIntBetween(0, sourceArraySize); + int elementsToCopy = randomIntBetween(0, Math.min(sourceArraySize - sourceOffset, targetArraySize - targetOffset)); - array.set(initialOffset, source, sourceOffset, elementsToCopy); + target.set(targetOffset, source, sourceOffset, elementsToCopy); - for (int i = 0; i < initialSize; i++) { - if (i < initialOffset || i >= initialOffset + elementsToCopy) { - assertThat(array.get(i), equalTo(0.0)); + for (int i = 0; i < targetArraySize; i++) { + if (i < targetOffset || i >= targetOffset + elementsToCopy) { + assertThat(target.get(i), equalTo(0.0)); } else { - assertThat(array.get(i), equalTo(value)); + assertThat(target.get(i), equalTo(value)); } } } From bcc5caf8009802fd7ddcc6161c6fbe559b77b554 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 2 Oct 2024 22:30:11 +1000 Subject: [PATCH 002/194] Mute org.elasticsearch.kibana.KibanaThreadPoolIT testBlockedThreadPoolsRejectUserRequests #113939 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 28d9b15af78cd..c84cdf9f9fc00 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -341,6 +341,9 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/113874 - class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT issue: https://github.com/elastic/elasticsearch/issues/113916 +- class: org.elasticsearch.kibana.KibanaThreadPoolIT + method: testBlockedThreadPoolsRejectUserRequests + issue: https://github.com/elastic/elasticsearch/issues/113939 # Examples: # From 6425c617ae531a79b2bb7beb8038d754fef3ec9c Mon Sep 17 00:00:00 2001 From: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com> Date: Wed, 2 Oct 2024 08:34:13 -0400 Subject: [PATCH 003/194] Bugfix by moving onResponse call into the new action listener; also add new test to catch this bug (#113898) --- .../services/elser/ElserInternalService.java | 19 +++++++------- .../elser/ElserInternalServiceTests.java | 25 +++++++++++++++++++ 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java index 1198be7ab7a3b..d36b8eca7661e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java @@ -101,17 +101,16 @@ public void parseRequestConfig( serviceSettingsBuilder.setModelId( selectDefaultModelVariantBasedOnClusterArchitecture(arch, ELSER_V2_MODEL_LINUX_X86, ELSER_V2_MODEL) ); + parsedModelListener.onResponse( + new ElserInternalModel( + inferenceEntityId, + taskType, + NAME, + new ElserInternalServiceSettings(serviceSettingsBuilder.build()), + taskSettings + ) + ); })); - - parsedModelListener.onResponse( - new ElserInternalModel( - inferenceEntityId, - taskType, - NAME, - new ElserInternalServiceSettings(serviceSettingsBuilder.build()), - taskSettings - ) - ); } else { parsedModelListener.onResponse( new ElserInternalModel( diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java index adf9c7b4f5bc5..09abeb9b9b389 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java @@ -120,6 +120,31 @@ public void testParseConfigStrict() { } + public void testParseConfigWithoutModelId() { + Client mockClient = mock(Client.class); + when(mockClient.threadPool()).thenReturn(threadPool); + var service = createService(mockClient); + + var settings = new HashMap(); + settings.put( + ModelConfigurations.SERVICE_SETTINGS, + new HashMap<>(Map.of(ElserInternalServiceSettings.NUM_ALLOCATIONS, 1, ElserInternalServiceSettings.NUM_THREADS, 4)) + ); + + var expectedModel = new ElserInternalModel( + "foo", + TaskType.SPARSE_EMBEDDING, + ElserInternalService.NAME, + new ElserInternalServiceSettings(1, 4, ".elser_model_2", null), + ElserMlNodeTaskSettings.DEFAULT + ); + + var modelVerificationListener = getModelVerificationListener(expectedModel); + + service.parseRequestConfig("foo", TaskType.SPARSE_EMBEDDING, settings, modelVerificationListener); + + } + public void testParseConfigLooseWithOldModelId() { var service = createService(mock(Client.class)); From 521e4341d7d310ecc4634c5a75a1690b7d9c4884 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Wed, 2 Oct 2024 15:02:43 +0200 Subject: [PATCH 004/194] `ignore_above` default to 8191 for `logsdb` (#113442) In LogsDB we would like to use a default value of `8191` for the index-level setting `index.mapping.ignore_above`. The value for `ignore_above` is the _character count_, but Lucene counts bytes. Here we set the limit to `32766 / 4 = 8191` since UTF-8 characters may occupy at most 4 bytes. --- .../logsdb/LogsIndexModeCustomSettingsIT.java | 61 +++++++++++++++++++ .../common/settings/Setting.java | 10 +++ .../elasticsearch/index/IndexSettings.java | 20 +++++- .../elasticsearch/index/IndexVersions.java | 2 +- 4 files changed, 90 insertions(+), 3 deletions(-) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java index 0637fc80f6644..db6c12c8bc565 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.hamcrest.Matchers; import org.junit.Before; import org.junit.ClassRule; @@ -357,6 +358,66 @@ public void testIgnoreMalformedSetting() throws IOException { } } + public void testIgnoreAboveSetting() throws IOException { + // with default template + { + assertOK(createDataStream(client, "logs-test-1")); + String logsIndex1 = getDataStreamBackingIndex(client, "logs-test-1", 0); + assertThat(getSetting(client, logsIndex1, "index.mapping.ignore_above"), equalTo("8191")); + for (String newValue : List.of("512", "2048", "12000", String.valueOf(Integer.MAX_VALUE))) { + closeIndex(logsIndex1); + updateIndexSettings(logsIndex1, Settings.builder().put("index.mapping.ignore_above", newValue)); + assertThat(getSetting(client, logsIndex1, "index.mapping.ignore_above"), equalTo(newValue)); + } + for (String newValue : List.of(String.valueOf((long) Integer.MAX_VALUE + 1), String.valueOf(Long.MAX_VALUE))) { + closeIndex(logsIndex1); + ResponseException ex = assertThrows( + ResponseException.class, + () -> updateIndexSettings(logsIndex1, Settings.builder().put("index.mapping.ignore_above", newValue)) + ); + assertThat( + ex.getMessage(), + Matchers.containsString("Failed to parse value [" + newValue + "] for setting [index.mapping.ignore_above]") + ); + } + } + // with override template + { + var template = """ + { + "template": { + "settings": { + "index": { + "mapping": { + "ignore_above": "128" + } + } + } + } + }"""; + assertOK(putComponentTemplate(client, "logs@custom", template)); + assertOK(createDataStream(client, "logs-custom-dev")); + String index = getDataStreamBackingIndex(client, "logs-custom-dev", 0); + assertThat(getSetting(client, index, "index.mapping.ignore_above"), equalTo("128")); + for (String newValue : List.of("64", "256", "12000", String.valueOf(Integer.MAX_VALUE))) { + closeIndex(index); + updateIndexSettings(index, Settings.builder().put("index.mapping.ignore_above", newValue)); + assertThat(getSetting(client, index, "index.mapping.ignore_above"), equalTo(newValue)); + } + } + // standard index + { + String index = "test-index"; + createIndex(index); + assertThat(getSetting(client, index, "index.mapping.ignore_above"), equalTo(Integer.toString(Integer.MAX_VALUE))); + for (String newValue : List.of("256", "512", "12000", String.valueOf(Integer.MAX_VALUE))) { + closeIndex(index); + updateIndexSettings(index, Settings.builder().put("index.mapping.ignore_above", newValue)); + assertThat(getSetting(client, index, "index.mapping.ignore_above"), equalTo(newValue)); + } + } + } + private static Map getMapping(final RestClient client, final String indexName) throws IOException { final Request request = new Request("GET", "/" + indexName + "/_mapping"); diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index feda67cb466b9..70bf88625dc29 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -1366,6 +1366,16 @@ public static Setting intSetting(String key, int defaultValue, int minV return new Setting<>(key, Integer.toString(defaultValue), intParser(key, minValue, properties), properties); } + public static Setting intSetting( + String key, + Function defaultValueFn, + int minValue, + int maxValue, + Property... properties + ) { + return new Setting<>(key, defaultValueFn, intParser(key, minValue, maxValue, properties), properties); + } + private static Function intParser(String key, int minValue, Property[] properties) { final boolean isFiltered = isFiltered(properties); return s -> parseInt(s, minValue, key, isFiltered); diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index f6ad5e22b9ed7..e82f9eff7d5e0 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -701,7 +701,7 @@ public Iterator> settings() { /** * The `index.mapping.ignore_above` setting defines the maximum length for the content of a field that will be indexed * or stored. If the length of the field’s content exceeds this limit, the field value will be ignored during indexing. - * This setting is useful for `keyword`, `flattened`, and `wildcard` fields where very large values are undesirable. + * This setting is useful for `keyword`, `flattened`, and `wildcard` fields where very large values are undesirable. * It allows users to manage the size of indexed data by skipping fields with excessively long content. As an index-level * setting, it applies to all `keyword` and `wildcard` fields, as well as to keyword values within `flattened` fields. * When it comes to arrays, the `ignore_above` setting applies individually to each element of the array. If any element's @@ -713,14 +713,30 @@ public Iterator> settings() { * 
      * "index.mapping.ignore_above": 256
      *
+ *

+ * NOTE: The value for `ignore_above` is the _character count_, but Lucene counts + * bytes. Here we set the limit to `32766 / 4 = 8191` since UTF-8 characters may + * occupy at most 4 bytes. */ + public static final Setting IGNORE_ABOVE_SETTING = Setting.intSetting( "index.mapping.ignore_above", - Integer.MAX_VALUE, + IndexSettings::getIgnoreAboveDefaultValue, 0, + Integer.MAX_VALUE, Property.IndexScope, Property.ServerlessPublic ); + + private static String getIgnoreAboveDefaultValue(final Settings settings) { + if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB + && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB)) { + return "8191"; + } else { + return String.valueOf(Integer.MAX_VALUE); + } + } + public static final NodeFeature IGNORE_ABOVE_INDEX_LEVEL_SETTING = new NodeFeature("mapper.ignore_above_index_level_setting"); private final Index index; diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index e9b1adabfd001..7e04a64e74cb5 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -117,7 +117,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion ENABLE_IGNORE_MALFORMED_LOGSDB = def(8_514_00_0, Version.LUCENE_9_11_1); public static final IndexVersion MERGE_ON_RECOVERY_VERSION = def(8_515_00_0, Version.LUCENE_9_11_1); public static final IndexVersion UPGRADE_TO_LUCENE_9_12 = def(8_516_00_0, Version.LUCENE_9_12_0); - + public static final IndexVersion ENABLE_IGNORE_ABOVE_LOGSDB = def(8_517_00_0, Version.LUCENE_9_12_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ From a5d033b9ae9128b429748d23c00c79820d99a9f3 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Wed, 2 Oct 2024 09:28:03 -0400 Subject: [PATCH 005/194] Move AbstractBWCSerializationTestCase To General Test Framework (#113907) --- .../AbstractBWCSerializationTestCase.java | 14 +++++---- .../org/elasticsearch/test/BWCVersions.java | 29 +++++++++++++++++++ .../AbstractBWCWireSerializationTestCase.java | 11 +------ ...stractChunkedBWCSerializationTestCase.java | 2 +- .../DataFrameAnalyticsConfigTests.java | 2 +- .../DataFrameAnalyticsDestTests.java | 2 +- .../DataFrameAnalyticsSourceTests.java | 2 +- .../analyses/BoostedTreeParamsTests.java | 2 +- .../analyses/ClassificationTests.java | 2 +- .../analyses/OutlierDetectionTests.java | 2 +- .../dataframe/analyses/RegressionTests.java | 2 +- .../ClassificationStatsTests.java | 2 +- .../classification/HyperparametersTests.java | 2 +- .../classification/TimingStatsTests.java | 2 +- .../classification/ValidationLossTests.java | 2 +- .../stats/common/DataCountsTests.java | 2 +- .../stats/common/FoldValuesTests.java | 2 +- .../OutlierDetectionStatsTests.java | 2 +- .../outlierdetection/ParametersTests.java | 2 +- .../outlierdetection/TimingStatsTests.java | 2 +- .../regression/HyperparametersTests.java | 2 +- .../regression/RegressionStatsTests.java | 2 +- .../stats/regression/TimingStatsTests.java | 2 +- .../stats/regression/ValidationLossTests.java | 2 +- .../InferenceConfigItemTestCase.java | 4 +-- .../ml/inference/TrainedModelConfigTests.java | 2 +- .../AbstractNlpConfigUpdateTestCase.java | 2 +- .../BertJapaneseTokenizationTests.java | 2 +- .../trainedmodel/BertTokenizationTests.java | 2 +- .../ClassificationConfigTests.java | 2 +- .../ClassificationConfigUpdateTests.java | 2 +- .../trainedmodel/MPNetTokenizationTests.java | 2 +- .../trainedmodel/ModelPackageConfigTests.java | 2 +- .../trainedmodel/RegressionConfigTests.java | 2 +- .../RegressionConfigUpdateTests.java | 2 +- .../RobertaTokenizationTests.java | 2 +- .../trainedmodel/VocabularyConfigTests.java | 2 +- .../XLMRobertaTokenizationTests.java | 2 +- .../FeatureImportanceBaselineTests.java | 2 +- .../metadata/HyperparametersTests.java | 2 +- .../metadata/TotalFeatureImportanceTests.java | 2 +- .../metadata/TrainedModelMetadataTests.java | 2 +- ...sCollectionRequestBWCSerializingTests.java | 2 +- ...sCollectionRequestBWCSerializingTests.java | 2 +- ...yticsEventResponseBWCSerializingTests.java | 2 +- ...sCollectionRequestBWCSerializingTests.java | 2 +- ...CollectionResponseBWCSerializingTests.java | 2 +- ...ectorActionRequestBWCSerializingTests.java | 2 +- ...ectorActionRequestBWCSerializingTests.java | 2 +- ...ectorActionRequestBWCSerializingTests.java | 2 +- ...ectorActionRequestBWCSerializingTests.java | 2 +- ...ectorActionRequestBWCSerializingTests.java | 2 +- ...eringActionRequestBWCSerializingTests.java | 2 +- ...KeyIdActionRequestBWCSerializingTests.java | 2 +- ...ationActionRequestBWCSerializingTests.java | 2 +- ...ErrorActionRequestBWCSerializingTests.java | 2 +- ...turesActionRequestBWCSerializingTests.java | 2 +- ...eringActionRequestBWCSerializingTests.java | 2 +- ...ationActionRequestBWCSerializingTests.java | 2 +- ...xNameActionRequestBWCSerializingTests.java | 2 +- ...StatsActionRequestBWCSerializingTests.java | 2 +- ...rNameActionRequestBWCSerializingTests.java | 2 +- ...ativeActionRequestBWCSerializingTests.java | 2 +- ...elineActionRequestBWCSerializingTests.java | 2 +- ...ulingActionRequestBWCSerializingTests.java | 2 +- ...eTypeActionRequestBWCSerializingTests.java | 2 +- ...tatusActionRequestBWCSerializingTests.java | 2 +- ...ncJobActionRequestBWCSerializingTests.java | 2 +- ...ncJobActionRequestBWCSerializingTests.java | 2 +- ...ncJobActionRequestBWCSerializingTests.java | 2 +- ...ncJobActionRequestBWCSerializingTests.java | 2 +- ...ncJobActionRequestBWCSerializingTests.java | 2 +- ...cJobsActionRequestBWCSerializingTests.java | 2 +- ...ncJobActionRequestBWCSerializingTests.java | 2 +- ...rorActionRequestBWCSerializationTests.java | 2 +- ...StatsActionRequestBWCSerializingTests.java | 2 +- ...yRuleActionRequestBWCSerializingTests.java | 2 +- ...lesetActionRequestBWCSerializingTests.java | 2 +- ...yRuleActionRequestBWCSerializingTests.java | 2 +- ...RuleActionResponseBWCSerializingTests.java | 4 +-- ...lesetActionRequestBWCSerializingTests.java | 2 +- ...esetActionResponseBWCSerializingTests.java | 2 +- ...esetsActionRequestBWCSerializingTests.java | 2 +- ...yRuleActionRequestBWCSerializingTests.java | 4 +-- ...lesetActionRequestBWCSerializingTests.java | 2 +- ...ationActionRequestBWCSerializingTests.java | 2 +- ...ationActionRequestBWCSerializingTests.java | 2 +- ...tionActionResponseBWCSerializingTests.java | 2 +- ...ationActionRequestBWCSerializingTests.java | 2 +- ...ationActionRequestBWCSerializingTests.java | 2 +- ...ationSearchRequestBWCSerializingTests.java | 2 +- ...RankRescorerBuilderSerializationTests.java | 2 +- .../job/results/CategoryDefinitionTests.java | 2 +- 93 files changed, 131 insertions(+), 109 deletions(-) rename {x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml => test/framework/src/main/java/org/elasticsearch/test}/AbstractBWCSerializationTestCase.java (83%) create mode 100644 test/framework/src/main/java/org/elasticsearch/test/BWCVersions.java diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractBWCSerializationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBWCSerializationTestCase.java similarity index 83% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractBWCSerializationTestCase.java rename to test/framework/src/main/java/org/elasticsearch/test/AbstractBWCSerializationTestCase.java index 380355303934f..d931340365cd6 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractBWCSerializationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBWCSerializationTestCase.java @@ -1,20 +1,22 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". */ -package org.elasticsearch.xpack.core.ml; + +package org.elasticsearch.test; import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.test.AbstractXContentSerializingTestCase; import org.elasticsearch.xcontent.ToXContent; import java.io.IOException; import java.util.List; -import static org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase.DEFAULT_BWC_VERSIONS; +import static org.elasticsearch.test.BWCVersions.DEFAULT_BWC_VERSIONS; public abstract class AbstractBWCSerializationTestCase extends AbstractXContentSerializingTestCase { diff --git a/test/framework/src/main/java/org/elasticsearch/test/BWCVersions.java b/test/framework/src/main/java/org/elasticsearch/test/BWCVersions.java new file mode 100644 index 0000000000000..568df8c06b942 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/BWCVersions.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.test; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; + +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.KnownTransportVersions.ALL_VERSIONS; + +public final class BWCVersions { + private BWCVersions() {} + + public static List getAllBWCVersions() { + int minCompatVersion = Collections.binarySearch(ALL_VERSIONS, TransportVersions.MINIMUM_COMPATIBLE); + return ALL_VERSIONS.subList(minCompatVersion, ALL_VERSIONS.size()); + } + + public static final List DEFAULT_BWC_VERSIONS = getAllBWCVersions(); +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractBWCWireSerializationTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractBWCWireSerializationTestCase.java index 2098a7ff904a1..451c85936f3cb 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractBWCWireSerializationTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractBWCWireSerializationTestCase.java @@ -7,26 +7,17 @@ package org.elasticsearch.xpack.core.ml; import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Strings; import org.elasticsearch.test.AbstractWireSerializingTestCase; import java.io.IOException; -import java.util.Collections; import java.util.List; -import static org.elasticsearch.KnownTransportVersions.ALL_VERSIONS; +import static org.elasticsearch.test.BWCVersions.DEFAULT_BWC_VERSIONS; public abstract class AbstractBWCWireSerializationTestCase extends AbstractWireSerializingTestCase { - public static List getAllBWCVersions() { - int minCompatVersion = Collections.binarySearch(ALL_VERSIONS, TransportVersions.MINIMUM_COMPATIBLE); - return ALL_VERSIONS.subList(minCompatVersion, ALL_VERSIONS.size()); - } - - static final List DEFAULT_BWC_VERSIONS = getAllBWCVersions(); - /** * Returns the expected instance if serialized from the given version. */ diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractChunkedBWCSerializationTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractChunkedBWCSerializationTestCase.java index a23ce2c107fe3..0254406a2c8ec 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractChunkedBWCSerializationTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/AbstractChunkedBWCSerializationTestCase.java @@ -15,7 +15,7 @@ import java.io.IOException; import java.util.List; -import static org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase.DEFAULT_BWC_VERSIONS; +import static org.elasticsearch.test.BWCVersions.DEFAULT_BWC_VERSIONS; public abstract class AbstractChunkedBWCSerializationTestCase extends AbstractChunkedSerializingTestCase { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java index e0318e3c63bc5..02924b6d15017 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfigTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ToXContent; @@ -34,7 +35,6 @@ import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.elasticsearch.xpack.core.ml.MlConfigVersion; import org.elasticsearch.xpack.core.ml.dataframe.analyses.Classification; import org.elasticsearch.xpack.core.ml.dataframe.analyses.ClassificationTests; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsDestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsDestTests.java index edf2cd2dc685b..c3e07e8bc9081 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsDestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsDestTests.java @@ -8,8 +8,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsSourceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsSourceTests.java index 609ff56944fdc..ceb7660ca8123 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsSourceTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsSourceTests.java @@ -14,9 +14,9 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.elasticsearch.xpack.core.ml.utils.QueryProvider; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParamsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParamsTests.java index 896b8d34daf88..2bcfe541a21b7 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParamsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/BoostedTreeParamsTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; import java.util.HashMap; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java index 62f73b0f2bccd..f12c754d105ac 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.DeprecationHandler; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ToXContent; @@ -27,7 +28,6 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.elasticsearch.xpack.core.ml.MlConfigVersion; import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider; import org.elasticsearch.xpack.core.ml.inference.preprocessing.FrequencyEncodingTests; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetectionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetectionTests.java index 3d842aff1faa7..f4f23131470ae 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetectionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetectionTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; import java.util.Collections; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/RegressionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/RegressionTests.java index 86a9a6a8877bb..98707f8f501d0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/RegressionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/RegressionTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.DeprecationHandler; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ToXContent; @@ -22,7 +23,6 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.elasticsearch.xpack.core.ml.MlConfigVersion; import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider; import org.elasticsearch.xpack.core.ml.inference.preprocessing.FrequencyEncodingTests; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStatsTests.java index b541be0c1d0ea..553fd3ef94192 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStatsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ClassificationStatsTests.java @@ -8,9 +8,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import org.junit.Before; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/HyperparametersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/HyperparametersTests.java index aa287197054b8..069bcb0874f5e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/HyperparametersTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/HyperparametersTests.java @@ -8,8 +8,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/TimingStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/TimingStatsTests.java index 714426125eaa0..719224aef4934 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/TimingStatsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/TimingStatsTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ValidationLossTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ValidationLossTests.java index 778c4e7a18233..79391539435a2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ValidationLossTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/classification/ValidationLossTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.elasticsearch.xpack.core.ml.dataframe.stats.common.FoldValuesTests; import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import org.junit.Before; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCountsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCountsTests.java index cdbb1fb4c5932..572e8db8c1abe 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCountsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/DataCountsTests.java @@ -8,9 +8,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import org.junit.Before; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/FoldValuesTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/FoldValuesTests.java index 67c4d8acb8339..7058ff381574e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/FoldValuesTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/common/FoldValuesTests.java @@ -8,8 +8,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStatsTests.java index bbf549d58d204..a8d2024c7f227 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStatsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/OutlierDetectionStatsTests.java @@ -8,9 +8,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import org.junit.Before; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/ParametersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/ParametersTests.java index 8e9d0e0a93169..4d0d658f4f2f2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/ParametersTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/ParametersTests.java @@ -8,8 +8,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/TimingStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/TimingStatsTests.java index 0d47369aa118c..6c5ff4158baeb 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/TimingStatsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/outlierdetection/TimingStatsTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/HyperparametersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/HyperparametersTests.java index 1d291e8d3f956..8dbecff254c94 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/HyperparametersTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/HyperparametersTests.java @@ -8,8 +8,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStatsTests.java index 705f160cfd81d..17f20004d208f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStatsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/RegressionStatsTests.java @@ -8,9 +8,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import org.junit.Before; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/TimingStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/TimingStatsTests.java index 32f01a57e98c5..262ec9d5b144e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/TimingStatsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/TimingStatsTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/ValidationLossTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/ValidationLossTests.java index 046469565a568..a18deb5fc4baf 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/ValidationLossTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/stats/regression/ValidationLossTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.elasticsearch.xpack.core.ml.dataframe.stats.common.FoldValuesTests; import org.elasticsearch.xpack.core.ml.utils.ToXContentParams; import org.junit.Before; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/InferenceConfigItemTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/InferenceConfigItemTestCase.java index a5ba16474fcbb..90ea6b1e385c6 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/InferenceConfigItemTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/InferenceConfigItemTestCase.java @@ -12,9 +12,9 @@ import org.elasticsearch.common.io.stream.VersionedNamedWriteable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ToXContent; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.FillMaskConfig; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.FillMaskConfigTests; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfig; @@ -42,7 +42,7 @@ import java.util.List; import java.util.stream.Collectors; -import static org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase.getAllBWCVersions; +import static org.elasticsearch.test.BWCVersions.getAllBWCVersions; public abstract class InferenceConfigItemTestCase extends AbstractBWCSerializationTestCase< T> { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfigTests.java index a17ca18aba622..84a4dc412e827 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/TrainedModelConfigTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.license.License; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.DeprecationHandler; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ToXContent; @@ -23,7 +24,6 @@ import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.elasticsearch.xpack.core.ml.MlConfigVersion; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ClassificationConfigTests; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.FillMaskConfigTests; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/AbstractNlpConfigUpdateTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/AbstractNlpConfigUpdateTestCase.java index d549e1fa6463c..c8cea1c9207dc 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/AbstractNlpConfigUpdateTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/AbstractNlpConfigUpdateTestCase.java @@ -10,8 +10,8 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.core.Tuple; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertJapaneseTokenizationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertJapaneseTokenizationTests.java index 9253469ecc49d..8c3c0c1ea2c40 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertJapaneseTokenizationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertJapaneseTokenizationTests.java @@ -10,8 +10,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationTests.java index 6d382bc0a5fe5..c9c4795df8f01 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/BertTokenizationTests.java @@ -10,8 +10,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigTests.java index f2f3bdbff713a..b084e65efff30 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigTests.java @@ -8,8 +8,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdateTests.java index 620036a040368..0f176e7b12d98 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdateTests.java @@ -10,8 +10,8 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; import java.util.Collections; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/MPNetTokenizationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/MPNetTokenizationTests.java index d6db1bf313bf0..da2b6dfd4525c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/MPNetTokenizationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/MPNetTokenizationTests.java @@ -10,8 +10,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ModelPackageConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ModelPackageConfigTests.java index 88a9c7fb7b0ea..f061d2976a8e9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ModelPackageConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ModelPackageConfigTests.java @@ -12,12 +12,12 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.elasticsearch.xpack.core.ml.inference.TrainedModelPrefixStrings; import org.elasticsearch.xpack.core.ml.inference.TrainedModelPrefixStringsTests; import org.elasticsearch.xpack.core.ml.inference.TrainedModelType; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigTests.java index 0c5e90fac9ae5..28a0310b79ddf 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigTests.java @@ -8,8 +8,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdateTests.java index 35d2cb7fda16f..60239a1c62f28 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdateTests.java @@ -10,8 +10,8 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; import java.util.Collections; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RobertaTokenizationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RobertaTokenizationTests.java index 8cedd20432a6e..f6d58fc94a706 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RobertaTokenizationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RobertaTokenizationTests.java @@ -10,8 +10,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/VocabularyConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/VocabularyConfigTests.java index e33e8c2784ad2..bbc21dca5ace5 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/VocabularyConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/VocabularyConfigTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/XLMRobertaTokenizationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/XLMRobertaTokenizationTests.java index c7525b1c571a2..646a3387be043 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/XLMRobertaTokenizationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/XLMRobertaTokenizationTests.java @@ -10,8 +10,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/FeatureImportanceBaselineTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/FeatureImportanceBaselineTests.java index 251768a77fcd8..c354c89c2391d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/FeatureImportanceBaselineTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/FeatureImportanceBaselineTests.java @@ -8,9 +8,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/HyperparametersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/HyperparametersTests.java index 3fb2cb3939f0e..c56058460c071 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/HyperparametersTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/HyperparametersTests.java @@ -8,8 +8,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/TotalFeatureImportanceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/TotalFeatureImportanceTests.java index 3830c60a61a56..3231da0da921a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/TotalFeatureImportanceTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/TotalFeatureImportanceTests.java @@ -8,9 +8,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/TrainedModelMetadataTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/TrainedModelMetadataTests.java index 3bed1cb3af10b..97f68f0ec4b2b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/TrainedModelMetadataTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/metadata/TrainedModelMetadataTests.java @@ -8,8 +8,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/DeleteAnalyticsCollectionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/DeleteAnalyticsCollectionRequestBWCSerializingTests.java index 7b6b78940f575..3734508681522 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/DeleteAnalyticsCollectionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/DeleteAnalyticsCollectionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/GetAnalyticsCollectionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/GetAnalyticsCollectionRequestBWCSerializingTests.java index 5e3e0bf91daee..dcd7583c3c9d5 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/GetAnalyticsCollectionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/GetAnalyticsCollectionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; import java.util.List; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PostAnalyticsEventResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PostAnalyticsEventResponseBWCSerializingTests.java index 23ee16c9861c5..f349f1913af07 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PostAnalyticsEventResponseBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PostAnalyticsEventResponseBWCSerializingTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionRequestBWCSerializingTests.java index 0f1f4cfa7e89f..27fa688556470 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionResponseBWCSerializingTests.java index f4b85af251c57..256915744079c 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionResponseBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/PutAnalyticsCollectionResponseBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.test.rest.TestResponseParsers; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorActionRequestBWCSerializingTests.java index 9d6388a709cb2..5ad7109a6b7c1 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorActionRequestBWCSerializingTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java index 124a068abce93..2b8e8735fa2cc 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java index c71fbaf6716e4..3390ece073e5f 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java @@ -9,10 +9,10 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.EnterpriseSearchModuleTestUtils; import org.elasticsearch.xpack.core.action.util.PageParams; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; import java.util.List; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PostConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PostConnectorActionRequestBWCSerializingTests.java index 0587ef7da8654..188a7bd6b4f8f 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PostConnectorActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PostConnectorActionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.ConnectorTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PutConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PutConnectorActionRequestBWCSerializingTests.java index f618b4562fdc9..6c9f888427f13 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PutConnectorActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/PutConnectorActionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.ConnectorTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringActionRequestBWCSerializingTests.java index 630cf019f34da..270708db9983e 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringActionRequestBWCSerializingTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdActionRequestBWCSerializingTests.java index a6671aedd9910..ace219c688573 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdActionRequestBWCSerializingTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationActionRequestBWCSerializingTests.java index 16383adba1729..598dcc9253913 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationActionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.ConnectorTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorActionRequestBWCSerializingTests.java index 94092cee61b40..31d31361e577a 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorActionRequestBWCSerializingTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesActionRequestBWCSerializingTests.java index 9a191dba2e525..6a229a620c420 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesActionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.ConnectorTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringActionRequestBWCSerializingTests.java index 6874f4b2a1b36..d1018cc604769 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringActionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.ConnectorTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; import java.util.List; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationActionRequestBWCSerializingTests.java index c8a15b164790a..ca09152a0d576 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationActionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.ConnectorTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameActionRequestBWCSerializingTests.java index 99bf15d20385f..c233df0c595e8 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameActionRequestBWCSerializingTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsActionRequestBWCSerializingTests.java index b324a43b46b81..607bd9ec80db0 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsActionRequestBWCSerializingTests.java @@ -10,9 +10,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Tuple; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.ConnectorTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameActionRequestBWCSerializingTests.java index 7ee377a7933bf..9002eace2b886 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameActionRequestBWCSerializingTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeActionRequestBWCSerializingTests.java index a680108e50055..83678cd3a2841 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeActionRequestBWCSerializingTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineActionRequestBWCSerializingTests.java index 14df1b704f995..6c5e7d7d79577 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineActionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.ConnectorTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingActionRequestBWCSerializingTests.java index ee2823a27400a..08d08f9ebefcd 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingActionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.ConnectorTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeActionRequestBWCSerializingTests.java index a30e0a6b8d493..f2fac3471a7e4 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeActionRequestBWCSerializingTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusActionRequestBWCSerializingTests.java index b0efe0d8483ea..1633e38b82284 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusActionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.ConnectorTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobActionRequestBWCSerializingTests.java index 81f59a130ac70..2f9a80af98faf 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobActionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobActionRequestBWCSerializingTests.java index 63f874b32f37c..8c75d03ebefed 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobActionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobActionRequestBWCSerializingTests.java index 4a3dc96bafc8a..85150392b2e41 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobActionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobActionRequestBWCSerializingTests.java index c9d2c446e028b..b4d78508290ce 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobActionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionRequestBWCSerializingTests.java index c0b7711474a0b..d9fa75aef925d 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsActionRequestBWCSerializingTests.java index 967994ebe57e0..8a47a5b4869c0 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsActionRequestBWCSerializingTests.java @@ -9,13 +9,13 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.EnterpriseSearchModuleTestUtils; import org.elasticsearch.xpack.application.connector.ConnectorSyncStatus; import org.elasticsearch.xpack.application.connector.ConnectorTestUtils; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobType; import org.elasticsearch.xpack.core.action.util.PageParams; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; import java.util.Collections; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobActionRequestBWCSerializingTests.java index 73e6036dd5148..3c3d5c03bd8e5 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobActionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorActionRequestBWCSerializationTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorActionRequestBWCSerializationTests.java index a6c52d8cbf62c..cd8a38e70b652 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorActionRequestBWCSerializationTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorActionRequestBWCSerializationTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsActionRequestBWCSerializingTests.java index ff586ae28109a..0a11600aab7fd 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsActionRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/DeleteQueryRuleActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/DeleteQueryRuleActionRequestBWCSerializingTests.java index a3882b8a5d9e4..1d70fa3035e3c 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/DeleteQueryRuleActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/DeleteQueryRuleActionRequestBWCSerializingTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/DeleteQueryRulesetActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/DeleteQueryRulesetActionRequestBWCSerializingTests.java index e1fbb57ed359e..a15170f93a988 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/DeleteQueryRulesetActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/DeleteQueryRulesetActionRequestBWCSerializingTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRuleActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRuleActionRequestBWCSerializingTests.java index 9e907b8f68996..dd02f8c13dd86 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRuleActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRuleActionRequestBWCSerializingTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRuleActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRuleActionResponseBWCSerializingTests.java index f364fc0c83ba7..b60747ce47321 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRuleActionResponseBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRuleActionResponseBWCSerializingTests.java @@ -10,16 +10,16 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.rules.QueryRule; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; import java.util.List; import java.util.stream.Collectors; +import static org.elasticsearch.test.BWCVersions.getAllBWCVersions; import static org.elasticsearch.xpack.application.EnterpriseSearchModuleTestUtils.randomQueryRule; -import static org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase.getAllBWCVersions; public class GetQueryRuleActionResponseBWCSerializingTests extends AbstractBWCSerializationTestCase { public QueryRule queryRule; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetActionRequestBWCSerializingTests.java index fb8bfc4fb258d..27a08ab90fb55 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetActionRequestBWCSerializingTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetActionResponseBWCSerializingTests.java index 4942f9fb076af..ca58e42ba6248 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetActionResponseBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/GetQueryRulesetActionResponseBWCSerializingTests.java @@ -10,11 +10,11 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.rules.QueryRule; import org.elasticsearch.xpack.application.rules.QueryRuleCriteria; import org.elasticsearch.xpack.application.rules.QueryRuleset; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; import java.util.ArrayList; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsActionRequestBWCSerializingTests.java index dfac7c57e01d3..4dcd4000c5e3b 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/ListQueryRulesetsActionRequestBWCSerializingTests.java @@ -9,10 +9,10 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.EnterpriseSearchModuleTestUtils; import org.elasticsearch.xpack.core.action.util.PageParams; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/PutQueryRuleActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/PutQueryRuleActionRequestBWCSerializingTests.java index a66d0c0aa5895..5b42a899feac8 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/PutQueryRuleActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/PutQueryRuleActionRequestBWCSerializingTests.java @@ -10,16 +10,16 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.EnterpriseSearchModuleTestUtils; import org.elasticsearch.xpack.application.rules.QueryRule; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; import java.util.List; import java.util.stream.Collectors; -import static org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase.getAllBWCVersions; +import static org.elasticsearch.test.BWCVersions.getAllBWCVersions; public class PutQueryRuleActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase { diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/PutQueryRulesetActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/PutQueryRulesetActionRequestBWCSerializingTests.java index 83702b0b0672c..f9b47f5bb2cd2 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/PutQueryRulesetActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/action/PutQueryRulesetActionRequestBWCSerializingTests.java @@ -10,12 +10,12 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.EnterpriseSearchModuleTestUtils; import org.elasticsearch.xpack.application.rules.QueryRule; import org.elasticsearch.xpack.application.rules.QueryRuleCriteria; import org.elasticsearch.xpack.application.rules.QueryRuleset; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; import java.util.ArrayList; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/DeleteSearchApplicationActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/DeleteSearchApplicationActionRequestBWCSerializingTests.java index 0711ce6834f28..e4e38ed0a8f32 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/DeleteSearchApplicationActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/DeleteSearchApplicationActionRequestBWCSerializingTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/GetSearchApplicationActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/GetSearchApplicationActionRequestBWCSerializingTests.java index 6a6efedade06c..7eecac1b6fe5e 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/GetSearchApplicationActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/GetSearchApplicationActionRequestBWCSerializingTests.java @@ -9,8 +9,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/GetSearchApplicationActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/GetSearchApplicationActionResponseBWCSerializingTests.java index 11c28f062d272..0570309896245 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/GetSearchApplicationActionResponseBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/GetSearchApplicationActionResponseBWCSerializingTests.java @@ -9,10 +9,10 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.EnterpriseSearchModuleTestUtils; import org.elasticsearch.xpack.application.search.SearchApplication; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/ListSearchApplicationActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/ListSearchApplicationActionRequestBWCSerializingTests.java index ba7b07441d8b1..770f62938acc5 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/ListSearchApplicationActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/ListSearchApplicationActionRequestBWCSerializingTests.java @@ -9,10 +9,10 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.EnterpriseSearchModuleTestUtils; import org.elasticsearch.xpack.core.action.util.PageParams; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/PutSearchApplicationActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/PutSearchApplicationActionRequestBWCSerializingTests.java index 88b752c80c26a..245207ebf96b2 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/PutSearchApplicationActionRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/PutSearchApplicationActionRequestBWCSerializingTests.java @@ -9,10 +9,10 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.EnterpriseSearchModuleTestUtils; import org.elasticsearch.xpack.application.search.SearchApplication; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/SearchApplicationSearchRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/SearchApplicationSearchRequestBWCSerializingTests.java index 7c3b504655bf3..747c3a4639e38 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/SearchApplicationSearchRequestBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/search/action/SearchApplicationSearchRequestBWCSerializingTests.java @@ -9,9 +9,9 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.EnterpriseSearchModuleTestUtils; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import java.io.IOException; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilderSerializationTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilderSerializationTests.java index 2f43e12a2e3c7..76efc0a071883 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilderSerializationTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankRescorerBuilderSerializationTests.java @@ -21,13 +21,13 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.rescore.QueryRescorerBuilder; import org.elasticsearch.search.rescore.RescorerBuilder; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvider; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.LearningToRankConfig; import org.elasticsearch.xpack.core.ml.ltr.MlLTRNamedXContentProvider; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/CategoryDefinitionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/CategoryDefinitionTests.java index 60fd1033753da..d4bffe0b2019f 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/CategoryDefinitionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/CategoryDefinitionTests.java @@ -8,10 +8,10 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractBWCSerializationTestCase; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.json.JsonXContent; -import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition; import java.io.IOException; From eb9b897e27bc245cf4124672fa8c7f6fbfe28538 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 2 Oct 2024 14:32:26 +0100 Subject: [PATCH 006/194] Migrate `ReferenceDocs` resource to plain text (#113866) Removes the dependency on `XContent` parsing so we can move this out of `:server` and into `:libs:core`. --- .../elasticsearch/common/ReferenceDocs.java | 94 +++++--- .../common/reference-docs-links.json | 51 +--- .../common/reference-docs-links.txt | 44 ++++ .../common/ReferenceDocsTests.java | 220 ++++++++++-------- 4 files changed, 233 insertions(+), 176 deletions(-) create mode 100644 server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt diff --git a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java index e301d786ce5a4..4b0a0c5e77ebb 100644 --- a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java +++ b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java @@ -11,29 +11,26 @@ import org.elasticsearch.Build; import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentParserConfiguration; -import org.elasticsearch.xcontent.XContentType; +import java.io.BufferedReader; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.net.URL; -import java.util.LinkedHashMap; +import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.regex.Pattern; /** * Encapsulates links to pages in the reference docs, so that for example we can include URLs in logs and API outputs. Each instance's * {@link #toString()} yields (a string representation of) a URL for the relevant docs. Links are defined in the resource file - * {@code reference-docs-links.json} which must include definitions for exactly the set of values of this enum. + * {@code reference-docs-links.txt} which must include definitions for exactly the set of values of this enum. */ public enum ReferenceDocs { /* - * Note that the docs subsystem parses {@code reference-docs-links.json} with regexes, not a JSON parser, so the whitespace in the file - * is important too. See {@code sub check_elasticsearch_links} in {@code https://github.com/elastic/docs/blob/master/build_docs.pl} for - * more details. + * Note that the docs subsystem parses {@code reference-docs-links.txt} differently. See {@code sub check_elasticsearch_links} in + * {@code https://github.com/elastic/docs/blob/master/build_docs.pl} for more details. * * Also note that the docs are built from the HEAD of each minor release branch, so in principle docs can move around independently of * the ES release process. To avoid breaking any links that have been baked into earlier patch releases, you may only add links in a @@ -89,7 +86,7 @@ public enum ReferenceDocs { private static final Map linksBySymbol; static { - try (var resourceStream = readFromJarResourceUrl(ReferenceDocs.class.getResource("reference-docs-links.json"))) { + try (var resourceStream = readFromJarResourceUrl(ReferenceDocs.class.getResource("reference-docs-links.txt"))) { linksBySymbol = Map.copyOf(readLinksBySymbol(resourceStream)); } catch (Exception e) { assert false : e; @@ -101,34 +98,69 @@ public enum ReferenceDocs { static final String CURRENT_VERSION_COMPONENT = "current"; static final String VERSION_COMPONENT = getVersionComponent(Build.current().version(), Build.current().isSnapshot()); - static Map readLinksBySymbol(InputStream inputStream) throws Exception { - try (var parser = XContentFactory.xContent(XContentType.JSON).createParser(XContentParserConfiguration.EMPTY, inputStream)) { - final var result = parser.map(LinkedHashMap::new, XContentParser::text); - final var iterator = result.keySet().iterator(); - for (int i = 0; i < values().length; i++) { - final var expected = values()[i].name(); - if (iterator.hasNext() == false) { - throw new IllegalStateException("ran out of values at index " + i + ": expecting " + expected); - } - final var actual = iterator.next(); - if (actual.equals(expected) == false) { - throw new IllegalStateException("mismatch at index " + i + ": found " + actual + " but expected " + expected); - } + static final int SYMBOL_COLUMN_WIDTH = 64; // increase as needed to accommodate yet longer symbols + + static Map readLinksBySymbol(InputStream inputStream) throws IOException { + final var padding = " ".repeat(SYMBOL_COLUMN_WIDTH); + + record LinksBySymbolEntry(String symbol, String link) implements Map.Entry { + @Override + public String getKey() { + return symbol; } - if (iterator.hasNext()) { - throw new IllegalStateException("found unexpected extra value: " + iterator.next()); + + @Override + public String getValue() { + return link; + } + + @Override + public String setValue(String value) { + assert false; + throw new UnsupportedOperationException(); } + } - // We must only link to anchors with fixed IDs (defined by [[fragment-name]] in the docs) because auto-generated fragment IDs - // depend on the heading text and are too easy to break inadvertently. Auto-generated fragment IDs begin with an underscore. - for (final var entry : result.entrySet()) { - if (entry.getValue().startsWith("_") || entry.getValue().contains("#_")) { - throw new IllegalStateException("found auto-generated fragment ID at " + entry.getKey()); + final var symbolCount = values().length; + final var linksBySymbolEntries = new LinksBySymbolEntry[symbolCount]; + + try (var reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + for (int i = 0; i < symbolCount; i++) { + final var currentLine = reader.readLine(); + final var symbol = values()[i].name(); + if (currentLine == null) { + throw new IllegalStateException("links resource truncated at line " + (i + 1)); + } + if (currentLine.startsWith(symbol + " ") == false) { + throw new IllegalStateException( + "unexpected symbol at line " + (i + 1) + ": expected line starting with [" + symbol + " ]" + ); + } + final var link = currentLine.substring(SYMBOL_COLUMN_WIDTH).trim(); + if (Strings.hasText(link) == false) { + throw new IllegalStateException("no link found for [" + symbol + "] at line " + (i + 1)); + } + final var expectedLine = (symbol + padding).substring(0, SYMBOL_COLUMN_WIDTH) + link; + if (currentLine.equals(expectedLine) == false) { + throw new IllegalStateException("unexpected content at line " + (i + 1) + ": expected [" + expectedLine + "]"); } + + // We must only link to anchors with fixed IDs (defined by [[fragment-name]] in the docs) because auto-generated fragment + // IDs depend on the heading text and are too easy to break inadvertently. Auto-generated fragment IDs begin with "_" + if (link.startsWith("_") || link.contains("#_")) { + throw new IllegalStateException( + "found auto-generated fragment ID in link [" + link + "] for [" + symbol + "] at line " + (i + 1) + ); + } + linksBySymbolEntries[i] = new LinksBySymbolEntry(symbol, link); } - return result; + if (reader.readLine() != null) { + throw new IllegalStateException("unexpected trailing content at line " + (symbolCount + 1)); + } } + + return Map.ofEntries(linksBySymbolEntries); } /** diff --git a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json index 1b5dc5b2f31e0..71be3d333ec3f 100644 --- a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json +++ b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json @@ -1,46 +1,5 @@ -{ - "INITIAL_MASTER_NODES": "important-settings.html#initial_master_nodes", - "DISCOVERY_TROUBLESHOOTING": "discovery-troubleshooting.html", - "UNSTABLE_CLUSTER_TROUBLESHOOTING": "troubleshooting-unstable-cluster.html", - "LAGGING_NODE_TROUBLESHOOTING": "troubleshooting-unstable-cluster.html#troubleshooting-unstable-cluster-lagging", - "SHARD_LOCK_TROUBLESHOOTING": "troubleshooting-unstable-cluster.html#troubleshooting-unstable-cluster-shardlockobtainfailedexception", - "NETWORK_DISCONNECT_TROUBLESHOOTING": "troubleshooting-unstable-cluster.html#troubleshooting-unstable-cluster-network", - "CONCURRENT_REPOSITORY_WRITERS": "diagnosing-corrupted-repositories.html", - "ARCHIVE_INDICES": "archive-indices.html", - "HTTP_TRACER": "modules-network.html#http-rest-request-tracer", - "LOGGING": "logging.html", - "BOOTSTRAP_CHECK_HEAP_SIZE": "bootstrap-checks-heap-size.html", - "BOOTSTRAP_CHECK_FILE_DESCRIPTOR": "bootstrap-checks-file-descriptor.html", - "BOOTSTRAP_CHECK_MEMORY_LOCK": "bootstrap-checks-memory-lock.html", - "BOOTSTRAP_CHECK_MAX_NUMBER_THREADS": "max-number-threads-check.html", - "BOOTSTRAP_CHECK_MAX_FILE_SIZE": "bootstrap-checks-max-file-size.html", - "BOOTSTRAP_CHECK_MAX_SIZE_VIRTUAL_MEMORY": "max-size-virtual-memory-check.html", - "BOOTSTRAP_CHECK_MAXIMUM_MAP_COUNT": "bootstrap-checks-max-map-count.html", - "BOOTSTRAP_CHECK_CLIENT_JVM": "bootstrap-checks-client-jvm.html", - "BOOTSTRAP_CHECK_USE_SERIAL_COLLECTOR": "bootstrap-checks-serial-collector.html", - "BOOTSTRAP_CHECK_SYSTEM_CALL_FILTER": "bootstrap-checks-syscall-filter.html", - "BOOTSTRAP_CHECK_ONERROR_AND_ONOUTOFMEMORYERROR": "bootstrap-checks-onerror.html", - "BOOTSTRAP_CHECK_EARLY_ACCESS": "bootstrap-checks-early-access.html", - "BOOTSTRAP_CHECK_ALL_PERMISSION": "bootstrap-checks-all-permission.html", - "BOOTSTRAP_CHECK_DISCOVERY_CONFIGURATION": "bootstrap-checks-discovery-configuration.html", - "BOOTSTRAP_CHECKS": "bootstrap-checks.html", - "BOOTSTRAP_CHECK_ENCRYPT_SENSITIVE_DATA": "bootstrap-checks-xpack.html#bootstrap-checks-xpack-encrypt-sensitive-data", - "BOOTSTRAP_CHECK_PKI_REALM": "bootstrap-checks-xpack.html#bootstrap-checks-xpack-pki-realm", - "BOOTSTRAP_CHECK_ROLE_MAPPINGS": "bootstrap-checks-xpack.html#bootstrap-checks-xpack-role-mappings", - "BOOTSTRAP_CHECK_TLS": "bootstrap-checks-xpack.html#bootstrap-checks-tls", - "BOOTSTRAP_CHECK_TOKEN_SSL": "bootstrap-checks-xpack.html#bootstrap-checks-xpack-token-ssl", - "BOOTSTRAP_CHECK_SECURITY_MINIMAL_SETUP": "security-minimal-setup.html", - "CONTACT_SUPPORT": "troubleshooting.html#troubleshooting-contact-support", - "UNASSIGNED_SHARDS": "red-yellow-cluster-status.html", - "EXECUTABLE_JNA_TMPDIR": "executable-jna-tmpdir.html", - "NETWORK_THREADING_MODEL": "modules-network.html#modules-network-threading-model", - "ALLOCATION_EXPLAIN_API": "cluster-allocation-explain.html", - "NETWORK_BINDING_AND_PUBLISHING": "modules-network.html#modules-network-binding-publishing", - "SNAPSHOT_REPOSITORY_ANALYSIS": "repo-analysis-api.html", - "S3_COMPATIBLE_REPOSITORIES": "repository-s3.html#repository-s3-compatible-services", - "LUCENE_MAX_DOCS_LIMIT": "size-your-shards.html#troubleshooting-max-docs-limit", - "MAX_SHARDS_PER_NODE": "size-your-shards.html#troubleshooting-max-shards-open", - "FLOOD_STAGE_WATERMARK": "fix-watermark-errors.html", - "X_OPAQUE_ID": "api-conventions.html#x-opaque-id", - "FORMING_SINGLE_NODE_CLUSTERS": "modules-discovery-bootstrap-cluster.html#modules-discovery-bootstrap-cluster-joining" -} +[ + "Content moved to reference-docs-links.txt", + "This is a temporary placeholder to satisfy sub check_elasticsearch_links in the docs build", + "Remove with @UpdateForV10 (if not before)" +] diff --git a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt new file mode 100644 index 0000000000000..190bbd3c319b4 --- /dev/null +++ b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt @@ -0,0 +1,44 @@ +INITIAL_MASTER_NODES important-settings.html#initial_master_nodes +DISCOVERY_TROUBLESHOOTING discovery-troubleshooting.html +UNSTABLE_CLUSTER_TROUBLESHOOTING troubleshooting-unstable-cluster.html +LAGGING_NODE_TROUBLESHOOTING troubleshooting-unstable-cluster.html#troubleshooting-unstable-cluster-lagging +SHARD_LOCK_TROUBLESHOOTING troubleshooting-unstable-cluster.html#troubleshooting-unstable-cluster-shardlockobtainfailedexception +NETWORK_DISCONNECT_TROUBLESHOOTING troubleshooting-unstable-cluster.html#troubleshooting-unstable-cluster-network +CONCURRENT_REPOSITORY_WRITERS diagnosing-corrupted-repositories.html +ARCHIVE_INDICES archive-indices.html +HTTP_TRACER modules-network.html#http-rest-request-tracer +LOGGING logging.html +BOOTSTRAP_CHECK_HEAP_SIZE bootstrap-checks-heap-size.html +BOOTSTRAP_CHECK_FILE_DESCRIPTOR bootstrap-checks-file-descriptor.html +BOOTSTRAP_CHECK_MEMORY_LOCK bootstrap-checks-memory-lock.html +BOOTSTRAP_CHECK_MAX_NUMBER_THREADS max-number-threads-check.html +BOOTSTRAP_CHECK_MAX_FILE_SIZE bootstrap-checks-max-file-size.html +BOOTSTRAP_CHECK_MAX_SIZE_VIRTUAL_MEMORY max-size-virtual-memory-check.html +BOOTSTRAP_CHECK_MAXIMUM_MAP_COUNT bootstrap-checks-max-map-count.html +BOOTSTRAP_CHECK_CLIENT_JVM bootstrap-checks-client-jvm.html +BOOTSTRAP_CHECK_USE_SERIAL_COLLECTOR bootstrap-checks-serial-collector.html +BOOTSTRAP_CHECK_SYSTEM_CALL_FILTER bootstrap-checks-syscall-filter.html +BOOTSTRAP_CHECK_ONERROR_AND_ONOUTOFMEMORYERROR bootstrap-checks-onerror.html +BOOTSTRAP_CHECK_EARLY_ACCESS bootstrap-checks-early-access.html +BOOTSTRAP_CHECK_ALL_PERMISSION bootstrap-checks-all-permission.html +BOOTSTRAP_CHECK_DISCOVERY_CONFIGURATION bootstrap-checks-discovery-configuration.html +BOOTSTRAP_CHECKS bootstrap-checks.html +BOOTSTRAP_CHECK_ENCRYPT_SENSITIVE_DATA bootstrap-checks-xpack.html#bootstrap-checks-xpack-encrypt-sensitive-data +BOOTSTRAP_CHECK_PKI_REALM bootstrap-checks-xpack.html#bootstrap-checks-xpack-pki-realm +BOOTSTRAP_CHECK_ROLE_MAPPINGS bootstrap-checks-xpack.html#bootstrap-checks-xpack-role-mappings +BOOTSTRAP_CHECK_TLS bootstrap-checks-xpack.html#bootstrap-checks-tls +BOOTSTRAP_CHECK_TOKEN_SSL bootstrap-checks-xpack.html#bootstrap-checks-xpack-token-ssl +BOOTSTRAP_CHECK_SECURITY_MINIMAL_SETUP security-minimal-setup.html +CONTACT_SUPPORT troubleshooting.html#troubleshooting-contact-support +UNASSIGNED_SHARDS red-yellow-cluster-status.html +EXECUTABLE_JNA_TMPDIR executable-jna-tmpdir.html +NETWORK_THREADING_MODEL modules-network.html#modules-network-threading-model +ALLOCATION_EXPLAIN_API cluster-allocation-explain.html +NETWORK_BINDING_AND_PUBLISHING modules-network.html#modules-network-binding-publishing +SNAPSHOT_REPOSITORY_ANALYSIS repo-analysis-api.html +S3_COMPATIBLE_REPOSITORIES repository-s3.html#repository-s3-compatible-services +LUCENE_MAX_DOCS_LIMIT size-your-shards.html#troubleshooting-max-docs-limit +MAX_SHARDS_PER_NODE size-your-shards.html#troubleshooting-max-shards-open +FLOOD_STAGE_WATERMARK fix-watermark-errors.html +X_OPAQUE_ID api-conventions.html#x-opaque-id +FORMING_SINGLE_NODE_CLUSTERS modules-discovery-bootstrap-cluster.html#modules-discovery-bootstrap-cluster-joining diff --git a/server/src/test/java/org/elasticsearch/common/ReferenceDocsTests.java b/server/src/test/java/org/elasticsearch/common/ReferenceDocsTests.java index a5efd578df36c..ae28b83ae12fc 100644 --- a/server/src/test/java/org/elasticsearch/common/ReferenceDocsTests.java +++ b/server/src/test/java/org/elasticsearch/common/ReferenceDocsTests.java @@ -9,16 +9,17 @@ package org.elasticsearch.common; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.bytes.CompositeBytesReference; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentParseException; -import java.io.ByteArrayInputStream; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; +import java.io.IOException; +import java.io.InputStream; import static org.elasticsearch.common.ReferenceDocs.getVersionComponent; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.startsWith; public class ReferenceDocsTests extends ESTestCase { @@ -39,119 +40,140 @@ public void testVersionComponent() { assertEquals("master", getVersionComponent("ABCDEF", true)); } - public void testReadsValidLinkDefinitions() throws Exception { - try (var builder = XContentFactory.jsonBuilder()) { - builder.startObject(); - for (ReferenceDocs link : ReferenceDocs.values()) { - builder.field(link.name(), "TEST"); - } - builder.endObject(); + private static final String TEST_LINK_PLACEHOLDER = "TEST_LINK"; - var map = ReferenceDocs.readLinksBySymbol(BytesReference.bytes(builder).streamInput()); - assertEquals(ReferenceDocs.values().length, map.size()); - for (ReferenceDocs link : ReferenceDocs.values()) { - assertEquals("TEST", map.get(link.name())); - } - } + private interface LinkSupplier { + String mutateLinkLine(int index, String lineWithPlaceholder); } - public void testRejectsInvalidJSON() throws Exception { - try (var stream = new ByteArrayInputStream("{\"invalid\":".getBytes(StandardCharsets.UTF_8))) { - expectThrows(XContentParseException.class, () -> ReferenceDocs.readLinksBySymbol(stream)); - } - } - - public void testRejectsBadStructure() throws Exception { - try (var builder = XContentFactory.jsonBuilder()) { - builder.startObject(); - for (ReferenceDocs link : ReferenceDocs.values()) { - builder.field(link.name(), "TEST"); - } - builder.startObject("UNEXPECTED").endObject().endObject(); - - try (var stream = BytesReference.bytes(builder).streamInput()) { - expectThrows(IllegalArgumentException.class, () -> ReferenceDocs.readLinksBySymbol(stream)); + private static InputStream getResourceStream(LinkSupplier linkSupplier) { + final var stringBuilder = new StringBuilder(); + for (int i = 0; i < ReferenceDocs.values().length; i++) { + final var symbol = ReferenceDocs.values()[i]; + final var lineWithPlaceholder = symbol.name() + " ".repeat(ReferenceDocs.SYMBOL_COLUMN_WIDTH - symbol.name().length()) + + TEST_LINK_PLACEHOLDER; + final var updatedLine = linkSupplier.mutateLinkLine(i, lineWithPlaceholder); + if (updatedLine == null) { + break; + } else { + stringBuilder.append(updatedLine).append('\n'); } } + return new BytesArray(stringBuilder.toString()).streamInput(); } - public void testRejectsExtraSymbol() throws Exception { - try (var builder = XContentFactory.jsonBuilder()) { - builder.startObject(); - for (ReferenceDocs link : ReferenceDocs.values()) { - builder.field(link.name(), "TEST"); - } - builder.field("EXTRA", "TEST").endObject(); - - try (var stream = BytesReference.bytes(builder).streamInput()) { - expectThrows(IllegalStateException.class, () -> ReferenceDocs.readLinksBySymbol(stream)); - } + public void testSuccess() throws IOException { + final var linksMap = ReferenceDocs.readLinksBySymbol(getResourceStream((i, l) -> l)); + assertEquals(ReferenceDocs.values().length, linksMap.size()); + for (ReferenceDocs link : ReferenceDocs.values()) { + assertEquals(TEST_LINK_PLACEHOLDER, linksMap.get(link.name())); } } - public void testRejectsMissingSymbol() throws Exception { - try (var builder = XContentFactory.jsonBuilder()) { - builder.startObject(); - var skipped = randomFrom(ReferenceDocs.values()); - for (ReferenceDocs link : ReferenceDocs.values()) { - if (link != skipped) { - builder.field(link.name(), "TEST"); - } - } - builder.endObject(); - - try (var stream = BytesReference.bytes(builder).streamInput()) { - expectThrows(IllegalStateException.class, () -> ReferenceDocs.readLinksBySymbol(stream)); - } - } + public void testTruncated() { + final var targetLine = between(0, ReferenceDocs.values().length - 1); + assertThat( + expectThrows( + IllegalStateException.class, + () -> ReferenceDocs.readLinksBySymbol(getResourceStream((i, l) -> i == targetLine ? null : l)) + ).getMessage(), + equalTo("links resource truncated at line " + (targetLine + 1)) + ); } - public void testRejectsIncorrectOrder() throws Exception { - try (var builder = XContentFactory.jsonBuilder()) { - var shuffled = Arrays.copyOf(ReferenceDocs.values(), ReferenceDocs.values().length); - var i = between(0, ReferenceDocs.values().length - 1); - var j = randomValueOtherThan(i, () -> between(0, ReferenceDocs.values().length - 1)); - var tmp = shuffled[i]; - shuffled[i] = shuffled[j]; - shuffled[j] = tmp; - - builder.startObject(); - for (ReferenceDocs link : shuffled) { - builder.field(link.name(), "TEST"); - } - builder.endObject(); + public void testMissingLink() { + final var targetLine = between(0, ReferenceDocs.values().length - 1); + assertThat( + expectThrows( + IllegalStateException.class, + () -> ReferenceDocs.readLinksBySymbol( + getResourceStream((i, l) -> i == targetLine ? l.replace(TEST_LINK_PLACEHOLDER, "") : l) + ) + ).getMessage(), + equalTo("no link found for [" + ReferenceDocs.values()[targetLine].name() + "] at line " + (targetLine + 1)) + ); + } - try (var stream = BytesReference.bytes(builder).streamInput()) { - expectThrows(IllegalStateException.class, () -> ReferenceDocs.readLinksBySymbol(stream)); - } - } + public void testUnexpectedSymbol() { + final var targetSymbol = randomFrom(ReferenceDocs.values()).name(); + final var replacement = "x".repeat(targetSymbol.length()); + assertThat( + expectThrows( + IllegalStateException.class, + () -> ReferenceDocs.readLinksBySymbol(getResourceStream((i, l) -> l.replace(targetSymbol, replacement))) + ).getMessage(), + startsWith("unexpected symbol at line ") + ); } - public void testRejectsAutoGeneratedFragment() throws Exception { - try (var builder = XContentFactory.jsonBuilder()) { - builder.startObject(); - for (ReferenceDocs link : ReferenceDocs.values()) { - builder.field(link.name(), "test.html#_auto_generated_fragment"); - } - builder.endObject(); + public void testWhitespace() { + final var leadingWhitespaceLine = between(0, ReferenceDocs.values().length - 1); + final var trailingWhitespaceLine = between(0, ReferenceDocs.values().length - 1); + assertThat( + expectThrows( + IllegalStateException.class, + () -> ReferenceDocs.readLinksBySymbol( + getResourceStream( + (i, l) -> l.replace( + TEST_LINK_PLACEHOLDER, + (i == leadingWhitespaceLine ? " " : "") + TEST_LINK_PLACEHOLDER + (i == trailingWhitespaceLine ? " " : "") + ) + ) + ) + ).getMessage(), + startsWith("unexpected content at line " + (Math.min(leadingWhitespaceLine, trailingWhitespaceLine) + 1) + ": expected [") + ); + } - try (var stream = BytesReference.bytes(builder).streamInput()) { - expectThrows(IllegalStateException.class, () -> ReferenceDocs.readLinksBySymbol(stream)); - } + public void testTrailingContent() throws IOException { + final byte[] validContent; + try (var stream = getResourceStream((i, l) -> l)) { + validContent = stream.readAllBytes(); } + final BytesReference contentWithTrailingData = CompositeBytesReference.of(new BytesArray(validContent), new BytesArray("x")); + + assertThat( + expectThrows(IllegalStateException.class, () -> ReferenceDocs.readLinksBySymbol(contentWithTrailingData.streamInput())) + .getMessage(), + equalTo("unexpected trailing content at line " + (ReferenceDocs.values().length + 1)) + ); } - public void testRejectsAutoGeneratedPageName() throws Exception { - try (var builder = XContentFactory.jsonBuilder()) { - builder.startObject(); - for (ReferenceDocs link : ReferenceDocs.values()) { - builder.field(link.name(), "_auto_generated_page.html"); - } - builder.endObject(); + public void testRejectsAutoGeneratedFragment() { + final var targetLine = between(0, ReferenceDocs.values().length - 1); + assertThat( + expectThrows( + IllegalStateException.class, + () -> ReferenceDocs.readLinksBySymbol( + getResourceStream( + (i, l) -> i == targetLine ? l.replace(TEST_LINK_PLACEHOLDER, "test.html#_auto_generated_fragment") : l + ) + ) + ).getMessage(), + equalTo( + "found auto-generated fragment ID in link [test.html#_auto_generated_fragment] for [" + + ReferenceDocs.values()[targetLine].name() + + "] at line " + + (targetLine + 1) + ) + ); + } - try (var stream = BytesReference.bytes(builder).streamInput()) { - expectThrows(IllegalStateException.class, () -> ReferenceDocs.readLinksBySymbol(stream)); - } - } + public void testRejectsAutoGeneratedPageName() { + final var targetLine = between(0, ReferenceDocs.values().length - 1); + assertThat( + expectThrows( + IllegalStateException.class, + () -> ReferenceDocs.readLinksBySymbol( + getResourceStream((i, l) -> i == targetLine ? l.replace(TEST_LINK_PLACEHOLDER, "_auto_generated_page.html") : l) + ) + ).getMessage(), + equalTo( + "found auto-generated fragment ID in link [_auto_generated_page.html] for [" + + ReferenceDocs.values()[targetLine].name() + + "] at line " + + (targetLine + 1) + ) + ); } } From a18b3313364b6b7c87c45bae9dc342dec314966d Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 2 Oct 2024 10:42:43 -0400 Subject: [PATCH 007/194] ESQL: Fix filtering all elements in aggs (#113804) This adds a test to *every* agg for when it's entirely filtered away and another when filtering is enabled but unused. I'll follow up with another test later for partial filtering. That test caught a bug where some aggs would think they'd been `seen` when they hadn't. This fixes that too. --- .../compute/operator/AggregatorBenchmark.java | 96 ++++++++++++++++--- .../compute/gen/AggregatorImplementer.java | 15 +-- .../compute/data/BooleanArrayVector.java | 26 +++++ .../compute/data/BooleanBigArrayVector.java | 26 +++++ .../compute/data/BooleanVector.java | 10 ++ .../compute/data/ConstantBooleanVector.java | 16 ++++ ...ountDistinctBooleanAggregatorFunction.java | 10 +- ...untDistinctBytesRefAggregatorFunction.java | 10 +- ...CountDistinctDoubleAggregatorFunction.java | 10 +- .../CountDistinctFloatAggregatorFunction.java | 10 +- .../CountDistinctIntAggregatorFunction.java | 10 +- .../CountDistinctLongAggregatorFunction.java | 10 +- .../MaxBooleanAggregatorFunction.java | 10 +- .../MaxBytesRefAggregatorFunction.java | 10 +- .../MaxDoubleAggregatorFunction.java | 10 +- .../MaxFloatAggregatorFunction.java | 10 +- .../aggregation/MaxIntAggregatorFunction.java | 10 +- .../aggregation/MaxIpAggregatorFunction.java | 10 +- .../MaxLongAggregatorFunction.java | 10 +- ...luteDeviationDoubleAggregatorFunction.java | 10 +- ...oluteDeviationFloatAggregatorFunction.java | 10 +- ...bsoluteDeviationIntAggregatorFunction.java | 10 +- ...soluteDeviationLongAggregatorFunction.java | 10 +- .../MinBooleanAggregatorFunction.java | 10 +- .../MinBytesRefAggregatorFunction.java | 10 +- .../MinDoubleAggregatorFunction.java | 10 +- .../MinFloatAggregatorFunction.java | 10 +- .../aggregation/MinIntAggregatorFunction.java | 10 +- .../aggregation/MinIpAggregatorFunction.java | 10 +- .../MinLongAggregatorFunction.java | 10 +- .../PercentileDoubleAggregatorFunction.java | 10 +- .../PercentileFloatAggregatorFunction.java | 10 +- .../PercentileIntAggregatorFunction.java | 10 +- .../PercentileLongAggregatorFunction.java | 10 +- .../SumDoubleAggregatorFunction.java | 10 +- .../SumFloatAggregatorFunction.java | 10 +- .../aggregation/SumIntAggregatorFunction.java | 10 +- .../SumLongAggregatorFunction.java | 10 +- .../TopBooleanAggregatorFunction.java | 10 +- .../TopBytesRefAggregatorFunction.java | 10 +- .../TopDoubleAggregatorFunction.java | 10 +- .../TopFloatAggregatorFunction.java | 10 +- .../aggregation/TopIntAggregatorFunction.java | 10 +- .../aggregation/TopIpAggregatorFunction.java | 10 +- .../TopLongAggregatorFunction.java | 10 +- .../ValuesBooleanAggregatorFunction.java | 10 +- .../ValuesBytesRefAggregatorFunction.java | 10 +- .../ValuesDoubleAggregatorFunction.java | 10 +- .../ValuesFloatAggregatorFunction.java | 10 +- .../ValuesIntAggregatorFunction.java | 10 +- .../ValuesLongAggregatorFunction.java | 10 +- ...esianPointDocValuesAggregatorFunction.java | 10 +- ...anPointSourceValuesAggregatorFunction.java | 10 +- ...idGeoPointDocValuesAggregatorFunction.java | 10 +- ...eoPointSourceValuesAggregatorFunction.java | 10 +- .../CountGroupingAggregatorFunction.java | 1 - .../aggregation/blockhash/AddPage.java | 2 +- .../aggregation/blockhash/BlockHash.java | 3 + .../compute/data/ConstantNullVector.java | 12 +++ .../compute/data/X-ArrayVector.java.st | 27 ++++++ .../compute/data/X-BigArrayVector.java.st | 27 ++++++ .../compute/data/X-ConstantVector.java.st | 17 ++++ .../compute/data/X-Vector.java.st | 11 +++ .../ConstantBooleanExpressionEvaluator.java | 42 ++++++++ .../AggregatorFunctionTestCase.java | 39 +++++++- ...ooleanGroupingAggregatorFunctionTests.java | 11 +++ ...tesRefGroupingAggregatorFunctionTests.java | 11 +++ ...DoubleGroupingAggregatorFunctionTests.java | 11 +++ ...tFloatGroupingAggregatorFunctionTests.java | 11 +++ ...nctIntGroupingAggregatorFunctionTests.java | 11 +++ ...ctLongGroupingAggregatorFunctionTests.java | 11 +++ .../CountGroupingAggregatorFunctionTests.java | 11 +++ .../FilteredAggregatorFunctionTests.java | 10 ++ .../GroupingAggregatorFunctionTestCase.java | 95 ++++++++++++++---- .../aggregation/blockhash/AddPageTests.java | 8 +- .../aggregation/blockhash/BlockHashTests.java | 12 ++- .../compute/data/BasicBlockTests.java | 24 ++++- .../compute/data/BigArrayVectorTests.java | 17 +++- 78 files changed, 804 insertions(+), 299 deletions(-) create mode 100644 x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/ConstantBooleanExpressionEvaluator.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java index f23a4b07d8719..27f4d68b0bc3f 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java @@ -17,6 +17,7 @@ import org.elasticsearch.compute.aggregation.CountAggregatorFunction; import org.elasticsearch.compute.aggregation.CountDistinctDoubleAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.CountDistinctLongAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.FilteredAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MaxDoubleAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MaxLongAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MinDoubleAggregatorFunctionSupplier; @@ -27,6 +28,7 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.ElementType; @@ -35,6 +37,7 @@ import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.AggregationOperator; import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.HashAggregationOperator; import org.elasticsearch.compute.operator.Operator; import org.openjdk.jmh.annotations.Benchmark; @@ -94,13 +97,20 @@ public class AggregatorBenchmark { private static final String NONE = "none"; + private static final String CONSTANT_TRUE = "constant_true"; + private static final String ALL_TRUE = "all_true"; + private static final String HALF_TRUE = "half_true"; + private static final String CONSTANT_FALSE = "constant_false"; + static { // Smoke test all the expected values and force loading subclasses more like prod try { for (String grouping : AggregatorBenchmark.class.getField("grouping").getAnnotationsByType(Param.class)[0].value()) { for (String op : AggregatorBenchmark.class.getField("op").getAnnotationsByType(Param.class)[0].value()) { for (String blockType : AggregatorBenchmark.class.getField("blockType").getAnnotationsByType(Param.class)[0].value()) { - run(grouping, op, blockType, 50); + for (String filter : AggregatorBenchmark.class.getField("filter").getAnnotationsByType(Param.class)[0].value()) { + run(grouping, op, blockType, filter, 10); + } } } } @@ -118,10 +128,14 @@ public class AggregatorBenchmark { @Param({ VECTOR_LONGS, HALF_NULL_LONGS, VECTOR_DOUBLES, HALF_NULL_DOUBLES }) public String blockType; - private static Operator operator(DriverContext driverContext, String grouping, String op, String dataType) { + @Param({ NONE, CONSTANT_TRUE, ALL_TRUE, HALF_TRUE, CONSTANT_FALSE }) + public String filter; + + private static Operator operator(DriverContext driverContext, String grouping, String op, String dataType, String filter) { + if (grouping.equals("none")) { return new AggregationOperator( - List.of(supplier(op, dataType, 0).aggregatorFactory(AggregatorMode.SINGLE).apply(driverContext)), + List.of(supplier(op, dataType, filter, 0).aggregatorFactory(AggregatorMode.SINGLE).apply(driverContext)), driverContext ); } @@ -144,14 +158,14 @@ private static Operator operator(DriverContext driverContext, String grouping, S default -> throw new IllegalArgumentException("unsupported grouping [" + grouping + "]"); }; return new HashAggregationOperator( - List.of(supplier(op, dataType, groups.size()).groupingAggregatorFactory(AggregatorMode.SINGLE)), + List.of(supplier(op, dataType, filter, groups.size()).groupingAggregatorFactory(AggregatorMode.SINGLE)), () -> BlockHash.build(groups, driverContext.blockFactory(), 16 * 1024, false), driverContext ); } - private static AggregatorFunctionSupplier supplier(String op, String dataType, int dataChannel) { - return switch (op) { + private static AggregatorFunctionSupplier supplier(String op, String dataType, String filter, int dataChannel) { + return filtered(switch (op) { case COUNT -> CountAggregatorFunction.supplier(List.of(dataChannel)); case COUNT_DISTINCT -> switch (dataType) { case LONGS -> new CountDistinctLongAggregatorFunctionSupplier(List.of(dataChannel), 3000); @@ -174,10 +188,22 @@ private static AggregatorFunctionSupplier supplier(String op, String dataType, i default -> throw new IllegalArgumentException("unsupported data type [" + dataType + "]"); }; default -> throw new IllegalArgumentException("unsupported op [" + op + "]"); - }; + }, filter); } - private static void checkExpected(String grouping, String op, String blockType, String dataType, Page page, int opCount) { + private static void checkExpected( + String grouping, + String op, + String blockType, + String filter, + String dataType, + Page page, + int opCount + ) { + if (filter.equals(CONSTANT_FALSE) || filter.equals(HALF_TRUE)) { + // We don't verify these because it's hard to get the right answer. + return; + } String prefix = String.format("[%s][%s][%s] ", grouping, op, blockType); if (grouping.equals("none")) { checkUngrouped(prefix, op, dataType, page, opCount); @@ -559,13 +585,59 @@ private static BytesRef bytesGroup(int group) { }); } + private static AggregatorFunctionSupplier filtered(AggregatorFunctionSupplier agg, String filter) { + if (filter.equals("none")) { + return agg; + } + BooleanBlock mask = mask(filter).asBlock(); + return new FilteredAggregatorFunctionSupplier(agg, context -> new EvalOperator.ExpressionEvaluator() { + @Override + public Block eval(Page page) { + mask.incRef(); + return mask; + } + + @Override + public void close() { + mask.close(); + } + }); + } + + private static BooleanVector mask(String filter) { + // Usually BLOCK_LENGTH is the count of positions, but sometimes the blocks are longer + int positionCount = BLOCK_LENGTH * 10; + return switch (filter) { + case CONSTANT_TRUE -> blockFactory.newConstantBooleanVector(true, positionCount); + case ALL_TRUE -> { + try (BooleanVector.Builder builder = blockFactory.newBooleanVectorFixedBuilder(positionCount)) { + for (int i = 0; i < positionCount; i++) { + builder.appendBoolean(true); + } + yield builder.build(); + } + } + case HALF_TRUE -> { + try (BooleanVector.Builder builder = blockFactory.newBooleanVectorFixedBuilder(positionCount)) { + for (int i = 0; i < positionCount; i++) { + builder.appendBoolean(i % 2 == 0); + } + yield builder.build(); + } + } + case CONSTANT_FALSE -> blockFactory.newConstantBooleanVector(false, positionCount); + default -> throw new IllegalArgumentException("unsupported filter [" + filter + "]"); + }; + } + @Benchmark @OperationsPerInvocation(OP_COUNT * BLOCK_LENGTH) public void run() { - run(grouping, op, blockType, OP_COUNT); + run(grouping, op, blockType, filter, OP_COUNT); } - private static void run(String grouping, String op, String blockType, int opCount) { + private static void run(String grouping, String op, String blockType, String filter, int opCount) { + // System.err.printf("[%s][%s][%s][%s][%s]\n", grouping, op, blockType, filter, opCount); String dataType = switch (blockType) { case VECTOR_LONGS, HALF_NULL_LONGS -> LONGS; case VECTOR_DOUBLES, HALF_NULL_DOUBLES -> DOUBLES; @@ -573,13 +645,13 @@ private static void run(String grouping, String op, String blockType, int opCoun }; DriverContext driverContext = driverContext(); - try (Operator operator = operator(driverContext, grouping, op, dataType)) { + try (Operator operator = operator(driverContext, grouping, op, dataType, filter)) { Page page = page(driverContext.blockFactory(), grouping, blockType); for (int i = 0; i < opCount; i++) { operator.addInput(page.shallowCopy()); } operator.finish(); - checkExpected(grouping, op, blockType, dataType, operator.getOutput(), opCount); + checkExpected(grouping, op, blockType, filter, dataType, operator.getOutput(), opCount); } } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java index 48269e7e2af9b..fe9576672cc2f 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java @@ -353,14 +353,14 @@ private MethodSpec addRawInput() { builder.addStatement("return"); builder.endControlFlow(); } - builder.beginControlFlow("if (mask.isConstant())"); + builder.beginControlFlow("if (mask.allFalse())"); + { + builder.addComment("Entire page masked away"); + builder.addStatement("return"); + } + builder.endControlFlow(); + builder.beginControlFlow("if (mask.allTrue())"); { - builder.beginControlFlow("if (mask.getBoolean(0) == false)"); - { - builder.addComment("Entire page masked away"); - builder.addStatement("return"); - } - builder.endControlFlow(); builder.addComment("No masking"); builder.addStatement("$T block = page.getBlock(channels.get(0))", valueBlockType(init, combine)); builder.addStatement("$T vector = block.asVector()", valueVectorType(init, combine)); @@ -372,6 +372,7 @@ private MethodSpec addRawInput() { builder.addStatement("return"); } builder.endControlFlow(); + builder.addComment("Some positions masked away, others kept"); builder.addStatement("$T block = page.getBlock(channels.get(0))", valueBlockType(init, combine)); builder.addStatement("$T vector = block.asVector()", valueVectorType(init, combine)); diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayVector.java index b44ad180a66ff..f761ed5806a06 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayVector.java @@ -128,6 +128,32 @@ public static long ramBytesEstimated(boolean[] values) { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values); } + /** + * Are all values {@code true}? This will scan all values to check and always answer accurately. + */ + @Override + public boolean allTrue() { + for (int i = 0; i < getPositionCount(); i++) { + if (values[i] == false) { + return false; + } + } + return true; + } + + /** + * Are all values {@code false}? This will scan all values to check and always answer accurately. + */ + @Override + public boolean allFalse() { + for (int i = 0; i < getPositionCount(); i++) { + if (values[i]) { + return false; + } + } + return true; + } + @Override public long ramBytesUsed() { return ramBytesEstimated(values); diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayVector.java index f6bd6e978bc7e..a1ccfc487cca9 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanBigArrayVector.java @@ -62,6 +62,32 @@ public boolean getBoolean(int position) { return values.get(position); } + /** + * Are all values {@code true}? This will scan all values to check and always answer accurately. + */ + @Override + public boolean allTrue() { + for (int i = 0; i < getPositionCount(); i++) { + if (values.get(i) == false) { + return false; + } + } + return true; + } + + /** + * Are all values {@code false}? This will scan all values to check and always answer accurately. + */ + @Override + public boolean allFalse() { + for (int i = 0; i < getPositionCount(); i++) { + if (values.get(i)) { + return false; + } + } + return true; + } + @Override public ElementType elementType() { return ElementType.BOOLEAN; diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVector.java index 5e3157a107fa5..f2d6b5fbd4ce9 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVector.java @@ -35,6 +35,16 @@ public sealed interface BooleanVector extends Vector permits ConstantBooleanVect @Override ReleasableIterator lookup(IntBlock positions, ByteSizeValue targetBlockSize); + /** + * Are all values {@code true}? This will scan all values to check and always answer accurately. + */ + boolean allTrue(); + + /** + * Are all values {@code false}? This will scan all values to check and always answer accurately. + */ + boolean allFalse(); + /** * Compares the given object with this vector for equality. Returns {@code true} if and only if the * given object is a BooleanVector, and both vectors are {@link #equals(BooleanVector, BooleanVector) equal}. diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBooleanVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBooleanVector.java index 1c886eb7c2dab..f36fbd7a20316 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBooleanVector.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantBooleanVector.java @@ -89,6 +89,22 @@ public ReleasableIterator lookup(IntBlock positions, ByteSizeValue return new BooleanLookup(asBlock(), positions, targetBlockSize); } + /** + * Are all values {@code true}? This will scan all values to check and always answer accurately. + */ + @Override + public boolean allTrue() { + return value; + } + + /** + * Are all values {@code false}? This will scan all values to check and always answer accurately. + */ + @Override + public boolean allFalse() { + return value == false; + } + @Override public ElementType elementType() { return ElementType.BOOLEAN; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBooleanAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBooleanAggregatorFunction.java index 37543714717de..ca5cd1bda44d0 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBooleanAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBooleanAggregatorFunction.java @@ -54,11 +54,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking BooleanBlock block = page.getBlock(channels.get(0)); BooleanVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefAggregatorFunction.java index 77d7e88cf9a93..38dadda1eba0c 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefAggregatorFunction.java @@ -58,11 +58,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking BytesRefBlock block = page.getBlock(channels.get(0)); BytesRefVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleAggregatorFunction.java index 4f0604b4f03c4..1d985fbd1dff6 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleAggregatorFunction.java @@ -60,11 +60,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking DoubleBlock block = page.getBlock(channels.get(0)); DoubleVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java index 00e5335138aa9..36d2aaf3e3d4f 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java @@ -60,11 +60,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking FloatBlock block = page.getBlock(channels.get(0)); FloatVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntAggregatorFunction.java index 90b4947b77d92..05bebca924f7e 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntAggregatorFunction.java @@ -60,11 +60,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking IntBlock block = page.getBlock(channels.get(0)); IntVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongAggregatorFunction.java index 99dc37d58a88c..9e62525fa2bb0 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongAggregatorFunction.java @@ -60,11 +60,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking LongBlock block = page.getBlock(channels.get(0)); LongVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBooleanAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBooleanAggregatorFunction.java index 38de18bea776a..01763200f2d2c 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBooleanAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBooleanAggregatorFunction.java @@ -54,11 +54,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking BooleanBlock block = page.getBlock(channels.get(0)); BooleanVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java index 1a4d440d2b8bc..73b927cd9c521 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java @@ -57,11 +57,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking BytesRefBlock block = page.getBlock(channels.get(0)); BytesRefVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxDoubleAggregatorFunction.java index 266977e2a689c..04d24d49cbff8 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxDoubleAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxDoubleAggregatorFunction.java @@ -56,11 +56,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking DoubleBlock block = page.getBlock(channels.get(0)); DoubleVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunction.java index 3a4dcaa3289fe..ce22983bff72b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunction.java @@ -56,11 +56,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking FloatBlock block = page.getBlock(channels.get(0)); FloatVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIntAggregatorFunction.java index d5c0cea243499..6a91b574da769 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIntAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIntAggregatorFunction.java @@ -56,11 +56,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking IntBlock block = page.getBlock(channels.get(0)); IntVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunction.java index 13c74775d2796..7f6d47ce1c876 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunction.java @@ -57,11 +57,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking BytesRefBlock block = page.getBlock(channels.get(0)); BytesRefVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxLongAggregatorFunction.java index d2acff0509dfe..97d12d1ef6852 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxLongAggregatorFunction.java @@ -56,11 +56,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking LongBlock block = page.getBlock(channels.get(0)); LongVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleAggregatorFunction.java index 4791767f4b43e..611314318eba7 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleAggregatorFunction.java @@ -57,11 +57,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking DoubleBlock block = page.getBlock(channels.get(0)); DoubleVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java index fd0e5d5fce1a8..e20badf2ce38a 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java @@ -57,11 +57,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking FloatBlock block = page.getBlock(channels.get(0)); FloatVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntAggregatorFunction.java index 67bc4b3bf0356..df0d24d442283 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntAggregatorFunction.java @@ -57,11 +57,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking IntBlock block = page.getBlock(channels.get(0)); IntVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongAggregatorFunction.java index 9255bb19cda70..e0ace94a1da49 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongAggregatorFunction.java @@ -57,11 +57,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking LongBlock block = page.getBlock(channels.get(0)); LongVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBooleanAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBooleanAggregatorFunction.java index 9bd63ed8efbd8..4d91d3794aecb 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBooleanAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBooleanAggregatorFunction.java @@ -54,11 +54,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking BooleanBlock block = page.getBlock(channels.get(0)); BooleanVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java index 2789f18a19dfc..01ee21f82ab53 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java @@ -57,11 +57,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking BytesRefBlock block = page.getBlock(channels.get(0)); BytesRefVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinDoubleAggregatorFunction.java index e736f91e0b38c..a436cdcdbef6d 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinDoubleAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinDoubleAggregatorFunction.java @@ -56,11 +56,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking DoubleBlock block = page.getBlock(channels.get(0)); DoubleVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunction.java index 9d67ccb8fb736..ec6757e59d074 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunction.java @@ -56,11 +56,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking FloatBlock block = page.getBlock(channels.get(0)); FloatVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIntAggregatorFunction.java index a5ead0bd635c0..f76dcec81d871 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIntAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIntAggregatorFunction.java @@ -56,11 +56,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking IntBlock block = page.getBlock(channels.get(0)); IntVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpAggregatorFunction.java index 60ba1993f45d8..795299d9332fc 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpAggregatorFunction.java @@ -57,11 +57,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking BytesRefBlock block = page.getBlock(channels.get(0)); BytesRefVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinLongAggregatorFunction.java index b7bab86d6423e..4fc968bab2eff 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinLongAggregatorFunction.java @@ -56,11 +56,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking LongBlock block = page.getBlock(channels.get(0)); LongVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleAggregatorFunction.java index 4de609a4c6044..9ece01135e0a9 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleAggregatorFunction.java @@ -60,11 +60,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking DoubleBlock block = page.getBlock(channels.get(0)); DoubleVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java index 095499fc9642a..434989adf47b2 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java @@ -60,11 +60,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking FloatBlock block = page.getBlock(channels.get(0)); FloatVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntAggregatorFunction.java index 33e06691fd367..eb4ae96f5dea5 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntAggregatorFunction.java @@ -60,11 +60,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking IntBlock block = page.getBlock(channels.get(0)); IntVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongAggregatorFunction.java index 0a20153e4a33d..837f7efb32441 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongAggregatorFunction.java @@ -60,11 +60,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking LongBlock block = page.getBlock(channels.get(0)); LongVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumDoubleAggregatorFunction.java index 7f0e0b4e15158..4d24579203df1 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumDoubleAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumDoubleAggregatorFunction.java @@ -57,11 +57,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking DoubleBlock block = page.getBlock(channels.get(0)); DoubleVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunction.java index d916b832d77ff..50f41b5edc05f 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunction.java @@ -59,11 +59,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking FloatBlock block = page.getBlock(channels.get(0)); FloatVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumIntAggregatorFunction.java index 5cd1abc35d28f..95bd95ac474ad 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumIntAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumIntAggregatorFunction.java @@ -58,11 +58,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking IntBlock block = page.getBlock(channels.get(0)); IntVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumLongAggregatorFunction.java index e7781f82b1021..fac21d99bf713 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumLongAggregatorFunction.java @@ -56,11 +56,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking LongBlock block = page.getBlock(channels.get(0)); LongVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBooleanAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBooleanAggregatorFunction.java index 0580dc297a362..b8d06787f7f68 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBooleanAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBooleanAggregatorFunction.java @@ -59,11 +59,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking BooleanBlock block = page.getBlock(channels.get(0)); BooleanVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefAggregatorFunction.java index 17b3d84ab0028..9ef460be5796b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefAggregatorFunction.java @@ -61,11 +61,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking BytesRefBlock block = page.getBlock(channels.get(0)); BytesRefVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopDoubleAggregatorFunction.java index 899af1a58851b..210bc76483a81 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopDoubleAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopDoubleAggregatorFunction.java @@ -60,11 +60,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking DoubleBlock block = page.getBlock(channels.get(0)); DoubleVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopFloatAggregatorFunction.java index 168e7685c5273..f7fdb406acadb 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopFloatAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopFloatAggregatorFunction.java @@ -60,11 +60,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking FloatBlock block = page.getBlock(channels.get(0)); FloatVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIntAggregatorFunction.java index 80964decf572d..1ea40134f7260 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIntAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIntAggregatorFunction.java @@ -60,11 +60,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking IntBlock block = page.getBlock(channels.get(0)); IntVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunction.java index 90d8d7c124244..8c216c90504c1 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunction.java @@ -61,11 +61,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking BytesRefBlock block = page.getBlock(channels.get(0)); BytesRefVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopLongAggregatorFunction.java index 18eef5a29b895..85df0f7edc843 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopLongAggregatorFunction.java @@ -60,11 +60,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking LongBlock block = page.getBlock(channels.get(0)); LongVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBooleanAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBooleanAggregatorFunction.java index d71d9a7b45bdb..abf73c07d4ab6 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBooleanAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBooleanAggregatorFunction.java @@ -53,11 +53,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking BooleanBlock block = page.getBlock(channels.get(0)); BooleanVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregatorFunction.java index 56e79b64e3b86..ecc6424ba8501 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregatorFunction.java @@ -55,11 +55,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking BytesRefBlock block = page.getBlock(channels.get(0)); BytesRefVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesDoubleAggregatorFunction.java index 3ec31b0fd5a4d..2fa8ed31ec427 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesDoubleAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesDoubleAggregatorFunction.java @@ -54,11 +54,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking DoubleBlock block = page.getBlock(channels.get(0)); DoubleVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunction.java index 00ab8db1c4ac6..8b61c6d07eed6 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunction.java @@ -54,11 +54,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking FloatBlock block = page.getBlock(channels.get(0)); FloatVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesIntAggregatorFunction.java index 5a0d7c893e607..7f12bbc18b202 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesIntAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesIntAggregatorFunction.java @@ -54,11 +54,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking IntBlock block = page.getBlock(channels.get(0)); IntVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesLongAggregatorFunction.java index ca9a8347e3a41..7e8c256d90f93 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesLongAggregatorFunction.java @@ -54,11 +54,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking LongBlock block = page.getBlock(channels.get(0)); LongVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidCartesianPointDocValuesAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidCartesianPointDocValuesAggregatorFunction.java index a427c75c63fff..a205c728db5fc 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidCartesianPointDocValuesAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidCartesianPointDocValuesAggregatorFunction.java @@ -62,11 +62,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking LongBlock block = page.getBlock(channels.get(0)); LongVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidCartesianPointSourceValuesAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidCartesianPointSourceValuesAggregatorFunction.java index c2086f2ab3d98..e20a3fb1cfa35 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidCartesianPointSourceValuesAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidCartesianPointSourceValuesAggregatorFunction.java @@ -65,11 +65,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking BytesRefBlock block = page.getBlock(channels.get(0)); BytesRefVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidGeoPointDocValuesAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidGeoPointDocValuesAggregatorFunction.java index 0509c03ebf77c..b2c237a904796 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidGeoPointDocValuesAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidGeoPointDocValuesAggregatorFunction.java @@ -62,11 +62,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking LongBlock block = page.getBlock(channels.get(0)); LongVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidGeoPointSourceValuesAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidGeoPointSourceValuesAggregatorFunction.java index 10a29c841b79f..db61420fb8cbe 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidGeoPointSourceValuesAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/spatial/SpatialCentroidGeoPointSourceValuesAggregatorFunction.java @@ -65,11 +65,11 @@ public int intermediateBlockCount() { @Override public void addRawInput(Page page, BooleanVector mask) { - if (mask.isConstant()) { - if (mask.getBoolean(0) == false) { - // Entire page masked away - return; - } + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { // No masking BytesRefBlock block = page.getBlock(channels.get(0)); BytesRefVector vector = block.asVector(); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountGroupingAggregatorFunction.java index f610abf271cfa..e107a73f7ab1e 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountGroupingAggregatorFunction.java @@ -142,7 +142,6 @@ private void addRawInput(IntVector groups) { */ private void addRawInput(IntBlock groups) { for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { - // TODO remove the check one we don't emit null anymore if (groups.isNull(groupPosition)) { continue; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/AddPage.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/AddPage.java index 4e051c73a3643..9a48ea9e5673a 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/AddPage.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/AddPage.java @@ -171,6 +171,6 @@ private void rollover(int position) { @Override public void close() { - Releasables.closeExpectNoException(ords, addInput); + Releasables.closeExpectNoException(ords); } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java index abd11f98e7376..fe1a07e8e16a6 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BlockHash.java @@ -47,6 +47,9 @@ public abstract sealed class BlockHash implements Releasable, SeenGroupIds // /** * Add all values for the "group by" columns in the page to the hash and * pass the ordinals to the provided {@link GroupingAggregatorFunction.AddInput}. + *

+ * This call will not {@link GroupingAggregatorFunction.AddInput#close} {@code addInput}. + *

*/ public abstract void add(Page page, GroupingAggregatorFunction.AddInput addInput); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullVector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullVector.java index 9bcd8a19c0132..236012a674ef6 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullVector.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullVector.java @@ -114,6 +114,18 @@ public int max() { throw new UnsupportedOperationException("null vector"); } + @Override + public boolean allTrue() { + assert false : "null vector"; + throw new UnsupportedOperationException("null vector"); + } + + @Override + public boolean allFalse() { + assert false : "null vector"; + throw new UnsupportedOperationException("null vector"); + } + @Override public ElementType elementType() { return ElementType.NULL; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayVector.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayVector.java.st index 9b5b8b65e8c66..3bb13674ce477 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayVector.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayVector.java.st @@ -240,6 +240,33 @@ $if(int)$ } return max; } + +$elseif(boolean)$ + /** + * Are all values {@code true}? This will scan all values to check and always answer accurately. + */ + @Override + public boolean allTrue() { + for (int i = 0; i < getPositionCount(); i++) { + if (values[i] == false) { + return false; + } + } + return true; + } + + /** + * Are all values {@code false}? This will scan all values to check and always answer accurately. + */ + @Override + public boolean allFalse() { + for (int i = 0; i < getPositionCount(); i++) { + if (values[i]) { + return false; + } + } + return true; + } $endif$ @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayVector.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayVector.java.st index b509313b079dd..106d0769ebb07 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayVector.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayVector.java.st @@ -119,6 +119,33 @@ $if(int)$ } return max; } + +$elseif(boolean)$ + /** + * Are all values {@code true}? This will scan all values to check and always answer accurately. + */ + @Override + public boolean allTrue() { + for (int i = 0; i < getPositionCount(); i++) { + if (values.get(i) == false) { + return false; + } + } + return true; + } + + /** + * Are all values {@code false}? This will scan all values to check and always answer accurately. + */ + @Override + public boolean allFalse() { + for (int i = 0; i < getPositionCount(); i++) { + if (values.get(i)) { + return false; + } + } + return true; + } $endif$ @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ConstantVector.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ConstantVector.java.st index 72999ffd96fae..5d0d4c8a956f3 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ConstantVector.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ConstantVector.java.st @@ -147,6 +147,23 @@ $if(int)$ public int max() { return value; } + +$elseif(boolean)$ + /** + * Are all values {@code true}? This will scan all values to check and always answer accurately. + */ + @Override + public boolean allTrue() { + return value; + } + + /** + * Are all values {@code false}? This will scan all values to check and always answer accurately. + */ + @Override + public boolean allFalse() { + return value == false; + } $endif$ @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st index e19c1788cdb6b..c556cba7ef2e4 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st @@ -72,6 +72,17 @@ $if(int)$ * The maximum value in the Vector. An empty Vector will return {@link Integer#MIN_VALUE}. */ int max(); + +$elseif(boolean)$ + /** + * Are all values {@code true}? This will scan all values to check and always answer accurately. + */ + boolean allTrue(); + + /** + * Are all values {@code false}? This will scan all values to check and always answer accurately. + */ + boolean allFalse(); $endif$ /** diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/ConstantBooleanExpressionEvaluator.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/ConstantBooleanExpressionEvaluator.java new file mode 100644 index 0000000000000..9700a0200f755 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/ConstantBooleanExpressionEvaluator.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.EvalOperator; + +import static org.elasticsearch.test.ESTestCase.randomBoolean; + +/** + * An {@link EvalOperator.ExpressionEvaluator} that evaluates to a constant boolean value. + */ +public record ConstantBooleanExpressionEvaluator(BlockFactory factory, boolean value) implements EvalOperator.ExpressionEvaluator { + public static EvalOperator.ExpressionEvaluator.Factory factory(boolean value) { + return ctx -> new ConstantBooleanExpressionEvaluator(ctx.blockFactory(), value); + } + + @Override + public Block eval(Page page) { + if (randomBoolean()) { + return factory.newConstantBooleanVector(value, page.getPositionCount()).asBlock(); + } + try (BooleanVector.Builder builder = factory.newBooleanVectorFixedBuilder(page.getPositionCount())) { + for (int p = 0; p < page.getPositionCount(); p++) { + builder.appendBoolean(value); + } + return builder.build().asBlock(); + } + } + + @Override + public void close() {} + +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java index 275038e6d2f02..a4eb252dbf35c 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java @@ -8,6 +8,7 @@ package org.elasticsearch.compute.aggregation; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.ConstantBooleanExpressionEvaluator; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BlockTestUtils; @@ -34,6 +35,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.function.Function; import java.util.stream.DoubleStream; import java.util.stream.IntStream; import java.util.stream.LongStream; @@ -58,8 +60,17 @@ protected final int aggregatorIntermediateBlockCount() { @Override protected Operator.OperatorFactory simpleWithMode(AggregatorMode mode) { + return simpleWithMode(mode, Function.identity()); + } + + private Operator.OperatorFactory simpleWithMode( + AggregatorMode mode, + Function wrap + ) { List channels = mode.isInputPartial() ? range(0, aggregatorIntermediateBlockCount()).boxed().toList() : List.of(0); - return new AggregationOperator.AggregationOperatorFactory(List.of(aggregatorFunction(channels).aggregatorFactory(mode)), mode); + AggregatorFunctionSupplier supplier = aggregatorFunction(channels); + Aggregator.Factory factory = wrap.apply(supplier).aggregatorFactory(mode); + return new AggregationOperator.AggregationOperatorFactory(List.of(factory), mode); } @Override @@ -141,6 +152,7 @@ public final void testEmptyInput() { List results = drive(simple().get(driverContext), List.of().iterator(), driverContext); assertThat(results, hasSize(1)); + assertOutputFromEmpty(results.get(0).getBlock(0)); } public final void testEmptyInputInitialFinal() { @@ -166,6 +178,31 @@ public final void testEmptyInputInitialIntermediateFinal() { assertOutputFromEmpty(results.get(0).getBlock(0)); } + public void testAllFiltered() { + Operator.OperatorFactory factory = simpleWithMode( + AggregatorMode.SINGLE, + agg -> new FilteredAggregatorFunctionSupplier(agg, ConstantBooleanExpressionEvaluator.factory(false)) + ); + DriverContext driverContext = driverContext(); + List input = CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), 10)); + List results = drive(factory.get(driverContext), input.iterator(), driverContext); + assertThat(results, hasSize(1)); + assertOutputFromEmpty(results.get(0).getBlock(0)); + } + + public void testNoneFiltered() { + Operator.OperatorFactory factory = simpleWithMode( + AggregatorMode.SINGLE, + agg -> new FilteredAggregatorFunctionSupplier(agg, ConstantBooleanExpressionEvaluator.factory(true)) + ); + DriverContext driverContext = driverContext(); + List input = CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), 10)); + List origInput = BlockTestUtils.deepCopyOf(input, TestBlockFactory.getNonBreakingInstance()); + List results = drive(factory.get(driverContext), input.iterator(), driverContext); + assertThat(results, hasSize(1)); + assertSimpleOutput(origInput, results); + } + // Returns an intermediate state that is equivalent to what the local execution planner will emit // if it determines that certain shards have no relevant data. List nullIntermediateState(BlockFactory blockFactory) { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctBooleanGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctBooleanGroupingAggregatorFunctionTests.java index 66ecbb6eb1130..c39fe32620ff9 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctBooleanGroupingAggregatorFunctionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctBooleanGroupingAggregatorFunctionTests.java @@ -9,7 +9,9 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.LongBooleanTupleBlockSourceOperator; import org.elasticsearch.compute.operator.SourceOperator; @@ -53,4 +55,13 @@ protected void assertOutputFromNullOnly(Block b, int position) { assertThat(b.getValueCount(position), equalTo(1)); assertThat(((LongBlock) b).getLong(b.getFirstValueIndex(position)), equalTo(0L)); } + + @Override + protected void assertOutputFromAllFiltered(Block b) { + assertThat(b.elementType(), equalTo(ElementType.LONG)); + LongVector v = (LongVector) b.asVector(); + for (int p = 0; p < v.getPositionCount(); p++) { + assertThat(v.getLong(p), equalTo(0L)); + } + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctBytesRefGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctBytesRefGroupingAggregatorFunctionTests.java index cbc2a5227d9ea..dd739d2189ba8 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctBytesRefGroupingAggregatorFunctionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctBytesRefGroupingAggregatorFunctionTests.java @@ -10,7 +10,9 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.LongBytesRefTupleBlockSourceOperator; import org.elasticsearch.compute.operator.SourceOperator; @@ -58,4 +60,13 @@ protected void assertOutputFromNullOnly(Block b, int position) { assertThat(b.getValueCount(position), equalTo(1)); assertThat(((LongBlock) b).getLong(b.getFirstValueIndex(position)), equalTo(0L)); } + + @Override + protected void assertOutputFromAllFiltered(Block b) { + assertThat(b.elementType(), equalTo(ElementType.LONG)); + LongVector v = (LongVector) b.asVector(); + for (int p = 0; p < v.getPositionCount(); p++) { + assertThat(v.getLong(p), equalTo(0L)); + } + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctDoubleGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctDoubleGroupingAggregatorFunctionTests.java index 56a0d863038bc..7b6f928d57ddb 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctDoubleGroupingAggregatorFunctionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctDoubleGroupingAggregatorFunctionTests.java @@ -9,7 +9,9 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.LongDoubleTupleBlockSourceOperator; import org.elasticsearch.compute.operator.SourceOperator; @@ -57,4 +59,13 @@ protected void assertOutputFromNullOnly(Block b, int position) { assertThat(b.getValueCount(position), equalTo(1)); assertThat(((LongBlock) b).getLong(b.getFirstValueIndex(position)), equalTo(0L)); } + + @Override + protected void assertOutputFromAllFiltered(Block b) { + assertThat(b.elementType(), equalTo(ElementType.LONG)); + LongVector v = (LongVector) b.asVector(); + for (int p = 0; p < v.getPositionCount(); p++) { + assertThat(v.getLong(p), equalTo(0L)); + } + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunctionTests.java index 03a11bb976b21..6b4a8f2900aaa 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunctionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunctionTests.java @@ -9,7 +9,9 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.LongFloatTupleBlockSourceOperator; import org.elasticsearch.compute.operator.SourceOperator; @@ -57,4 +59,13 @@ protected void assertOutputFromNullOnly(Block b, int position) { assertThat(b.getValueCount(position), equalTo(1)); assertThat(((LongBlock) b).getLong(b.getFirstValueIndex(position)), equalTo(0L)); } + + @Override + protected void assertOutputFromAllFiltered(Block b) { + assertThat(b.elementType(), equalTo(ElementType.LONG)); + LongVector v = (LongVector) b.asVector(); + for (int p = 0; p < v.getPositionCount(); p++) { + assertThat(v.getLong(p), equalTo(0L)); + } + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctIntGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctIntGroupingAggregatorFunctionTests.java index 229ec49bcffa8..cfd3357a14c03 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctIntGroupingAggregatorFunctionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctIntGroupingAggregatorFunctionTests.java @@ -9,7 +9,9 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.LongIntBlockSourceOperator; import org.elasticsearch.compute.operator.SourceOperator; @@ -57,4 +59,13 @@ protected void assertOutputFromNullOnly(Block b, int position) { assertThat(b.getValueCount(position), equalTo(1)); assertThat(((LongBlock) b).getLong(b.getFirstValueIndex(position)), equalTo(0L)); } + + @Override + protected void assertOutputFromAllFiltered(Block b) { + assertThat(b.elementType(), equalTo(ElementType.LONG)); + LongVector v = (LongVector) b.asVector(); + for (int p = 0; p < v.getPositionCount(); p++) { + assertThat(v.getLong(p), equalTo(0L)); + } + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctLongGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctLongGroupingAggregatorFunctionTests.java index 539ef35390663..55be7fe9a8ed3 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctLongGroupingAggregatorFunctionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctLongGroupingAggregatorFunctionTests.java @@ -9,7 +9,9 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.SourceOperator; import org.elasticsearch.compute.operator.TupleBlockSourceOperator; @@ -56,4 +58,13 @@ protected void assertOutputFromNullOnly(Block b, int position) { assertThat(b.getValueCount(position), equalTo(1)); assertThat(((LongBlock) b).getLong(b.getFirstValueIndex(position)), equalTo(0L)); } + + @Override + protected void assertOutputFromAllFiltered(Block b) { + assertThat(b.elementType(), equalTo(ElementType.LONG)); + LongVector v = (LongVector) b.asVector(); + for (int p = 0; p < v.getPositionCount(); p++) { + assertThat(v.getLong(p), equalTo(0L)); + } + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountGroupingAggregatorFunctionTests.java index 1d658f80c4e29..06c267ff2d6ab 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountGroupingAggregatorFunctionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountGroupingAggregatorFunctionTests.java @@ -9,7 +9,9 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.LongDoubleTupleBlockSourceOperator; import org.elasticsearch.compute.operator.SourceOperator; @@ -58,4 +60,13 @@ protected void assertOutputFromNullOnly(Block b, int position) { assertThat(b.getValueCount(position), equalTo(1)); assertThat(((LongBlock) b).getLong(b.getFirstValueIndex(position)), equalTo(0L)); } + + @Override + protected void assertOutputFromAllFiltered(Block b) { + assertThat(b.elementType(), equalTo(ElementType.LONG)); + LongVector v = (LongVector) b.asVector(); + for (int p = 0; p < v.getPositionCount(); p++) { + assertThat(v.getLong(p), equalTo(0L)); + } + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/FilteredAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/FilteredAggregatorFunctionTests.java index 6ad3251d3c120..7e1575fb81726 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/FilteredAggregatorFunctionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/FilteredAggregatorFunctionTests.java @@ -93,4 +93,14 @@ public void checkUnclosed() { } assertThat(unclosed, empty()); } + + @Override + public void testNoneFiltered() { + assumeFalse("can't double filter. tests already filter.", true); + } + + @Override + public void testAllFiltered() { + assumeFalse("can't double filter. tests already filter.", true); + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java index de9337f5fce2c..316058e57e089 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java @@ -10,6 +10,7 @@ import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.util.BitArray; +import org.elasticsearch.compute.ConstantBooleanExpressionEvaluator; import org.elasticsearch.compute.aggregation.blockhash.BlockHash; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; @@ -42,6 +43,7 @@ import java.util.List; import java.util.SortedSet; import java.util.TreeSet; +import java.util.function.Function; import java.util.stream.DoubleStream; import java.util.stream.IntStream; import java.util.stream.LongStream; @@ -82,10 +84,17 @@ protected DataType acceptedDataType() { @Override protected final Operator.OperatorFactory simpleWithMode(AggregatorMode mode) { + return simpleWithMode(mode, Function.identity()); + } + + private Operator.OperatorFactory simpleWithMode( + AggregatorMode mode, + Function wrap + ) { List channels = mode.isInputPartial() ? range(1, 1 + aggregatorIntermediateBlockCount()).boxed().toList() : List.of(1); int emitChunkSize = between(100, 200); - AggregatorFunctionSupplier supplier = aggregatorFunction(channels); + AggregatorFunctionSupplier supplier = wrap.apply(aggregatorFunction(channels)); if (randomBoolean()) { supplier = chunkGroups(emitChunkSize, supplier); } @@ -353,6 +362,49 @@ public final void testNullOnlyInputInitialIntermediateFinal() { ); } + public final void testEmptyInput() { + DriverContext driverContext = driverContext(); + List results = drive(simple().get(driverContext), List.of().iterator(), driverContext); + + assertThat(results, hasSize(0)); + } + + public final void testAllFiltered() { + Operator.OperatorFactory factory = simpleWithMode( + AggregatorMode.SINGLE, + agg -> new FilteredAggregatorFunctionSupplier(agg, ConstantBooleanExpressionEvaluator.factory(false)) + ); + DriverContext driverContext = driverContext(); + List input = CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), 10)); + List results = drive(factory.get(driverContext), input.iterator(), driverContext); + assertThat(results, hasSize(1)); + assertOutputFromAllFiltered(results.get(0).getBlock(1)); + } + + public final void testNoneFiltered() { + Operator.OperatorFactory factory = simpleWithMode( + AggregatorMode.SINGLE, + agg -> new FilteredAggregatorFunctionSupplier(agg, ConstantBooleanExpressionEvaluator.factory(true)) + ); + DriverContext driverContext = driverContext(); + List input = CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), 10)); + List origInput = BlockTestUtils.deepCopyOf(input, TestBlockFactory.getNonBreakingInstance()); + List results = drive(factory.get(driverContext), input.iterator(), driverContext); + assertThat(results, hasSize(1)); + assertSimpleOutput(origInput, results); + } + + /** + * Asserts that the output from an empty input is a {@link Block} containing + * only {@code null}. Override for {@code count} style aggregations that + * return other sorts of results. + */ + protected void assertOutputFromAllFiltered(Block b) { + assertThat(b.areAllValuesNull(), equalTo(true)); + assertThat(b.isNull(0), equalTo(true)); + assertThat(b.getValueCount(0), equalTo(0)); + } + /** * Run the aggregation passing only null values. */ @@ -560,31 +612,34 @@ public AddInput prepareProcessPage(SeenGroupIds ignoredSeenGroupIds, Page page) @Override public void add(int positionOffset, IntBlock groupIds) { for (int offset = 0; offset < groupIds.getPositionCount(); offset += emitChunkSize) { - IntBlock.Builder builder = blockFactory().newIntBlockBuilder(emitChunkSize); - int endP = Math.min(groupIds.getPositionCount(), offset + emitChunkSize); - for (int p = offset; p < endP; p++) { - int start = groupIds.getFirstValueIndex(p); - int count = groupIds.getValueCount(p); - switch (count) { - case 0 -> builder.appendNull(); - case 1 -> { - int group = groupIds.getInt(start); - seenGroupIds.set(group); - builder.appendInt(group); - } - default -> { - int end = start + count; - builder.beginPositionEntry(); - for (int i = start; i < end; i++) { - int group = groupIds.getInt(i); + try (IntBlock.Builder builder = blockFactory().newIntBlockBuilder(emitChunkSize)) { + int endP = Math.min(groupIds.getPositionCount(), offset + emitChunkSize); + for (int p = offset; p < endP; p++) { + int start = groupIds.getFirstValueIndex(p); + int count = groupIds.getValueCount(p); + switch (count) { + case 0 -> builder.appendNull(); + case 1 -> { + int group = groupIds.getInt(start); seenGroupIds.set(group); builder.appendInt(group); } - builder.endPositionEntry(); + default -> { + int end = start + count; + builder.beginPositionEntry(); + for (int i = start; i < end; i++) { + int group = groupIds.getInt(i); + seenGroupIds.set(group); + builder.appendInt(group); + } + builder.endPositionEntry(); + } } } + try (IntBlock chunked = builder.build()) { + delegateAddInput.add(positionOffset + offset, chunked); + } } - delegateAddInput.add(positionOffset + offset, builder.build()); } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/AddPageTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/AddPageTests.java index 810402d82c9d1..fb8b01e68c6cc 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/AddPageTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/AddPageTests.java @@ -41,7 +41,6 @@ public void testSv() { } expected.add(added(3, 4)); assertThat(result.added, equalTo(expected)); - assertThat(result.closed, equalTo(true)); } public void testMvBlockEndsOnBatchBoundary() { @@ -69,7 +68,6 @@ public void testMvBlockEndsOnBatchBoundary() { * about. */ assertThat(result.added, equalTo(expected)); - assertThat(result.closed, equalTo(true)); } public void testMvPositionEndOnBatchBoundary() { @@ -92,7 +90,6 @@ public void testMvPositionEndOnBatchBoundary() { // Because the first position ended on a block boundary we uselessly emit an empty position there expected.add(new Added(0, List.of(List.of(), List.of(0, 2)))); assertThat(result.added, equalTo(expected)); - assertThat(result.closed, equalTo(true)); } public void testMv() { @@ -114,7 +111,6 @@ public void testMv() { } expected.add(new Added(1, List.of(List.of(2)))); assertThat(result.added, equalTo(expected)); - assertThat(result.closed, equalTo(true)); } /** @@ -158,8 +154,6 @@ Added added(int positionOffset, int... ords) { } private class TestAddInput implements GroupingAggregatorFunction.AddInput { - private boolean closed = false; - private final List added = new ArrayList<>(); @Override @@ -185,7 +179,7 @@ public void add(int positionOffset, IntVector groupIds) { @Override public void close() { - closed = true; + fail("shouldn't close"); } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java index c4042ea15afc6..800683c696c0f 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/BlockHashTests.java @@ -1168,7 +1168,9 @@ public void add(int positionOffset, IntVector groupIds) { } @Override - public void close() {} + public void close() { + fail("hashes should not close AddInput"); + } }); hash2.add(page, new GroupingAggregatorFunction.AddInput() { @Override @@ -1184,7 +1186,9 @@ public void add(int positionOffset, IntVector groupIds) { } @Override - public void close() {} + public void close() { + fail("hashes should not close AddInput"); + } }); assertThat(output1.size(), equalTo(output1.size())); for (int i = 0; i < output1.size(); i++) { @@ -1305,7 +1309,9 @@ public void add(int positionOffset, IntVector groupIds) { } @Override - public void close() {} + public void close() { + fail("hashes should not close AddInput"); + } }); if (blockHash instanceof LongLongBlockHash == false && blockHash instanceof BytesRefLongBlockHash == false diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java index ad372da47d6b8..1fd670f836df3 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java @@ -829,10 +829,25 @@ public void testBooleanBlock() { BooleanVector.Builder vectorBuilder = blockFactory.newBooleanVectorBuilder( randomBoolean() ? randomIntBetween(1, positionCount) : positionCount ); - IntStream.range(0, positionCount).mapToObj(ii -> randomBoolean()).forEach(vectorBuilder::appendBoolean); + Boolean value = randomFrom(random(), null, true, false); + IntStream.range(0, positionCount).mapToObj(ii -> { + if (value == null) { + return randomBoolean(); + } + return value; + }).forEach(vectorBuilder::appendBoolean); BooleanVector vector = vectorBuilder.build(); assertSingleValueDenseBlock(vector.asBlock()); assertToMask(vector); + if (value != null) { + if (value) { + assertTrue(vector.allTrue()); + assertFalse(vector.allFalse()); + } else { + assertFalse(vector.allTrue()); + assertTrue(vector.allFalse()); + } + } releaseAndAssertBreaker(vector.asBlock()); } } @@ -867,6 +882,13 @@ public void testConstantBooleanBlock() { b -> assertThat(b, instanceOf(ConstantNullBlock.class)) ); assertEmptyLookup(blockFactory, block); + if (value) { + assertTrue(block.asVector().allTrue()); + assertFalse(block.asVector().allFalse()); + } else { + assertFalse(block.asVector().allTrue()); + assertTrue(block.asVector().allFalse()); + } releaseAndAssertBreaker(block); } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayVectorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayVectorTests.java index 21a7615491e03..6225aa1a6f2a0 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayVectorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayVectorTests.java @@ -37,7 +37,13 @@ public class BigArrayVectorTests extends SerializationTestCase { public void testBoolean() throws IOException { int positionCount = randomIntBetween(1, 16 * 1024); - Boolean[] values = IntStream.range(0, positionCount).mapToObj(i -> randomBoolean()).toArray(Boolean[]::new); + Boolean value = randomFrom(random(), null, true, false); + Boolean[] values = IntStream.range(0, positionCount).mapToObj(i -> { + if (value == null) { + return randomBoolean(); + } + return value; + }).toArray(Boolean[]::new); BitArray array = new BitArray(positionCount, bigArrays); IntStream.range(0, positionCount).filter(i -> values[i]).forEach(array::set); try (var vector = new BooleanBigArrayVector(array, positionCount, blockFactory)) { @@ -78,6 +84,15 @@ public void testBoolean() throws IOException { assertThat(mask.mask().getBoolean(p), equalTo(values[p])); } } + if (value != null) { + if (value) { + assertTrue(vector.allTrue()); + assertFalse(vector.allFalse()); + } else { + assertFalse(vector.allTrue()); + assertTrue(vector.allFalse()); + } + } } } From fbf62321ae294f314020ebf357ac4a47ccc20e56 Mon Sep 17 00:00:00 2001 From: john-wagster Date: Wed, 2 Oct 2024 10:17:03 -0500 Subject: [PATCH 008/194] Updated Date Range to Follow Documentation When Assuming Missing Values - Remove Skip Tests After Backport (#113951) * removed skip tests now that backports are in place * removed skip tests now that backports are in place --- rest-api-spec/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index a2afa13607dd7..ed1cf905f7e9d 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -57,6 +57,5 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTestTransform").configure({task -> task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") - task.skipTest("range/20_synthetic_source/Date range", "date range breaking change causes tests to produce incorrect values for compatibility") task.skipTestsByFilePattern("indices.create/synthetic_source*.yml", "@UpdateForV9 -> tests do not pass after bumping API version to 9 [ES-9597]") }) From c840ea38122c8c5f5759617a86a4ac0cf7a18c59 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Wed, 2 Oct 2024 18:21:54 +0300 Subject: [PATCH 009/194] Remove special handling for objects and arrays with `dynamic` overrides (#113924) * Remove special handling for objects and arrays with `dynamic` overrides * add test * add test --- .../indices.create/20_synthetic_source.yml | 22 ++-- .../index/mapper/DocumentParser.java | 22 ++-- .../mapper/IgnoredSourceFieldMapperTests.java | 107 +++++++++++++++++- 3 files changed, 127 insertions(+), 24 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index b5a9146bc54a6..a999bb7816065 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -535,6 +535,8 @@ object array in object with dynamic override: _source: mode: synthetic properties: + id: + type: integer path_no: dynamic: false properties: @@ -552,19 +554,25 @@ object array in object with dynamic override: refresh: true body: - '{ "create": { } }' - - '{ "path_no": [ { "some_int": 10 }, {"name": "foo"} ], "path_runtime": [ { "some_int": 20 }, {"name": "bar"} ], "name": "baz" }' + - '{ "id": 1, "path_no": [ { "some_int": 30 }, {"name": "baz"}, { "some_int": 20 }, {"name": "bar"} ], "name": "A" }' + - '{ "create": { } }' + - '{ "id": 2, "path_runtime": [ { "some_int": 30 }, {"name": "baz"}, { "some_int": 20 }, {"name": "bar"} ], "name": "B" }' - match: { errors: false } - do: search: index: test + sort: id - - match: { hits.total.value: 1 } - - match: { hits.hits.0._source.name: baz } - - match: { hits.hits.0._source.path_no.0.some_int: 10 } - - match: { hits.hits.0._source.path_no.1.name: foo } - - match: { hits.hits.0._source.path_runtime.0.some_int: 20 } - - match: { hits.hits.0._source.path_runtime.1.name: bar } + - match: { hits.hits.0._source.id: 1 } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.path_no.some_int: [ 30, 20 ] } + - match: { hits.hits.0._source.path_no.name: [ bar, baz ] } + + - match: { hits.hits.1._source.id: 2 } + - match: { hits.hits.1._source.name: B } + - match: { hits.hits.1._source.path_runtime.some_int: [ 30, 20 ] } + - match: { hits.hits.1._source.path_runtime.name: [ bar, baz ] } --- diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index ebe9f27f461cf..c82621baa717a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -401,7 +401,7 @@ static void parseObjectOrNested(DocumentParserContext context) throws IOExceptio context.addIgnoredField( new IgnoredSourceFieldMapper.NameValue( context.parent().fullPath(), - context.parent().fullPath().lastIndexOf(currentFieldName), + context.parent().fullPath().lastIndexOf(context.parent().leafName()), XContentDataHelper.encodeToken(parser), context.doc() ) @@ -803,27 +803,25 @@ private static void parseNonDynamicArray( boolean objectRequiresStoringSource = mapper instanceof ObjectMapper objectMapper && (objectMapper.storeArraySource() || (context.sourceKeepModeFromIndexSettings() == Mapper.SourceKeepMode.ARRAYS - && objectMapper instanceof NestedObjectMapper == false) - || objectMapper.dynamic == ObjectMapper.Dynamic.RUNTIME); + && objectMapper instanceof NestedObjectMapper == false)); boolean fieldWithFallbackSyntheticSource = mapper instanceof FieldMapper fieldMapper && fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK; boolean fieldWithStoredArraySource = mapper instanceof FieldMapper fieldMapper && getSourceKeepMode(context, fieldMapper.sourceKeepMode()) != Mapper.SourceKeepMode.NONE; - boolean dynamicRuntimeContext = context.dynamic() == ObjectMapper.Dynamic.RUNTIME; boolean copyToFieldHasValuesInDocument = context.isWithinCopyTo() == false && context.isCopyToDestinationField(fullPath); if (objectRequiresStoringSource || fieldWithFallbackSyntheticSource - || dynamicRuntimeContext || fieldWithStoredArraySource || copyToFieldHasValuesInDocument) { context = context.addIgnoredFieldFromContext(IgnoredSourceFieldMapper.NameValue.fromContext(context, fullPath, null)); - } else if (mapper instanceof ObjectMapper objectMapper - && (objectMapper.isEnabled() == false || objectMapper.dynamic == ObjectMapper.Dynamic.FALSE)) { - context.addIgnoredField( - IgnoredSourceFieldMapper.NameValue.fromContext(context, fullPath, XContentDataHelper.encodeToken(context.parser())) - ); - return; - } + } else if (mapper instanceof ObjectMapper objectMapper && (objectMapper.isEnabled() == false)) { + // No need to call #addIgnoredFieldFromContext as both singleton and array instances of this object + // get tracked through ignored source. + context.addIgnoredField( + IgnoredSourceFieldMapper.NameValue.fromContext(context, fullPath, XContentDataHelper.encodeToken(context.parser())) + ); + return; + } } // In synthetic source, if any array element requires storing its source as-is, it takes precedence over diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index eaa7bf6528203..4bc33558e104b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -902,7 +902,6 @@ public void testObjectArrayAndValue() throws IOException { } b.endObject(); })).documentMapper(); - // { "path": [ { "stored":[ { "leaf": 10 } ] }, { "stored": { "leaf": 20 } } ] } var syntheticSource = syntheticSource(documentMapper, b -> { b.startArray("path"); { @@ -927,6 +926,91 @@ public void testObjectArrayAndValue() throws IOException { {"path":{"stored":[{"leaf":10},{"leaf":20}]}}""", syntheticSource); } + public void testObjectArrayAndValueDisabledObject() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("path").field("type", "object").startObject("properties"); + { + b.startObject("regular"); + { + b.startObject("properties").startObject("leaf").field("type", "integer").endObject().endObject(); + } + b.endObject(); + b.startObject("disabled").field("type", "object").field("enabled", false); + { + b.startObject("properties").startObject("leaf").field("type", "integer").endObject().endObject(); + } + b.endObject(); + } + b.endObject().endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startArray("path"); + { + b.startObject().startArray("disabled").startObject().field("leaf", 10).endObject().endArray().endObject(); + b.startObject().startObject("disabled").field("leaf", 20).endObject().endObject(); + b.startObject().startArray("regular").startObject().field("leaf", 10).endObject().endArray().endObject(); + b.startObject().startObject("regular").field("leaf", 20).endObject().endObject(); + } + b.endArray(); + }); + assertEquals(""" + {"path":{"disabled":[{"leaf":10},{"leaf":20}],"regular":{"leaf":[10,20]}}}""", syntheticSource); + } + + public void testObjectArrayAndValueNonDynamicObject() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("path").field("type", "object").startObject("properties"); + { + b.startObject("regular"); + { + b.startObject("properties").startObject("leaf").field("type", "integer").endObject().endObject(); + } + b.endObject(); + b.startObject("disabled").field("type", "object").field("dynamic", "false").endObject(); + } + b.endObject().endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startArray("path"); + { + b.startObject().startArray("disabled").startObject().field("leaf", 10).endObject().endArray().endObject(); + b.startObject().startObject("disabled").field("leaf", 20).endObject().endObject(); + b.startObject().startArray("regular").startObject().field("leaf", 10).endObject().endArray().endObject(); + b.startObject().startObject("regular").field("leaf", 20).endObject().endObject(); + } + b.endArray(); + }); + assertEquals(""" + {"path":{"disabled":{"leaf":[10,20]},"regular":{"leaf":[10,20]}}}""", syntheticSource); + } + + public void testObjectArrayAndValueDynamicRuntimeObject() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("path").field("type", "object").startObject("properties"); + { + b.startObject("regular"); + { + b.startObject("properties").startObject("leaf").field("type", "integer").endObject().endObject(); + } + b.endObject(); + b.startObject("runtime").field("type", "object").field("dynamic", "runtime").endObject(); + } + b.endObject().endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startArray("path"); + { + b.startObject().startArray("runtime").startObject().field("leaf", 10).endObject().endArray().endObject(); + b.startObject().startObject("runtime").field("leaf", 20).endObject().endObject(); + b.startObject().startArray("regular").startObject().field("leaf", 10).endObject().endArray().endObject(); + b.startObject().startObject("regular").field("leaf", 20).endObject().endObject(); + } + b.endArray(); + }); + assertEquals(""" + {"path":{"regular":{"leaf":[10,20]},"runtime":{"leaf":[10,20]}}}""", syntheticSource); + } + public void testDisabledObjectWithinHigherLevelArray() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); @@ -1337,7 +1421,7 @@ public void testNoDynamicObjectSimpleArray() throws IOException { b.endArray(); }); assertEquals(""" - {"path":[{"name":"foo"},{"name":"bar"}]}""", syntheticSource); + {"path":{"name":["foo","bar"]}}""", syntheticSource); } public void testNoDynamicObjectSimpleValueArray() throws IOException { @@ -1365,7 +1449,20 @@ public void testNoDynamicObjectNestedArray() throws IOException { b.endArray(); }); assertEquals(""" - {"path":[{"to":{"foo":"A","bar":"B"}},{"to":{"foo":"C","bar":"D"}}]}""", syntheticSource); + {"path":{"to":[{"foo":"A","bar":"B"},{"foo":"C","bar":"D"}]}}""", syntheticSource); + } + + public void testNoDynamicRootObject() throws IOException { + DocumentMapper documentMapper = createMapperService(topMapping(b -> { + b.startObject("_source").field("mode", "synthetic").endObject().field("dynamic", "false"); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.field("foo", "bar"); + b.startObject("path").field("X", "Y").endObject(); + b.array("name", "A", "D", "C", "B"); + }); + assertEquals(""" + {"foo":"bar","name":["A","D","C","B"],"path":{"X":"Y"}}""", syntheticSource); } public void testRuntimeDynamicObjectSingleField() throws IOException { @@ -1445,7 +1542,7 @@ public void testRuntimeDynamicObjectSimpleArray() throws IOException { b.endArray(); }); assertEquals(""" - {"path":[{"name":"foo"},{"name":"bar"}]}""", syntheticSource); + {"path":{"name":["foo","bar"]}}""", syntheticSource); } public void testRuntimeDynamicObjectSimpleValueArray() throws IOException { @@ -1473,7 +1570,7 @@ public void testRuntimeDynamicObjectNestedArray() throws IOException { b.endArray(); }); assertEquals(""" - {"path":[{"to":{"foo":"A","bar":"B"}},{"to":{"foo":"C","bar":"D"}}]}""", syntheticSource); + {"path":{"to":[{"foo":"A","bar":"B"},{"foo":"C","bar":"D"}]}}""", syntheticSource); } public void testDisabledSubObjectWithNameOverlappingParentName() throws IOException { From 8c509c13c047497e16a503cd018fc1a29bb944e1 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Wed, 2 Oct 2024 17:32:33 +0200 Subject: [PATCH 010/194] Increase number of replicas to enable searching in serverless (#113859) --- .../resources/rest-api-spec/test/logsdb/10_settings.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml index 07cb154449a70..4439441efdd02 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml @@ -25,7 +25,6 @@ create logs index: settings: index: mode: logsdb - number_of_replicas: 0 number_of_shards: 2 mappings: properties: From 8099a4641bdd2c6279c9b5adcdd59b6f96a28305 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 3 Oct 2024 02:01:17 +1000 Subject: [PATCH 011/194] Mute org.elasticsearch.test.rest.ClientYamlTestSuiteIT test {yaml=indices.create/20_synthetic_source/object array in object with dynamic override} #113962 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index c84cdf9f9fc00..cedbe6a24f33e 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -344,6 +344,9 @@ tests: - class: org.elasticsearch.kibana.KibanaThreadPoolIT method: testBlockedThreadPoolsRejectUserRequests issue: https://github.com/elastic/elasticsearch/issues/113939 +- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT + method: test {yaml=indices.create/20_synthetic_source/object array in object with dynamic override} + issue: https://github.com/elastic/elasticsearch/issues/113962 # Examples: # From 6759ae2e892c2cf9d5df99f216aaf7c55c92efe7 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Wed, 2 Oct 2024 10:06:33 -0600 Subject: [PATCH 012/194] Introduce watermarks for indexing pressure backoff (#113912) Currently we have a relatively basic decider about when to throttling indexing. This commit adds two levels of watermarks with configurable bulk size deciders. Additionally, adds additional settings to control primary, coordinating, and replica rejection limits. --- .../kibana/KibanaThreadPoolIT.java | 3 +- .../http/IndexingPressureRestIT.java | 3 +- .../action/bulk/IncrementalBulkIT.java | 92 +++++++++++++++++-- .../index/IndexingPressureIT.java | 11 ++- .../seqno/RetentionLeaseSyncActionIT.java | 2 +- .../metrics/NodeIndexingMetricsIT.java | 15 +-- .../action/bulk/IncrementalBulkService.java | 23 ++--- .../common/settings/ClusterSettings.java | 7 ++ .../common/settings/Setting.java | 4 + .../elasticsearch/index/IndexingPressure.java | 87 ++++++++++++++---- .../index/IndexingPressureTests.java | 39 +++++++- .../elasticsearch/test/ESIntegTestCase.java | 5 +- .../persistence/LimitAwareBulkIndexer.java | 2 +- 13 files changed, 237 insertions(+), 56 deletions(-) diff --git a/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java index fa529fe07673e..61bd31fea3455 100644 --- a/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java +++ b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java @@ -57,7 +57,8 @@ public class KibanaThreadPoolIT extends ESIntegTestCase { protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal, otherSettings)) - .put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), "1KB") + .put(IndexingPressure.MAX_PRIMARY_BYTES.getKey(), "1KB") + .put(IndexingPressure.MAX_COORDINATING_BYTES.getKey(), "1KB") .put("thread_pool.search.size", 1) .put("thread_pool.search.queue_size", 1) .put("thread_pool.write.size", 1) diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndexingPressureRestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndexingPressureRestIT.java index f2627a339450e..a19ce67368dff 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndexingPressureRestIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndexingPressureRestIT.java @@ -42,7 +42,8 @@ public class IndexingPressureRestIT extends HttpSmokeTestCase { protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal, otherSettings)) - .put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), "1KB") + .put(IndexingPressure.MAX_COORDINATING_BYTES.getKey(), "1KB") + .put(IndexingPressure.MAX_PRIMARY_BYTES.getKey(), "1KB") .put(unboundedWriteQueue) .build(); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java index 75f914f76dd77..60ea4138e923d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java @@ -50,6 +50,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.lessThan; public class IncrementalBulkIT extends ESIntegTestCase { @@ -62,7 +63,10 @@ protected Collection> nodePlugins() { protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal, otherSettings)) - .put(IndexingPressure.SPLIT_BULK_THRESHOLD.getKey(), "512B") + .put(IndexingPressure.SPLIT_BULK_LOW_WATERMARK.getKey(), "512B") + .put(IndexingPressure.SPLIT_BULK_LOW_WATERMARK_SIZE.getKey(), "2048B") + .put(IndexingPressure.SPLIT_BULK_HIGH_WATERMARK.getKey(), "2KB") + .put(IndexingPressure.SPLIT_BULK_HIGH_WATERMARK_SIZE.getKey(), "1024B") .build(); } @@ -79,7 +83,7 @@ public void testSingleBulkRequest() { AbstractRefCounted refCounted = AbstractRefCounted.of(() -> {}); handler.lastItems(List.of(indexRequest), refCounted::decRef, future); - BulkResponse bulkResponse = future.actionGet(); + BulkResponse bulkResponse = safeGet(future); assertNoFailures(bulkResponse); refresh(index); @@ -142,7 +146,7 @@ public void testIndexingPressureRejection() { } } - public void testIncrementalBulkRequestMemoryBackOff() throws Exception { + public void testIncrementalBulkLowWatermarkBackOff() throws Exception { String index = "test"; createIndex(index); @@ -157,7 +161,7 @@ public void testIncrementalBulkRequestMemoryBackOff() throws Exception { IndexRequest indexRequest = indexRequest(index); long total = indexRequest.ramBytesUsed(); - while (total < 512) { + while (total < 2048) { refCounted.incRef(); handler.addItems(List.of(indexRequest), refCounted::decRef, () -> nextPage.set(true)); assertTrue(nextPage.get()); @@ -175,11 +179,73 @@ public void testIncrementalBulkRequestMemoryBackOff() throws Exception { PlainActionFuture future = new PlainActionFuture<>(); handler.lastItems(List.of(indexRequest), refCounted::decRef, future); - BulkResponse bulkResponse = future.actionGet(); + BulkResponse bulkResponse = safeGet(future); assertNoFailures(bulkResponse); assertFalse(refCounted.hasReferences()); } + public void testIncrementalBulkHighWatermarkBackOff() throws Exception { + String index = "test"; + createIndex(index); + + String nodeName = internalCluster().getRandomNodeName(); + IncrementalBulkService incrementalBulkService = internalCluster().getInstance(IncrementalBulkService.class, nodeName); + IndexingPressure indexingPressure = internalCluster().getInstance(IndexingPressure.class, nodeName); + ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, nodeName); + + AbstractRefCounted refCounted = AbstractRefCounted.of(() -> {}); + AtomicBoolean nextPage = new AtomicBoolean(false); + + ArrayList handlers = new ArrayList<>(); + for (int i = 0; i < 4; ++i) { + ArrayList> requests = new ArrayList<>(); + add512BRequests(requests, index); + IncrementalBulkService.Handler handler = incrementalBulkService.newBulkRequest(); + handlers.add(handler); + refCounted.incRef(); + handler.addItems(requests, refCounted::decRef, () -> nextPage.set(true)); + assertTrue(nextPage.get()); + nextPage.set(false); + } + + // Test that a request smaller than SPLIT_BULK_HIGH_WATERMARK_SIZE (1KB) is not throttled + ArrayList> requestsNoThrottle = new ArrayList<>(); + add512BRequests(requestsNoThrottle, index); + IncrementalBulkService.Handler handlerNoThrottle = incrementalBulkService.newBulkRequest(); + handlers.add(handlerNoThrottle); + refCounted.incRef(); + handlerNoThrottle.addItems(requestsNoThrottle, refCounted::decRef, () -> nextPage.set(true)); + assertTrue(nextPage.get()); + nextPage.set(false); + + ArrayList> requestsThrottle = new ArrayList<>(); + // Test that a request larger than SPLIT_BULK_HIGH_WATERMARK_SIZE (1KB) is throttled + add512BRequests(requestsThrottle, index); + add512BRequests(requestsThrottle, index); + + CountDownLatch finishLatch = new CountDownLatch(1); + blockWritePool(threadPool, finishLatch); + IncrementalBulkService.Handler handlerThrottled = incrementalBulkService.newBulkRequest(); + refCounted.incRef(); + handlerThrottled.addItems(requestsThrottle, refCounted::decRef, () -> nextPage.set(true)); + assertFalse(nextPage.get()); + finishLatch.countDown(); + + handlers.add(handlerThrottled); + + for (IncrementalBulkService.Handler h : handlers) { + refCounted.incRef(); + PlainActionFuture future = new PlainActionFuture<>(); + h.lastItems(List.of(indexRequest(index)), refCounted::decRef, future); + BulkResponse bulkResponse = safeGet(future); + assertNoFailures(bulkResponse); + } + + assertBusy(() -> assertThat(indexingPressure.stats().getCurrentCombinedCoordinatingAndPrimaryBytes(), equalTo(0L))); + refCounted.decRef(); + assertFalse(refCounted.hasReferences()); + } + public void testMultipleBulkPartsWithBackoff() { ExecutorService executorService = Executors.newFixedThreadPool(1); @@ -278,7 +344,7 @@ public void testBulkLevelBulkFailureAfterFirstIncrementalRequest() throws Except } // Should not throw because some succeeded - BulkResponse bulkResponse = future.actionGet(); + BulkResponse bulkResponse = safeGet(future); assertTrue(bulkResponse.hasFailures()); BulkItemResponse[] items = bulkResponse.getItems(); @@ -346,7 +412,7 @@ public void testShortCircuitShardLevelFailure() throws Exception { PlainActionFuture future = new PlainActionFuture<>(); handler.lastItems(List.of(indexRequest(index)), refCounted::decRef, future); - BulkResponse bulkResponse = future.actionGet(); + BulkResponse bulkResponse = safeGet(future); assertTrue(bulkResponse.hasFailures()); for (int i = 0; i < hits.get(); ++i) { assertFalse(bulkResponse.getItems()[i].isFailed()); @@ -439,7 +505,7 @@ public void testShortCircuitShardLevelFailureWithIngestNodeHop() throws Exceptio PlainActionFuture future = new PlainActionFuture<>(); handler.lastItems(List.of(indexRequest(index)), refCounted::decRef, future); - BulkResponse bulkResponse = future.actionGet(); + BulkResponse bulkResponse = safeGet(future); assertTrue(bulkResponse.hasFailures()); for (int i = 0; i < hits.get(); ++i) { assertFalse(bulkResponse.getItems()[i].isFailed()); @@ -538,6 +604,16 @@ public void run() { return bulkResponse; } + private static void add512BRequests(ArrayList> requests, String index) { + long total = 0; + while (total < 512) { + IndexRequest indexRequest = indexRequest(index); + requests.add(indexRequest); + total += indexRequest.ramBytesUsed(); + } + assertThat(total, lessThan(1024L)); + } + private static IndexRequest indexRequest(String index) { IndexRequest indexRequest = new IndexRequest(); indexRequest.index(index); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexingPressureIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexingPressureIT.java index 70701f85b25d8..1ee5dc8fc9ac0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/IndexingPressureIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/IndexingPressureIT.java @@ -248,7 +248,7 @@ public void testWriteCanBeRejectedAtCoordinatingLevel() throws Exception { final long bulkRequestSize = bulkRequest.ramBytesUsed(); final long bulkShardRequestSize = totalRequestSize; restartNodesWithSettings( - Settings.builder().put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), (long) (bulkShardRequestSize * 1.5) + "B").build() + Settings.builder().put(IndexingPressure.MAX_COORDINATING_BYTES.getKey(), (long) (bulkShardRequestSize * 1.5) + "B").build() ); assertAcked(prepareCreate(INDEX_NAME, indexSettings(1, 1))); @@ -312,7 +312,7 @@ public void testWriteCanBeRejectedAtPrimaryLevel() throws Exception { } final long bulkShardRequestSize = totalRequestSize; restartNodesWithSettings( - Settings.builder().put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), (long) (bulkShardRequestSize * 1.5) + "B").build() + Settings.builder().put(IndexingPressure.MAX_PRIMARY_BYTES.getKey(), (long) (bulkShardRequestSize * 1.5) + "B").build() ); assertAcked(prepareCreate(INDEX_NAME, indexSettings(1, 1))); @@ -358,7 +358,12 @@ public void testWriteCanBeRejectedAtPrimaryLevel() throws Exception { } public void testWritesWillSucceedIfBelowThreshold() throws Exception { - restartNodesWithSettings(Settings.builder().put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), "1MB").build()); + restartNodesWithSettings( + Settings.builder() + .put(IndexingPressure.MAX_COORDINATING_BYTES.getKey(), "1MB") + .put(IndexingPressure.MAX_PRIMARY_BYTES.getKey(), "1MB") + .build() + ); assertAcked(prepareCreate(INDEX_NAME, indexSettings(1, 1))); ensureGreen(INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionIT.java index e909a4c5b80e8..b9611973e5c63 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionIT.java @@ -92,7 +92,7 @@ private static Releasable fullyAllocatePrimaryIndexingCapacityOnNode(String targ return internalCluster().getInstance(IndexingPressure.class, targetNode) .markPrimaryOperationStarted( 1, - IndexingPressure.MAX_INDEXING_BYTES.get(internalCluster().getInstance(Settings.class, targetNode)).getBytes() + 1, + IndexingPressure.MAX_PRIMARY_BYTES.get(internalCluster().getInstance(Settings.class, targetNode)).getBytes() + 1, true ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java index 9a82d10bbc3c4..9364e7437141e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/NodeIndexingMetricsIT.java @@ -32,7 +32,8 @@ import java.util.Map; import java.util.function.Function; -import static org.elasticsearch.index.IndexingPressure.MAX_INDEXING_BYTES; +import static org.elasticsearch.index.IndexingPressure.MAX_COORDINATING_BYTES; +import static org.elasticsearch.index.IndexingPressure.MAX_PRIMARY_BYTES; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -198,7 +199,7 @@ public void testNodeIndexingMetricsArePublishing() { public void testCoordinatingRejectionMetricsArePublishing() { // lower Indexing Pressure limits to trigger coordinating rejections - final String dataNode = internalCluster().startNode(Settings.builder().put(MAX_INDEXING_BYTES.getKey(), "1KB")); + final String dataNode = internalCluster().startNode(Settings.builder().put(MAX_COORDINATING_BYTES.getKey(), "1KB")); ensureStableCluster(1); final TestTelemetryPlugin plugin = internalCluster().getInstance(PluginsService.class, dataNode) @@ -239,7 +240,7 @@ public void testCoordinatingRejectionMetricsArePublishing() { public void testCoordinatingRejectionMetricsSpiking() throws Exception { // lower Indexing Pressure limits to trigger coordinating rejections - final String dataNode = internalCluster().startNode(Settings.builder().put(MAX_INDEXING_BYTES.getKey(), "1KB")); + final String dataNode = internalCluster().startNode(Settings.builder().put(MAX_COORDINATING_BYTES.getKey(), "1KB")); ensureStableCluster(1); final TestTelemetryPlugin plugin = internalCluster().getInstance(PluginsService.class, dataNode) @@ -308,10 +309,10 @@ public void testCoordinatingRejectionMetricsSpiking() throws Exception { public void testPrimaryDocumentRejectionMetricsArePublishing() { // setting low Indexing Pressure limits to trigger primary rejections - final String dataNode = internalCluster().startNode(Settings.builder().put(MAX_INDEXING_BYTES.getKey(), "2KB").build()); + final String dataNode = internalCluster().startNode(Settings.builder().put(MAX_PRIMARY_BYTES.getKey(), "2KB").build()); // setting high Indexing Pressure limits to pass coordinating checks final String coordinatingNode = internalCluster().startCoordinatingOnlyNode( - Settings.builder().put(MAX_INDEXING_BYTES.getKey(), "10MB").build() + Settings.builder().put(MAX_COORDINATING_BYTES.getKey(), "10MB").build() ); ensureStableCluster(2); @@ -375,10 +376,10 @@ public void testPrimaryDocumentRejectionMetricsArePublishing() { public void testPrimaryDocumentRejectionMetricsFluctuatingOverTime() throws Exception { // setting low Indexing Pressure limits to trigger primary rejections - final String dataNode = internalCluster().startNode(Settings.builder().put(MAX_INDEXING_BYTES.getKey(), "4KB").build()); + final String dataNode = internalCluster().startNode(Settings.builder().put(MAX_PRIMARY_BYTES.getKey(), "4KB").build()); // setting high Indexing Pressure limits to pass coordinating checks final String coordinatingNode = internalCluster().startCoordinatingOnlyNode( - Settings.builder().put(MAX_INDEXING_BYTES.getKey(), "100MB").build() + Settings.builder().put(MAX_COORDINATING_BYTES.getKey(), "100MB").build() ); ensureStableCluster(2); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java b/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java index fc264de35f510..25e58a82f8e8b 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java @@ -107,6 +107,7 @@ public static class Handler implements Releasable { private boolean incrementalRequestSubmitted = false; private ThreadContext.StoredContext requestContext; private Exception bulkActionLevelFailure = null; + private long currentBulkSize = 0L; private BulkRequest bulkRequest = null; protected Handler( @@ -172,7 +173,7 @@ public void onFailure(Exception e) { } private boolean shouldBackOff() { - return indexingPressure.shouldSplitBulks(); + return indexingPressure.shouldSplitBulk(currentBulkSize); } public void lastItems(List> items, Releasable releasable, ActionListener listener) { @@ -235,6 +236,7 @@ private void errorResponse(ActionListener listener) { private void handleBulkSuccess(BulkResponse bulkResponse) { responses.add(bulkResponse); + currentBulkSize = 0L; bulkRequest = null; } @@ -243,6 +245,7 @@ private void handleBulkFailure(boolean isFirstRequest, Exception e) { globalFailure = isFirstRequest; bulkActionLevelFailure = e; addItemLevelFailures(bulkRequest.requests()); + currentBulkSize = 0; bulkRequest = null; } @@ -261,13 +264,9 @@ private boolean internalAddItems(List> items, Releasable rele try { bulkRequest.add(items); releasables.add(releasable); - releasables.add( - indexingPressure.markCoordinatingOperationStarted( - items.size(), - items.stream().mapToLong(Accountable::ramBytesUsed).sum(), - false - ) - ); + long size = items.stream().mapToLong(Accountable::ramBytesUsed).sum(); + releasables.add(indexingPressure.markCoordinatingOperationStarted(items.size(), size, false)); + currentBulkSize += size; return true; } catch (EsRejectedExecutionException e) { handleBulkFailure(incrementalRequestSubmitted == false, e); @@ -278,6 +277,8 @@ private boolean internalAddItems(List> items, Releasable rele } private void createNewBulkRequest(BulkRequest.IncrementalState incrementalState) { + assert currentBulkSize == 0L; + assert bulkRequest == null; bulkRequest = new BulkRequest(); bulkRequest.incrementalState(incrementalState); @@ -292,12 +293,6 @@ private void createNewBulkRequest(BulkRequest.IncrementalState incrementalState) } } - private void releaseCurrentReferences() { - bulkRequest = null; - releasables.forEach(Releasable::close); - releasables.clear(); - } - private BulkResponse combineResponses() { long tookInMillis = 0; long ingestTookInMillis = 0; diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 2ab0318490f7a..fbce913dac139 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -562,7 +562,14 @@ public void apply(Settings value, Settings current, Settings previous) { FsHealthService.REFRESH_INTERVAL_SETTING, FsHealthService.SLOW_PATH_LOGGING_THRESHOLD_SETTING, IndexingPressure.MAX_INDEXING_BYTES, + IndexingPressure.MAX_COORDINATING_BYTES, + IndexingPressure.MAX_PRIMARY_BYTES, + IndexingPressure.MAX_REPLICA_BYTES, IndexingPressure.SPLIT_BULK_THRESHOLD, + IndexingPressure.SPLIT_BULK_HIGH_WATERMARK, + IndexingPressure.SPLIT_BULK_HIGH_WATERMARK_SIZE, + IndexingPressure.SPLIT_BULK_LOW_WATERMARK, + IndexingPressure.SPLIT_BULK_LOW_WATERMARK_SIZE, ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE_FROZEN, DataTier.ENFORCE_DEFAULT_TIER_PREFERENCE_SETTING, CoordinationDiagnosticsService.IDENTITY_CHANGES_THRESHOLD_SETTING, diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index 70bf88625dc29..6ad20b9fc6d16 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -1671,6 +1671,10 @@ public static Setting memorySizeSetting(String key, ByteSizeValue return memorySizeSetting(key, defaultValue.toString(), properties); } + public static Setting memorySizeSetting(String key, Setting fallbackSetting, Property... properties) { + return new Setting<>(key, fallbackSetting, (s) -> MemorySizeValue.parseBytesSizeValueOrHeapRatio(s, key), properties); + } + /** * Creates a setting which specifies a memory size. This can either be * specified as an absolute bytes value or as a percentage of the heap diff --git a/server/src/main/java/org/elasticsearch/index/IndexingPressure.java b/server/src/main/java/org/elasticsearch/index/IndexingPressure.java index 14f8b92db3eaa..f80e8a89f5cf2 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexingPressure.java +++ b/server/src/main/java/org/elasticsearch/index/IndexingPressure.java @@ -30,12 +30,55 @@ public class IndexingPressure { Setting.Property.NodeScope ); + // TODO: Remove once it is no longer needed for BWC public static final Setting SPLIT_BULK_THRESHOLD = Setting.memorySizeSetting( "indexing_pressure.memory.split_bulk_threshold", "8.5%", Setting.Property.NodeScope ); + public static final Setting MAX_COORDINATING_BYTES = Setting.memorySizeSetting( + "indexing_pressure.memory.coordinating.limit", + MAX_INDEXING_BYTES, + Setting.Property.NodeScope + ); + + public static final Setting MAX_PRIMARY_BYTES = Setting.memorySizeSetting( + "indexing_pressure.memory.primary.limit", + MAX_INDEXING_BYTES, + Setting.Property.NodeScope + ); + + public static final Setting MAX_REPLICA_BYTES = Setting.memorySizeSetting( + "indexing_pressure.memory.replica.limit", + (s) -> ByteSizeValue.ofBytes((long) (MAX_PRIMARY_BYTES.get(s).getBytes() * 1.5)).getStringRep(), + Setting.Property.NodeScope + ); + + public static final Setting SPLIT_BULK_HIGH_WATERMARK = Setting.memorySizeSetting( + "indexing_pressure.memory.split_bulk.watermark.high", + "7.5%", + Setting.Property.NodeScope + ); + + public static final Setting SPLIT_BULK_HIGH_WATERMARK_SIZE = Setting.byteSizeSetting( + "indexing_pressure.memory.split_bulk.watermark.high.bulk_size", + ByteSizeValue.ofMb(1), + Setting.Property.NodeScope + ); + + public static final Setting SPLIT_BULK_LOW_WATERMARK = Setting.memorySizeSetting( + "indexing_pressure.memory.split_bulk.watermark.low", + "5.0%", + Setting.Property.NodeScope + ); + + public static final Setting SPLIT_BULK_LOW_WATERMARK_SIZE = Setting.byteSizeSetting( + "indexing_pressure.memory.split_bulk.watermark.low.bulk_size", + ByteSizeValue.ofMb(4), + Setting.Property.NodeScope + ); + private static final Logger logger = LogManager.getLogger(IndexingPressure.class); private final AtomicLong currentCombinedCoordinatingAndPrimaryBytes = new AtomicLong(0); @@ -62,14 +105,22 @@ public class IndexingPressure { private final AtomicLong replicaRejections = new AtomicLong(0); private final AtomicLong primaryDocumentRejections = new AtomicLong(0); - private final long primaryAndCoordinatingLimits; - private final long splitBulkThreshold; - private final long replicaLimits; + private final long lowWatermark; + private final long lowWatermarkSize; + private final long highWatermark; + private final long highWatermarkSize; + private final long coordinatingLimit; + private final long primaryLimit; + private final long replicaLimit; public IndexingPressure(Settings settings) { - this.primaryAndCoordinatingLimits = MAX_INDEXING_BYTES.get(settings).getBytes(); - this.splitBulkThreshold = SPLIT_BULK_THRESHOLD.get(settings).getBytes(); - this.replicaLimits = (long) (this.primaryAndCoordinatingLimits * 1.5); + this.lowWatermark = SPLIT_BULK_LOW_WATERMARK.get(settings).getBytes(); + this.lowWatermarkSize = SPLIT_BULK_LOW_WATERMARK_SIZE.get(settings).getBytes(); + this.highWatermark = SPLIT_BULK_HIGH_WATERMARK.get(settings).getBytes(); + this.highWatermarkSize = SPLIT_BULK_HIGH_WATERMARK_SIZE.get(settings).getBytes(); + this.coordinatingLimit = MAX_COORDINATING_BYTES.get(settings).getBytes(); + this.primaryLimit = MAX_PRIMARY_BYTES.get(settings).getBytes(); + this.replicaLimit = MAX_REPLICA_BYTES.get(settings).getBytes(); } private static Releasable wrapReleasable(Releasable releasable) { @@ -88,7 +139,7 @@ public Releasable markCoordinatingOperationStarted(int operations, long bytes, b long combinedBytes = this.currentCombinedCoordinatingAndPrimaryBytes.addAndGet(bytes); long replicaWriteBytes = this.currentReplicaBytes.get(); long totalBytes = combinedBytes + replicaWriteBytes; - if (forceExecution == false && totalBytes > primaryAndCoordinatingLimits) { + if (forceExecution == false && totalBytes > coordinatingLimit) { long bytesWithoutOperation = combinedBytes - bytes; long totalBytesWithoutOperation = totalBytes - bytes; this.currentCombinedCoordinatingAndPrimaryBytes.getAndAdd(-bytes); @@ -107,8 +158,8 @@ public Releasable markCoordinatingOperationStarted(int operations, long bytes, b + "coordinating_operation_bytes=" + bytes + ", " - + "max_coordinating_and_primary_bytes=" - + primaryAndCoordinatingLimits + + "max_coordinating_bytes=" + + coordinatingLimit + "]", false ); @@ -143,7 +194,7 @@ public Releasable markPrimaryOperationStarted(int operations, long bytes, boolea long combinedBytes = this.currentCombinedCoordinatingAndPrimaryBytes.addAndGet(bytes); long replicaWriteBytes = this.currentReplicaBytes.get(); long totalBytes = combinedBytes + replicaWriteBytes; - if (forceExecution == false && totalBytes > primaryAndCoordinatingLimits) { + if (forceExecution == false && totalBytes > primaryLimit) { long bytesWithoutOperation = combinedBytes - bytes; long totalBytesWithoutOperation = totalBytes - bytes; this.currentCombinedCoordinatingAndPrimaryBytes.getAndAdd(-bytes); @@ -163,8 +214,8 @@ public Releasable markPrimaryOperationStarted(int operations, long bytes, boolea + "primary_operation_bytes=" + bytes + ", " - + "max_coordinating_and_primary_bytes=" - + primaryAndCoordinatingLimits + + "max_primary_bytes=" + + primaryLimit + "]", false ); @@ -185,7 +236,7 @@ public Releasable markPrimaryOperationStarted(int operations, long bytes, boolea public Releasable markReplicaOperationStarted(int operations, long bytes, boolean forceExecution) { long replicaWriteBytes = this.currentReplicaBytes.addAndGet(bytes); - if (forceExecution == false && replicaWriteBytes > replicaLimits) { + if (forceExecution == false && replicaWriteBytes > replicaLimit) { long replicaBytesWithoutOperation = replicaWriteBytes - bytes; this.currentReplicaBytes.getAndAdd(-bytes); this.replicaRejections.getAndIncrement(); @@ -198,7 +249,7 @@ public Releasable markReplicaOperationStarted(int operations, long bytes, boolea + bytes + ", " + "max_replica_bytes=" - + replicaLimits + + replicaLimit + "]", false ); @@ -212,11 +263,13 @@ public Releasable markReplicaOperationStarted(int operations, long bytes, boolea }); } - public boolean shouldSplitBulks() { - return currentCombinedCoordinatingAndPrimaryBytes.get() >= splitBulkThreshold; + public boolean shouldSplitBulk(long size) { + long currentUsage = (currentCombinedCoordinatingAndPrimaryBytes.get() + currentReplicaBytes.get()); + return (currentUsage >= lowWatermark && size >= lowWatermarkSize) || (currentUsage >= highWatermark && size >= highWatermarkSize); } public IndexingPressureStats stats() { + // TODO: Update stats with new primary/replica/coordinating limits and add throttling stats return new IndexingPressureStats( totalCombinedCoordinatingAndPrimaryBytes.get(), totalCoordinatingBytes.get(), @@ -229,7 +282,7 @@ public IndexingPressureStats stats() { coordinatingRejections.get(), primaryRejections.get(), replicaRejections.get(), - primaryAndCoordinatingLimits, + coordinatingLimit, totalCoordinatingOps.get(), totalPrimaryOps.get(), totalReplicaOps.get(), diff --git a/server/src/test/java/org/elasticsearch/index/IndexingPressureTests.java b/server/src/test/java/org/elasticsearch/index/IndexingPressureTests.java index 1b7a77e082da0..b4130120372a1 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexingPressureTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexingPressureTests.java @@ -10,14 +10,49 @@ package org.elasticsearch.index; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.Releasable; import org.elasticsearch.index.stats.IndexingPressureStats; import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; public class IndexingPressureTests extends ESTestCase { - private final Settings settings = Settings.builder().put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), "10KB").build(); + private final Settings settings = Settings.builder() + .put(IndexingPressure.MAX_COORDINATING_BYTES.getKey(), "10KB") + .put(IndexingPressure.MAX_PRIMARY_BYTES.getKey(), "12KB") + .put(IndexingPressure.MAX_REPLICA_BYTES.getKey(), "15KB") + .put(IndexingPressure.SPLIT_BULK_LOW_WATERMARK.getKey(), "8KB") + .put(IndexingPressure.SPLIT_BULK_LOW_WATERMARK_SIZE.getKey(), "1KB") + .put(IndexingPressure.SPLIT_BULK_HIGH_WATERMARK.getKey(), "9KB") + .put(IndexingPressure.SPLIT_BULK_HIGH_WATERMARK_SIZE.getKey(), "128B") + .build(); + + public void testMemoryLimitSettingsFallbackToOldSingleLimitSetting() { + Settings settings = Settings.builder().put(IndexingPressure.MAX_INDEXING_BYTES.getKey(), "20KB").build(); + + assertThat(IndexingPressure.MAX_COORDINATING_BYTES.get(settings), Matchers.equalTo(ByteSizeValue.ofKb(20))); + assertThat(IndexingPressure.MAX_PRIMARY_BYTES.get(settings), Matchers.equalTo(ByteSizeValue.ofKb(20))); + assertThat(IndexingPressure.MAX_REPLICA_BYTES.get(settings), Matchers.equalTo(ByteSizeValue.ofKb(30))); + } + + public void testHighAndLowWatermarkSettings() { + IndexingPressure indexingPressure = new IndexingPressure(settings); + + try ( + Releasable ignored1 = indexingPressure.markCoordinatingOperationStarted(10, ByteSizeValue.ofKb(6).getBytes(), false); + Releasable ignored2 = indexingPressure.markCoordinatingOperationStarted(10, ByteSizeValue.ofKb(2).getBytes(), false) + ) { + assertFalse(indexingPressure.shouldSplitBulk(randomIntBetween(1, 1000))); + assertTrue(indexingPressure.shouldSplitBulk(randomIntBetween(1025, 10000))); + + try (Releasable ignored3 = indexingPressure.markPrimaryOperationStarted(10, ByteSizeValue.ofKb(1).getBytes(), false)) { + assertFalse(indexingPressure.shouldSplitBulk(randomIntBetween(1, 127))); + assertTrue(indexingPressure.shouldSplitBulk(randomIntBetween(129, 1000))); + } + } + } public void testMemoryBytesAndOpsMarkedAndReleased() { IndexingPressure indexingPressure = new IndexingPressure(settings); @@ -95,7 +130,7 @@ public void testCoordinatingPrimaryRejections() { assertEquals(1, stats.getCoordinatingRejections()); assertEquals(1024 * 6, stats.getCurrentCombinedCoordinatingAndPrimaryBytes()); } else { - expectThrows(EsRejectedExecutionException.class, () -> indexingPressure.markPrimaryOperationStarted(1, 1024 * 2, false)); + expectThrows(EsRejectedExecutionException.class, () -> indexingPressure.markPrimaryOperationStarted(1, 1024 * 4, false)); IndexingPressureStats stats = indexingPressure.stats(); assertEquals(1, stats.getPrimaryRejections()); assertEquals(1024 * 6, stats.getCurrentCombinedCoordinatingAndPrimaryBytes()); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index cca3443c28e3a..5a40816c94beb 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -2064,7 +2064,10 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { randomFrom(1, 2, SearchRequest.DEFAULT_PRE_FILTER_SHARD_SIZE) ); if (randomBoolean()) { - builder.put(IndexingPressure.SPLIT_BULK_THRESHOLD.getKey(), randomFrom("256B", "1KB", "64KB")); + builder.put(IndexingPressure.SPLIT_BULK_LOW_WATERMARK.getKey(), randomFrom("256B", "512B")); + builder.put(IndexingPressure.SPLIT_BULK_LOW_WATERMARK_SIZE.getKey(), "1KB"); + builder.put(IndexingPressure.SPLIT_BULK_HIGH_WATERMARK.getKey(), randomFrom("1KB", "16KB", "64KB")); + builder.put(IndexingPressure.SPLIT_BULK_HIGH_WATERMARK_SIZE.getKey(), "256B"); } return builder.build(); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/LimitAwareBulkIndexer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/LimitAwareBulkIndexer.java index b69fc5944021c..ead18ea55abbf 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/LimitAwareBulkIndexer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/utils/persistence/LimitAwareBulkIndexer.java @@ -34,7 +34,7 @@ public class LimitAwareBulkIndexer implements AutoCloseable { private long currentRamBytes; public LimitAwareBulkIndexer(Settings settings, Consumer executor) { - this((long) Math.ceil(0.5 * IndexingPressure.MAX_INDEXING_BYTES.get(settings).getBytes()), executor); + this((long) Math.ceil(0.5 * IndexingPressure.MAX_COORDINATING_BYTES.get(settings).getBytes()), executor); } LimitAwareBulkIndexer(long bytesLimit, Consumer executor) { From c0fae087efa34697932cd359c1e87420fc215bc2 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 3 Oct 2024 02:25:03 +1000 Subject: [PATCH 013/194] Mute org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT test {p0=indices.create/20_synthetic_source/object array in object with dynamic override} #113966 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index cedbe6a24f33e..2c6d30369063f 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -347,6 +347,9 @@ tests: - class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT method: test {yaml=indices.create/20_synthetic_source/object array in object with dynamic override} issue: https://github.com/elastic/elasticsearch/issues/113962 +- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT + method: test {p0=indices.create/20_synthetic_source/object array in object with dynamic override} + issue: https://github.com/elastic/elasticsearch/issues/113966 # Examples: # From 35cdce5d10e526f75be9c429b6e077cff30de0a9 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Wed, 2 Oct 2024 19:29:37 +0300 Subject: [PATCH 014/194] Restore ClientYamlTestSuiteIT (#113965) --- muted-tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 2c6d30369063f..3e8c887a3fe6b 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -344,9 +344,6 @@ tests: - class: org.elasticsearch.kibana.KibanaThreadPoolIT method: testBlockedThreadPoolsRejectUserRequests issue: https://github.com/elastic/elasticsearch/issues/113939 -- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT - method: test {yaml=indices.create/20_synthetic_source/object array in object with dynamic override} - issue: https://github.com/elastic/elasticsearch/issues/113962 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=indices.create/20_synthetic_source/object array in object with dynamic override} issue: https://github.com/elastic/elasticsearch/issues/113966 From fca267ebc3a6a95a400d0ef2f2527550a37bdcb2 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 2 Oct 2024 18:40:12 +0200 Subject: [PATCH 015/194] Minimize Automaton used in APMTracer (#113775) This thing was taking up 400k of heap since it wasn't minimized, turns into ~2k after minimization. --- .../telemetry/apm/internal/tracing/APMTracer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java index ed2ce47d11dc3..8f1c0cf515e14 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java @@ -24,6 +24,7 @@ import org.apache.lucene.util.automaton.Automata; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.apache.lucene.util.automaton.MinimizationOperations; import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.Build; @@ -439,7 +440,7 @@ private static CharacterRunAutomaton buildAutomaton(List includePatterns ? includeAutomaton : Operations.minus(includeAutomaton, excludeAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); - return new CharacterRunAutomaton(finalAutomaton); + return new CharacterRunAutomaton(MinimizationOperations.minimize(finalAutomaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT)); } private static Automaton patternsToAutomaton(List patterns) { From c85c2d92040f4e6ca8ffb2b156f8ef2696326adb Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Wed, 2 Oct 2024 20:00:09 +0200 Subject: [PATCH 016/194] [DOCS][101] Update first quick start with mappings examples (#113558) --- docs/reference/mapping.asciidoc | 22 +- .../quickstart/getting-started.asciidoc | 407 ++++++++++++++++-- docs/reference/quickstart/index.asciidoc | 4 +- 3 files changed, 384 insertions(+), 49 deletions(-) diff --git a/docs/reference/mapping.asciidoc b/docs/reference/mapping.asciidoc index 239614345d782..5d6245a964104 100644 --- a/docs/reference/mapping.asciidoc +++ b/docs/reference/mapping.asciidoc @@ -75,9 +75,29 @@ reindexing. You can use runtime fields in conjunction with indexed fields to balance resource usage and performance. Your index will be smaller, but with slower search performance. +[discrete] +[[mapping-manage-update]] +== Managing and updating mappings + +Explicit mappings should be defined at index creation for fields you know in advance. +You can still add _new fields_ to mappings at any time, as your data evolves. + +Use the <> to update an existing mapping. + +In most cases, you can't change mappings for fields that are already mapped. +These changes require <>. + +However, you can _update_ mappings under certain conditions: + +* You can add new fields to an existing mapping at any time, explicitly or dynamically. +* You can add new <> for existing fields. +** Documents indexed before the mapping update will not have values for the new multi-fields until they are updated or reindexed. Documents indexed after the mapping change will automatically have values for the new multi-fields. +* Some <> can be updated for existing fields of certain <>. + [discrete] [[mapping-limit-settings]] -== Settings to prevent mapping explosion +== Prevent mapping explosions + Defining too many fields in an index can lead to a mapping explosion, which can cause out of memory errors and difficult situations to recover from. diff --git a/docs/reference/quickstart/getting-started.asciidoc b/docs/reference/quickstart/getting-started.asciidoc index e674dda147bcc..a6d233d8b8abc 100644 --- a/docs/reference/quickstart/getting-started.asciidoc +++ b/docs/reference/quickstart/getting-started.asciidoc @@ -1,83 +1,150 @@ [[getting-started]] -== Quick start: Add data using Elasticsearch APIs +== Index and search data using {es} APIs ++++ -Basics: Add data using APIs +Basics: Index and search using APIs ++++ -In this quick start guide, you'll learn how to do the following tasks: +This quick start guide is a hands-on introduction to the fundamental concepts of Elasticsearch: <>. -* Add a small, non-timestamped dataset to {es} using Elasticsearch REST APIs. -* Run basic searches. +You'll learn how to create an index, add data as documents, work with dynamic and explicit mappings, and perform your first basic searches. -[discrete] -[[add-data]] -=== Add data - -You add data to {es} as JSON objects called documents. -{es} stores these -documents in searchable indices. +[TIP] +==== +The code examples in this tutorial are in {kibana-ref}/console-kibana.html[Console] syntax by default. +You can {kibana-ref}/console-kibana.html#import-export-console-requests[convert into other programming languages] in the Console UI. +==== [discrete] -[[add-single-document]] -==== Add a single document +[[getting-started-prerequisites]] +=== Prerequisites -Submit the following indexing request to add a single document to the -`books` index. -The request automatically creates the index. +Before you begin, you need to have a running {es} cluster. +The fastest way to get started is with a <>. +Refer to <> for other deployment options. //// [source,console] ---- PUT books +PUT my-explicit-mappings-books ---- // TESTSETUP [source,console] -------------------------------------------------- DELETE books +DELETE my-explicit-mappings-books -------------------------------------------------- // TEARDOWN //// +[discrete] +[[getting-started-index-creation]] +=== Step 1: Create an index + +Create a new index named `books`: + +[source,console] +---- +PUT /books +---- +// TEST[skip: index already setup] + +The following response indicates the index was created successfully. + +.Example response +[%collapsible] +=============== +[source,console-result] +---- +{ + "acknowledged": true, + "shards_acknowledged": true, + "index": "books" +} +---- +// TEST[skip: index already setup] +=============== + +[discrete] +[[getting-started-add-documents]] +=== Step 2: Add data to your index + +[TIP] +==== +This tutorial uses {es} APIs, but there are many other ways to +<>. +==== + +You add data to {es} as JSON objects called documents. +{es} stores these +documents in searchable indices. + +[discrete] +[[getting-started-add-single-document]] +==== Add a single document + +Submit the following indexing request to add a single document to the +`books` index. + +[TIP] +==== +If the index didn't already exist, this request would automatically create it. +==== + [source,console] ---- POST books/_doc -{"name": "Snow Crash", "author": "Neal Stephenson", "release_date": "1992-06-01", "page_count": 470} +{ + "name": "Snow Crash", + "author": "Neal Stephenson", + "release_date": "1992-06-01", + "page_count": 470 +} ---- -// TEST[s/_doc/_doc?refresh=wait_for/] +// TEST[continued] -The response includes metadata that {es} generates for the document including a unique `_id` for the document within the index. +The response includes metadata that {es} generates for the document, including a unique `_id` for the document within the index. -.Expand to see example response +.Example response [%collapsible] =============== [source,console-result] ---- { - "_index": "books", - "_id": "O0lG2IsBaSa7VYx_rEia", - "_version": 1, - "result": "created", - "_shards": { - "total": 2, - "successful": 2, - "failed": 0 + "_index": "books", <1> + "_id": "O0lG2IsBaSa7VYx_rEia", <2> + "_version": 1, <3> + "result": "created", <4> + "_shards": { <5> + "total": 2, <6> + "successful": 2, <7> + "failed": 0 <8> }, - "_seq_no": 0, - "_primary_term": 1 + "_seq_no": 0, <9> + "_primary_term": 1 <10> } ---- -// TEST[skip:TODO] +// TEST[s/O0lG2IsBaSa7VYx_rEia/*/] +<1> The `_index` field indicates the index the document was added to. +<2> The `_id` field is the unique identifier for the document. +<3> The `_version` field indicates the version of the document. +<4> The `result` field indicates the result of the indexing operation. +<5> The `_shards` field contains information about the number of <> that the indexing operation was executed on and the number that succeeded. +<6> The `total` field indicates the total number of shards for the index. +<7> The `successful` field indicates the number of shards that the indexing operation was executed on. +<8> The `failed` field indicates the number of shards that failed during the indexing operation. '0' indicates no failures. +<9> The `_seq_no` field holds a monotonically increasing number incremented for each indexing operation on a shard. +<10> The `_primary_term` field is a monotonically increasing number incremented each time a primary shard is assigned to a different node. =============== [discrete] -[[add-multiple-documents]] +[[getting-started-add-multiple-documents]] ==== Add multiple documents -Use the `_bulk` endpoint to add multiple documents in one request. Bulk data -must be newline-delimited JSON (NDJSON). Each line must end in a newline -character (`\n`), including the last line. +Use the <> to add multiple documents in one request. Bulk data +must be formatted as newline-delimited JSON (NDJSON). [source,console] ---- @@ -97,7 +164,7 @@ POST /_bulk You should receive a response indicating there were no errors. -.Expand to see example response +.Example response [%collapsible] =============== [source,console-result] @@ -193,31 +260,218 @@ You should receive a response indicating there were no errors. =============== [discrete] -[[qs-search-data]] -=== Search data +[[getting-started-mappings-and-data-types]] +=== Step 3: Define mappings and data types + +<> define how data is stored and indexed in {es}, like a schema in a relational database. + +[discrete] +[[getting-started-dynamic-mapping]] +==== Use dynamic mapping + +When using dynamic mapping, {es} automatically creates mappings for new fields by default. +The documents we've added so far have used dynamic mapping, because we didn't specify a mapping when creating the index. + +To see how dynamic mapping works, add a new document to the `books` index with a field that doesn't appear in the existing documents. + +[source,console] +---- +POST /books/_doc +{ + "name": "The Great Gatsby", + "author": "F. Scott Fitzgerald", + "release_date": "1925-04-10", + "page_count": 180, + "language": "EN" <1> +} +---- +// TEST[continued] +<1> The new field. + +View the mapping for the `books` index with the <>. The new field `new_field` has been added to the mapping with a `text` data type. + +[source,console] +---- +GET /books/_mapping +---- +// TEST[continued] + +.Example response +[%collapsible] +=============== +[source,console-result] +---- +{ + "books": { + "mappings": { + "properties": { + "author": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "new_field": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "page_count": { + "type": "long" + }, + "release_date": { + "type": "date" + } + } + } + } +} +---- +// TEST[continued] +=============== + +[discrete] +[[getting-started-explicit-mapping]] +==== Define explicit mapping -Indexed documents are available for search in near real-time. +Create an index named `my-explicit-mappings-books` with explicit mappings. +Pass each field's properties as a JSON object. This object should contain the <> and any additional <>. + +[source,console] +---- +PUT /my-explicit-mappings-books +{ + "mappings": { + "dynamic": false, <1> + "properties": { <2> + "name": { "type": "text" }, + "author": { "type": "text" }, + "release_date": { "type": "date", "format": "yyyy-MM-dd" }, + "page_count": { "type": "integer" } + } + } +} +---- +// TEST[continued] +<1> Disables dynamic mapping for the index. Documents containing fields not defined in the mapping will be rejected. +<2> The `properties` object defines the fields and their data types for documents in this index. + +.Example response +[%collapsible] +=============== +[source,console-result] +---- +{ + "acknowledged": true, + "shards_acknowledged": true, + "index": "my-explicit-mappings-books" +} +---- +// TEST[skip:already created in setup] +=============== + +[discrete] +[[getting-started-combined-mapping]] +==== Combine dynamic and explicit mappings + +Explicit mappings are defined at index creation, and documents must conform to these mappings. +You can also use the <>. +When an index has the `dynamic` flag set to `true`, you can add new fields to documents without updating the mapping. + +This allows you to combine explicit and dynamic mappings. +Learn more about <>. + +[discrete] +[[getting-started-search-data]] +=== Step 4: Search your index + +Indexed documents are available for search in near real-time, using the <>. +// TODO: You'll find more detailed quick start guides in TODO [discrete] -[[search-all-documents]] +[[getting-started-search-all-documents]] ==== Search all documents Run the following command to search the `books` index for all documents: + [source,console] ---- GET books/_search ---- // TEST[continued] -The `_source` of each hit contains the original -JSON object submitted during indexing. +.Example response +[%collapsible] +=============== +[source,console-result] +---- +{ + "took": 2, <1> + "timed_out": false, <2> + "_shards": { <3> + "total": 5, + "successful": 5, + "skipped": 0, + "failed": 0 + }, + "hits": { <4> + "total": { <5> + "value": 7, + "relation": "eq" + }, + "max_score": 1, <6> + "hits": [ + { + "_index": "books", <7> + "_id": "CwICQpIBO6vvGGiC_3Ls", <8> + "_score": 1, <9> + "_source": { <10> + "name": "Brave New World", + "author": "Aldous Huxley", + "release_date": "1932-06-01", + "page_count": 268 + } + }, + ... (truncated) + ] + } +} +---- +// TEST[continued] +<1> The `took` field indicates the time in milliseconds for {es} to execute the search +<2> The `timed_out` field indicates whether the search timed out +<3> The `_shards` field contains information about the number of <> that the search was executed on and the number that succeeded +<4> The `hits` object contains the search results +<5> The `total` object provides information about the total number of matching documents +<6> The `max_score` field indicates the highest relevance score among all matching documents +<7> The `_index` field indicates the index the document belongs to +<8> The `_id` field is the document's unique identifier +<9> The `_score` field indicates the relevance score of the document +<10> The `_source` field contains the original JSON object submitted during indexing +=============== [discrete] -[[qs-match-query]] +[[getting-started-match-query]] ==== `match` query You can use the <> to search for documents that contain a specific value in a specific field. -This is the standard query for performing full-text search, including fuzzy matching and phrase searches. +This is the standard query for full-text searches. Run the following command to search the `books` index for documents containing `brave` in the `name` field: @@ -232,4 +486,65 @@ GET books/_search } } ---- -// TEST[continued] \ No newline at end of file +// TEST[continued] + +.Example response +[%collapsible] +=============== +[source,console-result] +---- +{ + "took": 9, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 0.6931471, <1> + "hits": [ + { + "_index": "books", + "_id": "CwICQpIBO6vvGGiC_3Ls", + "_score": 0.6931471, + "_source": { + "name": "Brave New World", + "author": "Aldous Huxley", + "release_date": "1932-06-01", + "page_count": 268 + } + } + ] + } +} +---- +// TEST[continued] +<1> The `max_score` is the score of the highest-scoring document in the results. In this case, there is only one matching document, so the `max_score` is the score of that document. +=============== + +[discrete] +[[getting-started-delete-indices]] +=== Step 5: Delete your indices (optional) + +When following along with examples, you might want to delete an index to start from scratch. +You can delete indices using the <>. + +For example, run the following command to delete the indices created in this tutorial: + +[source,console] +---- +DELETE /books +DELETE /my-explicit-mappings-books +---- +// TEST[skip:handled by setup/teardown] + +[CAUTION] +==== +Deleting an index permanently deletes its documents, shards, and metadata. +==== diff --git a/docs/reference/quickstart/index.asciidoc b/docs/reference/quickstart/index.asciidoc index 6bfed4c198c75..2d9114882254f 100644 --- a/docs/reference/quickstart/index.asciidoc +++ b/docs/reference/quickstart/index.asciidoc @@ -15,7 +15,7 @@ Get started <> , or see our <>. Learn how to add data to {es} and perform basic searches. +* <>. Learn about indices, documents, and mappings, and perform a basic search. [discrete] [[quickstart-python-links]] @@ -26,4 +26,4 @@ If you're interested in using {es} with Python, check out Elastic Search Labs: * https://github.com/elastic/elasticsearch-labs[`elasticsearch-labs` repository]: Contains a range of Python https://github.com/elastic/elasticsearch-labs/tree/main/notebooks[notebooks] and https://github.com/elastic/elasticsearch-labs/tree/main/example-apps[example apps]. * https://www.elastic.co/search-labs/tutorials/search-tutorial/welcome[Tutorial]: This walks you through building a complete search solution with {es} from the ground up using Flask. -include::getting-started.asciidoc[] \ No newline at end of file +include::getting-started.asciidoc[] From 43c15580b5ea9b569fbe98dd26911a0a7eff61be Mon Sep 17 00:00:00 2001 From: Brendan Cully Date: Wed, 2 Oct 2024 11:50:49 -0700 Subject: [PATCH 017/194] Add GetAliases test case (followup from #113571) (#113901) As requested in #113571, augmented testFindAliasWithExclusion to validate that exclusions work for multiple indices sharing an alias. --- .../cluster/metadata/MetadataTests.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index 4818291710a09..892b6aa3bf176 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -268,10 +268,19 @@ public void testFindAliasWithExclusion() { .putAlias(AliasMetadata.builder("alias1").build()) .putAlias(AliasMetadata.builder("alias2").build()) ) + .put( + IndexMetadata.builder("index2") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetadata.builder("alias1").build()) + .putAlias(AliasMetadata.builder("alias3").build()) + ) .build(); GetAliasesRequest request = new GetAliasesRequest().aliases("*", "-alias1"); - List aliases = metadata.findAliases(request.aliases(), new String[] { "index" }).get("index"); - assertThat(aliases, transformedItemsMatch(AliasMetadata::alias, contains("alias2"))); + Map> aliases = metadata.findAliases(request.aliases(), new String[] { "index", "index2" }); + assertThat(aliases.get("index"), transformedItemsMatch(AliasMetadata::alias, contains("alias2"))); + assertThat(aliases.get("index2"), transformedItemsMatch(AliasMetadata::alias, contains("alias3"))); } public void testFindDataStreams() { From 8db6a65758eaa97207b9211b49a1f173b185cfa1 Mon Sep 17 00:00:00 2001 From: Mikhail Berezovskiy Date: Wed, 2 Oct 2024 12:03:09 -0700 Subject: [PATCH 018/194] enable incremental bulk (#113971) Enable incremental bulk processing by default. --- .../org/elasticsearch/action/bulk/IncrementalBulkService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java b/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java index 25e58a82f8e8b..d8c2389dd7d69 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java @@ -36,7 +36,7 @@ public class IncrementalBulkService { public static final Setting INCREMENTAL_BULK = boolSetting( "rest.incremental_bulk", - false, + true, Setting.Property.NodeScope, Setting.Property.Dynamic ); From 1964d3c5ead26f7ad9acf9bf08de6b05cc9543a1 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 2 Oct 2024 20:59:39 +0100 Subject: [PATCH 019/194] Ensure that supported params are computed once (#113953) A few of today's REST handler implementations compute a new set of supported parameters on each request. This is needlessly inefficient since the set never changes. This commit fixes those implementations and adds assertions to verify that we are returning the exact same instance each time. --- .../rest/RestGetDataStreamsAction.java | 32 +++++++++++-------- .../elasticsearch/rest/BaseRestHandler.java | 1 + .../elasticsearch/rest/RestController.java | 2 ++ .../admin/cluster/RestClusterStatsAction.java | 5 +-- .../cluster/RestNodesCapabilitiesAction.java | 10 +++++- 5 files changed, 33 insertions(+), 17 deletions(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java index 23472b4830fa3..da55376fb403b 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java @@ -29,6 +29,23 @@ @ServerlessScope(Scope.PUBLIC) public class RestGetDataStreamsAction extends BaseRestHandler { + private static final Set SUPPORTED_QUERY_PARAMETERS = Set.copyOf( + Sets.union( + RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS, + Set.of( + "name", + "include_defaults", + "timeout", + "master_timeout", + IndicesOptions.WildcardOptions.EXPAND_WILDCARDS, + IndicesOptions.ConcreteTargetOptions.IGNORE_UNAVAILABLE, + IndicesOptions.WildcardOptions.ALLOW_NO_INDICES, + IndicesOptions.GatekeeperOptions.IGNORE_THROTTLED, + "verbose" + ) + ) + ); + @Override public String getName() { return "get_data_streams_action"; @@ -63,19 +80,6 @@ public Set supportedCapabilities() { @Override public Set supportedQueryParameters() { - return Sets.union( - RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS, - Set.of( - "name", - "include_defaults", - "timeout", - "master_timeout", - IndicesOptions.WildcardOptions.EXPAND_WILDCARDS, - IndicesOptions.ConcreteTargetOptions.IGNORE_UNAVAILABLE, - IndicesOptions.WildcardOptions.ALLOW_NO_INDICES, - IndicesOptions.GatekeeperOptions.IGNORE_THROTTLED, - "verbose" - ) - ); + return SUPPORTED_QUERY_PARAMETERS; } } diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java index 5f12a2bdd6783..99fa3e0166963 100644 --- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java @@ -85,6 +85,7 @@ public final long getUsageCount() { public final void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception { // check if the query has any parameters that are not in the supported set (if declared) Set supported = allSupportedParameters(); + assert supported == allSupportedParameters() : getName() + ": did not return same instance from allSupportedParameters()"; if (supported != null) { var allSupported = Sets.union( RestResponse.RESPONSE_PARAMS, diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java index 7fcc75ad65872..924cd361c671d 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestController.java +++ b/server/src/main/java/org/elasticsearch/rest/RestController.java @@ -397,6 +397,8 @@ public boolean checkSupported( if (handler != null) { var supportedParams = handler.supportedQueryParameters(); + assert supportedParams == handler.supportedQueryParameters() + : handler.getName() + ": did not return same instance from supportedQueryParameters()"; return (supportedParams == null || supportedParams.containsAll(parameters)) && handler.supportedCapabilities().containsAll(capabilities); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java index 014360477bee1..6379f6594e7c5 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java @@ -35,8 +35,9 @@ public class RestClusterStatsAction extends BaseRestHandler { "human-readable-total-docs-size", "verbose-dense-vector-mapping-stats" ); - private static final Set SUPPORTED_CAPABILITIES_CCS_STATS = Sets.union(SUPPORTED_CAPABILITIES, Set.of("ccs-stats")); + private static final Set SUPPORTED_CAPABILITIES_CCS_STATS = Set.copyOf(Sets.union(SUPPORTED_CAPABILITIES, Set.of("ccs-stats"))); public static final FeatureFlag CCS_TELEMETRY_FEATURE_FLAG = new FeatureFlag("ccs_telemetry"); + private static final Set SUPPORTED_QUERY_PARAMETERS = Set.of("include_remotes", "nodeId", REST_TIMEOUT_PARAM); @Override public List routes() { @@ -50,7 +51,7 @@ public String getName() { @Override public Set supportedQueryParameters() { - return Set.of("include_remotes", "nodeId", REST_TIMEOUT_PARAM); + return SUPPORTED_QUERY_PARAMETERS; } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesCapabilitiesAction.java index 5c8e5928678c3..fcc98c1b5caa1 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesCapabilitiesAction.java @@ -32,6 +32,14 @@ public class RestNodesCapabilitiesAction extends BaseRestHandler { public static final NodeFeature CAPABILITIES_ACTION = new NodeFeature("rest.capabilities_action"); public static final NodeFeature LOCAL_ONLY_CAPABILITIES = new NodeFeature("rest.local_only_capabilities"); + private static final Set SUPPORTED_QUERY_PARAMETERS = Set.of( + "timeout", + "method", + "path", + "parameters", + "capabilities", + "local_only" + ); @Override public List routes() { @@ -40,7 +48,7 @@ public List routes() { @Override public Set supportedQueryParameters() { - return Set.of("timeout", "method", "path", "parameters", "capabilities", "local_only"); + return SUPPORTED_QUERY_PARAMETERS; } @Override From 91dca8dab9f3478447de3b47a5a243a61bd68ee9 Mon Sep 17 00:00:00 2001 From: Fang Xing <155562079+fang-xing-esql@users.noreply.github.com> Date: Wed, 2 Oct 2024 16:27:03 -0400 Subject: [PATCH 020/194] [ES|QL] Validate index name in parser (#112081) * validate index name in parser --- docs/changelog/112081.yaml | 5 + .../xpack/esql/core/util/StringUtils.java | 1 + .../xpack/esql/parser/IdentifierBuilder.java | 50 ++++ .../parser/AbstractStatementParserTests.java | 13 + .../esql/parser/StatementParserTests.java | 227 +++++++++++++++++- 5 files changed, 284 insertions(+), 12 deletions(-) create mode 100644 docs/changelog/112081.yaml diff --git a/docs/changelog/112081.yaml b/docs/changelog/112081.yaml new file mode 100644 index 0000000000000..a4009e01fca71 --- /dev/null +++ b/docs/changelog/112081.yaml @@ -0,0 +1,5 @@ +pr: 112081 +summary: "[ES|QL] Validate index name in parser" +area: ES|QL +type: enhancement +issues: [] diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java index 1bfd94730c4fc..288612c9a593d 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java @@ -39,6 +39,7 @@ private StringUtils() {} public static final String NEW_LINE = "\n"; public static final String SQL_WILDCARD = "%"; public static final String WILDCARD = "*"; + public static final String EXCLUSION = "-"; private static final String[] INTEGER_ORDINALS = new String[] { "th", "st", "nd", "rd", "th", "th", "th", "th", "th", "th" }; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java index 9ccbb00ea4b5b..51c2eef3d75c3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/IdentifierBuilder.java @@ -8,7 +8,12 @@ package org.elasticsearch.xpack.esql.parser; import org.antlr.v4.runtime.tree.TerminalNode; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.common.Strings; +import org.elasticsearch.indices.InvalidIndexNameException; +import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.parser.EsqlBaseParser.IdentifierContext; import org.elasticsearch.xpack.esql.parser.EsqlBaseParser.IndexStringContext; @@ -16,6 +21,9 @@ import java.util.List; import static org.elasticsearch.transport.RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR; +import static org.elasticsearch.xpack.esql.core.util.StringUtils.EXCLUSION; +import static org.elasticsearch.xpack.esql.core.util.StringUtils.WILDCARD; +import static org.elasticsearch.xpack.esql.parser.ParserUtils.source; abstract class IdentifierBuilder extends AbstractBuilder { @@ -46,12 +54,54 @@ public String visitIndexString(IndexStringContext ctx) { public String visitIndexPattern(List ctx) { List patterns = new ArrayList<>(ctx.size()); + Holder hasSeenStar = new Holder<>(false); ctx.forEach(c -> { String indexPattern = visitIndexString(c.indexString()); + hasSeenStar.set(indexPattern.contains(WILDCARD) || hasSeenStar.get()); + validateIndexPattern(indexPattern, c, hasSeenStar.get()); patterns.add( c.clusterString() != null ? c.clusterString().getText() + REMOTE_CLUSTER_INDEX_SEPARATOR + indexPattern : indexPattern ); }); return Strings.collectionToDelimitedString(patterns, ","); } + + private static void validateIndexPattern(String indexPattern, EsqlBaseParser.IndexPatternContext ctx, boolean hasSeenStar) { + // multiple index names can be in the same double quote, e.g. indexPattern = "idx1, *, -idx2" + String[] indices = indexPattern.split(","); + boolean hasExclusion = false; + for (String index : indices) { + hasSeenStar = index.contains(WILDCARD) || hasSeenStar; + index = index.replace(WILDCARD, "").strip(); + if (index.isBlank()) { + continue; + } + hasExclusion = index.startsWith(EXCLUSION); + index = removeExclusion(index); + String tempName; + try { + // remove the exclusion outside of <>, from index names with DateMath expression, + // e.g. -<-logstash-{now/d}> becomes <-logstash-{now/d}> before calling resolveDateMathExpression + tempName = IndexNameExpressionResolver.resolveDateMathExpression(index); + } catch (ElasticsearchParseException e) { + // throws exception if the DateMath expression is invalid, resolveDateMathExpression does not complain about exclusions + throw new ParsingException(e, source(ctx), e.getMessage()); + } + hasExclusion = tempName.startsWith(EXCLUSION) || hasExclusion; + index = tempName.equals(index) ? index : removeExclusion(tempName); + try { + MetadataCreateIndexService.validateIndexOrAliasName(index, InvalidIndexNameException::new); + } catch (InvalidIndexNameException e) { + // ignore invalid index name if it has exclusions and there is an index with wildcard before it + if (hasSeenStar && hasExclusion) { + continue; + } + throw new ParsingException(e, source(ctx), e.getMessage()); + } + } + } + + private static String removeExclusion(String indexPattern) { + return indexPattern.charAt(0) == EXCLUSION.charAt(0) ? indexPattern.substring(1) : indexPattern; + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java index 697dfcf0a8e6b..e3574576e478f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.parser; +import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -38,6 +39,10 @@ void assertStatement(String statement, LogicalPlan expected) { assertThat(statement, actual, equalTo(expected)); } + LogicalPlan statement(String query, String arg) { + return statement(LoggerMessageFormat.format(null, query, arg), new QueryParams()); + } + LogicalPlan statement(String e) { return statement(e, new QueryParams()); } @@ -124,4 +129,12 @@ void expectError(String query, List params, String errorMessage) { ); assertThat(e.getMessage(), containsString(errorMessage)); } + + void expectInvalidIndexNameErrorWithLineNumber(String query, String arg, String lineNumber, String indexString) { + expectError(LoggerMessageFormat.format(null, query, arg), lineNumber + "Invalid index name [" + indexString); + } + + void expectDateMathErrorWithLineNumber(String query, String arg, String lineNumber, String error) { + expectError(LoggerMessageFormat.format(null, query, arg), lineNumber + error); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index c5a5bfac023c1..adf31ca983067 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -50,6 +50,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Row; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -377,24 +378,23 @@ public void testStringAsIndexPattern() { ",", command + " , \"\"" ); - + assertStringAsIndexPattern( + "-,-<-logstash-{now/M{yyyy.MM}}>," + + "-,-<-logstash-{now/d{yyyy.MM.dd|+12:00}}>", + command + + " -, -<-logstash-{now/M{yyyy.MM}}>, " + + "\"-\", \"-<-logstash-{now/d{yyyy.MM.dd|+12:00}}>\"" + ); assertStringAsIndexPattern("foo,test,xyz", command + " \"\"\"foo\"\"\", test,\"xyz\""); - assertStringAsIndexPattern("`backtick`,``multiple`back``ticks```", command + " `backtick`, ``multiple`back``ticks```"); - assertStringAsIndexPattern("test,metadata,metaata,.metadata", command + " test,\"metadata\", metaata, .metadata"); - assertStringAsIndexPattern(".dot", command + " .dot"); - assertStringAsIndexPattern("cluster:index", command + " cluster:index"); - assertStringAsIndexPattern("cluster:index|pattern", command + " cluster:\"index|pattern\""); assertStringAsIndexPattern("cluster:.index", command + " cluster:.index"); assertStringAsIndexPattern("cluster*:index*", command + " cluster*:index*"); assertStringAsIndexPattern("cluster*:*", command + " cluster*:*"); assertStringAsIndexPattern("*:index*", command + " *:index*"); - assertStringAsIndexPattern("*:index|pattern", command + " *:\"index|pattern\""); assertStringAsIndexPattern("*:*", command + " *:*"); - assertStringAsIndexPattern("*:*,cluster*:index|pattern,i|p", command + " *:*, cluster*:\"index|pattern\", \"i|p\""); } } @@ -416,19 +416,222 @@ public void testStringAsLookupIndexPattern() { ); assertStringAsLookupIndexPattern("foo", "ROW x = 1 | LOOKUP \"\"\"foo\"\"\" ON j"); - assertStringAsLookupIndexPattern("`backtick`", "ROW x = 1 | LOOKUP `backtick` ON j"); assertStringAsLookupIndexPattern("``multiple`back``ticks```", "ROW x = 1 | LOOKUP ``multiple`back``ticks``` ON j"); - assertStringAsLookupIndexPattern(".dot", "ROW x = 1 | LOOKUP .dot ON j"); - assertStringAsLookupIndexPattern("cluster:index", "ROW x = 1 | LOOKUP cluster:index ON j"); assertStringAsLookupIndexPattern("cluster:.index", "ROW x = 1 | LOOKUP cluster:.index ON j"); assertStringAsLookupIndexPattern("cluster*:index*", "ROW x = 1 | LOOKUP cluster*:index* ON j"); assertStringAsLookupIndexPattern("cluster*:*", "ROW x = 1 | LOOKUP cluster*:* ON j"); assertStringAsLookupIndexPattern("*:index*", "ROW x = 1 | LOOKUP *:index* ON j"); assertStringAsLookupIndexPattern("*:*", "ROW x = 1 | LOOKUP *:* ON j"); + } + public void testInvalidCharacterInIndexPattern() { + Map commands = new HashMap<>(); + commands.put("FROM {}", "line 1:8: "); + if (Build.current().isSnapshot()) { + commands.put("METRICS {}", "line 1:11: "); + commands.put("ROW x = 1 | LOOKUP {} ON j", "line 1:22: "); + } + List clusterStrings = List.of(" ", " *:", " cluster:"); + String lineNumber; + for (String command : commands.keySet()) { + lineNumber = commands.get(command); + for (String clusterString : clusterStrings) { + expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index|pattern\"", lineNumber, "index|pattern"); + expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index pattern\"", lineNumber, "index pattern"); + expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index#pattern\"", lineNumber, "index#pattern"); + expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "index#pattern", lineNumber, "index#pattern"); + expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index?pattern\"", lineNumber, "index?pattern"); + expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "index?pattern", lineNumber, "index?pattern"); + expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index>pattern\"", lineNumber, "index>pattern"); + expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "index>pattern", lineNumber, "index>pattern"); + expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"index\"", + lineNumber, + "-logstash-" + ); + expectInvalidIndexNameErrorWithLineNumber( + command, + clusterString + "--", + lineNumber, + "-" + ); + expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"\"", lineNumber, "logstash#"); + expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "", lineNumber, "logstash#"); + expectInvalidIndexNameErrorWithLineNumber( + command, + clusterString + "\"+\"", + lineNumber, + "+" + ); + expectInvalidIndexNameErrorWithLineNumber( + command, + clusterString + "+", + lineNumber, + "+" + ); + expectInvalidIndexNameErrorWithLineNumber( + command, + clusterString + "\"_\"", + lineNumber, + "_" + ); + expectInvalidIndexNameErrorWithLineNumber( + command, + clusterString + "_", + lineNumber, + "_" + ); + expectInvalidIndexNameErrorWithLineNumber(command, clusterString + "\"<>\"", lineNumber, ">", lineNumber, ">>\"", lineNumber, ">>", lineNumber, "\"", + lineNumber, + "logstash- " + ); + } + } + + // comma separated indices + // Invalid index names after removing exclusion fail, when there is no index name with wildcard before it + for (String command : commands.keySet()) { + if (command.contains("LOOKUP")) { + continue; + } + for (String clusterString : clusterStrings) { + lineNumber = command.contains("FROM") + ? "line 1:" + (22 + clusterString.length() - 1) + ": " + : "line 1:" + (25 + clusterString.length() - 1) + ": "; + expectInvalidIndexNameErrorWithLineNumber( + command, + clusterString + "indexpattern, --indexpattern", + lineNumber, + "-indexpattern" + ); + expectInvalidIndexNameErrorWithLineNumber( + command, + clusterString + "indexpattern, \"--indexpattern\"", + lineNumber, + "-indexpattern" + ); + expectInvalidIndexNameErrorWithLineNumber( + command, + clusterString + "\"indexpattern, --indexpattern\"", + commands.get(command), + "-indexpattern" + ); + assertEquals( + unresolvedRelation(clusterString.strip() + "indexpattern,-indexpattern"), + statement(command, clusterString + "indexpattern, -indexpattern") + ); + assertEquals( + unresolvedRelation(clusterString.strip() + "indexpattern,-indexpattern"), + statement(command, clusterString + "indexpattern, \"-indexpattern\"") + ); + assertEquals( + unresolvedRelation(clusterString.strip() + "indexpattern, -indexpattern"), + statement(command, clusterString + "\"indexpattern, -indexpattern\"") + ); + } + } + + // Invalid index names, except invalid DateMath, are ignored if there is an index name with wildcard before it + for (String command : commands.keySet()) { + if (command.contains("LOOKUP")) { + continue; + } + for (String clusterString : clusterStrings) { + lineNumber = command.contains("FROM") + ? "line 1:" + (11 + clusterString.length() - 1) + ": " + : "line 1:" + (14 + clusterString.length() - 1) + ": "; + assertEquals( + unresolvedRelation(clusterString.strip() + "*,-index#pattern"), + statement(command, clusterString + "*, \"-index#pattern\"") + ); + assertEquals( + unresolvedRelation(clusterString.strip() + "*,-index#pattern"), + statement(command, clusterString + "*, -index#pattern") + ); + assertEquals( + unresolvedRelation(clusterString.strip() + "*, -index#pattern"), + statement(command, clusterString + "\"*, -index#pattern\"") + ); + assertEquals( + unresolvedRelation(clusterString.strip() + "index*,-index#pattern"), + statement(command, clusterString + "index*, \"-index#pattern\"") + ); + assertEquals( + unresolvedRelation(clusterString.strip() + "index*,-index#pattern"), + statement(command, clusterString + "index*, -index#pattern") + ); + assertEquals( + unresolvedRelation(clusterString.strip() + "index*, -index#pattern"), + statement(command, clusterString + "\"index*, -index#pattern\"") + ); + assertEquals( + unresolvedRelation(clusterString.strip() + "*,-<--logstash-{now/M{yyyy.MM}}>"), + statement(command, clusterString + "*, \"-<--logstash-{now/M{yyyy.MM}}>\"") + ); + assertEquals( + unresolvedRelation(clusterString.strip() + "*,-<--logstash-{now/M{yyyy.MM}}>"), + statement(command, clusterString + "*, -<--logstash-{now/M{yyyy.MM}}>") + ); + assertEquals( + unresolvedRelation(clusterString.strip() + "*, -<--logstash-{now/M{yyyy.MM}}>"), + statement(command, clusterString + "\"*, -<--logstash-{now/M{yyyy.MM}}>\"") + ); + assertEquals( + unresolvedRelation(clusterString.strip() + "index*,-<--logstash#-{now/M{yyyy.MM}}>"), + statement(command, clusterString + "index*, \"-<--logstash#-{now/M{yyyy.MM}}>\"") + ); + assertEquals( + unresolvedRelation(clusterString.strip() + "index*,-<--logstash#-{now/M{yyyy.MM}}>"), + statement(command, clusterString + "index*, -<--logstash#-{now/M{yyyy.MM}}>") + ); + assertEquals( + unresolvedRelation(clusterString.strip() + "index*, -<--logstash#-{now/M{yyyy.MM}}>"), + statement(command, clusterString + "\"index*, -<--logstash#-{now/M{yyyy.MM}}>\"") + ); + expectDateMathErrorWithLineNumber( + command, + clusterString + "*, \"-<-logstash-{now/D}>\"", + lineNumber, + "unit [D] not supported for date math [/D]" + ); + expectDateMathErrorWithLineNumber( + command, + clusterString + "*, -<-logstash-{now/D}>", + lineNumber, + "unit [D] not supported for date math [/D]" + ); + expectDateMathErrorWithLineNumber( + command, + clusterString + "\"*, -<-logstash-{now/D}>\"", + commands.get(command), + "unit [D] not supported for date math [/D]" + ); + } + } } public void testInvalidQuotingAsFromIndexPattern() { @@ -1569,7 +1772,7 @@ public void testMetricsIdentifiers() { Map.entry("metrics foo,test-*", "foo,test-*"), Map.entry("metrics 123-test@foo_bar+baz1", "123-test@foo_bar+baz1"), Map.entry("metrics foo, test,xyz", "foo,test,xyz"), - Map.entry("metrics >", ">") + Map.entry("metrics ", "") ); for (Map.Entry e : patterns.entrySet()) { assertStatement(e.getKey(), unresolvedRelation(e.getValue())); From 2f3bf74e6e7018a87f88145a27971bbc3966e960 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Wed, 2 Oct 2024 17:15:55 -0400 Subject: [PATCH 021/194] Revert semantic query passage ranking documentation (#113982) --- .../query-dsl/semantic-query.asciidoc | 200 ------------------ 1 file changed, 200 deletions(-) diff --git a/docs/reference/query-dsl/semantic-query.asciidoc b/docs/reference/query-dsl/semantic-query.asciidoc index f3f6aca3fd07a..11e19d6356081 100644 --- a/docs/reference/query-dsl/semantic-query.asciidoc +++ b/docs/reference/query-dsl/semantic-query.asciidoc @@ -40,209 +40,9 @@ The `semantic_text` field to perform the query on. (Required, string) The query text to be searched for on the field. -`inner_hits`:: -(Optional, object) -Retrieves the specific passages that match the query. -See <> for more information. -+ -.Properties of `inner_hits` -[%collapsible%open] -==== -`from`:: -(Optional, integer) -The offset from the first matching passage to fetch. -Used to paginate through the passages. -Defaults to `0`. - -`size`:: -(Optional, integer) -The maximum number of matching passages to return. -Defaults to `3`. -==== Refer to <> to learn more about semantic search using `semantic_text` and `semantic` query. -[discrete] -[[semantic-query-passage-ranking]] -==== Passage ranking with the `semantic` query -The `inner_hits` parameter can be used for _passage ranking_, which allows you to determine which passages in the document best match the query. -For example, if you have a document that covers varying topics: - -[source,console] ------------------------------------------------------------- -POST my-index/_doc/lake_tahoe -{ - "inference_field": [ - "Lake Tahoe is the largest alpine lake in North America", - "When hiking in the area, please be on alert for bears" - ] -} ------------------------------------------------------------- -// TEST[skip: Requires inference endpoints] - -You can use passage ranking to find the passage that best matches your query: - -[source,console] ------------------------------------------------------------- -GET my-index/_search -{ - "query": { - "semantic": { - "field": "inference_field", - "query": "mountain lake", - "inner_hits": { } - } - } -} ------------------------------------------------------------- -// TEST[skip: Requires inference endpoints] - -[source,console-result] ------------------------------------------------------------- -{ - "took": 67, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": { - "value": 1, - "relation": "eq" - }, - "max_score": 10.844536, - "hits": [ - { - "_index": "my-index", - "_id": "lake_tahoe", - "_score": 10.844536, - "_source": { - ... - }, - "inner_hits": { <1> - "inference_field": { - "hits": { - "total": { - "value": 2, - "relation": "eq" - }, - "max_score": 10.844536, - "hits": [ - { - "_index": "my-index", - "_id": "lake_tahoe", - "_nested": { - "field": "inference_field.inference.chunks", - "offset": 0 - }, - "_score": 10.844536, - "_source": { - "text": "Lake Tahoe is the largest alpine lake in North America" - } - }, - { - "_index": "my-index", - "_id": "lake_tahoe", - "_nested": { - "field": "inference_field.inference.chunks", - "offset": 1 - }, - "_score": 3.2726858, - "_source": { - "text": "When hiking in the area, please be on alert for bears" - } - } - ] - } - } - } - } - ] - } -} ------------------------------------------------------------- -<1> Ranked passages will be returned using the <>, with `` set to the `semantic_text` field name. - -By default, the top three matching passages will be returned. -You can use the `size` parameter to control the number of passages returned and the `from` parameter to page through the matching passages: - -[source,console] ------------------------------------------------------------- -GET my-index/_search -{ - "query": { - "semantic": { - "field": "inference_field", - "query": "mountain lake", - "inner_hits": { - "from": 1, - "size": 1 - } - } - } -} ------------------------------------------------------------- -// TEST[skip: Requires inference endpoints] - -[source,console-result] ------------------------------------------------------------- -{ - "took": 42, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": { - "value": 1, - "relation": "eq" - }, - "max_score": 10.844536, - "hits": [ - { - "_index": "my-index", - "_id": "lake_tahoe", - "_score": 10.844536, - "_source": { - ... - }, - "inner_hits": { - "inference_field": { - "hits": { - "total": { - "value": 2, - "relation": "eq" - }, - "max_score": 10.844536, - "hits": [ - { - "_index": "my-index", - "_id": "lake_tahoe", - "_nested": { - "field": "inference_field.inference.chunks", - "offset": 1 - }, - "_score": 3.2726858, - "_source": { - "text": "When hiking in the area, please be on alert for bears" - } - } - ] - } - } - } - } - ] - } -} ------------------------------------------------------------- - [discrete] [[hybrid-search-semantic]] ==== Hybrid search with the `semantic` query From 72c164ce191512d632489eb2b9e2bfc864dc0095 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 3 Oct 2024 07:50:13 +1000 Subject: [PATCH 022/194] Mute org.elasticsearch.xpack.inference.TextEmbeddingCrudIT testPutE5Small_withPlatformAgnosticVariant #113983 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 3e8c887a3fe6b..7c1f40e5e5639 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -347,6 +347,9 @@ tests: - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=indices.create/20_synthetic_source/object array in object with dynamic override} issue: https://github.com/elastic/elasticsearch/issues/113966 +- class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT + method: testPutE5Small_withPlatformAgnosticVariant + issue: https://github.com/elastic/elasticsearch/issues/113983 # Examples: # From 6d7ae82180fa30ee4b48bd58721cae2cda809b15 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 3 Oct 2024 06:44:28 +0100 Subject: [PATCH 023/194] Assert that REST params are consumed iff supported (#113933) REST APIs which declare their supported parameters must consume exactly those parameters: consuming an unsupported parameter means that requests including that parameter will be rejected, whereas failing to consume a supported parameter means that this parameter has no effect and should be removed. This commit adds an assertion to verify that we are consuming the correct parameters. Closes #113854 --- .../rest/RestGetDataStreamsAction.java | 3 ++- .../org/elasticsearch/rest/BaseRestHandler.java | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java index da55376fb403b..7a27eddfaf8c7 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.set.Sets; @@ -35,12 +36,12 @@ public class RestGetDataStreamsAction extends BaseRestHandler { Set.of( "name", "include_defaults", - "timeout", "master_timeout", IndicesOptions.WildcardOptions.EXPAND_WILDCARDS, IndicesOptions.ConcreteTargetOptions.IGNORE_UNAVAILABLE, IndicesOptions.WildcardOptions.ALLOW_NO_INDICES, IndicesOptions.GatekeeperOptions.IGNORE_THROTTLED, + DataStream.isFailureStoreFeatureFlagEnabled() ? IndicesOptions.FailureStoreOptions.FAILURE_STORE : "name", "verbose" ) ) diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java index 99fa3e0166963..2f7bb80a8d46a 100644 --- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.RestApiVersion; @@ -104,6 +105,8 @@ public final void handleRequest(RestRequest request, RestChannel channel, NodeCl // prepare the request for execution; has the side effect of touching the request parameters try (var action = prepareRequest(request, client)) { + assert assertConsumesSupportedParams(supported, request); + // validate unconsumed params, but we must exclude params used to format the response // use a sorted set so the unconsumed parameters appear in a reliable sorted order final SortedSet unconsumedParams = request.unconsumedParams() @@ -148,6 +151,20 @@ public void close() { } } + private boolean assertConsumesSupportedParams(@Nullable Set supported, RestRequest request) { + if (supported != null) { + final var supportedAndCommon = new TreeSet<>(supported); + supportedAndCommon.add("error_trace"); + supportedAndCommon.addAll(ALWAYS_SUPPORTED); + supportedAndCommon.removeAll(RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS); + final var consumed = new TreeSet<>(request.consumedParams()); + consumed.removeAll(RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS); + assert supportedAndCommon.equals(consumed) + : getName() + ": consumed params " + consumed + " while supporting " + supportedAndCommon; + } + return true; + } + protected static String unrecognized(RestRequest request, Set invalids, Set candidates, String detail) { StringBuilder message = new StringBuilder().append("request [") .append(request.path()) From 539d4fdff56492ca7f6b8e59384006650a1ed946 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Thu, 3 Oct 2024 09:24:20 +0300 Subject: [PATCH 024/194] Restore 20_synthetic_source/object array in object with dynamic override (#113990) Fixes #113966 --- muted-tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 7c1f40e5e5639..68422264221f2 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -344,9 +344,6 @@ tests: - class: org.elasticsearch.kibana.KibanaThreadPoolIT method: testBlockedThreadPoolsRejectUserRequests issue: https://github.com/elastic/elasticsearch/issues/113939 -- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT - method: test {p0=indices.create/20_synthetic_source/object array in object with dynamic override} - issue: https://github.com/elastic/elasticsearch/issues/113966 - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT method: testPutE5Small_withPlatformAgnosticVariant issue: https://github.com/elastic/elasticsearch/issues/113983 From dc8c20d3b63dc667f20db14b87f16e3dc2db7b8a Mon Sep 17 00:00:00 2001 From: Panagiotis Bailis Date: Thu, 3 Oct 2024 12:39:13 +0300 Subject: [PATCH 025/194] Rework RRF to be evaluated during rewrite phase (#112648) --- docs/reference/search/rrf.asciidoc | 7 +- .../retriever/RankDocRetrieverBuilderIT.java | 90 +- .../org/elasticsearch/TransportVersions.java | 1 + .../action/search/TransportSearchAction.java | 2 + .../elasticsearch/common/lucene/Lucene.java | 3 - .../uhighlight/CustomUnifiedHighlighter.java | 2 +- .../elasticsearch/search/SearchModule.java | 2 - .../search/builder/SearchSourceBuilder.java | 23 +- .../search/fetch/StoredFieldsContext.java | 2 +- .../search/fetch/subphase/ExplainPhase.java | 12 +- .../elasticsearch/search/rank/RankDoc.java | 23 +- .../search/rank/feature/RankFeatureDoc.java | 2 +- .../retriever/CompoundRetrieverBuilder.java | 255 ++++ .../search/retriever/KnnRetrieverBuilder.java | 28 +- .../retriever/RankDocsRetrieverBuilder.java | 102 +- .../search/retriever/RetrieverBuilder.java | 61 +- .../retriever/StandardRetrieverBuilder.java | 49 +- .../retriever/rankdoc/RankDocsQuery.java | 396 ++++-- .../rankdoc/RankDocsQueryBuilder.java | 60 +- .../rankdoc/RankDocsSortBuilder.java | 114 -- .../retriever/rankdoc/RankDocsSortField.java | 102 -- .../search/sort/ShardDocSortField.java | 14 + .../action/search/SearchRequestTests.java | 87 +- .../search/rank/RankDocTests.java | 5 - .../KnnRetrieverBuilderParsingTests.java | 19 +- .../RankDocsRetrieverBuilderTests.java | 74 +- .../retriever/RetrieverBuilderErrorTests.java | 18 +- .../rankdoc/RankDocsQueryBuilderTests.java | 119 +- .../rankdoc/RankDocsSortBuilderTests.java | 72 -- .../TestCompoundRetrieverBuilder.java | 52 + .../TextSimilarityRankRetrieverBuilder.java | 10 +- ...xtSimilarityRankRetrieverBuilderTests.java | 3 +- .../xpack/rank/rrf/RRFRetrieverBuilderIT.java | 656 ++++++++++ .../rrf/RRFRetrieverBuilderNestedDocsIT.java | 171 +++ .../xpack/rank/rrf/RRFFeatures.java | 4 +- .../RRFQueryPhaseRankCoordinatorContext.java | 6 +- .../rrf/RRFQueryPhaseRankShardContext.java | 6 +- .../xpack/rank/rrf/RRFRankDoc.java | 57 +- .../xpack/rank/rrf/RRFRetrieverBuilder.java | 150 ++- .../xpack/rank/rrf/RRFRankContextTests.java | 108 +- .../xpack/rank/rrf/RRFRankDocTests.java | 77 +- .../rrf/RRFRetrieverBuilderParsingTests.java | 18 +- .../rank/rrf/RRFRetrieverBuilderTests.java | 87 +- .../rest-api-spec/test/rrf/100_rank_rrf.yml | 8 - .../test/rrf/150_rank_rrf_pagination.yml | 15 - .../test/rrf/200_rank_rrf_script.yml | 20 - .../test/rrf/300_rrf_retriever.yml | 87 +- .../test/rrf/350_rrf_retriever_pagination.yml | 1112 +++++++++++++++++ .../test/rrf/400_rrf_retriever_script.yml | 25 +- .../test/rrf/500_rrf_retriever_explain.yml | 16 +- .../test/rrf/600_rrf_retriever_profile.yml | 36 +- ...rrf_retriever_search_api_compatibility.yml | 541 ++++++++ 52 files changed, 3902 insertions(+), 1107 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java delete mode 100644 server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortBuilder.java delete mode 100644 server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortField.java delete mode 100644 server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortBuilderTests.java create mode 100644 test/framework/src/main/java/org/elasticsearch/search/retriever/TestCompoundRetrieverBuilder.java create mode 100644 x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java create mode 100644 x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java create mode 100644 x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/350_rrf_retriever_pagination.yml create mode 100644 x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml diff --git a/docs/reference/search/rrf.asciidoc b/docs/reference/search/rrf.asciidoc index 2525dfff23b94..2a676e5fba336 100644 --- a/docs/reference/search/rrf.asciidoc +++ b/docs/reference/search/rrf.asciidoc @@ -300,13 +300,12 @@ We have both the ranker's `score` and the `_rank` option to show our top-ranked "value" : 5, "relation" : "eq" }, - "max_score" : null, + "max_score" : ..., "hits" : [ { "_index" : "example-index", "_id" : "3", "_score" : 0.8333334, - "_rank" : 1, "_source" : { "integer" : 1, "vector" : [ @@ -319,7 +318,6 @@ We have both the ranker's `score` and the `_rank` option to show our top-ranked "_index" : "example-index", "_id" : "2", "_score" : 0.5833334, - "_rank" : 2, "_source" : { "integer" : 2, "vector" : [ @@ -332,7 +330,6 @@ We have both the ranker's `score` and the `_rank` option to show our top-ranked "_index" : "example-index", "_id" : "4", "_score" : 0.5, - "_rank" : 3, "_source" : { "integer" : 2, "text" : "rrf rrf rrf rrf" @@ -499,7 +496,6 @@ Working with the example above, and by adding `explain=true` to the search reque "_index": "example-index", "_id": "3", "_score": 0.8333334, - "_rank": 1, "_explanation": { "value": 0.8333334, <1> @@ -608,7 +604,6 @@ The response would now include the named query in the explanation: "_index": "example-index", "_id": "3", "_score": 0.8333334, - "_rank": 1, "_explanation": { "value": 0.8333334, diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java index 26af82cf021f2..891096dfa67a9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java @@ -36,6 +36,7 @@ import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; +import org.elasticsearch.search.sort.ShardDocSortField; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; @@ -189,8 +190,10 @@ public void testRankDocsRetrieverBasicWithPagination() { SearchSourceBuilder source = new SearchSourceBuilder(); StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD)) - .boost(10L); + standard0.queryBuilder = QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(9L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(8L)); StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); // this one retrieves docs 2 and 6 due to prefilter standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); @@ -205,8 +208,8 @@ public void testRankDocsRetrieverBasicWithPagination() { null ); // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 4, 7, 3 and with pagination, we'd just omit the first result + // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results) + // so ideal rank would be: 6, 2, 1, 3, 4, 7 and with pagination, we'd just omit the first result source.retriever( new CompoundRetrieverWithRankDocs( rankWindowSize, @@ -227,9 +230,9 @@ public void testRankDocsRetrieverBasicWithPagination() { assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2")); assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_4")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_7")); - assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_3")); + assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_3")); + assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_4")); + assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_7")); }); } @@ -242,8 +245,10 @@ public void testRankDocsRetrieverWithAggs() { SearchSourceBuilder source = new SearchSourceBuilder(); StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD)) - .boost(10L); + standard0.queryBuilder = QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(9L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(8L)); StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); // this one retrieves docs 2 and 6 due to prefilter standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); @@ -267,13 +272,15 @@ public void testRankDocsRetrieverWithAggs() { ) ) ); + source.size(1); source.aggregation(new TermsAggregationBuilder("topic").field(TOPIC_FIELD)); SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); ElasticsearchAssertions.assertResponse(req, resp -> { assertNull(resp.pointInTimeId()); assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(1L)); + assertThat(resp.getHits().getTotalHits().value, equalTo(5L)); assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); + assertThat(resp.getHits().getHits().length, equalTo(1)); assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2")); assertNotNull(resp.getAggregations()); assertNotNull(resp.getAggregations().get("topic")); @@ -291,8 +298,10 @@ public void testRankDocsRetrieverWithCollapse() { SearchSourceBuilder source = new SearchSourceBuilder(); StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD)) - .boost(10L); + standard0.queryBuilder = QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(9L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(8L)); StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); // this one retrieves docs 2 and 6 due to prefilter standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); @@ -307,8 +316,8 @@ public void testRankDocsRetrieverWithCollapse() { null ); // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 4, 7, 3 + // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results) + // so ideal rank would be: 6, 2, 1, 3, 4, 7 // with collapsing on topic field we would have 6, 2, 1, 7 source.retriever( new CompoundRetrieverWithRankDocs( @@ -338,7 +347,6 @@ public void testRankDocsRetrieverWithCollapse() { assertThat(resp.getHits().getAt(1).field(TOPIC_FIELD).getValue().toString(), equalTo("astronomy")); assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_1")); assertThat(resp.getHits().getAt(2).field(TOPIC_FIELD).getValue().toString(), equalTo("technology")); - assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getHits().length, equalTo(3)); assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(0).getId(), equalTo("doc_4")); assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(1).getId(), equalTo("doc_3")); assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(2).getId(), equalTo("doc_1")); @@ -347,17 +355,15 @@ public void testRankDocsRetrieverWithCollapse() { }); } - public void testRankDocsRetrieverWithCollapseAndAggs() { - // same as above, but we only want to bring back the top result from each subsearch - // so that would be 1, 2, and 7 - // and final rank would be (based on score): 2, 1, 7 - // aggs should still account for the same docs as the testRankDocsRetriever test, i.e. all but doc_5 + public void testRankDocsRetrieverWithNestedCollapseAndAggs() { final int rankWindowSize = 10; SearchSourceBuilder source = new SearchSourceBuilder(); StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); // this one retrieves docs 1 and 6 as doc_4 is collapsed to doc_1 - standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD)) - .boost(10L); + standard0.queryBuilder = QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(9L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(8L)); standard0.collapseBuilder = new CollapseBuilder(TOPIC_FIELD).setInnerHits( new InnerHitBuilder("a").addSort(new FieldSortBuilder(DOC_FIELD).order(SortOrder.DESC)).setSize(10) ); @@ -375,8 +381,8 @@ public void testRankDocsRetrieverWithCollapseAndAggs() { null ); // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 4, 7, 3 + // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results) + // so ideal rank would be: 6, 2, 1, 3, 4, 7 source.retriever( new CompoundRetrieverWithRankDocs( rankWindowSize, @@ -392,7 +398,7 @@ public void testRankDocsRetrieverWithCollapseAndAggs() { ElasticsearchAssertions.assertResponse(req, resp -> { assertNull(resp.pointInTimeId()); assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(5L)); + assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6")); assertNotNull(resp.getAggregations()); @@ -427,8 +433,8 @@ public void testRankDocsRetrieverWithNestedQuery() { null ); // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 4, 3, 7 + // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results) + // so ideal rank would be: 6, 2, 1, 3, 4, 7 source.retriever( new CompoundRetrieverWithRankDocs( rankWindowSize, @@ -460,8 +466,10 @@ public void testRankDocsRetrieverMultipleCompoundRetrievers() { SearchSourceBuilder source = new SearchSourceBuilder(); StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD)) - .boost(10L); + standard0.queryBuilder = QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(9L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(8L)); StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); // this one retrieves docs 2 and 6 due to prefilter standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); @@ -506,11 +514,11 @@ public void testRankDocsRetrieverMultipleCompoundRetrievers() { assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_4")); - assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_6")); + assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1")); assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_2")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_7")); - assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_3")); + assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_3")); + assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_6")); + assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_7")); }); } @@ -545,9 +553,9 @@ public void testRankDocsRetrieverDifferentNestedSorting() { assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_4")); assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_7")); + assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_2")); assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_6")); - assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_2")); + assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_7")); }); } @@ -673,22 +681,14 @@ private RankDoc[] getRankDocs(SearchResponse searchResponse) { for (int i = 0; i < size; i++) { var hit = searchResponse.getHits().getAt(i); long sortValue = (long) hit.getRawSortValues()[hit.getRawSortValues().length - 1]; - int doc = decodeDoc(sortValue); - int shardRequestIndex = decodeShardRequestIndex(sortValue); + int doc = ShardDocSortField.decodeDoc(sortValue); + int shardRequestIndex = ShardDocSortField.decodeShardRequestIndex(sortValue); docs[i] = new RankDoc(doc, hit.getScore(), shardRequestIndex); docs[i].rank = i + 1; } return docs; } - public static int decodeDoc(long value) { - return (int) value; - } - - public static int decodeShardRequestIndex(long value) { - return (int) (value >> 32); - } - record RankDocAndHitRatio(RankDoc rankDoc, float hitRatio) {} /** diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 0ced472ea310c..7ff0ed1bbe82c 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -231,6 +231,7 @@ static TransportVersion def(int id) { public static final TransportVersion CCS_REMOTE_TELEMETRY_STATS = def(8_755_00_0); public static final TransportVersion ESQL_CCS_EXECUTION_INFO = def(8_756_00_0); public static final TransportVersion REGEX_AND_RANGE_INTERVAL_QUERIES = def(8_757_00_0); + public static final TransportVersion RRF_QUERY_REWRITE = def(8_758_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index e3d663ec13618..553ee1d05a052 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -502,6 +502,8 @@ void executeRequest( }); final SearchSourceBuilder source = original.source(); if (shouldOpenPIT(source)) { + // disabling shard reordering for request + original.setPreFilterShardSize(Integer.MAX_VALUE); openPIT(client, original, searchService.getDefaultKeepAliveInMillis(), listener.delegateFailureAndWrap((delegate, resp) -> { // We set the keep alive to -1 to indicate that we don't need the pit id in the response. // This is needed since we delete the pit prior to sending the response so the id doesn't exist anymore. diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java index c526652fc4e67..5043508c781f0 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -74,7 +74,6 @@ import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.lucene.grouping.TopFieldGroups; -import org.elasticsearch.search.retriever.rankdoc.RankDocsSortField; import org.elasticsearch.search.sort.ShardDocSortField; import java.io.IOException; @@ -553,8 +552,6 @@ private static SortField rewriteMergeSortField(SortField sortField) { return newSortField; } else if (sortField.getClass() == ShardDocSortField.class) { return new SortField(sortField.getField(), SortField.Type.LONG, sortField.getReverse()); - } else if (sortField.getClass() == RankDocsSortField.class) { - return new SortField(sortField.getField(), SortField.Type.INT, sortField.getReverse()); } else { return sortField; } diff --git a/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java b/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java index 27e3b264a17e8..d1c7d0415ad15 100644 --- a/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java +++ b/server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java @@ -260,7 +260,7 @@ public void visitLeaf(Query leafQuery) { * KnnScoreDocQuery and RankDocsQuery requires the same reader that built the docs * When using {@link HighlightFlag#WEIGHT_MATCHES} different readers are used and isn't supported by this query */ - if (leafQuery instanceof KnnScoreDocQuery || leafQuery instanceof RankDocsQuery) { + if (leafQuery instanceof KnnScoreDocQuery || leafQuery instanceof RankDocsQuery.TopQuery) { hasUnknownLeaf[0] = true; } super.visitLeaf(query); diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index 6308b19358410..0bb914a9dbf97 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -239,7 +239,6 @@ import org.elasticsearch.search.retriever.RetrieverParserContext; import org.elasticsearch.search.retriever.StandardRetrieverBuilder; import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder; -import org.elasticsearch.search.retriever.rankdoc.RankDocsSortBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.GeoDistanceSortBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; @@ -868,7 +867,6 @@ private void registerSorts() { namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, ScoreSortBuilder.NAME, ScoreSortBuilder::new)); namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, ScriptSortBuilder.NAME, ScriptSortBuilder::new)); namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, FieldSortBuilder.NAME, FieldSortBuilder::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, RankDocsSortBuilder.NAME, RankDocsSortBuilder::new)); } private static void registerFromPlugin(List plugins, Function> producer, Consumer consumer) { diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 26780f85a15e0..fc0cb72bb82e0 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -2207,12 +2207,7 @@ public ActionRequestValidationException validate( boolean allowPartialSearchResults ) { if (retriever() != null) { - if (allowPartialSearchResults && retriever().isCompound()) { - validationException = addValidationError( - "cannot specify a compound retriever and [allow_partial_search_results]", - validationException - ); - } + validationException = retriever().validate(this, validationException, allowPartialSearchResults); List specified = new ArrayList<>(); if (subSearches().isEmpty() == false) { specified.add(QUERY_FIELD.getPreferredName()); @@ -2229,9 +2224,6 @@ public ActionRequestValidationException validate( if (sorts() != null) { specified.add(SORT_FIELD.getPreferredName()); } - if (minScore() != null) { - specified.add(MIN_SCORE_FIELD.getPreferredName()); - } if (rankBuilder() != null) { specified.add(RANK_FIELD.getPreferredName()); } @@ -2331,21 +2323,10 @@ public ActionRequestValidationException validate( if (rescores() != null && rescores().isEmpty() == false) { validationException = addValidationError("[rank] cannot be used with [rescore]", validationException); } - if (sorts() != null && sorts().isEmpty() == false) { - validationException = addValidationError("[rank] cannot be used with [sort]", validationException); - } - if (collapse() != null) { - validationException = addValidationError("[rank] cannot be used with [collapse]", validationException); - } + if (suggest() != null && suggest().getSuggestions().isEmpty() == false) { validationException = addValidationError("[rank] cannot be used with [suggest]", validationException); } - if (highlighter() != null) { - validationException = addValidationError("[rank] cannot be used with [highlighter]", validationException); - } - if (pointInTimeBuilder() != null) { - validationException = addValidationError("[rank] cannot be used with [point in time]", validationException); - } } if (rescores() != null) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsContext.java b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsContext.java index 62eaadfb15690..3076337d43c84 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsContext.java @@ -34,7 +34,7 @@ public class StoredFieldsContext implements Writeable { private final List fieldNames; private final boolean fetchFields; - private StoredFieldsContext(boolean fetchFields) { + public StoredFieldsContext(boolean fetchFields) { this.fetchFields = fetchFields; this.fieldNames = null; } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java index 7a2913ce56128..0e6172323277d 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Explanation; +import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; @@ -45,8 +46,17 @@ public void process(HitContext hitContext) throws IOException { for (RescoreContext rescore : context.rescore()) { explanation = rescore.rescorer().explain(topLevelDocId, context.searcher(), rescore, explanation); } + if (context.rankBuilder() != null) { - explanation = context.rankBuilder().explainHit(explanation, hitContext.rankDoc(), queryNames); + // if we have nested fields, then the query is wrapped using an additional filter on the _primary_term field + // through the DefaultSearchContext#buildFilteredQuery so we have to extract the actual query + if (context.getSearchExecutionContext().nestedLookup() != NestedLookup.EMPTY) { + explanation = explanation.getDetails()[0]; + } + + if (context.rankBuilder() != null) { + explanation = context.rankBuilder().explainHit(explanation, hitContext.rankDoc(), queryNames); + } } // we use the top level doc id, since we work with the top level searcher hitContext.hit().explanation(explanation); diff --git a/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java b/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java index 94b584607878a..b16a234931115 100644 --- a/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java +++ b/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java @@ -24,7 +24,7 @@ * {@code RankDoc} is the base class for all ranked results. * Subclasses should extend this with additional information required for their global ranking method. */ -public class RankDoc extends ScoreDoc implements NamedWriteable, ToXContentFragment { +public class RankDoc extends ScoreDoc implements NamedWriteable, ToXContentFragment, Comparable { public static final String NAME = "rank_doc"; @@ -40,6 +40,17 @@ public String getWriteableName() { return NAME; } + @Override + public final int compareTo(RankDoc other) { + if (score != other.score) { + return score < other.score ? 1 : -1; + } + if (shardIndex != other.shardIndex) { + return shardIndex < other.shardIndex ? -1 : 1; + } + return doc < other.doc ? -1 : 1; + } + public record RankKey(int doc, int shardIndex) {} public RankDoc(int doc, float score, int shardIndex) { @@ -65,8 +76,12 @@ public final void writeTo(StreamOutput out) throws IOException { /** * Explain the ranking of this document. */ - public Explanation explain() { - return Explanation.match(rank, "doc [" + doc + "] with an original score of [" + score + "] is at rank [" + rank + "]."); + public Explanation explain(Explanation[] sourceExplanations, String[] queryNames) { + return Explanation.match( + rank, + "doc [" + doc + "] with an original score of [" + score + "] is at rank [" + rank + "] from the following source queries.", + sourceExplanations + ); } @Override @@ -104,6 +119,6 @@ protected int doHashCode() { @Override public String toString() { - return "RankDoc{" + "_rank=" + rank + ", _doc=" + doc + ", _shard=" + shardIndex + ", _score=" + score + '}'; + return "RankDoc{" + "_rank=" + rank + ", _doc=" + doc + ", _shard=" + shardIndex + ", _score=" + score + "}"; } } diff --git a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureDoc.java b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureDoc.java index aadcb94c4b242..cd8d9392aced8 100644 --- a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureDoc.java +++ b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureDoc.java @@ -38,7 +38,7 @@ public RankFeatureDoc(StreamInput in) throws IOException { } @Override - public Explanation explain() { + public Explanation explain(Explanation[] sources, String[] queryNames) { throw new UnsupportedOperationException("explain is not supported for {" + getClass() + "}"); } diff --git a/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java new file mode 100644 index 0000000000000..1962145d7336d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java @@ -0,0 +1,255 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.search.retriever; + +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.search.MultiSearchRequest; +import org.elasticsearch.action.search.MultiSearchResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.TransportMultiSearchAction; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.search.builder.PointInTimeBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.StoredFieldsContext; +import org.elasticsearch.search.rank.RankDoc; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; +import org.elasticsearch.search.sort.ShardDocSortField; +import org.elasticsearch.search.sort.SortBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +/** + * This abstract retriever defines a compound retriever. The idea is that it is not a leaf-retriever, i.e. it does not + * perform actual searches itself. Instead, it is a container for a set of child retrievers and is responsible for combining + * the results of the child retrievers according to the implementation of {@code combineQueryPhaseResults}. + */ +public abstract class CompoundRetrieverBuilder> extends RetrieverBuilder { + + public record RetrieverSource(RetrieverBuilder retriever, SearchSourceBuilder source) {} + + protected final int rankWindowSize; + protected final List innerRetrievers; + + protected CompoundRetrieverBuilder(List innerRetrievers, int rankWindowSize) { + this.rankWindowSize = rankWindowSize; + this.innerRetrievers = innerRetrievers; + } + + @SuppressWarnings("unchecked") + public T addChild(RetrieverBuilder retrieverBuilder) { + innerRetrievers.add(new RetrieverSource(retrieverBuilder, null)); + return (T) this; + } + + /** + * Returns a clone of the original retriever, replacing the sub-retrievers with + * the provided {@code newChildRetrievers}. + */ + protected abstract T clone(List newChildRetrievers); + + /** + * Combines the provided {@code rankResults} to return the final top documents. + */ + protected abstract RankDoc[] combineInnerRetrieverResults(List rankResults); + + @Override + public final boolean isCompound() { + return true; + } + + @Override + public final RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException { + if (ctx.getPointInTimeBuilder() == null) { + throw new IllegalStateException("PIT is required"); + } + + // Rewrite prefilters + boolean hasChanged = false; + var newPreFilters = rewritePreFilters(ctx); + hasChanged |= newPreFilters != preFilterQueryBuilders; + + // Rewrite retriever sources + List newRetrievers = new ArrayList<>(); + for (var entry : innerRetrievers) { + RetrieverBuilder newRetriever = entry.retriever.rewrite(ctx); + if (newRetriever != entry.retriever) { + newRetrievers.add(new RetrieverSource(newRetriever, null)); + hasChanged |= true; + } else { + var sourceBuilder = entry.source != null + ? entry.source + : createSearchSourceBuilder(ctx.getPointInTimeBuilder(), newRetriever); + var rewrittenSource = sourceBuilder.rewrite(ctx); + newRetrievers.add(new RetrieverSource(newRetriever, rewrittenSource)); + hasChanged |= rewrittenSource != entry.source; + } + } + if (hasChanged) { + return clone(newRetrievers); + } + + // execute searches + final SetOnce results = new SetOnce<>(); + final MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); + for (var entry : innerRetrievers) { + SearchRequest searchRequest = new SearchRequest().source(entry.source); + // The can match phase can reorder shards, so we disable it to ensure the stable ordering + searchRequest.setPreFilterShardSize(Integer.MAX_VALUE); + multiSearchRequest.add(searchRequest); + } + ctx.registerAsyncAction((client, listener) -> { + client.execute(TransportMultiSearchAction.TYPE, multiSearchRequest, new ActionListener<>() { + @Override + public void onResponse(MultiSearchResponse items) { + List topDocs = new ArrayList<>(); + List failures = new ArrayList<>(); + for (int i = 0; i < items.getResponses().length; i++) { + var item = items.getResponses()[i]; + if (item.isFailure()) { + failures.add(item.getFailure()); + } else { + assert item.getResponse() != null; + var rankDocs = getRankDocs(item.getResponse()); + innerRetrievers.get(i).retriever().setRankDocs(rankDocs); + topDocs.add(rankDocs); + } + } + if (false == failures.isEmpty()) { + IllegalStateException ex = new IllegalStateException("Search failed - some nested retrievers returned errors."); + failures.forEach(ex::addSuppressed); + listener.onFailure(ex); + } else { + results.set(combineInnerRetrieverResults(topDocs)); + listener.onResponse(null); + } + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + }); + + return new RankDocsRetrieverBuilder( + rankWindowSize, + newRetrievers.stream().map(s -> s.retriever).toList(), + results::get, + newPreFilters + ); + } + + @Override + public final QueryBuilder topDocsQuery() { + throw new IllegalStateException(getName() + " cannot be nested"); + } + + @Override + public final void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) { + throw new IllegalStateException("Should not be called, missing a rewrite?"); + } + + @Override + public ActionRequestValidationException validate( + SearchSourceBuilder source, + ActionRequestValidationException validationException, + boolean allowPartialSearchResults + ) { + validationException = super.validate(source, validationException, allowPartialSearchResults); + if (source.size() > rankWindowSize) { + validationException = addValidationError( + "[" + + this.getName() + + "] requires [rank_window_size: " + + rankWindowSize + + "]" + + " be greater than or equal to [size: " + + source.size() + + "]", + validationException + ); + } + if (allowPartialSearchResults) { + validationException = addValidationError( + "cannot specify a compound retriever and [allow_partial_search_results]", + validationException + ); + } + return validationException; + } + + @Override + public boolean doEquals(Object o) { + CompoundRetrieverBuilder that = (CompoundRetrieverBuilder) o; + return rankWindowSize == that.rankWindowSize && Objects.equals(innerRetrievers, that.innerRetrievers); + } + + @Override + public int doHashCode() { + return Objects.hash(innerRetrievers); + } + + private SearchSourceBuilder createSearchSourceBuilder(PointInTimeBuilder pit, RetrieverBuilder retrieverBuilder) { + var sourceBuilder = new SearchSourceBuilder().pointInTimeBuilder(pit) + .trackTotalHits(false) + .storedFields(new StoredFieldsContext(false)) + .size(rankWindowSize); + if (preFilterQueryBuilders.isEmpty() == false) { + retrieverBuilder.getPreFilterQueryBuilders().addAll(preFilterQueryBuilders); + } + retrieverBuilder.extractToSearchSourceBuilder(sourceBuilder, true); + + // apply the pre-filters + if (preFilterQueryBuilders.size() > 0) { + QueryBuilder query = sourceBuilder.query(); + BoolQueryBuilder newQuery = new BoolQueryBuilder(); + if (query != null) { + newQuery.must(query); + } + preFilterQueryBuilders.forEach(newQuery::filter); + sourceBuilder.query(newQuery); + } + + // Record the shard id in the sort result + List> sortBuilders = sourceBuilder.sorts() != null ? new ArrayList<>(sourceBuilder.sorts()) : new ArrayList<>(); + if (sortBuilders.isEmpty()) { + sortBuilders.add(new ScoreSortBuilder()); + } + sortBuilders.add(new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME)); + sourceBuilder.sort(sortBuilders); + return sourceBuilder; + } + + private RankDoc[] getRankDocs(SearchResponse searchResponse) { + int size = searchResponse.getHits().getHits().length; + RankDoc[] docs = new RankDoc[size]; + for (int i = 0; i < size; i++) { + var hit = searchResponse.getHits().getAt(i); + long sortValue = (long) hit.getRawSortValues()[hit.getRawSortValues().length - 1]; + int doc = ShardDocSortField.decodeDoc(sortValue); + int shardRequestIndex = ShardDocSortField.decodeShardRequestIndex(sortValue); + docs[i] = new RankDoc(doc, hit.getScore(), shardRequestIndex); + docs[i].rank = i + 1; + } + return docs; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java index ceab04ebb55e4..8e564430ef57a 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java @@ -127,12 +127,30 @@ public String getName() { @Override public QueryBuilder topDocsQuery() { - assert rankDocs != null : "{rankDocs} should have been materialized at this point"; + assert rankDocs != null : "rankDocs should have been materialized by now"; + var rankDocsQuery = new RankDocsQueryBuilder(rankDocs, null, true); + if (preFilterQueryBuilders.isEmpty()) { + return rankDocsQuery.queryName(retrieverName); + } + BoolQueryBuilder res = new BoolQueryBuilder().must(rankDocsQuery); + preFilterQueryBuilders.forEach(res::filter); + return res.queryName(retrieverName); + } - BoolQueryBuilder knnTopResultsQuery = new BoolQueryBuilder().filter(new RankDocsQueryBuilder(rankDocs)) - .should(new ExactKnnQueryBuilder(VectorData.fromFloats(queryVector), field, similarity)); - preFilterQueryBuilders.forEach(knnTopResultsQuery::filter); - return knnTopResultsQuery; + @Override + public QueryBuilder explainQuery() { + assert rankDocs != null : "rankDocs should have been materialized by now"; + var rankDocsQuery = new RankDocsQueryBuilder( + rankDocs, + new QueryBuilder[] { new ExactKnnQueryBuilder(VectorData.fromFloats(queryVector), field, similarity) }, + true + ); + if (preFilterQueryBuilders.isEmpty()) { + return rankDocsQuery.queryName(retrieverName); + } + BoolQueryBuilder res = new BoolQueryBuilder().must(rankDocsQuery); + preFilterQueryBuilders.forEach(res::filter); + return res.queryName(retrieverName); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilder.java index 89daafdf05b4b..535db5c8fe28e 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilder.java @@ -10,14 +10,11 @@ package org.elasticsearch.search.retriever; import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.DisMaxQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.rank.RankDoc; import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder; -import org.elasticsearch.search.retriever.rankdoc.RankDocsSortBuilder; -import org.elasticsearch.search.sort.ScoreSortBuilder; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -32,7 +29,7 @@ public class RankDocsRetrieverBuilder extends RetrieverBuilder { public static final String NAME = "rank_docs_retriever"; - private final int rankWindowSize; + final int rankWindowSize; final List sources; final Supplier rankDocs; @@ -44,6 +41,9 @@ public RankDocsRetrieverBuilder( ) { this.rankWindowSize = rankWindowSize; this.rankDocs = rankDocs; + if (sources == null || sources.isEmpty()) { + throw new IllegalArgumentException("sources must not be null or empty"); + } this.sources = sources; this.preFilterQueryBuilders = preFilterQueryBuilders; } @@ -53,6 +53,10 @@ public String getName() { return NAME; } + private boolean sourceHasMinScore() { + return minScore != null || sources.stream().anyMatch(x -> x.minScore() != null); + } + private boolean sourceShouldRewrite(QueryRewriteContext ctx) throws IOException { for (var source : sources) { if (source.isCompound()) { @@ -80,64 +84,92 @@ public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException { public QueryBuilder topDocsQuery() { // this is used to fetch all documents form the parent retrievers (i.e. sources) // so that we can use all the matched documents to compute aggregations, nested hits etc - DisMaxQueryBuilder disMax = new DisMaxQueryBuilder().tieBreaker(0f); + BoolQueryBuilder boolQuery = new BoolQueryBuilder(); for (var retriever : sources) { var query = retriever.topDocsQuery(); if (query != null) { if (retriever.retrieverName() != null) { query.queryName(retriever.retrieverName()); } - disMax.add(query); + boolQuery.should(query); } } // ignore prefilters of this level, they are already propagated to children - return disMax; + return boolQuery; + } + + @Override + public QueryBuilder explainQuery() { + return new RankDocsQueryBuilder( + rankDocs.get(), + sources.stream().map(RetrieverBuilder::explainQuery).toArray(QueryBuilder[]::new), + true + ); } @Override public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) { - // here we force a custom sort based on the rank of the documents - // TODO: should we adjust to account for other fields sort options just for the top ranked docs? - if (searchSourceBuilder.rescores() == null || searchSourceBuilder.rescores().isEmpty()) { - searchSourceBuilder.sort(Arrays.asList(new RankDocsSortBuilder(rankDocs.get()), new ScoreSortBuilder())); - } - if (searchSourceBuilder.explain() != null && searchSourceBuilder.explain()) { - searchSourceBuilder.trackScores(true); - } - BoolQueryBuilder boolQuery = new BoolQueryBuilder(); - RankDocsQueryBuilder rankQuery = new RankDocsQueryBuilder(rankDocs.get()); + final RankDocsQueryBuilder rankQuery; // if we have aggregations we need to compute them based on all doc matches, not just the top hits - // so we just post-filter the top hits based on the rank queries we have - if (searchSourceBuilder.aggregations() != null) { - boolQuery.should(rankQuery); - // compute a disjunction of all the query sources that were executed to compute the top rank docs - QueryBuilder disjunctionOfSources = topDocsQuery(); - if (disjunctionOfSources != null) { - boolQuery.should(disjunctionOfSources); + // similarly, for profile and explain we re-run all parent queries to get all needed information + RankDoc[] rankDocResults = rankDocs.get(); + if (hasAggregations(searchSourceBuilder) + || isExplainRequest(searchSourceBuilder) + || isProfileRequest(searchSourceBuilder) + || shouldTrackTotalHits(searchSourceBuilder)) { + if (false == isExplainRequest(searchSourceBuilder)) { + rankQuery = new RankDocsQueryBuilder( + rankDocResults, + sources.stream().map(RetrieverBuilder::topDocsQuery).toArray(QueryBuilder[]::new), + false + ); + } else { + rankQuery = new RankDocsQueryBuilder( + rankDocResults, + sources.stream().map(RetrieverBuilder::explainQuery).toArray(QueryBuilder[]::new), + false + ); } - // post filter the results so that the top docs are still the same - searchSourceBuilder.postFilter(rankQuery); } else { - boolQuery.must(rankQuery); + rankQuery = new RankDocsQueryBuilder(rankDocResults, null, false); } - // add any prefilters present in the retriever - for (var preFilterQueryBuilder : preFilterQueryBuilders) { - boolQuery.filter(preFilterQueryBuilder); + // ignore prefilters of this level, they are already propagated to children + searchSourceBuilder.query(rankQuery); + if (sourceHasMinScore()) { + searchSourceBuilder.minScore(this.minScore() == null ? Float.MIN_VALUE : this.minScore()); + } + if (searchSourceBuilder.size() + searchSourceBuilder.from() > rankDocResults.length) { + searchSourceBuilder.size(Math.max(0, rankDocResults.length - searchSourceBuilder.from())); } - searchSourceBuilder.query(boolQuery); + } + + private boolean hasAggregations(SearchSourceBuilder searchSourceBuilder) { + return searchSourceBuilder.aggregations() != null; + } + + private boolean isExplainRequest(SearchSourceBuilder searchSourceBuilder) { + return searchSourceBuilder.explain() != null && searchSourceBuilder.explain(); + } + + private boolean isProfileRequest(SearchSourceBuilder searchSourceBuilder) { + return searchSourceBuilder.profile(); + } + + private boolean shouldTrackTotalHits(SearchSourceBuilder searchSourceBuilder) { + return searchSourceBuilder.trackTotalHitsUpTo() == null || searchSourceBuilder.trackTotalHitsUpTo() > rankDocs.get().length; } @Override protected boolean doEquals(Object o) { RankDocsRetrieverBuilder other = (RankDocsRetrieverBuilder) o; - return Arrays.equals(rankDocs.get(), other.rankDocs.get()) - && sources.equals(other.sources) - && rankWindowSize == other.rankWindowSize; + return rankWindowSize == other.rankWindowSize + && Arrays.equals(rankDocs.get(), other.rankDocs.get()) + && sources.equals(other.sources); } @Override protected int doHashCode() { - return Objects.hash(super.hashCode(), Arrays.hashCode(rankDocs.get()), sources, rankWindowSize); + return Objects.hash(super.hashCode(), rankWindowSize, Arrays.hashCode(rankDocs.get()), sources); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java index e8f6a2d795724..1328106896bcb 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java @@ -9,6 +9,7 @@ package org.elasticsearch.search.retriever; +import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.SuggestingErrorOnUnknown; @@ -53,6 +54,8 @@ public abstract class RetrieverBuilder implements Rewriteable, public static final ParseField PRE_FILTER_FIELD = new ParseField("filter"); + public static final ParseField MIN_SCORE_FIELD = new ParseField("min_score"); + public static final ParseField NAME_FIELD = new ParseField("_name"); protected static void declareBaseParserFields( @@ -65,41 +68,25 @@ protected static void declareBaseParserFields( return preFilterQueryBuilder; }, PRE_FILTER_FIELD); parser.declareString(RetrieverBuilder::retrieverName, NAME_FIELD); + parser.declareFloat(RetrieverBuilder::minScore, MIN_SCORE_FIELD); } - private void retrieverName(String retrieverName) { + public RetrieverBuilder retrieverName(String retrieverName) { this.retrieverName = retrieverName; + return this; + } + + public RetrieverBuilder minScore(Float minScore) { + this.minScore = minScore; + return this; } - /** - * This method parsers a top-level retriever within a search and tracks its own depth. Currently, the - * maximum depth allowed is limited to 2 as a compound retriever cannot currently contain another - * compound retriever. - */ public static RetrieverBuilder parseTopLevelRetrieverBuilder(XContentParser parser, RetrieverParserContext context) throws IOException { parser = new FilterXContentParserWrapper(parser) { - int nestedDepth = 0; - @Override public T namedObject(Class categoryClass, String name, Object context) throws IOException { - if (categoryClass.equals(RetrieverBuilder.class)) { - nestedDepth++; - - if (nestedDepth > 2) { - throw new IllegalArgumentException( - "the nested depth of the [" + name + "] retriever exceeds the maximum nested depth [2] for retrievers" - ); - } - } - - T namedObject = getXContentRegistry().parseNamedObject(categoryClass, name, this, context); - - if (categoryClass.equals(RetrieverBuilder.class)) { - nestedDepth--; - } - - return namedObject; + return getXContentRegistry().parseNamedObject(categoryClass, name, this, context); } }; @@ -186,6 +173,8 @@ protected static RetrieverBuilder parseInnerRetrieverBuilder(XContentParser pars protected String retrieverName; + protected Float minScore; + /** * Determines if this retriever contains sub-retrievers that need to be executed prior to search. */ @@ -217,6 +206,14 @@ protected final List rewritePreFilters(QueryRewriteContext ctx) th */ public abstract QueryBuilder topDocsQuery(); + public QueryBuilder explainQuery() { + return topDocsQuery(); + } + + public Float minScore() { + return minScore; + } + public void setRankDocs(RankDoc[] rankDocs) { this.rankDocs = rankDocs; } @@ -239,6 +236,14 @@ public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException { */ public abstract void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed); + public ActionRequestValidationException validate( + SearchSourceBuilder source, + ActionRequestValidationException validationException, + boolean allowPartialSearchResults + ) { + return validationException; + } + // ---- FOR TESTING XCONTENT PARSING ---- public abstract String getName(); @@ -267,14 +272,16 @@ public final boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; RetrieverBuilder that = (RetrieverBuilder) o; - return Objects.equals(preFilterQueryBuilders, that.preFilterQueryBuilders) && doEquals(o); + return Objects.equals(preFilterQueryBuilders, that.preFilterQueryBuilders) + && Objects.equals(minScore, that.minScore) + && doEquals(o); } protected abstract boolean doEquals(Object o); @Override public final int hashCode() { - return Objects.hash(getClass(), preFilterQueryBuilders, doHashCode()); + return Objects.hash(getClass(), preFilterQueryBuilders, minScore, doHashCode()); } protected abstract int doHashCode(); diff --git a/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java index 57381bdf558c9..ac329eb293e90 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java @@ -43,7 +43,6 @@ public final class StandardRetrieverBuilder extends RetrieverBuilder implements public static final ParseField SEARCH_AFTER_FIELD = new ParseField("search_after"); public static final ParseField TERMINATE_AFTER_FIELD = new ParseField("terminate_after"); public static final ParseField SORT_FIELD = new ParseField("sort"); - public static final ParseField MIN_SCORE_FIELD = new ParseField("min_score"); public static final ParseField COLLAPSE_FIELD = new ParseField("collapse"); public static final ObjectParser PARSER = new ObjectParser<>( @@ -76,12 +75,6 @@ public final class StandardRetrieverBuilder extends RetrieverBuilder implements return sortBuilders; }, SORT_FIELD, ObjectParser.ValueType.OBJECT_ARRAY); - PARSER.declareField((r, v) -> r.minScore = v, (p, c) -> { - float minScore = p.floatValue(); - c.trackSectionUsage(NAME + ":" + MIN_SCORE_FIELD.getPreferredName()); - return minScore; - }, MIN_SCORE_FIELD, ObjectParser.ValueType.FLOAT); - PARSER.declareField((r, v) -> r.collapseBuilder = v, (p, c) -> { CollapseBuilder collapseBuilder = CollapseBuilder.fromXContent(p); if (collapseBuilder.getField() != null) { @@ -104,23 +97,29 @@ public static StandardRetrieverBuilder fromXContent(XContentParser parser, Retri SearchAfterBuilder searchAfterBuilder; int terminateAfter = SearchContext.DEFAULT_TERMINATE_AFTER; List> sortBuilders; - Float minScore; CollapseBuilder collapseBuilder; + public StandardRetrieverBuilder() {} + + public StandardRetrieverBuilder(QueryBuilder queryBuilder) { + this.queryBuilder = queryBuilder; + } + @Override public QueryBuilder topDocsQuery() { - // TODO: for compound retrievers this will have to be reworked as queries like knn could be executed twice if (preFilterQueryBuilders.isEmpty()) { - return queryBuilder; + QueryBuilder qb = queryBuilder; + qb.queryName(this.retrieverName); + return qb; } - var ret = new BoolQueryBuilder().filter(queryBuilder); + var ret = new BoolQueryBuilder().filter(queryBuilder).queryName(this.retrieverName); preFilterQueryBuilders.stream().forEach(ret::filter); return ret; } @Override public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) { - if (preFilterQueryBuilders.isEmpty() == false) { + if (preFilterQueryBuilders.isEmpty() == false || minScore != null) { BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); for (QueryBuilder preFilterQueryBuilder : preFilterQueryBuilders) { @@ -130,7 +129,6 @@ public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder if (queryBuilder != null) { boolQueryBuilder.must(queryBuilder); } - searchSourceBuilder.subSearches().add(new SubSearchSourceBuilder(boolQueryBuilder)); } else if (queryBuilder != null) { searchSourceBuilder.subSearches().add(new SubSearchSourceBuilder(queryBuilder)); @@ -157,32 +155,14 @@ public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder } if (sortBuilders != null) { - if (compoundUsed) { - throw new IllegalArgumentException( - "[" + SORT_FIELD.getPreferredName() + "] cannot be used in children of compound retrievers" - ); - } - searchSourceBuilder.sort(sortBuilders); } if (minScore != null) { - if (compoundUsed) { - throw new IllegalArgumentException( - "[" + MIN_SCORE_FIELD.getPreferredName() + "] cannot be used in children of compound retrievers" - ); - } - searchSourceBuilder.minScore(minScore); } if (collapseBuilder != null) { - if (compoundUsed) { - throw new IllegalArgumentException( - "[" + COLLAPSE_FIELD.getPreferredName() + "] cannot be used in children of compound retrievers" - ); - } - searchSourceBuilder.collapse(collapseBuilder); } } @@ -212,10 +192,6 @@ public void doToXContent(XContentBuilder builder, ToXContent.Params params) thro builder.field(SORT_FIELD.getPreferredName(), sortBuilders); } - if (minScore != null) { - builder.field(MIN_SCORE_FIELD.getPreferredName(), minScore); - } - if (collapseBuilder != null) { builder.field(COLLAPSE_FIELD.getPreferredName(), collapseBuilder); } @@ -228,13 +204,12 @@ public boolean doEquals(Object o) { && Objects.equals(queryBuilder, that.queryBuilder) && Objects.equals(searchAfterBuilder, that.searchAfterBuilder) && Objects.equals(sortBuilders, that.sortBuilders) - && Objects.equals(minScore, that.minScore) && Objects.equals(collapseBuilder, that.collapseBuilder); } @Override public int doHashCode() { - return Objects.hash(queryBuilder, searchAfterBuilder, terminateAfter, sortBuilders, minScore, collapseBuilder); + return Objects.hash(queryBuilder, searchAfterBuilder, terminateAfter, sortBuilders, collapseBuilder); } // ---- END FOR TESTING ---- diff --git a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java index 079e725dd375b..fb5015a82dbdb 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQuery.java @@ -9,20 +9,27 @@ package org.elasticsearch.search.retriever.rankdoc; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Matches; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; import org.apache.lucene.search.Weight; import org.elasticsearch.search.rank.RankDoc; import java.io.IOException; import java.util.Arrays; +import java.util.Comparator; import java.util.Objects; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; @@ -33,142 +40,315 @@ * after performing any reranking or filtering. */ public class RankDocsQuery extends Query { + /** + * A {@link Query} that matches only the specified {@link RankDoc}, using the provided {@link Query} sources + * solely for the purpose of explainability. + */ + public static class TopQuery extends Query { + private final RankDoc[] docs; + private final Query[] sources; + private final String[] queryNames; + private final int[] segmentStarts; + private final Object contextIdentity; + + TopQuery(RankDoc[] docs, Query[] sources, String[] queryNames, int[] segmentStarts, Object contextIdentity) { + assert sources.length == queryNames.length; + this.docs = docs; + this.sources = sources; + this.queryNames = queryNames; + this.segmentStarts = segmentStarts; + this.contextIdentity = contextIdentity; + } + + @Override + public Query rewrite(IndexSearcher searcher) throws IOException { + if (docs.length == 0) { + return new MatchNoDocsQuery(); + } + boolean changed = false; + Query[] newSources = new Query[sources.length]; + for (int i = 0; i < sources.length; i++) { + newSources[i] = sources[i].rewrite(searcher); + changed |= newSources[i] != sources[i]; + } + if (changed) { + return new TopQuery(docs, newSources, queryNames, segmentStarts, contextIdentity); + } + return this; + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + if (searcher.getIndexReader().getContext().id() != contextIdentity) { + throw new IllegalStateException("This RankDocsDocQuery was created by a different reader"); + } + Weight[] weights = new Weight[sources.length]; + for (int i = 0; i < sources.length; i++) { + weights[i] = sources[i].createWeight(searcher, scoreMode, boost); + } + return new Weight(this) { + @Override + public int count(LeafReaderContext context) { + return segmentStarts[context.ord + 1] - segmentStarts[context.ord]; + } + + @Override + public Explanation explain(LeafReaderContext context, int doc) throws IOException { + int found = binarySearch(docs, 0, docs.length, doc + context.docBase); + if (found < 0) { + return Explanation.noMatch("doc not found in top " + docs.length + " rank docs"); + } + Explanation[] sourceExplanations = new Explanation[sources.length]; + for (int i = 0; i < sources.length; i++) { + sourceExplanations[i] = weights[i].explain(context, doc); + } + return docs[found].explain(sourceExplanations, queryNames); + } + + @Override + public Scorer scorer(LeafReaderContext context) { + // Segment starts indicate how many docs are in the segment, + // upper equalling lower indicates no documents for this segment + if (segmentStarts[context.ord] == segmentStarts[context.ord + 1]) { + return null; + } + return new Scorer(this) { + final int lower = segmentStarts[context.ord]; + final int upper = segmentStarts[context.ord + 1]; + int upTo = -1; + float score; + + @Override + public DocIdSetIterator iterator() { + return new DocIdSetIterator() { + @Override + public int docID() { + return currentDocId(); + } + + @Override + public int nextDoc() { + if (upTo == -1) { + upTo = lower; + } else { + ++upTo; + } + return currentDocId(); + } + + @Override + public int advance(int target) throws IOException { + return slowAdvance(target); + } + + @Override + public long cost() { + return upper - lower; + } + }; + } + + @Override + public float getMaxScore(int docId) { + if (docId != NO_MORE_DOCS) { + docId += context.docBase; + } + float maxScore = 0; + for (int idx = Math.max(lower, upTo); idx < upper && docs[idx].doc <= docId; idx++) { + maxScore = Math.max(maxScore, docs[idx].score); + } + return maxScore; + } + + @Override + public float score() { + return docs[upTo].score; + } + + @Override + public int docID() { + return currentDocId(); + } + + private int currentDocId() { + if (upTo == -1) { + return -1; + } + if (upTo >= upper) { + return NO_MORE_DOCS; + } + return docs[upTo].doc - context.docBase; + } + + }; + } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return true; + } + }; + } + + @Override + public String toString(String field) { + return this.getClass().getSimpleName() + "{rank_docs:" + Arrays.toString(docs) + "}"; + } + + @Override + public void visit(QueryVisitor visitor) { + visitor.visitLeaf(this); + } + + @Override + public boolean equals(Object obj) { + if (sameClassAs(obj) == false) { + return false; + } + TopQuery other = (TopQuery) obj; + return Arrays.equals(docs, other.docs) + && Arrays.equals(segmentStarts, other.segmentStarts) + && contextIdentity == other.contextIdentity; + } + + @Override + public int hashCode() { + return Objects.hash(classHash(), Arrays.hashCode(docs), Arrays.hashCode(segmentStarts), contextIdentity); + } + } private final RankDoc[] docs; - private final int[] segmentStarts; - private final Object contextIdentity; + // topQuery is used to match just the top docs from all the original queries. This match is based on the RankDoc array + // provided when constructing the object. This is the only clause that actually contributes to scoring + private final Query topQuery; + // tailQuery is used to match all the original documents that were used to compute the top docs. + // This is useful if we want to compute aggregations, total hits etc based on all matching documents, and not just the top + // RankDocs provided. This query does not contribute to scoring, as it is set as filter when creating the weight + private final Query tailQuery; + private final boolean onlyRankDocs; /** * Creates a {@code RankDocsQuery} based on the provided docs. * - * @param docs the global doc IDs of documents that match, in ascending order - * @param segmentStarts the indexes in docs and scores corresponding to the first matching - * document in each segment. If a segment has no matching documents, it should be assigned - * the index of the next segment that does. There should be a final entry that is always - * docs.length-1. - * @param contextIdentity an object identifying the reader context that was used to build this - * query + * @param rankDocs The global doc IDs of documents that match, in ascending order + * @param sources The original queries that were used to compute the top documents + * @param queryNames The names (if present) of the original retrievers + * @param onlyRankDocs Whether the query should only match the provided rank docs */ - RankDocsQuery(RankDoc[] docs, int[] segmentStarts, Object contextIdentity) { + public RankDocsQuery(IndexReader reader, RankDoc[] rankDocs, Query[] sources, String[] queryNames, boolean onlyRankDocs) { + assert sources.length == queryNames.length; + // clone to avoid side-effect after sorting + this.docs = rankDocs.clone(); + // sort rank docs by doc id + Arrays.sort(docs, Comparator.comparingInt(a -> a.doc)); + this.topQuery = new TopQuery(docs, sources, queryNames, findSegmentStarts(reader, docs), reader.getContext().id()); + if (sources.length > 0 && false == onlyRankDocs) { + var bq = new BooleanQuery.Builder(); + for (var source : sources) { + bq.add(source, BooleanClause.Occur.SHOULD); + } + this.tailQuery = bq.build(); + } else { + this.tailQuery = null; + } + this.onlyRankDocs = onlyRankDocs; + } + + private RankDocsQuery(RankDoc[] docs, Query topQuery, Query tailQuery, boolean onlyRankDocs) { this.docs = docs; - this.segmentStarts = segmentStarts; - this.contextIdentity = contextIdentity; + this.topQuery = topQuery; + this.tailQuery = tailQuery; + this.onlyRankDocs = onlyRankDocs; } - @Override - public Query rewrite(IndexSearcher searcher) throws IOException { - if (docs.length == 0) { - return new MatchNoDocsQuery(); + private static int binarySearch(RankDoc[] docs, int fromIndex, int toIndex, int key) { + return Arrays.binarySearch(docs, fromIndex, toIndex, new RankDoc(key, Float.NaN, -1), Comparator.comparingInt(a -> a.doc)); + } + + private static int[] findSegmentStarts(IndexReader reader, RankDoc[] docs) { + int[] starts = new int[reader.leaves().size() + 1]; + starts[starts.length - 1] = docs.length; + if (starts.length == 2) { + return starts; + } + int resultIndex = 0; + for (int i = 1; i < starts.length - 1; i++) { + int upper = reader.leaves().get(i).docBase; + + resultIndex = binarySearch(docs, resultIndex, docs.length, upper); + if (resultIndex < 0) { + resultIndex = -1 - resultIndex; + } + starts[i] = resultIndex; } - return this; + return starts; } RankDoc[] rankDocs() { return docs; } + @Override + public Query rewrite(IndexSearcher searcher) throws IOException { + if (tailQuery == null) { + return topQuery; + } + boolean hasChanged = false; + var topRewrite = topQuery.rewrite(searcher); + if (topRewrite != topQuery) { + hasChanged = true; + } + var tailRewrite = tailQuery.rewrite(searcher); + if (tailRewrite != tailQuery) { + hasChanged = true; + } + return hasChanged ? new RankDocsQuery(docs, topRewrite, tailRewrite, onlyRankDocs) : this; + } + @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { - if (searcher.getIndexReader().getContext().id() != contextIdentity) { - throw new IllegalStateException("This RankDocsDocQuery was created by a different reader"); + if (tailQuery == null) { + throw new IllegalArgumentException("[tailQuery] should not be null; maybe missing a rewrite?"); } + var combined = new BooleanQuery.Builder().add(topQuery, onlyRankDocs ? BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD) + .add(tailQuery, BooleanClause.Occur.FILTER) + .build(); + var topWeight = topQuery.createWeight(searcher, scoreMode, boost); + var combinedWeight = searcher.rewrite(combined).createWeight(searcher, scoreMode, boost); return new Weight(this) { - @Override - public int count(LeafReaderContext context) { - return segmentStarts[context.ord + 1] - segmentStarts[context.ord]; + public int count(LeafReaderContext context) throws IOException { + return combinedWeight.count(context); } @Override - public Explanation explain(LeafReaderContext context, int doc) { - int found = Arrays.binarySearch(docs, doc + context.docBase, (a, b) -> Integer.compare(((RankDoc) a).doc, (int) b)); - if (found < 0) { - return Explanation.noMatch("doc not found in top " + docs.length + " rank docs"); - } - return docs[found].explain(); + public Explanation explain(LeafReaderContext context, int doc) throws IOException { + return topWeight.explain(context, doc); } @Override - public Scorer scorer(LeafReaderContext context) { - // Segment starts indicate how many docs are in the segment, - // upper equalling lower indicates no documents for this segment - if (segmentStarts[context.ord] == segmentStarts[context.ord + 1]) { - return null; - } - return new Scorer(this) { - final int lower = segmentStarts[context.ord]; - final int upper = segmentStarts[context.ord + 1]; - int upTo = -1; - float score; - - @Override - public DocIdSetIterator iterator() { - return new DocIdSetIterator() { - @Override - public int docID() { - return currentDocId(); - } - - @Override - public int nextDoc() { - if (upTo == -1) { - upTo = lower; - } else { - ++upTo; - } - return currentDocId(); - } - - @Override - public int advance(int target) throws IOException { - return slowAdvance(target); - } - - @Override - public long cost() { - return upper - lower; - } - }; - } - - @Override - public float getMaxScore(int docId) { - if (docId != NO_MORE_DOCS) { - docId += context.docBase; - } - float maxScore = 0; - for (int idx = Math.max(lower, upTo); idx < upper && docs[idx].doc <= docId; idx++) { - maxScore = Math.max(maxScore, docs[idx].score); - } - return maxScore; - } - - @Override - public float score() { - return docs[upTo].score; - } + public Scorer scorer(LeafReaderContext context) throws IOException { + return combinedWeight.scorer(context); + } - @Override - public int docID() { - return currentDocId(); - } + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return combinedWeight.isCacheable(ctx); + } - private int currentDocId() { - if (upTo == -1) { - return -1; - } - if (upTo >= upper) { - return NO_MORE_DOCS; - } - return docs[upTo].doc - context.docBase; - } + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return combinedWeight.matches(context, doc); + } - }; + @Override + public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { + return combinedWeight.bulkScorer(context); } @Override - public boolean isCacheable(LeafReaderContext ctx) { - return true; + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + return combinedWeight.scorerSupplier(context); } }; } @@ -180,7 +360,10 @@ public String toString(String field) { @Override public void visit(QueryVisitor visitor) { - visitor.visitLeaf(this); + topQuery.visit(visitor); + if (tailQuery != null) { + tailQuery.visit(visitor); + } } @Override @@ -188,13 +371,12 @@ public boolean equals(Object obj) { if (sameClassAs(obj) == false) { return false; } - return Arrays.equals(docs, ((RankDocsQuery) obj).docs) - && Arrays.equals(segmentStarts, ((RankDocsQuery) obj).segmentStarts) - && contextIdentity == ((RankDocsQuery) obj).contextIdentity; + RankDocsQuery other = (RankDocsQuery) obj; + return Objects.equals(topQuery, other.topQuery) && Objects.equals(tailQuery, other.tailQuery) && onlyRankDocs == other.onlyRankDocs; } @Override public int hashCode() { - return Objects.hash(classHash(), Arrays.hashCode(docs), Arrays.hashCode(segmentStarts), contextIdentity); + return Objects.hash(classHash(), topQuery, tailQuery, onlyRankDocs); } } diff --git a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilder.java index 2b77ae543a86c..86cb27cb7ba7e 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilder.java @@ -13,30 +13,45 @@ import org.apache.lucene.search.Query; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.rank.RankDoc; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.Arrays; -import java.util.Comparator; +import java.util.Objects; + +import static org.elasticsearch.TransportVersions.RRF_QUERY_REWRITE; public class RankDocsQueryBuilder extends AbstractQueryBuilder { public static final String NAME = "rank_docs_query"; private final RankDoc[] rankDocs; + private final QueryBuilder[] queryBuilders; + private final boolean onlyRankDocs; - public RankDocsQueryBuilder(RankDoc[] rankDocs) { + public RankDocsQueryBuilder(RankDoc[] rankDocs, QueryBuilder[] queryBuilders, boolean onlyRankDocs) { this.rankDocs = rankDocs; + this.queryBuilders = queryBuilders; + this.onlyRankDocs = onlyRankDocs; } public RankDocsQueryBuilder(StreamInput in) throws IOException { super(in); this.rankDocs = in.readArray(c -> c.readNamedWriteable(RankDoc.class), RankDoc[]::new); + if (in.getTransportVersion().onOrAfter(RRF_QUERY_REWRITE)) { + this.queryBuilders = in.readOptionalArray(c -> c.readNamedWriteable(QueryBuilder.class), QueryBuilder[]::new); + this.onlyRankDocs = in.readBoolean(); + } else { + this.queryBuilders = null; + this.onlyRankDocs = false; + } } RankDoc[] rankDocs() { @@ -46,6 +61,10 @@ RankDoc[] rankDocs() { @Override protected void doWriteTo(StreamOutput out) throws IOException { out.writeArray(StreamOutput::writeNamedWriteable, rankDocs); + if (out.getTransportVersion().onOrAfter(RRF_QUERY_REWRITE)) { + out.writeOptionalArray(StreamOutput::writeNamedWriteable, queryBuilders); + out.writeBoolean(onlyRankDocs); + } } @Override @@ -57,29 +76,22 @@ public String getWriteableName() { protected Query doToQuery(SearchExecutionContext context) throws IOException { RankDoc[] shardRankDocs = Arrays.stream(rankDocs) .filter(r -> r.shardIndex == context.getShardRequestIndex()) - .sorted(Comparator.comparingInt(r -> r.doc)) .toArray(RankDoc[]::new); IndexReader reader = context.getIndexReader(); - int[] segmentStarts = findSegmentStarts(reader, shardRankDocs); - return new RankDocsQuery(shardRankDocs, segmentStarts, reader.getContext().id()); - } - - private static int[] findSegmentStarts(IndexReader reader, RankDoc[] docs) { - int[] starts = new int[reader.leaves().size() + 1]; - starts[starts.length - 1] = docs.length; - if (starts.length == 2) { - return starts; - } - int resultIndex = 0; - for (int i = 1; i < starts.length - 1; i++) { - int upper = reader.leaves().get(i).docBase; - resultIndex = Arrays.binarySearch(docs, resultIndex, docs.length, upper, (a, b) -> Integer.compare(((RankDoc) a).doc, (int) b)); - if (resultIndex < 0) { - resultIndex = -1 - resultIndex; + final Query[] queries; + final String[] queryNames; + if (queryBuilders != null) { + queries = new Query[queryBuilders.length]; + queryNames = new String[queryBuilders.length]; + for (int i = 0; i < queryBuilders.length; i++) { + queries[i] = queryBuilders[i].toQuery(context); + queryNames[i] = queryBuilders[i].queryName(); } - starts[i] = resultIndex; + } else { + queries = new Query[0]; + queryNames = Strings.EMPTY_ARRAY; } - return starts; + return new RankDocsQuery(reader, shardRankDocs, queries, queryNames, onlyRankDocs); } @Override @@ -97,12 +109,14 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep @Override protected boolean doEquals(RankDocsQueryBuilder other) { - return Arrays.equals(rankDocs, other.rankDocs); + return Arrays.equals(rankDocs, other.rankDocs) + && Arrays.equals(queryBuilders, other.queryBuilders) + && onlyRankDocs == other.onlyRankDocs; } @Override protected int doHashCode() { - return Arrays.hashCode(rankDocs); + return Objects.hash(Arrays.hashCode(rankDocs), Arrays.hashCode(queryBuilders), onlyRankDocs); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortBuilder.java deleted file mode 100644 index cfe307af1767a..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortBuilder.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.search.retriever.rankdoc; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.index.query.QueryRewriteContext; -import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.rank.RankDoc; -import org.elasticsearch.search.sort.BucketedSort; -import org.elasticsearch.search.sort.SortBuilder; -import org.elasticsearch.search.sort.SortFieldAndFormat; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Objects; - -/** - * Builds a {@code RankDocsSortField} that sorts documents by their rank as computed through the {@code RankDocsRetrieverBuilder}. - */ -public class RankDocsSortBuilder extends SortBuilder { - public static final String NAME = "rank_docs_sort"; - - private RankDoc[] rankDocs; - - public RankDocsSortBuilder(RankDoc[] rankDocs) { - this.rankDocs = rankDocs; - } - - public RankDocsSortBuilder(StreamInput in) throws IOException { - this.rankDocs = in.readArray(c -> c.readNamedWriteable(RankDoc.class), RankDoc[]::new); - } - - public RankDocsSortBuilder(RankDocsSortBuilder original) { - this.rankDocs = original.rankDocs; - } - - public RankDocsSortBuilder rankDocs(RankDoc[] rankDocs) { - this.rankDocs = rankDocs; - return this; - } - - public RankDoc[] rankDocs() { - return this.rankDocs; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeArray(StreamOutput::writeNamedWriteable, rankDocs); - } - - @Override - public String getWriteableName() { - return NAME; - } - - @Override - public SortBuilder rewrite(QueryRewriteContext ctx) throws IOException { - return this; - } - - @Override - protected SortFieldAndFormat build(SearchExecutionContext context) throws IOException { - RankDoc[] shardRankDocs = Arrays.stream(rankDocs) - .filter(r -> r.shardIndex == context.getShardRequestIndex()) - .toArray(RankDoc[]::new); - return new SortFieldAndFormat(new RankDocsSortField(shardRankDocs), DocValueFormat.RAW); - } - - @Override - public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.RANK_DOCS_RETRIEVER; - } - - @Override - public BucketedSort buildBucketedSort(SearchExecutionContext context, BigArrays bigArrays, int bucketSize, BucketedSort.ExtraData extra) - throws IOException { - throw new UnsupportedOperationException("buildBucketedSort() is not supported for " + this.getClass()); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - throw new UnsupportedOperationException("toXContent() is not supported for " + this.getClass()); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - RankDocsSortBuilder that = (RankDocsSortBuilder) obj; - return Arrays.equals(rankDocs, that.rankDocs) && this.order.equals(that.order); - } - - @Override - public int hashCode() { - return Objects.hash(Arrays.hashCode(this.rankDocs), this.order); - } -} diff --git a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortField.java b/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortField.java deleted file mode 100644 index 9fd2aceaf7949..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortField.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.search.retriever.rankdoc; - -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.FieldComparator; -import org.apache.lucene.search.FieldComparatorSource; -import org.apache.lucene.search.LeafFieldComparator; -import org.apache.lucene.search.Pruning; -import org.apache.lucene.search.Scorable; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.comparators.NumericComparator; -import org.apache.lucene.util.hnsw.IntToIntFunction; -import org.elasticsearch.search.rank.RankDoc; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * A {@link SortField} that sorts documents by their rank as computed through the {@code RankDocsRetrieverBuilder}. - * This is used when we want to score and rank the documents irrespective of their original scores, - * but based on the provided rank they were assigned, e.g. through an RRF retriever. - **/ -public class RankDocsSortField extends SortField { - - public static final String NAME = "_rank"; - - public RankDocsSortField(RankDoc[] rankDocs) { - super(NAME, new FieldComparatorSource() { - @Override - public FieldComparator newComparator(String fieldname, int numHits, Pruning pruning, boolean reversed) { - return new RankDocsComparator(numHits, rankDocs); - } - }); - } - - private static class RankDocsComparator extends NumericComparator { - private final int[] values; - private final Map rankDocMap; - private int topValue; - private int bottom; - - private RankDocsComparator(int numHits, RankDoc[] rankDocs) { - super(NAME, Integer.MAX_VALUE, false, Pruning.NONE, Integer.BYTES); - this.values = new int[numHits]; - this.rankDocMap = Arrays.stream(rankDocs).collect(Collectors.toMap(k -> k.doc, v -> v.rank)); - } - - @Override - public int compare(int slot1, int slot2) { - return Integer.compare(values[slot1], values[slot2]); - } - - @Override - public Integer value(int slot) { - return Integer.valueOf(values[slot]); - } - - @Override - public void setTopValue(Integer value) { - topValue = value; - } - - @Override - public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { - IntToIntFunction docToRank = doc -> rankDocMap.getOrDefault(context.docBase + doc, Integer.MAX_VALUE); - return new LeafFieldComparator() { - @Override - public void setBottom(int slot) throws IOException { - bottom = values[slot]; - } - - @Override - public int compareBottom(int doc) { - return Integer.compare(bottom, docToRank.apply(doc)); - } - - @Override - public int compareTop(int doc) { - return Integer.compare(topValue, docToRank.apply(doc)); - } - - @Override - public void copy(int slot, int doc) { - values[slot] = docToRank.apply(doc); - } - - @Override - public void setScorer(Scorable scorer) {} - }; - } - } -} diff --git a/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java b/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java index 0bdb3ea0cd247..a38a24eb75fca 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java +++ b/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java @@ -73,4 +73,18 @@ public LeafFieldComparator getLeafComparator(LeafReaderContext context) { } }; } + + /** + * Get the doc id encoded in the sort value. + */ + public static int decodeDoc(long value) { + return (int) value; + } + + /** + * Get the shard request index encoded in the sort value. + */ + public static int decodeShardRequestIndex(long value) { + return (int) (value >> 32); + } } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java index 1d430f2ae1079..23c956e6e52f2 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java @@ -28,10 +28,10 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SubSearchSourceBuilder; import org.elasticsearch.search.collapse.CollapseBuilder; -import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.rank.TestRankBuilder; import org.elasticsearch.search.rescore.QueryRescorerBuilder; import org.elasticsearch.search.retriever.RetrieverBuilder; +import org.elasticsearch.search.retriever.TestCompoundRetrieverBuilder; import org.elasticsearch.search.slice.SliceBuilder; import org.elasticsearch.search.suggest.SuggestBuilder; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; @@ -263,40 +263,9 @@ public void testValidate() throws IOException { } { // allow_partial_results and compound retriever - SearchRequest searchRequest = createSearchRequest().source(new SearchSourceBuilder().retriever(new RetrieverBuilder() { - @Override - public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) { - // no-op - } - - @Override - public String getName() { - return "compound_retriever"; - } - - @Override - protected void doToXContent(XContentBuilder builder, Params params) throws IOException {} - - @Override - protected boolean doEquals(Object o) { - return false; - } - - @Override - protected int doHashCode() { - return 0; - } - - @Override - public boolean isCompound() { - return true; - } - - @Override - public QueryBuilder topDocsQuery() { - return null; - } - })); + SearchRequest searchRequest = createSearchRequest().source( + new SearchSourceBuilder().retriever(new TestCompoundRetrieverBuilder(randomIntBetween(1, 10))) + ); searchRequest.allowPartialSearchResults(true); searchRequest.scroll((Scroll) null); ActionRequestValidationException validationErrors = searchRequest.validate(); @@ -583,30 +552,6 @@ public QueryBuilder topDocsQuery() { assertEquals(1, validationErrors.validationErrors().size()); assertEquals("[rank] cannot be used with [rescore]", validationErrors.validationErrors().get(0)); } - { - SearchRequest searchRequest = new SearchRequest().source( - new SearchSourceBuilder().rankBuilder(new TestRankBuilder(100)) - .query(QueryBuilders.termQuery("field", "term")) - .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, null))) - .sort("test") - ); - ActionRequestValidationException validationErrors = searchRequest.validate(); - assertNotNull(validationErrors); - assertEquals(1, validationErrors.validationErrors().size()); - assertEquals("[rank] cannot be used with [sort]", validationErrors.validationErrors().get(0)); - } - { - SearchRequest searchRequest = new SearchRequest().source( - new SearchSourceBuilder().rankBuilder(new TestRankBuilder(100)) - .query(QueryBuilders.termQuery("field", "term")) - .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, null))) - .collapse(new CollapseBuilder("field")) - ); - ActionRequestValidationException validationErrors = searchRequest.validate(); - assertNotNull(validationErrors); - assertEquals(1, validationErrors.validationErrors().size()); - assertEquals("[rank] cannot be used with [collapse]", validationErrors.validationErrors().get(0)); - } { SearchRequest searchRequest = new SearchRequest().source( new SearchSourceBuilder().rankBuilder(new TestRankBuilder(100)) @@ -619,30 +564,6 @@ public QueryBuilder topDocsQuery() { assertEquals(1, validationErrors.validationErrors().size()); assertEquals("[rank] cannot be used with [suggest]", validationErrors.validationErrors().get(0)); } - { - SearchRequest searchRequest = new SearchRequest().source( - new SearchSourceBuilder().rankBuilder(new TestRankBuilder(100)) - .query(QueryBuilders.termQuery("field", "term")) - .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, null))) - .highlighter(new HighlightBuilder().field("field")) - ); - ActionRequestValidationException validationErrors = searchRequest.validate(); - assertNotNull(validationErrors); - assertEquals(1, validationErrors.validationErrors().size()); - assertEquals("[rank] cannot be used with [highlighter]", validationErrors.validationErrors().get(0)); - } - { - SearchRequest searchRequest = new SearchRequest().source( - new SearchSourceBuilder().rankBuilder(new TestRankBuilder(100)) - .query(QueryBuilders.termQuery("field", "term")) - .knnSearch(List.of(new KnnSearchBuilder("vector", new float[] { 0f }, 10, 100, null))) - .pointInTimeBuilder(new PointInTimeBuilder(new BytesArray("test"))) - ); - ActionRequestValidationException validationErrors = searchRequest.validate(); - assertNotNull(validationErrors); - assertEquals(1, validationErrors.validationErrors().size()); - assertEquals("[rank] cannot be used with [point in time]", validationErrors.validationErrors().get(0)); - } { SearchRequest searchRequest = new SearchRequest("test").source( new SearchSourceBuilder().pointInTimeBuilder(new PointInTimeBuilder(BytesArray.EMPTY)) diff --git a/server/src/test/java/org/elasticsearch/search/rank/RankDocTests.java b/server/src/test/java/org/elasticsearch/search/rank/RankDocTests.java index db419b4019acf..d190139309c31 100644 --- a/server/src/test/java/org/elasticsearch/search/rank/RankDocTests.java +++ b/server/src/test/java/org/elasticsearch/search/rank/RankDocTests.java @@ -50,9 +50,4 @@ protected RankDoc mutateInstance(RankDoc instance) throws IOException { } return mutated; } - - public void testExplain() { - RankDoc instance = createTestRankDoc(); - assertEquals(instance.explain().toString(), instance.explain().toString()); - } } diff --git a/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java b/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java index 23a6357fa61be..f3dd86e0b1fa2 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java +++ b/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java @@ -22,7 +22,6 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.rank.RankDoc; import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder; -import org.elasticsearch.search.vectors.ExactKnnQueryBuilder; import org.elasticsearch.test.AbstractXContentTestCase; import org.elasticsearch.usage.SearchUsage; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -122,17 +121,15 @@ public void testTopDocsQuery() { final int preFilters = knnRetriever.preFilterQueryBuilders.size(); QueryBuilder topDocsQuery = knnRetriever.topDocsQuery(); assertNotNull(topDocsQuery); - assertThat(topDocsQuery, instanceOf(BoolQueryBuilder.class)); - assertThat(((BoolQueryBuilder) topDocsQuery).filter().size(), equalTo(1 + preFilters)); - assertThat(((BoolQueryBuilder) topDocsQuery).filter().get(0), instanceOf(RankDocsQueryBuilder.class)); - for (int i = 0; i < preFilters; i++) { - assertThat( - ((BoolQueryBuilder) topDocsQuery).filter().get(i + 1), - instanceOf(knnRetriever.preFilterQueryBuilders.get(i).getClass()) - ); + assertThat(topDocsQuery, anyOf(instanceOf(RankDocsQueryBuilder.class), instanceOf(BoolQueryBuilder.class))); + if (topDocsQuery instanceof BoolQueryBuilder bq) { + assertThat(bq.must().size(), equalTo(1)); + assertThat(bq.must().get(0), instanceOf(RankDocsQueryBuilder.class)); + assertThat(bq.filter().size(), equalTo(preFilters)); + for (int i = 0; i < preFilters; i++) { + assertThat(bq.filter().get(i), instanceOf(knnRetriever.preFilterQueryBuilders.get(i).getClass())); + } } - assertThat(((BoolQueryBuilder) topDocsQuery).should().size(), equalTo(1)); - assertThat(((BoolQueryBuilder) topDocsQuery).should().get(0), instanceOf(ExactKnnQueryBuilder.class)); } @Override diff --git a/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java b/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java index e8ad7d128dac2..bcb93b100ea48 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java @@ -10,9 +10,6 @@ package org.elasticsearch.search.retriever; import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.DisMaxQueryBuilder; -import org.elasticsearch.index.query.MatchAllQueryBuilder; -import org.elasticsearch.index.query.MatchNoneQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.RandomQueryBuilder; @@ -21,8 +18,6 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.rank.RankDoc; import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder; -import org.elasticsearch.search.retriever.rankdoc.RankDocsSortBuilder; -import org.elasticsearch.search.sort.ScoreSortBuilder; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -30,10 +25,10 @@ import java.util.List; import java.util.function.Supplier; +import static org.elasticsearch.search.SearchService.DEFAULT_SIZE; import static org.elasticsearch.search.vectors.KnnSearchBuilderTests.randomVector; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Mockito.mock; @@ -93,7 +88,7 @@ private List preFilters() { } private RankDocsRetrieverBuilder createRandomRankDocsRetrieverBuilder() { - return new RankDocsRetrieverBuilder(randomInt(100), innerRetrievers(), rankDocsSupplier(), preFilters()); + return new RankDocsRetrieverBuilder(randomIntBetween(1, 100), innerRetrievers(), rankDocsSupplier(), preFilters()); } public void testExtractToSearchSourceBuilder() { @@ -102,32 +97,30 @@ public void testExtractToSearchSourceBuilder() { if (randomBoolean()) { source.aggregation(new TermsAggregationBuilder("name").field("field")); } + source.explain(randomBoolean()); + source.profile(randomBoolean()); + source.trackTotalHits(randomBoolean()); + final int preFilters = retriever.preFilterQueryBuilders.size(); retriever.extractToSearchSourceBuilder(source, randomBoolean()); - assertThat(source.sorts().size(), equalTo(2)); - assertThat(source.sorts().get(0), instanceOf(RankDocsSortBuilder.class)); - assertThat(source.sorts().get(1), instanceOf(ScoreSortBuilder.class)); - assertThat(source.query(), instanceOf(BoolQueryBuilder.class)); - BoolQueryBuilder bq = (BoolQueryBuilder) source.query(); - if (source.aggregations() != null) { - assertThat(bq.must().size(), equalTo(0)); - assertThat(bq.should().size(), greaterThanOrEqualTo(1)); - assertThat(bq.should().get(0), instanceOf(RankDocsQueryBuilder.class)); - assertNotNull(source.postFilter()); - assertThat(source.postFilter(), instanceOf(RankDocsQueryBuilder.class)); - } else { + assertNull(source.sorts()); + assertThat(source.query(), anyOf(instanceOf(BoolQueryBuilder.class), instanceOf(RankDocsQueryBuilder.class))); + if (source.query() instanceof BoolQueryBuilder bq) { assertThat(bq.must().size(), equalTo(1)); assertThat(bq.must().get(0), instanceOf(RankDocsQueryBuilder.class)); - assertNull(source.postFilter()); + assertThat(bq.filter().size(), equalTo(preFilters)); + for (int i = 0; i < preFilters; i++) { + assertThat(bq.filter().get(i), instanceOf(retriever.preFilterQueryBuilders.get(i).getClass())); + } } - assertThat(bq.filter().size(), equalTo(retriever.preFilterQueryBuilders.size())); + assertNull(source.postFilter()); } public void testTopDocsQuery() { RankDocsRetrieverBuilder retriever = createRandomRankDocsRetrieverBuilder(); QueryBuilder topDocs = retriever.topDocsQuery(); assertNotNull(topDocs); - assertThat(topDocs, instanceOf(DisMaxQueryBuilder.class)); - assertThat(((DisMaxQueryBuilder) topDocs).innerQueries(), hasSize(retriever.sources.size())); + assertThat(topDocs, instanceOf(BoolQueryBuilder.class)); + assertThat(((BoolQueryBuilder) topDocs).should(), hasSize(retriever.sources.size())); } public void testRewrite() throws IOException { @@ -144,22 +137,27 @@ public boolean isCompound() { } SearchSourceBuilder source = new SearchSourceBuilder().retriever(retriever); QueryRewriteContext queryRewriteContext = mock(QueryRewriteContext.class); - if (compoundAdded) { - expectThrows(AssertionError.class, () -> Rewriteable.rewrite(source, queryRewriteContext)); + int size = source.size() < 0 ? DEFAULT_SIZE : source.size(); + if (retriever.rankWindowSize < size) { + if (compoundAdded) { + expectThrows(AssertionError.class, () -> Rewriteable.rewrite(source, queryRewriteContext)); + } } else { - SearchSourceBuilder rewrittenSource = Rewriteable.rewrite(source, queryRewriteContext); - assertNull(rewrittenSource.retriever()); - assertTrue(rewrittenSource.knnSearch().isEmpty()); - assertThat( - rewrittenSource.query(), - anyOf(instanceOf(BoolQueryBuilder.class), instanceOf(MatchAllQueryBuilder.class), instanceOf(MatchNoneQueryBuilder.class)) - ); - if (rewrittenSource.query() instanceof BoolQueryBuilder) { - BoolQueryBuilder bq = (BoolQueryBuilder) rewrittenSource.query(); - assertThat(bq.filter().size(), equalTo(retriever.preFilterQueryBuilders.size())); - // we don't have any aggregations so the RankDocs query is set as a must clause - assertThat(bq.must().size(), equalTo(1)); - assertThat(bq.must().get(0), instanceOf(RankDocsQueryBuilder.class)); + if (compoundAdded) { + expectThrows(AssertionError.class, () -> Rewriteable.rewrite(source, queryRewriteContext)); + } else { + SearchSourceBuilder rewrittenSource = Rewriteable.rewrite(source, queryRewriteContext); + assertNull(rewrittenSource.retriever()); + assertTrue(rewrittenSource.knnSearch().isEmpty()); + assertThat(rewrittenSource.query(), instanceOf(RankDocsQueryBuilder.class)); + if (rewrittenSource.query() instanceof BoolQueryBuilder) { + BoolQueryBuilder bq = (BoolQueryBuilder) rewrittenSource.query(); + assertThat(bq.filter().size(), equalTo(retriever.preFilterQueryBuilders.size())); + assertThat(bq.must().size(), equalTo(1)); + assertThat(bq.must().get(0), instanceOf(BoolQueryBuilder.class)); + assertThat(bq.should().size(), equalTo(1)); + assertThat(bq.should().get(0), instanceOf(RankDocsQueryBuilder.class)); + } } } } diff --git a/server/src/test/java/org/elasticsearch/search/retriever/RetrieverBuilderErrorTests.java b/server/src/test/java/org/elasticsearch/search/retriever/RetrieverBuilderErrorTests.java index cc8f5fe85d09a..66240e205e26b 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/RetrieverBuilderErrorTests.java +++ b/server/src/test/java/org/elasticsearch/search/retriever/RetrieverBuilderErrorTests.java @@ -73,14 +73,6 @@ public void testRetrieverExtractionErrors() throws IOException { assertThat(iae.getMessage(), containsString("cannot specify [retriever] and [terminate_after]")); } - try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"sort\": [\"field\"], \"retriever\":{\"standard\":{}}}")) { - SearchSourceBuilder ssb = new SearchSourceBuilder(); - ssb.parseXContent(parser, true, nf -> true); - ActionRequestValidationException iae = ssb.validate(null, false, false); - assertNotNull(iae); - assertThat(iae.getMessage(), containsString("cannot specify [retriever] and [sort]")); - } - try ( XContentParser parser = createParser( JsonXContent.jsonXContent, @@ -94,14 +86,6 @@ public void testRetrieverExtractionErrors() throws IOException { assertThat(iae.getMessage(), containsString("cannot specify [retriever] and [rescore]")); } - try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"min_score\": 2, \"retriever\":{\"standard\":{}}}")) { - SearchSourceBuilder ssb = new SearchSourceBuilder(); - ssb.parseXContent(parser, true, nf -> true); - ActionRequestValidationException iae = ssb.validate(null, false, false); - assertNotNull(iae); - assertThat(iae.getMessage(), containsString("cannot specify [retriever] and [min_score]")); - } - try ( XContentParser parser = createParser( JsonXContent.jsonXContent, @@ -112,7 +96,7 @@ public void testRetrieverExtractionErrors() throws IOException { ssb.parseXContent(parser, true, nf -> true); ActionRequestValidationException iae = ssb.validate(null, false, false); assertNotNull(iae); - assertThat(iae.getMessage(), containsString("cannot specify [retriever] and [query, terminate_after, min_score]")); + assertThat(iae.getMessage(), containsString("cannot specify [retriever] and [query, terminate_after]")); } } diff --git a/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilderTests.java index 01c915530bc4c..ca05c57b7d733 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilderTests.java @@ -10,8 +10,16 @@ package org.elasticsearch.search.retriever.rankdoc; import org.apache.lucene.document.Document; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopScoreDocCollectorManager; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; import org.elasticsearch.index.query.QueryBuilder; @@ -21,6 +29,10 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Random; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; public class RankDocsQueryBuilderTests extends AbstractQueryTestCase { @@ -39,7 +51,7 @@ private RankDoc[] generateRandomRankDocs() { @Override protected RankDocsQueryBuilder doCreateTestQueryBuilder() { RankDoc[] rankDocs = generateRandomRankDocs(); - return new RankDocsQueryBuilder(rankDocs); + return new RankDocsQueryBuilder(rankDocs, null, false); } @Override @@ -104,6 +116,111 @@ public void testMustRewrite() throws IOException { } } + public void testRankDocsQueryEarlyTerminate() throws IOException { + try (Directory directory = newDirectory()) { + IndexWriterConfig config = new IndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE); + try (IndexWriter iw = new IndexWriter(directory, config)) { + int seg = atLeast(5); + int numDocs = atLeast(20); + for (int i = 0; i < seg; i++) { + for (int j = 0; j < numDocs; j++) { + Document doc = new Document(); + doc.add(new NumericDocValuesField("active", 1)); + iw.addDocument(doc); + } + if (frequently()) { + iw.flush(); + } + } + } + try (IndexReader reader = DirectoryReader.open(directory)) { + int topSize = randomIntBetween(1, reader.maxDoc() / 5); + RankDoc[] rankDocs = new RankDoc[topSize]; + int index = 0; + for (int r : randomSample(random(), reader.maxDoc(), topSize)) { + rankDocs[index++] = new RankDoc(r, randomFloat(), randomIntBetween(0, 5)); + } + Arrays.sort(rankDocs); + for (int i = 0; i < rankDocs.length; i++) { + rankDocs[i].rank = i; + } + IndexSearcher searcher = new IndexSearcher(reader); + for (int totalHitsThreshold = 0; totalHitsThreshold < reader.maxDoc(); totalHitsThreshold += randomIntBetween(1, 10)) { + // Tests that the query matches only the {@link RankDoc} when the hit threshold is reached. + RankDocsQuery q = new RankDocsQuery( + reader, + rankDocs, + new Query[] { NumericDocValuesField.newSlowExactQuery("active", 1) }, + new String[1], + false + ); + var topDocsManager = new TopScoreDocCollectorManager(topSize, null, totalHitsThreshold); + var col = searcher.search(q, topDocsManager); + // depending on the doc-ids of the RankDocs (i.e. the actual docs to have score) we could visit them last, + // so worst case is we could end up collecting up to 1 + max(topSize , totalHitsThreshold) + rankDocs.length documents + // as we could have already filled the priority queue with non-optimal docs + assertThat( + col.totalHits.value, + lessThanOrEqualTo((long) (1 + Math.max(topSize, totalHitsThreshold) + rankDocs.length)) + ); + assertEqualTopDocs(col.scoreDocs, rankDocs); + } + + { + // Return all docs (rank + tail) + RankDocsQuery q = new RankDocsQuery( + reader, + rankDocs, + new Query[] { NumericDocValuesField.newSlowExactQuery("active", 1) }, + new String[1], + false + ); + var topDocsManager = new TopScoreDocCollectorManager(topSize, null, Integer.MAX_VALUE); + var col = searcher.search(q, topDocsManager); + assertThat(col.totalHits.value, equalTo((long) reader.maxDoc())); + assertEqualTopDocs(col.scoreDocs, rankDocs); + } + + { + // Only rank docs + RankDocsQuery q = new RankDocsQuery( + reader, + rankDocs, + new Query[] { NumericDocValuesField.newSlowExactQuery("active", 1) }, + new String[1], + true + ); + var topDocsManager = new TopScoreDocCollectorManager(topSize, null, Integer.MAX_VALUE); + var col = searcher.search(q, topDocsManager); + assertThat(col.totalHits.value, equalTo((long) topSize)); + assertEqualTopDocs(col.scoreDocs, rankDocs); + } + } + } + } + + private static int[] randomSample(Random rand, int n, int k) { + int[] reservoir = new int[k]; + for (int i = 0; i < k; i++) { + reservoir[i] = i; + } + for (int i = k; i < n; i++) { + int j = rand.nextInt(i + 1); + if (j < k) { + reservoir[j] = i; + } + } + return reservoir; + } + + private static void assertEqualTopDocs(ScoreDoc[] scoreDocs, RankDoc[] rankDocs) { + for (int i = 0; i < scoreDocs.length; i++) { + assertEquals(rankDocs[i].doc, scoreDocs[i].doc); + assertEquals(rankDocs[i].score, scoreDocs[i].score, 0f); + assertEquals(-1, scoreDocs[i].shardIndex); + } + } + @Override public void testFromXContent() throws IOException { // no-op since RankDocsQueryBuilder is an internal only API diff --git a/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortBuilderTests.java deleted file mode 100644 index 2c12126769c35..0000000000000 --- a/server/src/test/java/org/elasticsearch/search/retriever/rankdoc/RankDocsSortBuilderTests.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.search.retriever.rankdoc; - -import org.apache.lucene.search.SortField; -import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.rank.RankDoc; -import org.elasticsearch.search.sort.AbstractSortTestCase; -import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.xcontent.XContentParser; - -import java.io.IOException; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; - -public class RankDocsSortBuilderTests extends AbstractSortTestCase { - - @Override - protected RankDocsSortBuilder createTestItem() { - return randomRankDocsSortBuulder(); - } - - private RankDocsSortBuilder randomRankDocsSortBuulder() { - RankDoc[] rankDocs = randomRankDocs(randomInt(100)); - return new RankDocsSortBuilder(rankDocs); - } - - private RankDoc[] randomRankDocs(int totalDocs) { - RankDoc[] rankDocs = new RankDoc[totalDocs]; - for (int i = 0; i < totalDocs; i++) { - rankDocs[i] = new RankDoc(randomNonNegativeInt(), randomFloat(), randomIntBetween(0, 1)); - rankDocs[i].rank = i + 1; - } - return rankDocs; - } - - @Override - protected RankDocsSortBuilder mutate(RankDocsSortBuilder original) throws IOException { - RankDocsSortBuilder mutated = new RankDocsSortBuilder(original); - mutated.rankDocs(randomRankDocs(original.rankDocs().length + randomIntBetween(10, 100))); - return mutated; - } - - @Override - public void testFromXContent() throws IOException { - // no-op - } - - @Override - protected RankDocsSortBuilder fromXContent(XContentParser parser, String fieldName) throws IOException { - throw new UnsupportedOperationException( - "{" + RankDocsSortBuilder.class.getSimpleName() + "} does not support parsing from XContent" - ); - } - - @Override - protected void sortFieldAssertions(RankDocsSortBuilder builder, SortField sortField, DocValueFormat format) throws IOException { - assertThat(builder.order(), equalTo(SortOrder.ASC)); - assertThat(sortField, instanceOf(RankDocsSortField.class)); - assertThat(sortField.getField(), equalTo(RankDocsSortField.NAME)); - assertThat(sortField.getType(), equalTo(SortField.Type.CUSTOM)); - assertThat(sortField.getReverse(), equalTo(false)); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/search/retriever/TestCompoundRetrieverBuilder.java b/test/framework/src/main/java/org/elasticsearch/search/retriever/TestCompoundRetrieverBuilder.java new file mode 100644 index 0000000000000..9f199aa7f3ef8 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/search/retriever/TestCompoundRetrieverBuilder.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.search.retriever; + +import org.apache.lucene.search.ScoreDoc; +import org.elasticsearch.search.rank.RankDoc; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.net.UnknownServiceException; +import java.util.ArrayList; +import java.util.List; + +public class TestCompoundRetrieverBuilder extends CompoundRetrieverBuilder { + + public static final String NAME = "test_compound_retriever_builder"; + + public TestCompoundRetrieverBuilder(int rankWindowSize) { + this(new ArrayList<>(), rankWindowSize); + } + + TestCompoundRetrieverBuilder(List childRetrievers, int rankWindowSize) { + super(childRetrievers, rankWindowSize); + } + + @Override + protected TestCompoundRetrieverBuilder clone(List newChildRetrievers) { + return new TestCompoundRetrieverBuilder(newChildRetrievers, rankWindowSize); + } + + @Override + protected RankDoc[] combineInnerRetrieverResults(List rankResults) { + return new RankDoc[0]; + } + + @Override + public String getName() { + return NAME; + } + + @Override + protected void doToXContent(XContentBuilder builder, Params params) throws IOException { + throw new UnknownServiceException("should not be called"); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java index 927c708268a49..ab013e0275a69 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java @@ -43,7 +43,6 @@ public class TextSimilarityRankRetrieverBuilder extends RetrieverBuilder { public static final ParseField INFERENCE_TEXT_FIELD = new ParseField("inference_text"); public static final ParseField FIELD_FIELD = new ParseField("field"); public static final ParseField RANK_WINDOW_SIZE_FIELD = new ParseField("rank_window_size"); - public static final ParseField MIN_SCORE_FIELD = new ParseField("min_score"); public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(TextSimilarityRankBuilder.NAME, args -> { @@ -52,9 +51,8 @@ public class TextSimilarityRankRetrieverBuilder extends RetrieverBuilder { String inferenceText = (String) args[2]; String field = (String) args[3]; int rankWindowSize = args[4] == null ? DEFAULT_RANK_WINDOW_SIZE : (int) args[4]; - Float minScore = (Float) args[5]; - return new TextSimilarityRankRetrieverBuilder(retrieverBuilder, inferenceId, inferenceText, field, rankWindowSize, minScore); + return new TextSimilarityRankRetrieverBuilder(retrieverBuilder, inferenceId, inferenceText, field, rankWindowSize); }); static { @@ -63,7 +61,6 @@ public class TextSimilarityRankRetrieverBuilder extends RetrieverBuilder { PARSER.declareString(constructorArg(), INFERENCE_TEXT_FIELD); PARSER.declareString(constructorArg(), FIELD_FIELD); PARSER.declareInt(optionalConstructorArg(), RANK_WINDOW_SIZE_FIELD); - PARSER.declareFloat(optionalConstructorArg(), MIN_SCORE_FIELD); RetrieverBuilder.declareBaseParserFields(TextSimilarityRankBuilder.NAME, PARSER); } @@ -84,22 +81,19 @@ public static TextSimilarityRankRetrieverBuilder fromXContent(XContentParser par private final String inferenceText; private final String field; private final int rankWindowSize; - private final Float minScore; public TextSimilarityRankRetrieverBuilder( RetrieverBuilder retrieverBuilder, String inferenceId, String inferenceText, String field, - int rankWindowSize, - Float minScore + int rankWindowSize ) { this.retrieverBuilder = retrieverBuilder; this.inferenceId = inferenceId; this.inferenceText = inferenceText; this.field = field; this.rankWindowSize = rankWindowSize; - this.minScore = minScore; } public TextSimilarityRankRetrieverBuilder( diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java index c834f58f1134b..1a72cb0da2899 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java @@ -62,8 +62,7 @@ public static TextSimilarityRankRetrieverBuilder createRandomTextSimilarityRankR randomAlphaOfLength(10), randomAlphaOfLength(20), randomAlphaOfLength(50), - randomIntBetween(100, 10000), - randomBoolean() ? null : randomFloatBetween(-1.0f, 1.0f, true) + randomIntBetween(100, 10000) ); } diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java new file mode 100644 index 0000000000000..8b924af48c631 --- /dev/null +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java @@ -0,0 +1,656 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.rank.rrf; + +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.InnerHitBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.collapse.CollapseBuilder; +import org.elasticsearch.search.retriever.CompoundRetrieverBuilder; +import org.elasticsearch.search.retriever.KnnRetrieverBuilder; +import org.elasticsearch.search.retriever.StandardRetrieverBuilder; +import org.elasticsearch.search.retriever.TestRetrieverBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import org.elasticsearch.xcontent.XContentType; +import org.junit.Before; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +@ESIntegTestCase.ClusterScope(minNumDataNodes = 3) +public class RRFRetrieverBuilderIT extends ESIntegTestCase { + + protected static String INDEX = "test_index"; + protected static final String ID_FIELD = "_id"; + protected static final String DOC_FIELD = "doc"; + protected static final String TEXT_FIELD = "text"; + protected static final String VECTOR_FIELD = "vector"; + protected static final String TOPIC_FIELD = "topic"; + + @Override + protected Collection> nodePlugins() { + return List.of(RRFRankPlugin.class); + } + + @Before + public void setup() throws Exception { + setupIndex(); + } + + protected void setupIndex() { + String mapping = """ + { + "properties": { + "vector": { + "type": "dense_vector", + "dims": 1, + "element_type": "float", + "similarity": "l2_norm", + "index": true, + "index_options": { + "type": "hnsw" + } + }, + "text": { + "type": "text" + }, + "doc": { + "type": "keyword" + }, + "topic": { + "type": "keyword" + }, + "views": { + "type": "nested", + "properties": { + "last30d": { + "type": "integer" + }, + "all": { + "type": "integer" + } + } + } + } + } + """; + createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 5).build()); + admin().indices().preparePutMapping(INDEX).setSource(mapping, XContentType.JSON).get(); + indexDoc(INDEX, "doc_1", DOC_FIELD, "doc_1", TOPIC_FIELD, "technology", TEXT_FIELD, "term"); + indexDoc( + INDEX, + "doc_2", + DOC_FIELD, + "doc_2", + TOPIC_FIELD, + "astronomy", + TEXT_FIELD, + "search term term", + VECTOR_FIELD, + new float[] { 2.0f } + ); + indexDoc(INDEX, "doc_3", DOC_FIELD, "doc_3", TOPIC_FIELD, "technology", VECTOR_FIELD, new float[] { 3.0f }); + indexDoc(INDEX, "doc_4", DOC_FIELD, "doc_4", TOPIC_FIELD, "technology", TEXT_FIELD, "term term term term"); + indexDoc(INDEX, "doc_5", DOC_FIELD, "doc_5", TOPIC_FIELD, "science", TEXT_FIELD, "irrelevant stuff"); + indexDoc( + INDEX, + "doc_6", + DOC_FIELD, + "doc_6", + TEXT_FIELD, + "search term term term term term term", + VECTOR_FIELD, + new float[] { 6.0f } + ); + indexDoc( + INDEX, + "doc_7", + DOC_FIELD, + "doc_7", + TOPIC_FIELD, + "biology", + TEXT_FIELD, + "term term term term term term term", + VECTOR_FIELD, + new float[] { 7.0f } + ); + refresh(INDEX); + } + + public void testRRFPagination() { + final int rankWindowSize = 100; + final int rankConstant = 10; + final List expectedDocIds = List.of("doc_2", "doc_6", "doc_7", "doc_1", "doc_3", "doc_4"); + final int totalDocs = expectedDocIds.size(); + for (int i = 0; i < randomIntBetween(1, 5); i++) { + int from = randomIntBetween(0, totalDocs - 1); + int size = randomIntBetween(1, totalDocs - from); + for (int docs_to_fetch = from; docs_to_fetch < totalDocs; docs_to_fetch += size) { + SearchSourceBuilder source = new SearchSourceBuilder(); + source.from(docs_to_fetch); + source.size(size); + // this one retrieves docs 1, 2, 4, 6, and 7 + StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder( + QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(9L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(8L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(7L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_7")).boost(6L)) + ); + // this one retrieves docs 2 and 6 due to prefilter + StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( + QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + ); + standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); + // this one retrieves docs 3, 2, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + source.retriever( + new RRFRetrieverBuilder( + Arrays.asList( + new CompoundRetrieverBuilder.RetrieverSource(standard0, null), + new CompoundRetrieverBuilder.RetrieverSource(standard1, null), + new CompoundRetrieverBuilder.RetrieverSource(knnRetrieverBuilder, null) + ), + rankWindowSize, + rankConstant + ) + ); + SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); + int fDocs_to_fetch = docs_to_fetch; + ElasticsearchAssertions.assertResponse(req, resp -> { + assertNull(resp.pointInTimeId()); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); + assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); + assertThat(resp.getHits().getHits().length, lessThanOrEqualTo(size)); + for (int k = 0; k < Math.min(size, resp.getHits().getHits().length); k++) { + assertThat(resp.getHits().getAt(k).getId(), equalTo(expectedDocIds.get(k + fDocs_to_fetch))); + } + }); + } + } + } + + public void testRRFWithAggs() { + final int rankWindowSize = 100; + final int rankConstant = 10; + SearchSourceBuilder source = new SearchSourceBuilder(); + // this one retrieves docs 1, 2, 4, 6, and 7 + StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder( + QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(9L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(8L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(7L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_7")).boost(6L)) + ); + // this one retrieves docs 2 and 6 due to prefilter + StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( + QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + ); + standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); + // this one retrieves docs 3, 2, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + source.retriever( + new RRFRetrieverBuilder( + Arrays.asList( + new CompoundRetrieverBuilder.RetrieverSource(standard0, null), + new CompoundRetrieverBuilder.RetrieverSource(standard1, null), + new CompoundRetrieverBuilder.RetrieverSource(knnRetrieverBuilder, null) + ), + rankWindowSize, + rankConstant + ) + ); + source.size(1); + source.aggregation(AggregationBuilders.terms("topic_agg").field(TOPIC_FIELD)); + SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); + ElasticsearchAssertions.assertResponse(req, resp -> { + assertNull(resp.pointInTimeId()); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); + assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); + assertThat(resp.getHits().getHits().length, equalTo(1)); + assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2")); + + assertNotNull(resp.getAggregations()); + assertNotNull(resp.getAggregations().get("topic_agg")); + Terms terms = resp.getAggregations().get("topic_agg"); + + assertThat(terms.getBucketByKey("technology").getDocCount(), equalTo(3L)); + assertThat(terms.getBucketByKey("astronomy").getDocCount(), equalTo(1L)); + assertThat(terms.getBucketByKey("biology").getDocCount(), equalTo(1L)); + }); + } + + public void testRRFWithCollapse() { + final int rankWindowSize = 100; + final int rankConstant = 10; + SearchSourceBuilder source = new SearchSourceBuilder(); + // this one retrieves docs 1, 2, 4, 6, and 7 + StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder( + QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(9L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(8L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(7L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_7")).boost(6L)) + ); + // this one retrieves docs 2 and 6 due to prefilter + StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( + QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + ); + standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); + // this one retrieves docs 3, 2, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + source.retriever( + new RRFRetrieverBuilder( + Arrays.asList( + new CompoundRetrieverBuilder.RetrieverSource(standard0, null), + new CompoundRetrieverBuilder.RetrieverSource(standard1, null), + new CompoundRetrieverBuilder.RetrieverSource(knnRetrieverBuilder, null) + ), + rankWindowSize, + rankConstant + ) + ); + source.collapse( + new CollapseBuilder(TOPIC_FIELD).setInnerHits( + new InnerHitBuilder("a").addSort(new FieldSortBuilder(DOC_FIELD).order(SortOrder.DESC)).setSize(10) + ) + ); + source.fetchField(TOPIC_FIELD); + SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); + ElasticsearchAssertions.assertResponse(req, resp -> { + assertNull(resp.pointInTimeId()); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); + assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); + assertThat(resp.getHits().getHits().length, equalTo(4)); + assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2")); + assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_6")); + assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_7")); + assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_1")); + assertThat(resp.getHits().getAt(3).getInnerHits().get("a").getAt(0).getId(), equalTo("doc_4")); + assertThat(resp.getHits().getAt(3).getInnerHits().get("a").getAt(1).getId(), equalTo("doc_3")); + assertThat(resp.getHits().getAt(3).getInnerHits().get("a").getAt(2).getId(), equalTo("doc_1")); + }); + } + + public void testRankDocsRetrieverWithCollapseAndAggs() { + final int rankWindowSize = 100; + final int rankConstant = 10; + SearchSourceBuilder source = new SearchSourceBuilder(); + // this one retrieves docs 1, 2, 4, 6, and 7 + StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder( + QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(9L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(8L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(7L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_7")).boost(6L)) + ); + // this one retrieves docs 2 and 6 due to prefilter + StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( + QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + ); + standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); + // this one retrieves docs 3, 2, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + source.retriever( + new RRFRetrieverBuilder( + Arrays.asList( + new CompoundRetrieverBuilder.RetrieverSource(standard0, null), + new CompoundRetrieverBuilder.RetrieverSource(standard1, null), + new CompoundRetrieverBuilder.RetrieverSource(knnRetrieverBuilder, null) + ), + rankWindowSize, + rankConstant + ) + ); + source.collapse( + new CollapseBuilder(TOPIC_FIELD).setInnerHits( + new InnerHitBuilder("a").addSort(new FieldSortBuilder(DOC_FIELD).order(SortOrder.DESC)).setSize(10) + ) + ); + source.fetchField(TOPIC_FIELD); + source.aggregation(AggregationBuilders.terms("topic_agg").field(TOPIC_FIELD)); + SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); + ElasticsearchAssertions.assertResponse(req, resp -> { + assertNull(resp.pointInTimeId()); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); + assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); + assertThat(resp.getHits().getHits().length, equalTo(4)); + assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2")); + assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_6")); + assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_7")); + assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_1")); + assertThat(resp.getHits().getAt(3).getInnerHits().get("a").getAt(0).getId(), equalTo("doc_4")); + assertThat(resp.getHits().getAt(3).getInnerHits().get("a").getAt(1).getId(), equalTo("doc_3")); + assertThat(resp.getHits().getAt(3).getInnerHits().get("a").getAt(2).getId(), equalTo("doc_1")); + + assertNotNull(resp.getAggregations()); + assertNotNull(resp.getAggregations().get("topic_agg")); + Terms terms = resp.getAggregations().get("topic_agg"); + + assertThat(terms.getBucketByKey("technology").getDocCount(), equalTo(3L)); + assertThat(terms.getBucketByKey("astronomy").getDocCount(), equalTo(1L)); + assertThat(terms.getBucketByKey("biology").getDocCount(), equalTo(1L)); + }); + } + + public void testMultipleRRFRetrievers() { + final int rankWindowSize = 100; + final int rankConstant = 10; + SearchSourceBuilder source = new SearchSourceBuilder(); + // this one retrieves docs 1, 2, 4, 6, and 7 + StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder( + QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(9L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(8L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(7L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_7")).boost(6L)) + ); + // this one retrieves docs 2 and 6 due to prefilter + StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( + QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + ); + standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); + // this one retrieves docs 3, 2, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + source.retriever( + new RRFRetrieverBuilder( + Arrays.asList( + new CompoundRetrieverBuilder.RetrieverSource( + // this one returns docs 6, 7, 1, 3, and 4 + new RRFRetrieverBuilder( + Arrays.asList( + new CompoundRetrieverBuilder.RetrieverSource(standard0, null), + new CompoundRetrieverBuilder.RetrieverSource(standard1, null), + new CompoundRetrieverBuilder.RetrieverSource(knnRetrieverBuilder, null) + ), + rankWindowSize, + rankConstant + ), + null + ), + // this one bring just doc 7 which should be ranked first eventually + new CompoundRetrieverBuilder.RetrieverSource( + new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 7.0f }, null, 1, 100, null), + null + ) + ), + rankWindowSize, + rankConstant + ) + ); + + SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); + ElasticsearchAssertions.assertResponse(req, resp -> { + assertNull(resp.pointInTimeId()); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); + assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); + assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_7")); + assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_2")); + assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_6")); + assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_1")); + assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_3")); + assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_4")); + }); + } + + public void testRRFExplainWithNamedRetrievers() { + final int rankWindowSize = 100; + final int rankConstant = 10; + SearchSourceBuilder source = new SearchSourceBuilder(); + // this one retrieves docs 1, 2, 4, 6, and 7 + StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder( + QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(9L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(8L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(7L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_7")).boost(6L)) + ); + standard0.retrieverName("my_custom_retriever"); + // this one retrieves docs 2 and 6 due to prefilter + StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( + QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + ); + standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); + // this one retrieves docs 3, 2, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + source.retriever( + new RRFRetrieverBuilder( + Arrays.asList( + new CompoundRetrieverBuilder.RetrieverSource(standard0, null), + new CompoundRetrieverBuilder.RetrieverSource(standard1, null), + new CompoundRetrieverBuilder.RetrieverSource(knnRetrieverBuilder, null) + ), + rankWindowSize, + rankConstant + ) + ); + source.explain(true); + source.size(1); + SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); + ElasticsearchAssertions.assertResponse(req, resp -> { + assertNull(resp.pointInTimeId()); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); + assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); + assertThat(resp.getHits().getHits().length, equalTo(1)); + assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2")); + assertThat(resp.getHits().getAt(0).getExplanation().isMatch(), equalTo(true)); + assertThat(resp.getHits().getAt(0).getExplanation().getDescription(), containsString("sum of:")); + assertThat(resp.getHits().getAt(0).getExplanation().getDetails().length, equalTo(2)); + var rrfDetails = resp.getHits().getAt(0).getExplanation().getDetails()[0]; + assertThat(rrfDetails.getDetails().length, equalTo(3)); + assertThat(rrfDetails.getDescription(), containsString("computed for initial ranks [2, 1, 2]")); + + assertThat(rrfDetails.getDetails()[0].getDescription(), containsString("for rank [2] in query at index [0]")); + assertThat(rrfDetails.getDetails()[0].getDescription(), containsString("for rank [2] in query at index [0]")); + assertThat(rrfDetails.getDetails()[0].getDescription(), containsString("[my_custom_retriever]")); + assertThat(rrfDetails.getDetails()[1].getDescription(), containsString("for rank [1] in query at index [1]")); + assertThat(rrfDetails.getDetails()[2].getDescription(), containsString("for rank [2] in query at index [2]")); + }); + } + + public void testRRFExplainWithAnotherNestedRRF() { + final int rankWindowSize = 100; + final int rankConstant = 10; + SearchSourceBuilder source = new SearchSourceBuilder(); + // this one retrieves docs 1, 2, 4, 6, and 7 + StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder( + QueryBuilders.boolQuery() + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2")).boost(9L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(8L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(7L)) + .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_7")).boost(6L)) + ); + standard0.retrieverName("my_custom_retriever"); + // this one retrieves docs 2 and 6 due to prefilter + StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( + QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + ); + standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); + // this one retrieves docs 3, 2, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + + RRFRetrieverBuilder nestedRRF = new RRFRetrieverBuilder( + Arrays.asList( + new CompoundRetrieverBuilder.RetrieverSource(standard0, null), + new CompoundRetrieverBuilder.RetrieverSource(standard1, null), + new CompoundRetrieverBuilder.RetrieverSource(knnRetrieverBuilder, null) + ), + rankWindowSize, + rankConstant + ); + StandardRetrieverBuilder standard2 = new StandardRetrieverBuilder( + QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(20L) + ); + source.retriever( + new RRFRetrieverBuilder( + Arrays.asList( + new CompoundRetrieverBuilder.RetrieverSource(nestedRRF, null), + new CompoundRetrieverBuilder.RetrieverSource(standard2, null) + ), + rankWindowSize, + rankConstant + ) + ); + source.explain(true); + source.size(1); + SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); + ElasticsearchAssertions.assertResponse(req, resp -> { + assertNull(resp.pointInTimeId()); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); + assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); + assertThat(resp.getHits().getHits().length, equalTo(1)); + assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6")); + assertThat(resp.getHits().getAt(0).getExplanation().isMatch(), equalTo(true)); + assertThat(resp.getHits().getAt(0).getExplanation().getDescription(), containsString("sum of:")); + assertThat(resp.getHits().getAt(0).getExplanation().getDetails().length, equalTo(2)); + var rrfTopLevel = resp.getHits().getAt(0).getExplanation().getDetails()[0]; + assertThat(rrfTopLevel.getDetails().length, equalTo(2)); + assertThat(rrfTopLevel.getDescription(), containsString("computed for initial ranks [2, 1]")); + assertThat(rrfTopLevel.getDetails()[0].getDetails()[0].getDescription(), containsString("rrf score")); + assertThat(rrfTopLevel.getDetails()[1].getDetails()[0].getDescription(), containsString("ConstantScore")); + + var rrfDetails = rrfTopLevel.getDetails()[0].getDetails()[0]; + assertThat(rrfDetails.getDetails().length, equalTo(3)); + assertThat(rrfDetails.getDescription(), containsString("computed for initial ranks [4, 2, 3]")); + + assertThat(rrfDetails.getDetails()[0].getDescription(), containsString("for rank [4] in query at index [0]")); + assertThat(rrfDetails.getDetails()[0].getDescription(), containsString("for rank [4] in query at index [0]")); + assertThat(rrfDetails.getDetails()[0].getDescription(), containsString("[my_custom_retriever]")); + assertThat(rrfDetails.getDetails()[1].getDescription(), containsString("for rank [2] in query at index [1]")); + assertThat(rrfDetails.getDetails()[2].getDescription(), containsString("for rank [3] in query at index [2]")); + }); + } + + public void testRRFInnerRetrieverSearchError() { + final int rankWindowSize = 100; + final int rankConstant = 10; + SearchSourceBuilder source = new SearchSourceBuilder(); + // this will throw an error during evaluation + StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder( + QueryBuilders.constantScoreQuery(QueryBuilders.rangeQuery(VECTOR_FIELD).gte(10)) + ); + StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( + QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + ); + standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); + source.retriever( + new RRFRetrieverBuilder( + Arrays.asList( + new CompoundRetrieverBuilder.RetrieverSource(standard0, null), + new CompoundRetrieverBuilder.RetrieverSource(standard1, null) + ), + rankWindowSize, + rankConstant + ) + ); + SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); + Exception ex = expectThrows(IllegalStateException.class, req::get); + assertThat(ex, instanceOf(IllegalStateException.class)); + assertThat(ex.getMessage(), containsString("Search failed - some nested retrievers returned errors")); + assertThat(ex.getSuppressed().length, greaterThan(0)); + } + + public void testRRFInnerRetrieverErrorWhenExtractingToSource() { + final int rankWindowSize = 100; + final int rankConstant = 10; + SearchSourceBuilder source = new SearchSourceBuilder(); + TestRetrieverBuilder failingRetriever = new TestRetrieverBuilder("some value") { + @Override + public QueryBuilder topDocsQuery() { + return QueryBuilders.matchAllQuery(); + } + + @Override + public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) { + throw new UnsupportedOperationException("simulated failure"); + } + }; + StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( + QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + ); + standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); + source.retriever( + new RRFRetrieverBuilder( + Arrays.asList( + new CompoundRetrieverBuilder.RetrieverSource(failingRetriever, null), + new CompoundRetrieverBuilder.RetrieverSource(standard1, null) + ), + rankWindowSize, + rankConstant + ) + ); + source.size(1); + expectThrows(UnsupportedOperationException.class, () -> client().prepareSearch(INDEX).setSource(source).get()); + } + + public void testRRFInnerRetrieverErrorOnTopDocs() { + final int rankWindowSize = 100; + final int rankConstant = 10; + SearchSourceBuilder source = new SearchSourceBuilder(); + TestRetrieverBuilder failingRetriever = new TestRetrieverBuilder("some value") { + @Override + public QueryBuilder topDocsQuery() { + throw new UnsupportedOperationException("simulated failure"); + } + + @Override + public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) { + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + } + }; + StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( + QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) + ); + standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); + source.retriever( + new RRFRetrieverBuilder( + Arrays.asList( + new CompoundRetrieverBuilder.RetrieverSource(failingRetriever, null), + new CompoundRetrieverBuilder.RetrieverSource(standard1, null) + ), + rankWindowSize, + rankConstant + ) + ); + source.size(1); + source.aggregation(AggregationBuilders.terms("topic_agg").field(TOPIC_FIELD)); + expectThrows(UnsupportedOperationException.class, () -> client().prepareSearch(INDEX).setSource(source).get()); + } +} diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java new file mode 100644 index 0000000000000..3a4ace9b6754a --- /dev/null +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java @@ -0,0 +1,171 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.rank.rrf; + +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.retriever.CompoundRetrieverBuilder; +import org.elasticsearch.search.retriever.KnnRetrieverBuilder; +import org.elasticsearch.search.retriever.StandardRetrieverBuilder; +import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import org.elasticsearch.xcontent.XContentType; + +import java.util.Arrays; + +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.hamcrest.Matchers.equalTo; + +public class RRFRetrieverBuilderNestedDocsIT extends RRFRetrieverBuilderIT { + + private static final String LAST_30D_FIELD = "views.last30d"; + private static final String ALL_TIME_FIELD = "views.all"; + + @Override + protected void setupIndex() { + String mapping = """ + { + "properties": { + "vector": { + "type": "dense_vector", + "dims": 1, + "element_type": "float", + "similarity": "l2_norm", + "index": true, + "index_options": { + "type": "hnsw" + } + }, + "text": { + "type": "text" + }, + "doc": { + "type": "keyword" + }, + "topic": { + "type": "keyword" + }, + "views": { + "type": "nested", + "properties": { + "last30d": { + "type": "integer" + }, + "all": { + "type": "integer" + } + } + } + } + } + """; + createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 5).build()); + admin().indices().preparePutMapping(INDEX).setSource(mapping, XContentType.JSON).get(); + indexDoc(INDEX, "doc_1", DOC_FIELD, "doc_1", TOPIC_FIELD, "technology", TEXT_FIELD, "term", LAST_30D_FIELD, 100); + indexDoc( + INDEX, + "doc_2", + DOC_FIELD, + "doc_2", + TOPIC_FIELD, + "astronomy", + TEXT_FIELD, + "search term term", + VECTOR_FIELD, + new float[] { 2.0f }, + LAST_30D_FIELD, + 3 + ); + indexDoc(INDEX, "doc_3", DOC_FIELD, "doc_3", TOPIC_FIELD, "technology", VECTOR_FIELD, new float[] { 3.0f }); + indexDoc( + INDEX, + "doc_4", + DOC_FIELD, + "doc_4", + TOPIC_FIELD, + "technology", + TEXT_FIELD, + "term term term term", + ALL_TIME_FIELD, + 100, + LAST_30D_FIELD, + 40 + ); + indexDoc(INDEX, "doc_5", DOC_FIELD, "doc_5", TOPIC_FIELD, "science", TEXT_FIELD, "irrelevant stuff"); + indexDoc( + INDEX, + "doc_6", + DOC_FIELD, + "doc_6", + TEXT_FIELD, + "search term term term term term term", + VECTOR_FIELD, + new float[] { 6.0f }, + LAST_30D_FIELD, + 15 + ); + indexDoc( + INDEX, + "doc_7", + DOC_FIELD, + "doc_7", + TOPIC_FIELD, + "biology", + TEXT_FIELD, + "term term term term term term term", + VECTOR_FIELD, + new float[] { 7.0f }, + ALL_TIME_FIELD, + 1000 + ); + refresh(INDEX); + } + + public void testRRFRetrieverWithNestedQuery() { + final int rankWindowSize = 100; + final int rankConstant = 10; + SearchSourceBuilder source = new SearchSourceBuilder(); + // this one retrieves docs 1, 4 + StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder( + QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(LAST_30D_FIELD).gte(30L), ScoreMode.Avg) + ); + // this one retrieves docs 2 and 6 due to prefilter + StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( + QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L) + ); + standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); + // this one retrieves docs 6 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 6.0f }, null, 1, 100, null); + source.retriever( + new RRFRetrieverBuilder( + Arrays.asList( + new CompoundRetrieverBuilder.RetrieverSource(standard0, null), + new CompoundRetrieverBuilder.RetrieverSource(standard1, null), + new CompoundRetrieverBuilder.RetrieverSource(knnRetrieverBuilder, null) + ), + rankWindowSize, + rankConstant + ) + ); + source.fetchField(TOPIC_FIELD); + SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); + ElasticsearchAssertions.assertResponse(req, resp -> { + assertNull(resp.pointInTimeId()); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value, equalTo(4L)); + assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); + assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6")); + assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1")); + assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_2")); + assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_4")); + }); + } +} diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFFeatures.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFFeatures.java index 816b25d53d375..bbc0f622724a3 100644 --- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFFeatures.java +++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFFeatures.java @@ -12,6 +12,8 @@ import java.util.Set; +import static org.elasticsearch.xpack.rank.rrf.RRFRetrieverBuilder.RRF_RETRIEVER_COMPOSITION_SUPPORTED; + /** * A set of features specifically for the rrf plugin. */ @@ -19,6 +21,6 @@ public class RRFFeatures implements FeatureSpecification { @Override public Set getFeatures() { - return Set.of(RRFRetrieverBuilder.RRF_RETRIEVER_SUPPORTED); + return Set.of(RRFRetrieverBuilder.RRF_RETRIEVER_SUPPORTED, RRF_RETRIEVER_COMPOSITION_SUPPORTED); } } diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFQueryPhaseRankCoordinatorContext.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFQueryPhaseRankCoordinatorContext.java index b6a1ad52d5d15..56054955d25e7 100644 --- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFQueryPhaseRankCoordinatorContext.java +++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFQueryPhaseRankCoordinatorContext.java @@ -115,7 +115,7 @@ protected boolean lessThan(RRFRankDoc a, RRFRankDoc b) { final int frank = rank; results.compute(new RankKey(rrfRankDoc.doc, rrfRankDoc.shardIndex), (key, value) -> { if (value == null) { - value = new RRFRankDoc(rrfRankDoc.doc, rrfRankDoc.shardIndex, fqc); + value = new RRFRankDoc(rrfRankDoc.doc, rrfRankDoc.shardIndex, fqc, rankConstant); } value.score += 1.0f / (rankConstant + frank); @@ -171,4 +171,8 @@ protected boolean lessThan(RRFRankDoc a, RRFRankDoc b) { // and completion suggesters are not allowed return topResults; } + + public int rankConstant() { + return rankConstant; + } } diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFQueryPhaseRankShardContext.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFQueryPhaseRankShardContext.java index 9843b14df6903..62e261d752d3e 100644 --- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFQueryPhaseRankShardContext.java +++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFQueryPhaseRankShardContext.java @@ -48,7 +48,7 @@ public RRFRankShardResult combineQueryPhaseResults(List rankResults) { final int frank = rank; docsToRankResults.compute(scoreDoc.doc, (key, value) -> { if (value == null) { - value = new RRFRankDoc(scoreDoc.doc, scoreDoc.shardIndex, queries); + value = new RRFRankDoc(scoreDoc.doc, scoreDoc.shardIndex, queries, rankConstant); } // calculate the current rrf score for this document @@ -100,4 +100,8 @@ public RRFRankShardResult combineQueryPhaseResults(List rankResults) { } return new RRFRankShardResult(rankResults.size(), topResults); } + + public int rankConstant() { + return rankConstant; + } } diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java index 4dbc9a6a54dcf..500ed17395127 100644 --- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java +++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.rank.rrf; import org.apache.lucene.search.Explanation; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.search.rank.RankDoc; @@ -15,12 +16,15 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Objects; + +import static org.elasticsearch.xpack.rank.rrf.RRFRankBuilder.DEFAULT_RANK_CONSTANT; /** * {@code RRFRankDoc} supports additional ranking information * required for RRF. */ -public class RRFRankDoc extends RankDoc { +public final class RRFRankDoc extends RankDoc { static final String NAME = "rrf_rank_doc"; @@ -42,11 +46,14 @@ public class RRFRankDoc extends RankDoc { */ public final float[] scores; - public RRFRankDoc(int doc, int shardIndex, int queryCount) { + public final int rankConstant; + + public RRFRankDoc(int doc, int shardIndex, int queryCount, int rankConstant) { super(doc, 0f, shardIndex); positions = new int[queryCount]; Arrays.fill(positions, NO_RANK); scores = new float[queryCount]; + this.rankConstant = rankConstant; } public RRFRankDoc(StreamInput in) throws IOException { @@ -54,21 +61,43 @@ public RRFRankDoc(StreamInput in) throws IOException { rank = in.readVInt(); positions = in.readIntArray(); scores = in.readFloatArray(); + if (in.getTransportVersion().onOrAfter(TransportVersions.RRF_QUERY_REWRITE)) { + this.rankConstant = in.readVInt(); + } else { + this.rankConstant = DEFAULT_RANK_CONSTANT; + } } @Override - public Explanation explain() { - // ideally we'd need access to the rank constant to provide score info for this one + public Explanation explain(Explanation[] sources, String[] queryNames) { + assert sources.length == scores.length; int queries = positions.length; Explanation[] details = new Explanation[queries]; for (int i = 0; i < queries; i++) { - final String queryIndex = "at index [" + i + "]"; + final String queryAlias = queryNames[i] == null ? "" : " [" + queryNames[i] + "]"; + final String queryIdentifier = "at index [" + i + "]" + queryAlias; if (positions[i] == RRFRankDoc.NO_RANK) { - final String description = "rrf score: [0], result not found in query " + queryIndex; + final String description = "rrf score: [0], result not found in query " + queryIdentifier; details[i] = Explanation.noMatch(description); } else { final int rank = positions[i] + 1; - details[i] = Explanation.match(rank, "rank [" + (rank) + "] in query " + queryIndex); + final float rrfScore = (1f / (rank + rankConstant)); + details[i] = Explanation.match( + rank, + "rrf score: [" + + rrfScore + + "], " + + "for rank [" + + (rank) + + "] in query " + + queryIdentifier + + " computed as [1 / (" + + (rank) + + " + " + + rankConstant + + ")], for matching query with score", + sources[i] + ); } } return Explanation.match( @@ -77,6 +106,8 @@ public Explanation explain() { + score + "] computed for initial ranks " + Arrays.toString(Arrays.stream(positions).map(x -> x + 1).toArray()) + + " with rankConstant: [" + + rankConstant + "] as sum of [1 / (rank + rankConstant)] for each query", details ); @@ -87,17 +118,22 @@ public void doWriteTo(StreamOutput out) throws IOException { out.writeVInt(rank); out.writeIntArray(positions); out.writeFloatArray(scores); + if (out.getTransportVersion().onOrAfter(TransportVersions.RRF_QUERY_REWRITE)) { + out.writeVInt(rankConstant); + } } @Override public boolean doEquals(RankDoc rd) { RRFRankDoc rrfrd = (RRFRankDoc) rd; - return Arrays.equals(positions, rrfrd.positions) && Arrays.equals(scores, rrfrd.scores); + return Arrays.equals(positions, rrfrd.positions) + && Arrays.equals(scores, rrfrd.scores) + && Objects.equals(rankConstant, rrfrd.rankConstant); } @Override public int doHashCode() { - int result = Arrays.hashCode(positions); + int result = Arrays.hashCode(positions) + Objects.hash(rankConstant); result = 31 * result + Arrays.hashCode(scores); return result; } @@ -117,6 +153,8 @@ public String toString() { + doc + ", shardIndex=" + shardIndex + + ", rankConstant=" + + rankConstant + '}'; } @@ -129,5 +167,6 @@ public String getWriteableName() { protected void doToXContent(XContentBuilder builder, Params params) throws IOException { builder.field("positions", positions); builder.field("scores", scores); + builder.field("rankConstant", rankConstant); } } diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java index 0d6208e474eea..496af99574431 100644 --- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java +++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java @@ -7,24 +7,30 @@ package org.elasticsearch.xpack.rank.rrf; +import org.apache.lucene.search.ScoreDoc; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.util.Maps; import org.elasticsearch.features.NodeFeature; -import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.license.LicenseUtils; -import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.rank.RankDoc; +import org.elasticsearch.search.retriever.CompoundRetrieverBuilder; import org.elasticsearch.search.retriever.RetrieverBuilder; import org.elasticsearch.search.retriever.RetrieverParserContext; -import org.elasticsearch.xcontent.ObjectParser; +import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.core.XPackPlugin; import java.io.IOException; -import java.util.Collections; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Objects; +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; import static org.elasticsearch.xpack.rank.rrf.RRFRankPlugin.NAME; /** @@ -34,30 +40,39 @@ * top docs that will then be combined and ranked according to the rrf * formula. */ -public final class RRFRetrieverBuilder extends RetrieverBuilder { +public final class RRFRetrieverBuilder extends CompoundRetrieverBuilder { + public static final String NAME = "rrf"; public static final NodeFeature RRF_RETRIEVER_SUPPORTED = new NodeFeature("rrf_retriever_supported"); + public static final NodeFeature RRF_RETRIEVER_COMPOSITION_SUPPORTED = new NodeFeature("rrf_retriever_composition_supported"); public static final ParseField RETRIEVERS_FIELD = new ParseField("retrievers"); public static final ParseField RANK_WINDOW_SIZE_FIELD = new ParseField("rank_window_size"); public static final ParseField RANK_CONSTANT_FIELD = new ParseField("rank_constant"); - public static final ObjectParser PARSER = new ObjectParser<>( + @SuppressWarnings("unchecked") + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( NAME, - RRFRetrieverBuilder::new + false, + args -> { + List childRetrievers = (List) args[0]; + List innerRetrievers = childRetrievers.stream().map(r -> new RetrieverSource(r, null)).toList(); + int rankWindowSize = args[1] == null ? RRFRankBuilder.DEFAULT_RANK_WINDOW_SIZE : (int) args[1]; + int rankConstant = args[2] == null ? RRFRankBuilder.DEFAULT_RANK_CONSTANT : (int) args[2]; + return new RRFRetrieverBuilder(innerRetrievers, rankWindowSize, rankConstant); + } ); static { - PARSER.declareObjectArray((r, v) -> r.retrieverBuilders = v, (p, c) -> { + PARSER.declareObjectArray(constructorArg(), (p, c) -> { p.nextToken(); String name = p.currentName(); RetrieverBuilder retrieverBuilder = p.namedObject(RetrieverBuilder.class, name, c); p.nextToken(); return retrieverBuilder; }, RETRIEVERS_FIELD); - PARSER.declareInt((r, v) -> r.rankWindowSize = v, RANK_WINDOW_SIZE_FIELD); - PARSER.declareInt((r, v) -> r.rankConstant = v, RANK_CONSTANT_FIELD); - + PARSER.declareInt(optionalConstructorArg(), RANK_WINDOW_SIZE_FIELD); + PARSER.declareInt(optionalConstructorArg(), RANK_CONSTANT_FIELD); RetrieverBuilder.declareBaseParserFields(NAME, PARSER); } @@ -65,76 +80,115 @@ public static RRFRetrieverBuilder fromXContent(XContentParser parser, RetrieverP if (context.clusterSupportsFeature(RRF_RETRIEVER_SUPPORTED) == false) { throw new ParsingException(parser.getTokenLocation(), "unknown retriever [" + NAME + "]"); } + if (context.clusterSupportsFeature(RRF_RETRIEVER_COMPOSITION_SUPPORTED) == false) { + throw new UnsupportedOperationException("[rrf] retriever composition feature is not supported by all nodes in the cluster"); + } if (RRFRankPlugin.RANK_RRF_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) { throw LicenseUtils.newComplianceException("Reciprocal Rank Fusion (RRF)"); } return PARSER.apply(parser, context); } - List retrieverBuilders = Collections.emptyList(); - int rankWindowSize = RRFRankBuilder.DEFAULT_RANK_WINDOW_SIZE; - int rankConstant = RRFRankBuilder.DEFAULT_RANK_CONSTANT; + private final int rankConstant; + + public RRFRetrieverBuilder(int rankWindowSize, int rankConstant) { + this(new ArrayList<>(), rankWindowSize, rankConstant); + } + + RRFRetrieverBuilder(List childRetrievers, int rankWindowSize, int rankConstant) { + super(childRetrievers, rankWindowSize); + this.rankConstant = rankConstant; + } @Override - public QueryBuilder topDocsQuery() { - throw new IllegalStateException("{" + getName() + "} cannot be nested"); + public String getName() { + return NAME; } @Override - public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) { - if (compoundUsed) { - throw new IllegalArgumentException("[rank] cannot be used in children of compound retrievers"); - } + protected RRFRetrieverBuilder clone(List newRetrievers) { + return new RRFRetrieverBuilder(newRetrievers, this.rankWindowSize, this.rankConstant); + } - for (RetrieverBuilder retrieverBuilder : retrieverBuilders) { - if (preFilterQueryBuilders.isEmpty() == false) { - retrieverBuilder.getPreFilterQueryBuilders().addAll(preFilterQueryBuilders); + @Override + protected RRFRankDoc[] combineInnerRetrieverResults(List rankResults) { + // combine the disjointed sets of TopDocs into a single set or RRFRankDocs + // each RRFRankDoc will have both the position and score for each query where + // it was within the result set for that query + // if a doc isn't part of a result set its position will be NO_RANK [0] and + // its score is [0f] + int queries = rankResults.size(); + Map docsToRankResults = Maps.newMapWithExpectedSize(rankWindowSize); + int index = 0; + for (var rrfRankResult : rankResults) { + int rank = 1; + for (ScoreDoc scoreDoc : rrfRankResult) { + final int findex = index; + final int frank = rank; + docsToRankResults.compute(new RankDoc.RankKey(scoreDoc.doc, scoreDoc.shardIndex), (key, value) -> { + if (value == null) { + value = new RRFRankDoc(scoreDoc.doc, scoreDoc.shardIndex, queries, rankConstant); + } + + // calculate the current rrf score for this document + // later used to sort and covert to a rank + value.score += 1.0f / (rankConstant + frank); + + // record the position for each query + // for explain and debugging + value.positions[findex] = frank - 1; + + // record the score for each query + // used to later re-rank on the coordinator + value.scores[findex] = scoreDoc.score; + + return value; + }); + ++rank; } - - retrieverBuilder.extractToSearchSourceBuilder(searchSourceBuilder, true); + ++index; } - searchSourceBuilder.rankBuilder(new RRFRankBuilder(rankWindowSize, rankConstant)); + // sort the results based on rrf score, tiebreaker based on smaller doc id + RRFRankDoc[] sortedResults = docsToRankResults.values().toArray(RRFRankDoc[]::new); + Arrays.sort(sortedResults); + // trim the results if needed, otherwise each shard will always return `rank_window_sieze` results. + RRFRankDoc[] topResults = new RRFRankDoc[Math.min(rankWindowSize, sortedResults.length)]; + for (int rank = 0; rank < topResults.length; ++rank) { + topResults[rank] = sortedResults[rank]; + topResults[rank].rank = rank + 1; + } + return topResults; } // ---- FOR TESTING XCONTENT PARSING ---- @Override - public String getName() { - return NAME; + public boolean doEquals(Object o) { + RRFRetrieverBuilder that = (RRFRetrieverBuilder) o; + return super.doEquals(o) && rankConstant == that.rankConstant; + } + + @Override + public int doHashCode() { + return Objects.hash(super.doHashCode(), rankConstant); } @Override public void doToXContent(XContentBuilder builder, Params params) throws IOException { - if (retrieverBuilders.isEmpty() == false) { + if (innerRetrievers.isEmpty() == false) { builder.startArray(RETRIEVERS_FIELD.getPreferredName()); - for (RetrieverBuilder retrieverBuilder : retrieverBuilders) { + for (var entry : innerRetrievers) { builder.startObject(); - builder.field(retrieverBuilder.getName()); - retrieverBuilder.toXContent(builder, params); + builder.field(entry.retriever().getName()); + entry.retriever().toXContent(builder, params); builder.endObject(); } - builder.endArray(); } builder.field(RANK_WINDOW_SIZE_FIELD.getPreferredName(), rankWindowSize); builder.field(RANK_CONSTANT_FIELD.getPreferredName(), rankConstant); } - - @Override - public boolean doEquals(Object o) { - RRFRetrieverBuilder that = (RRFRetrieverBuilder) o; - return rankWindowSize == that.rankWindowSize - && rankConstant == that.rankConstant - && Objects.equals(retrieverBuilders, that.retrieverBuilders); - } - - @Override - public int doHashCode() { - return Objects.hash(retrieverBuilders, rankWindowSize, rankConstant); - } - - // ---- END FOR TESTING ---- } diff --git a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankContextTests.java b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankContextTests.java index 61859e280acdf..cd6883e1e54fd 100644 --- a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankContextTests.java +++ b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankContextTests.java @@ -71,7 +71,7 @@ public void testShardCombine() { assertEquals(2, result.queryCount); assertEquals(10, result.rrfRankDocs.length); - RRFRankDoc expected = new RRFRankDoc(8, -1, 2); + RRFRankDoc expected = new RRFRankDoc(8, -1, 2, context.rankConstant()); expected.rank = 1; expected.positions[0] = 7; expected.positions[1] = 0; @@ -80,7 +80,7 @@ public void testShardCombine() { expected.score = Float.NaN; assertRDEquals(expected, result.rrfRankDocs[0]); - expected = new RRFRankDoc(1, -1, 2); + expected = new RRFRankDoc(1, -1, 2, context.rankConstant()); expected.rank = 2; expected.positions[0] = 0; expected.positions[1] = NO_RANK; @@ -89,7 +89,7 @@ public void testShardCombine() { expected.score = Float.NaN; assertRDEquals(expected, result.rrfRankDocs[1]); - expected = new RRFRankDoc(9, -1, 2); + expected = new RRFRankDoc(9, -1, 2, context.rankConstant()); expected.rank = 3; expected.positions[0] = 8; expected.positions[1] = 1; @@ -98,7 +98,7 @@ public void testShardCombine() { expected.score = Float.NaN; assertRDEquals(expected, result.rrfRankDocs[2]); - expected = new RRFRankDoc(10, -1, 2); + expected = new RRFRankDoc(10, -1, 2, context.rankConstant()); expected.rank = 4; expected.positions[0] = 9; expected.positions[1] = 2; @@ -107,7 +107,7 @@ public void testShardCombine() { expected.score = Float.NaN; assertRDEquals(expected, result.rrfRankDocs[3]); - expected = new RRFRankDoc(2, -1, 2); + expected = new RRFRankDoc(2, -1, 2, context.rankConstant()); expected.rank = 5; expected.positions[0] = 1; expected.positions[1] = NO_RANK; @@ -116,7 +116,7 @@ public void testShardCombine() { expected.score = Float.NaN; assertRDEquals(expected, result.rrfRankDocs[4]); - expected = new RRFRankDoc(3, -1, 2); + expected = new RRFRankDoc(3, -1, 2, context.rankConstant()); expected.rank = 6; expected.positions[0] = 2; expected.positions[1] = NO_RANK; @@ -125,7 +125,7 @@ public void testShardCombine() { expected.score = Float.NaN; assertRDEquals(expected, result.rrfRankDocs[5]); - expected = new RRFRankDoc(4, -1, 2); + expected = new RRFRankDoc(4, -1, 2, context.rankConstant()); expected.rank = 7; expected.positions[0] = 3; expected.positions[1] = NO_RANK; @@ -134,7 +134,7 @@ public void testShardCombine() { expected.score = Float.NaN; assertRDEquals(expected, result.rrfRankDocs[6]); - expected = new RRFRankDoc(11, -1, 2); + expected = new RRFRankDoc(11, -1, 2, context.rankConstant()); expected.rank = 8; expected.positions[0] = NO_RANK; expected.positions[1] = 3; @@ -143,7 +143,7 @@ public void testShardCombine() { expected.score = Float.NaN; assertRDEquals(expected, result.rrfRankDocs[7]); - expected = new RRFRankDoc(5, -1, 2); + expected = new RRFRankDoc(5, -1, 2, context.rankConstant()); expected.rank = 9; expected.positions[0] = 4; expected.positions[1] = NO_RANK; @@ -152,7 +152,7 @@ public void testShardCombine() { expected.score = Float.NaN; assertRDEquals(expected, result.rrfRankDocs[8]); - expected = new RRFRankDoc(12, -1, 2); + expected = new RRFRankDoc(12, -1, 2, context.rankConstant()); expected.rank = 10; expected.positions[0] = NO_RANK; expected.positions[1] = 4; @@ -166,27 +166,27 @@ public void testCoordinatorRank() { RRFQueryPhaseRankCoordinatorContext context = new RRFQueryPhaseRankCoordinatorContext(4, 0, 5, 1); QuerySearchResult qsr0 = new QuerySearchResult(); qsr0.setShardIndex(1); - RRFRankDoc rd11 = new RRFRankDoc(1, -1, 2); + RRFRankDoc rd11 = new RRFRankDoc(1, -1, 2, context.rankConstant()); rd11.positions[0] = 2; rd11.positions[1] = 0; rd11.scores[0] = 3.0f; rd11.scores[1] = 8.0f; - RRFRankDoc rd12 = new RRFRankDoc(2, -1, 2); + RRFRankDoc rd12 = new RRFRankDoc(2, -1, 2, context.rankConstant()); rd12.positions[0] = 3; rd12.positions[1] = 1; rd12.scores[0] = 2.0f; rd12.scores[1] = 7.0f; - RRFRankDoc rd13 = new RRFRankDoc(3, -1, 2); + RRFRankDoc rd13 = new RRFRankDoc(3, -1, 2, context.rankConstant()); rd13.positions[0] = 0; rd13.positions[1] = NO_RANK; rd13.scores[0] = 10.0f; rd13.scores[1] = 0.0f; - RRFRankDoc rd14 = new RRFRankDoc(4, -1, 2); + RRFRankDoc rd14 = new RRFRankDoc(4, -1, 2, context.rankConstant()); rd14.positions[0] = 4; rd14.positions[1] = 2; rd14.scores[0] = 1.0f; rd14.scores[1] = 6.0f; - RRFRankDoc rd15 = new RRFRankDoc(5, -1, 2); + RRFRankDoc rd15 = new RRFRankDoc(5, -1, 2, context.rankConstant()); rd15.positions[0] = 1; rd15.positions[1] = NO_RANK; rd15.scores[0] = 9.0f; @@ -195,27 +195,27 @@ public void testCoordinatorRank() { QuerySearchResult qsr1 = new QuerySearchResult(); qsr1.setShardIndex(2); - RRFRankDoc rd21 = new RRFRankDoc(1, -1, 2); + RRFRankDoc rd21 = new RRFRankDoc(1, -1, 2, context.rankConstant()); rd21.positions[0] = 0; rd21.positions[1] = 0; rd21.scores[0] = 9.5f; rd21.scores[1] = 7.5f; - RRFRankDoc rd22 = new RRFRankDoc(2, -1, 2); + RRFRankDoc rd22 = new RRFRankDoc(2, -1, 2, context.rankConstant()); rd22.positions[0] = 1; rd22.positions[1] = 1; rd22.scores[0] = 8.5f; rd22.scores[1] = 6.5f; - RRFRankDoc rd23 = new RRFRankDoc(3, -1, 2); + RRFRankDoc rd23 = new RRFRankDoc(3, -1, 2, context.rankConstant()); rd23.positions[0] = 2; rd23.positions[1] = 2; rd23.scores[0] = 7.5f; rd23.scores[1] = 4.5f; - RRFRankDoc rd24 = new RRFRankDoc(4, -1, 2); + RRFRankDoc rd24 = new RRFRankDoc(4, -1, 2, context.rankConstant()); rd24.positions[0] = 3; rd24.positions[1] = NO_RANK; rd24.scores[0] = 5.5f; rd24.scores[1] = 0.0f; - RRFRankDoc rd25 = new RRFRankDoc(5, -1, 2); + RRFRankDoc rd25 = new RRFRankDoc(5, -1, 2, context.rankConstant()); rd25.positions[0] = NO_RANK; rd25.positions[1] = 3; rd25.scores[0] = 0.0f; @@ -228,7 +228,7 @@ public void testCoordinatorRank() { assertEquals(4, tds.fetchHits); assertEquals(4, scoreDocs.length); - RRFRankDoc expected = new RRFRankDoc(1, 2, 2); + RRFRankDoc expected = new RRFRankDoc(1, 2, 2, context.rankConstant()); expected.rank = 1; expected.positions[0] = 1; expected.positions[1] = 1; @@ -237,7 +237,7 @@ public void testCoordinatorRank() { expected.score = 0.6666667f; assertRDEquals(expected, (RRFRankDoc) scoreDocs[0]); - expected = new RRFRankDoc(3, 1, 2); + expected = new RRFRankDoc(3, 1, 2, context.rankConstant()); expected.rank = 2; expected.positions[0] = 0; expected.positions[1] = NO_RANK; @@ -246,7 +246,7 @@ public void testCoordinatorRank() { expected.score = 0.5f; assertRDEquals(expected, (RRFRankDoc) scoreDocs[1]); - expected = new RRFRankDoc(1, 1, 2); + expected = new RRFRankDoc(1, 1, 2, context.rankConstant()); expected.rank = 3; expected.positions[0] = NO_RANK; expected.positions[1] = 0; @@ -255,7 +255,7 @@ public void testCoordinatorRank() { expected.score = 0.5f; assertRDEquals(expected, (RRFRankDoc) scoreDocs[2]); - expected = new RRFRankDoc(2, 2, 2); + expected = new RRFRankDoc(2, 2, 2, context.rankConstant()); expected.rank = 4; expected.positions[0] = 3; expected.positions[1] = 3; @@ -277,7 +277,7 @@ public void testShardTieBreaker() { assertEquals(2, result.queryCount); assertEquals(2, result.rrfRankDocs.length); - RRFRankDoc expected = new RRFRankDoc(1, -1, 2); + RRFRankDoc expected = new RRFRankDoc(1, -1, 2, context.rankConstant()); expected.rank = 1; expected.positions[0] = 0; expected.positions[1] = 1; @@ -286,7 +286,7 @@ public void testShardTieBreaker() { expected.score = Float.NaN; assertRDEquals(expected, result.rrfRankDocs[0]); - expected = new RRFRankDoc(2, -1, 2); + expected = new RRFRankDoc(2, -1, 2, context.rankConstant()); expected.rank = 2; expected.positions[0] = 1; expected.positions[1] = 0; @@ -304,7 +304,7 @@ public void testShardTieBreaker() { assertEquals(2, result.queryCount); assertEquals(4, result.rrfRankDocs.length); - expected = new RRFRankDoc(3, -1, 2); + expected = new RRFRankDoc(3, -1, 2, context.rankConstant()); expected.rank = 1; expected.positions[0] = 2; expected.positions[1] = 1; @@ -313,7 +313,7 @@ public void testShardTieBreaker() { expected.score = Float.NaN; assertRDEquals(expected, result.rrfRankDocs[0]); - expected = new RRFRankDoc(2, -1, 2); + expected = new RRFRankDoc(2, -1, 2, context.rankConstant()); expected.rank = 2; expected.positions[0] = 1; expected.positions[1] = 2; @@ -322,7 +322,7 @@ public void testShardTieBreaker() { expected.score = Float.NaN; assertRDEquals(expected, result.rrfRankDocs[1]); - expected = new RRFRankDoc(1, -1, 2); + expected = new RRFRankDoc(1, -1, 2, context.rankConstant()); expected.rank = 3; expected.positions[0] = 0; expected.positions[1] = -1; @@ -331,7 +331,7 @@ public void testShardTieBreaker() { expected.score = Float.NaN; assertRDEquals(expected, result.rrfRankDocs[2]); - expected = new RRFRankDoc(4, -1, 2); + expected = new RRFRankDoc(4, -1, 2, context.rankConstant()); expected.rank = 4; expected.positions[0] = -1; expected.positions[1] = 0; @@ -349,7 +349,7 @@ public void testShardTieBreaker() { assertEquals(2, result.queryCount); assertEquals(4, result.rrfRankDocs.length); - expected = new RRFRankDoc(1, -1, 2); + expected = new RRFRankDoc(1, -1, 2, context.rankConstant()); expected.rank = 1; expected.positions[0] = 0; expected.positions[1] = -1; @@ -358,7 +358,7 @@ public void testShardTieBreaker() { expected.score = Float.NaN; assertRDEquals(expected, result.rrfRankDocs[0]); - expected = new RRFRankDoc(2, -1, 2); + expected = new RRFRankDoc(2, -1, 2, context.rankConstant()); expected.rank = 2; expected.positions[0] = -1; expected.positions[1] = 0; @@ -367,7 +367,7 @@ public void testShardTieBreaker() { expected.score = Float.NaN; assertRDEquals(expected, result.rrfRankDocs[1]); - expected = new RRFRankDoc(3, -1, 2); + expected = new RRFRankDoc(3, -1, 2, context.rankConstant()); expected.rank = 3; expected.positions[0] = 1; expected.positions[1] = -1; @@ -376,7 +376,7 @@ public void testShardTieBreaker() { expected.score = Float.NaN; assertRDEquals(expected, result.rrfRankDocs[2]); - expected = new RRFRankDoc(4, -1, 2); + expected = new RRFRankDoc(4, -1, 2, context.rankConstant()); expected.rank = 4; expected.positions[0] = -1; expected.positions[1] = 1; @@ -391,7 +391,7 @@ public void testCoordinatorRankTieBreaker() { QuerySearchResult qsr0 = new QuerySearchResult(); qsr0.setShardIndex(1); - RRFRankDoc rd11 = new RRFRankDoc(1, -1, 2); + RRFRankDoc rd11 = new RRFRankDoc(1, -1, 2, context.rankConstant()); rd11.positions[0] = 0; rd11.positions[1] = 0; rd11.scores[0] = 10.0f; @@ -400,7 +400,7 @@ public void testCoordinatorRankTieBreaker() { QuerySearchResult qsr1 = new QuerySearchResult(); qsr1.setShardIndex(2); - RRFRankDoc rd21 = new RRFRankDoc(1, -1, 2); + RRFRankDoc rd21 = new RRFRankDoc(1, -1, 2, context.rankConstant()); rd21.positions[0] = 0; rd21.positions[1] = 0; rd21.scores[0] = 9.0f; @@ -413,7 +413,7 @@ public void testCoordinatorRankTieBreaker() { assertEquals(2, tds.fetchHits); assertEquals(2, scoreDocs.length); - RRFRankDoc expected = new RRFRankDoc(1, 1, 2); + RRFRankDoc expected = new RRFRankDoc(1, 1, 2, context.rankConstant()); expected.rank = 1; expected.positions[0] = 0; expected.positions[1] = 1; @@ -422,7 +422,7 @@ public void testCoordinatorRankTieBreaker() { expected.score = 0.8333333730697632f; assertRDEquals(expected, (RRFRankDoc) scoreDocs[0]); - expected = new RRFRankDoc(1, 2, 2); + expected = new RRFRankDoc(1, 2, 2, context.rankConstant()); expected.rank = 2; expected.positions[0] = 1; expected.positions[1] = 0; @@ -433,12 +433,12 @@ public void testCoordinatorRankTieBreaker() { qsr0 = new QuerySearchResult(); qsr0.setShardIndex(1); - rd11 = new RRFRankDoc(1, -1, 2); + rd11 = new RRFRankDoc(1, -1, 2, context.rankConstant()); rd11.positions[0] = 0; rd11.positions[1] = -1; rd11.scores[0] = 10.0f; rd11.scores[1] = 0.0f; - RRFRankDoc rd12 = new RRFRankDoc(2, -1, 2); + RRFRankDoc rd12 = new RRFRankDoc(2, -1, 2, context.rankConstant()); rd12.positions[0] = 0; rd12.positions[1] = 1; rd12.scores[0] = 9.0f; @@ -447,12 +447,12 @@ public void testCoordinatorRankTieBreaker() { qsr1 = new QuerySearchResult(); qsr1.setShardIndex(2); - rd21 = new RRFRankDoc(1, -1, 2); + rd21 = new RRFRankDoc(1, -1, 2, context.rankConstant()); rd21.positions[0] = -1; rd21.positions[1] = 0; rd21.scores[0] = 0.0f; rd21.scores[1] = 11.0f; - RRFRankDoc rd22 = new RRFRankDoc(2, -1, 2); + RRFRankDoc rd22 = new RRFRankDoc(2, -1, 2, context.rankConstant()); rd22.positions[0] = 0; rd22.positions[1] = 1; rd22.scores[0] = 9.0f; @@ -465,7 +465,7 @@ public void testCoordinatorRankTieBreaker() { assertEquals(4, tds.fetchHits); assertEquals(4, scoreDocs.length); - expected = new RRFRankDoc(2, 2, 2); + expected = new RRFRankDoc(2, 2, 2, context.rankConstant()); expected.rank = 1; expected.positions[0] = 2; expected.positions[1] = 1; @@ -474,7 +474,7 @@ public void testCoordinatorRankTieBreaker() { expected.score = 0.5833333730697632f; assertRDEquals(expected, (RRFRankDoc) scoreDocs[0]); - expected = new RRFRankDoc(2, 1, 2); + expected = new RRFRankDoc(2, 1, 2, context.rankConstant()); expected.rank = 2; expected.positions[0] = 1; expected.positions[1] = 2; @@ -483,7 +483,7 @@ public void testCoordinatorRankTieBreaker() { expected.score = 0.5833333730697632f; assertRDEquals(expected, (RRFRankDoc) scoreDocs[1]); - expected = new RRFRankDoc(1, 1, 2); + expected = new RRFRankDoc(1, 1, 2, context.rankConstant()); expected.rank = 3; expected.positions[0] = 0; expected.positions[1] = -1; @@ -492,7 +492,7 @@ public void testCoordinatorRankTieBreaker() { expected.score = 0.5f; assertRDEquals(expected, (RRFRankDoc) scoreDocs[2]); - expected = new RRFRankDoc(1, 2, 2); + expected = new RRFRankDoc(1, 2, 2, context.rankConstant()); expected.rank = 4; expected.positions[0] = -1; expected.positions[1] = 0; @@ -503,12 +503,12 @@ public void testCoordinatorRankTieBreaker() { qsr0 = new QuerySearchResult(); qsr0.setShardIndex(1); - rd11 = new RRFRankDoc(1, -1, 2); + rd11 = new RRFRankDoc(1, -1, 2, context.rankConstant()); rd11.positions[0] = 0; rd11.positions[1] = -1; rd11.scores[0] = 10.0f; rd11.scores[1] = 0.0f; - rd12 = new RRFRankDoc(2, -1, 2); + rd12 = new RRFRankDoc(2, -1, 2, context.rankConstant()); rd12.positions[0] = -1; rd12.positions[1] = 0; rd12.scores[0] = 0.0f; @@ -517,12 +517,12 @@ public void testCoordinatorRankTieBreaker() { qsr1 = new QuerySearchResult(); qsr1.setShardIndex(2); - rd21 = new RRFRankDoc(1, -1, 2); + rd21 = new RRFRankDoc(1, -1, 2, context.rankConstant()); rd21.positions[0] = 0; rd21.positions[1] = -1; rd21.scores[0] = 3.0f; rd21.scores[1] = 0.0f; - rd22 = new RRFRankDoc(2, -1, 2); + rd22 = new RRFRankDoc(2, -1, 2, context.rankConstant()); rd22.positions[0] = -1; rd22.positions[1] = 0; rd22.scores[0] = 0.0f; @@ -535,7 +535,7 @@ public void testCoordinatorRankTieBreaker() { assertEquals(4, tds.fetchHits); assertEquals(4, scoreDocs.length); - expected = new RRFRankDoc(1, 1, 2); + expected = new RRFRankDoc(1, 1, 2, context.rankConstant()); expected.rank = 1; expected.positions[0] = 0; expected.positions[1] = -1; @@ -544,7 +544,7 @@ public void testCoordinatorRankTieBreaker() { expected.score = 0.5f; assertRDEquals(expected, (RRFRankDoc) scoreDocs[0]); - expected = new RRFRankDoc(2, 1, 2); + expected = new RRFRankDoc(2, 1, 2, context.rankConstant()); expected.rank = 2; expected.positions[0] = -1; expected.positions[1] = 0; @@ -553,7 +553,7 @@ public void testCoordinatorRankTieBreaker() { expected.score = 0.5f; assertRDEquals(expected, (RRFRankDoc) scoreDocs[1]); - expected = new RRFRankDoc(1, 2, 2); + expected = new RRFRankDoc(1, 2, 2, context.rankConstant()); expected.rank = 3; expected.positions[0] = 1; expected.positions[1] = -1; @@ -562,7 +562,7 @@ public void testCoordinatorRankTieBreaker() { expected.score = 0.3333333333333333f; assertRDEquals(expected, (RRFRankDoc) scoreDocs[2]); - expected = new RRFRankDoc(2, 2, 2); + expected = new RRFRankDoc(2, 2, 2, context.rankConstant()); expected.rank = 4; expected.positions[0] = -1; expected.positions[1] = 1; diff --git a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankDocTests.java b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankDocTests.java index 0b8ee30fe0680..4b64b6c173c92 100644 --- a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankDocTests.java +++ b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankDocTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -17,7 +18,12 @@ public class RRFRankDocTests extends AbstractWireSerializingTestCase { static RRFRankDoc createTestRRFRankDoc(int queryCount) { - RRFRankDoc instance = new RRFRankDoc(randomNonNegativeInt(), randomBoolean() ? -1 : randomNonNegativeInt(), queryCount); + RRFRankDoc instance = new RRFRankDoc( + randomNonNegativeInt(), + randomBoolean() ? -1 : randomNonNegativeInt(), + queryCount, + randomIntBetween(1, 100) + ); instance.score = randomFloat(); instance.rank = randomBoolean() ? NO_RANK : randomIntBetween(1, 10000); for (int qi = 0; qi < queryCount; ++qi) { @@ -46,34 +52,49 @@ protected RRFRankDoc createTestInstance() { @Override protected RRFRankDoc mutateInstance(RRFRankDoc instance) throws IOException { - RRFRankDoc mutated = new RRFRankDoc(instance.doc, instance.shardIndex, instance.positions.length); - mutated.score = instance.score; - mutated.rank = instance.rank; - System.arraycopy(instance.positions, 0, mutated.positions, 0, instance.positions.length); - System.arraycopy(instance.scores, 0, mutated.scores, 0, instance.positions.length); - mutated.rank = mutated.rank == NO_RANK ? randomIntBetween(1, 10000) : NO_RANK; - if (rarely()) { - int ri = randomInt(mutated.positions.length - 1); - mutated.positions[ri] = mutated.positions[ri] == NO_RANK ? randomIntBetween(1, 10000) : NO_RANK; - } - if (rarely()) { - int ri = randomInt(mutated.positions.length - 1); - mutated.scores[ri] = randomFloat(); - } - if (rarely()) { - mutated.doc = randomNonNegativeInt(); - } - if (rarely()) { - mutated.score = randomFloat(); - } - if (frequently()) { - mutated.shardIndex = mutated.shardIndex == -1 ? randomNonNegativeInt() : -1; + int doc = instance.doc; + int shardIndex = instance.shardIndex; + float score = instance.score; + int rankConstant = instance.rankConstant; + int rank = instance.rank; + int queries = instance.positions.length; + int[] positions = new int[queries]; + float[] scores = new float[queries]; + + switch (randomInt(6)) { + case 0: + doc = randomValueOtherThan(doc, ESTestCase::randomNonNegativeInt); + break; + case 1: + shardIndex = shardIndex == -1 ? randomNonNegativeInt() : -1; + break; + case 2: + score = randomValueOtherThan(score, ESTestCase::randomFloat); + break; + case 3: + rankConstant = randomValueOtherThan(rankConstant, () -> randomIntBetween(1, 100)); + break; + case 4: + rank = rank == NO_RANK ? randomIntBetween(1, 10000) : NO_RANK; + break; + case 5: + for (int i = 0; i < queries; i++) { + positions[i] = instance.positions[i] == NO_RANK ? randomIntBetween(1, 10000) : NO_RANK; + } + break; + case 6: + for (int i = 0; i < queries; i++) { + scores[i] = randomValueOtherThan(scores[i], ESTestCase::randomFloat); + } + break; + default: + throw new AssertionError(); } + RRFRankDoc mutated = new RRFRankDoc(doc, shardIndex, queries, rankConstant); + System.arraycopy(positions, 0, mutated.positions, 0, instance.positions.length); + System.arraycopy(scores, 0, mutated.scores, 0, instance.scores.length); + mutated.rank = rank; + mutated.score = score; return mutated; } - - public void testExplain() { - RRFRankDoc instance = createTestRRFRankDoc(); - assertEquals(instance.explain().toString(), instance.explain().toString()); - } } diff --git a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java index 330c936327b81..e360237371a82 100644 --- a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java +++ b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java @@ -29,25 +29,21 @@ public class RRFRetrieverBuilderParsingTests extends AbstractXContentTestCase(retrieverCount); - while (retrieverCount > 0) { - rrfRetrieverBuilder.retrieverBuilders.add(TestRetrieverBuilder.createRandomTestRetrieverBuilder()); + ret.addChild(TestRetrieverBuilder.createRandomTestRetrieverBuilder()); --retrieverCount; } - - return rrfRetrieverBuilder; + return ret; } @Override diff --git a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderTests.java b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderTests.java index f5a9f4e9b0c3e..d20f0f88aeb16 100644 --- a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderTests.java +++ b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderTests.java @@ -8,9 +8,12 @@ package org.elasticsearch.xpack.rank.rrf; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.retriever.RetrieverBuilder; import org.elasticsearch.search.retriever.RetrieverParserContext; @@ -50,7 +53,8 @@ public void testRetrieverExtractionErrors() throws IOException { SearchSourceBuilder ssb = new SearchSourceBuilder(); IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, - () -> ssb.parseXContent(parser, true, nf -> true).rewrite(null) + () -> ssb.parseXContent(parser, true, nf -> true) + .rewrite(new QueryRewriteContext(parserConfig(), null, null, null, new PointInTimeBuilder(new BytesArray("pitid")))) ); assertEquals("[search_after] cannot be used in children of compound retrievers", iae.getMessage()); } @@ -65,88 +69,11 @@ public void testRetrieverExtractionErrors() throws IOException { SearchSourceBuilder ssb = new SearchSourceBuilder(); IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, - () -> ssb.parseXContent(parser, true, nf -> true).rewrite(null) + () -> ssb.parseXContent(parser, true, nf -> true) + .rewrite(new QueryRewriteContext(parserConfig(), null, null, null, new PointInTimeBuilder(new BytesArray("pitid")))) ); assertEquals("[terminate_after] cannot be used in children of compound retrievers", iae.getMessage()); } - - try ( - XContentParser parser = createParser( - JsonXContent.jsonXContent, - "{\"retriever\":{\"rrf_nl\":{\"retrievers\":" + "[{\"standard\":{\"sort\":[\"f1\"]}},{\"standard\":{\"sort\":[\"f2\"]}}]}}}" - ) - ) { - SearchSourceBuilder ssb = new SearchSourceBuilder(); - IllegalArgumentException iae = expectThrows( - IllegalArgumentException.class, - () -> ssb.parseXContent(parser, true, nf -> true).rewrite(null) - ); - assertEquals("[sort] cannot be used in children of compound retrievers", iae.getMessage()); - } - - try ( - XContentParser parser = createParser( - JsonXContent.jsonXContent, - "{\"retriever\":{\"rrf_nl\":{\"retrievers\":" + "[{\"standard\":{\"min_score\":1}},{\"standard\":{\"min_score\":2}}]}}}" - ) - ) { - SearchSourceBuilder ssb = new SearchSourceBuilder(); - IllegalArgumentException iae = expectThrows( - IllegalArgumentException.class, - () -> ssb.parseXContent(parser, true, nf -> true).rewrite(null) - ); - assertEquals("[min_score] cannot be used in children of compound retrievers", iae.getMessage()); - } - - try ( - XContentParser parser = createParser( - JsonXContent.jsonXContent, - "{\"retriever\":{\"rrf_nl\":{\"retrievers\":" - + "[{\"standard\":{\"collapse\":{\"field\":\"f0\"}}},{\"standard\":{\"collapse\":{\"field\":\"f1\"}}}]}}}" - ) - ) { - SearchSourceBuilder ssb = new SearchSourceBuilder(); - IllegalArgumentException iae = expectThrows( - IllegalArgumentException.class, - () -> ssb.parseXContent(parser, true, nf -> true).rewrite(null) - ); - assertEquals("[collapse] cannot be used in children of compound retrievers", iae.getMessage()); - } - - try ( - XContentParser parser = createParser( - JsonXContent.jsonXContent, - "{\"retriever\":{\"rrf_nl\":{\"retrievers\":[{\"rrf_nl\":{}}]}}}" - ) - ) { - SearchSourceBuilder ssb = new SearchSourceBuilder(); - IllegalArgumentException iae = expectThrows( - IllegalArgumentException.class, - () -> ssb.parseXContent(parser, true, nf -> true).rewrite(null) - ); - assertEquals("[rank] cannot be used in children of compound retrievers", iae.getMessage()); - } - } - - /** Tests max depth errors related to compound retrievers. These tests require a compound retriever which is why they are here. */ - public void testRetrieverBuilderParsingMaxDepth() throws IOException { - try ( - XContentParser parser = createParser( - JsonXContent.jsonXContent, - "{\"retriever\":{\"rrf_nl\":{\"retrievers\":[{\"rrf_nl\":{\"retrievers\":[{\"standard\":{}}]}}]}}}" - ) - ) { - SearchSourceBuilder ssb = new SearchSourceBuilder(); - IllegalArgumentException iae = expectThrows( - IllegalArgumentException.class, - () -> ssb.parseXContent(parser, true, nf -> true).rewrite(null) - ); - assertEquals("[1:65] [rrf] failed to parse field [retrievers]", iae.getMessage()); - assertEquals( - "the nested depth of the [standard] retriever exceeds the maximum nested depth [2] for retrievers", - iae.getCause().getCause().getMessage() - ); - } } @Override diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml index 4f76f52409810..647540644ce9e 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/100_rank_rrf.yml @@ -80,17 +80,14 @@ setup: size: 10 - match: { hits.hits.0._id: "1" } - - match: { hits.hits.0._rank: 1 } - match: { hits.hits.0.fields.text.0: "term term" } - match: { hits.hits.0.fields.keyword.0: "other" } - match: { hits.hits.1._id: "3" } - - match: { hits.hits.1._rank: 2 } - match: { hits.hits.1.fields.text.0: "term" } - match: { hits.hits.1.fields.keyword.0: "keyword" } - match: { hits.hits.2._id: "2" } - - match: { hits.hits.2._rank: 3 } - match: { hits.hits.2.fields.text.0: "other" } - match: { hits.hits.2.fields.keyword.0: "other" } @@ -128,12 +125,10 @@ setup: - match: { hits.total.value: 2 } - match: { hits.hits.0._id: "3" } - - match: { hits.hits.0._rank: 1 } - match: { hits.hits.0.fields.text.0: "term" } - match: { hits.hits.0.fields.keyword.0: "keyword" } - match: { hits.hits.1._id: "1" } - - match: { hits.hits.1._rank: 2 } - match: { hits.hits.1.fields.text.0: "term term" } - match: { hits.hits.1.fields.keyword.0: "other" } @@ -176,17 +171,14 @@ setup: - match: { hits.total.value: 3 } - match: { hits.hits.0._id: "3" } - - match: { hits.hits.0._rank: 1 } - match: { hits.hits.0.fields.text.0: "term" } - match: { hits.hits.0.fields.keyword.0: "keyword" } - match: { hits.hits.1._id: "1" } - - match: { hits.hits.1._rank: 2 } - match: { hits.hits.1.fields.text.0: "term term" } - match: { hits.hits.1.fields.keyword.0: "other" } - match: { hits.hits.2._id: "2" } - - match: { hits.hits.2._rank: 3 } - match: { hits.hits.2.fields.text.0: "other" } - match: { hits.hits.2.fields.keyword.0: "other" } diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/150_rank_rrf_pagination.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/150_rank_rrf_pagination.yml index 575723853f0aa..b4893bfec0849 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/150_rank_rrf_pagination.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/150_rank_rrf_pagination.yml @@ -164,11 +164,8 @@ setup: - match: { hits.total.value : 4 } - length: { hits.hits : 3 } - match: { hits.hits.0._id: "2" } - - match: { hits.hits.0._rank: 2 } - match: { hits.hits.1._id: "3" } - - match: { hits.hits.1._rank: 3 } - match: { hits.hits.2._id: "4" } - - match: { hits.hits.2._rank: 4 } --- "Standard pagination outside rank_window_size": @@ -378,7 +375,6 @@ setup: - match: { hits.total.value : 4 } - length: { hits.hits : 1 } - match: { hits.hits.0._id: "3" } - - match: { hits.hits.0._rank: 3 } --- @@ -489,9 +485,7 @@ setup: - match: { hits.total.value : 4 } - length: { hits.hits : 2 } - match: { hits.hits.0._id: "1" } - - match: { hits.hits.0._rank: 1 } - match: { hits.hits.1._id: "4" } - - match: { hits.hits.1._rank: 2 } - do: search: @@ -594,9 +588,7 @@ setup: - match: { hits.total.value : 4 } - length: { hits.hits : 2 } - match: { hits.hits.0._id: "3" } - - match: { hits.hits.0._rank: 3 } - match: { hits.hits.1._id: "2" } - - match: { hits.hits.1._rank: 4 } --- "Pagination within interleaved results, different result set sizes, rank_window_size covering all results": @@ -690,9 +682,7 @@ setup: - match: { hits.total.value : 5 } - length: { hits.hits : 2 } - match: { hits.hits.0._id: "1" } - - match: { hits.hits.0._rank: 1 } - match: { hits.hits.1._id: "5" } - - match: { hits.hits.1._rank: 2 } - do: search: @@ -779,9 +769,7 @@ setup: - match: { hits.total.value : 5 } - length: { hits.hits : 2 } - match: { hits.hits.0._id: "4" } - - match: { hits.hits.0._rank: 3 } - match: { hits.hits.1._id: "3" } - - match: { hits.hits.1._rank: 4 } - do: search: @@ -868,7 +856,6 @@ setup: - match: { hits.total.value: 5 } - length: { hits.hits: 1 } - match: { hits.hits.0._id: "2" } - - match: { hits.hits.0._rank: 5 } --- @@ -965,9 +952,7 @@ setup: - match: { hits.total.value : 5 } - length: { hits.hits : 2 } - match: { hits.hits.0._id: "5" } - - match: { hits.hits.0._rank: 1 } - match: { hits.hits.1._id: "4" } - - match: { hits.hits.1._rank: 2 } - do: search: diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/200_rank_rrf_script.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/200_rank_rrf_script.yml index 76cedf44d3dbe..bca39dea4ae57 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/200_rank_rrf_script.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/200_rank_rrf_script.yml @@ -129,19 +129,10 @@ setup: size: 5 - match: { hits.hits.0._id: "5" } - - match: { hits.hits.0._rank: 1 } - - match: { hits.hits.1._id: "6" } - - match: { hits.hits.1._rank: 2 } - - match: { hits.hits.2._id: "4" } - - match: { hits.hits.2._rank: 3 } - - match: { hits.hits.3._id: "7" } - - match: { hits.hits.3._rank: 4 } - - match: { hits.hits.4._id: "3" } - - match: { hits.hits.4._rank: 5 } - close_to: { aggregations.sums.value.asc_total: { value: 25.0, error: 0.001 }} - match: { aggregations.sums.value.text_total: 25 } @@ -196,7 +187,6 @@ setup: - match: { hits.total.value: 6 } - match: { hits.hits.0._id: "5" } - - match: { hits.hits.0._rank: 1 } - close_to: { aggregations.sums.value.asc_total: { value: 33.0, error: 0.001 }} - close_to: { aggregations.sums.value.desc_total: { value: 39.0, error: 0.001 }} @@ -272,20 +262,10 @@ setup: size: 5 - match: { hits.hits.0._id: "6" } - - match: { hits.hits.0._rank: 1 } - - match: { hits.hits.1._id: "5" } - - match: { hits.hits.1._rank: 2 } - - match: { hits.hits.2._id: "7" } - - match: { hits.hits.2._rank: 3 } - - match: { hits.hits.3._id: "4" } - - match: { hits.hits.3._rank: 4 } - - match: { hits.hits.4._id: "8" } - - match: { hits.hits.4._rank: 5 } - - close_to: { aggregations.sums.value.asc_total: { value: 30.0, error: 0.001 }} - close_to: { aggregations.sums.value.desc_total: { value: 30.0, error: 0.001 }} - match: { aggregations.sums.value.text_total: 30 } diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml index d3d45ef2b18e8..258ab70cd09bd 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/300_rrf_retriever.yml @@ -91,17 +91,14 @@ setup: size: 10 - match: { hits.hits.0._id: "1" } - - match: { hits.hits.0._rank: 1 } - match: { hits.hits.0.fields.text.0: "term term" } - match: { hits.hits.0.fields.keyword.0: "other" } - match: { hits.hits.1._id: "3" } - - match: { hits.hits.1._rank: 2 } - match: { hits.hits.1.fields.text.0: "term" } - match: { hits.hits.1.fields.keyword.0: "keyword" } - match: { hits.hits.2._id: "2" } - - match: { hits.hits.2._rank: 3 } - match: { hits.hits.2.fields.text.0: "other" } - match: { hits.hits.2.fields.keyword.0: "other" } @@ -143,12 +140,10 @@ setup: - match: { hits.total.value : 2 } - match: { hits.hits.0._id: "3" } - - match: { hits.hits.0._rank: 1 } - match: { hits.hits.0.fields.text.0: "term" } - match: { hits.hits.0.fields.keyword.0: "keyword" } - match: { hits.hits.1._id: "1" } - - match: { hits.hits.1._rank: 2 } - match: { hits.hits.1.fields.text.0: "term term" } - match: { hits.hits.1.fields.keyword.0: "other" } @@ -198,17 +193,14 @@ setup: - match: { hits.total.value : 3 } - match: { hits.hits.0._id: "3" } - - match: { hits.hits.0._rank: 1 } - match: { hits.hits.0.fields.text.0: "term" } - match: { hits.hits.0.fields.keyword.0: "keyword" } - match: { hits.hits.1._id: "1" } - - match: { hits.hits.1._rank: 2 } - match: { hits.hits.1.fields.text.0: "term term" } - match: { hits.hits.1.fields.keyword.0: "other" } - match: { hits.hits.2._id: "2" } - - match: { hits.hits.2._rank: 3 } - match: { hits.hits.2.fields.text.0: "other" } - match: { hits.hits.2.fields.keyword.0: "other" } @@ -267,7 +259,6 @@ setup: - length: { hits.hits: 1 } - match: { hits.hits.0._id: "3" } - - match: { hits.hits.0._rank: 1 } - match: { hits.hits.0.fields.text.0: "term" } - match: { hits.hits.0.fields.keyword.0: "keyword" } @@ -330,6 +321,82 @@ setup: - length: { hits.hits: 1 } - match: { hits.hits.0._id: "3" } - - match: { hits.hits.0._rank: 1 } - match: { hits.hits.0.fields.text.0: "term" } - match: { hits.hits.0.fields.keyword.0: "keyword" } + +--- +"rrf retriever with nested rrf retriever and pagination": + + - do: + search: + index: test + body: + track_total_hits: true + fields: [ "text", "keyword" ] + retriever: + rrf: + retrievers: [ + { + "rrf": + { + "retrievers": [ + { + "knn": { + "field": "vector", + "query_vector": [ 0.0 ], + "k": 3, + "num_candidates": 3 + } + }, + { + "standard": + { + "query": + { + "term": + { + "text": "term" + } + } + } + }, + { + "standard": + { + "query": + { + "match": + { + "keyword": "keyword" + } + } + } + } + ], + "rank_window_size": 100, + "rank_constant": 1 + } + }, + { + "knn": { + "field": vector, + "query_vector": [ 0.0 ], + "k": 2, + "num_candidates": 2 + } + } + ] + "rank_window_size": 10 + "rank_constant": 1 + size: 10 + from: 1 + + - match: { hits.total.value : 3 } + + - match: { hits.hits.0._id: "2" } + - match: { hits.hits.0.fields.text.0: "other" } + - match: { hits.hits.0.fields.keyword.0: "other" } + + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.1.fields.text.0: "term" } + - match: { hits.hits.1.fields.keyword.0: "keyword" } diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/350_rrf_retriever_pagination.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/350_rrf_retriever_pagination.yml new file mode 100644 index 0000000000000..47ba3658bb38d --- /dev/null +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/350_rrf_retriever_pagination.yml @@ -0,0 +1,1112 @@ +setup: + - skip: + features: close_to + + - requires: + cluster_features: 'rrf_retriever_composition_supported' + reason: 'test requires rrf retriever composition support' + + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 1 + mappings: + properties: + number_val: + type: keyword + char_val: + type: keyword + + - do: + index: + index: test + id: 1 + body: + number_val: "1" + char_val: "A" + + - do: + index: + index: test + id: 2 + body: + number_val: "2" + char_val: "B" + + - do: + index: + index: test + id: 3 + body: + number_val: "3" + char_val: "C" + + - do: + index: + index: test + id: 4 + body: + number_val: "4" + char_val: "D" + + - do: + index: + index: test + id: 5 + body: + number_val: "5" + char_val: "E" + + - do: + indices.refresh: {} + +--- +"Standard pagination within rank_window_size": + # this test retrieves the same results from two queries, and applies a simple pagination skipping the first result + - do: + search: + index: test + body: + track_total_hits: true + retriever: + rrf: { + retrievers: [ + { + # this should clause would generate the result set [1, 2, 3, 4] + standard: + { + query: { + bool: { + should: [ + { + term: { + number_val: { + value: "1", + boost: 10.0 + } + } + }, + { + term: { + number_val: { + value: "2", + boost: 9.0 + } + } + }, + { + term: { + number_val: { + value: "3", + boost: 8.0 + } + } + }, + { + term: { + number_val: { + value: "4", + boost: 7.0 + } + } + } + ] + } + } + } + }, + { + # this should clause would generate the result set [1, 2, 3, 4] + standard: { + query: { + bool: { + should: [ + { + term: { + char_val: { + value: "A", + boost: 10.0 + } + } + }, + { + term: { + char_val: { + value: "B", + boost: 9.0 + } + } + }, + { + term: { + char_val: { + value: "C", + boost: 8.0 + } + } + }, + { + term: { + char_val: { + value: "D", + boost: 7.0 + } + } + } + ] + } + } + } + } + ], + rank_window_size: 10, + rank_constant: 10 + } + from : 1 + size : 10 + + - match: { hits.total.value : 4 } + - length: { hits.hits : 3 } + - match: { hits.hits.0._id: "2" } + # score for doc 2 is (1/12 + 1/12) + - close_to: {hits.hits.0._score: {value: 0.1666, error: 0.001}} + - match: { hits.hits.1._id: "3" } + # score for doc 3 is (1/13 + 1/13) + - close_to: {hits.hits.1._score: {value: 0.1538, error: 0.001}} + - match: { hits.hits.2._id: "4" } + # score for doc 4 is (1/14 + 1/14) + - close_to: {hits.hits.2._score: {value: 0.1428, error: 0.001}} + +--- +"Standard pagination outside rank_window_size": + # in this example, from starts *after* rank_window_size so, we expect 0 results to be returned + - do: + search: + index: test + body: + track_total_hits: true + retriever: + rrf: + { + retrievers: [ + { + # this should clause would generate the result set [1, 2, 3, 4] + standard: { + query: { + bool: { + should: [ + { + term: { + number_val: { + value: "1", + boost: 10.0 + } + } + }, + { + term: { + number_val: { + value: "2", + boost: 9.0 + } + } + }, + { + term: { + number_val: { + value: "3", + boost: 8.0 + } + } + }, + { + term: { + number_val: { + value: "4", + boost: 7.0 + } + } + } + ] + } + } + } + }, + { + # this should clause would generate the result set [1, 2, 3, 4] + standard: { + query: { + bool: { + should: [ + { + term: { + char_val: { + value: "A", + boost: 10.0 + } + } + }, + { + term: { + char_val: { + value: "B", + boost: 9.0 + } + } + }, + { + term: { + char_val: { + value: "C", + boost: 8.0 + } + } + }, + { + term: { + char_val: { + value: "D", + boost: 7.0 + } + } + } + ] + } + } + } + } + ], + rank_window_size: 2, + rank_constant: 10 + } + from : 10 + size : 2 + + - match: { hits.total.value : 4 } + - length: { hits.hits : 0 } + +--- +"Standard pagination partially outside rank_window_size": + # in this example we have that from starts *within* rank_window_size, but "from + size" goes over + - do: + search: + index: test + body: + track_total_hits: true + retriever: + rrf: + { + retrievers: [ + { + # this should clause would generate the result set [1, 2, 3, 4] + standard: { + query: { + bool: { + should: [ + { + term: { + number_val: { + value: "1", + boost: 10.0 + } + } + }, + { + term: { + number_val: { + value: "2", + boost: 9.0 + } + } + }, + { + term: { + number_val: { + value: "3", + boost: 8.0 + } + } + }, + { + term: { + number_val: { + value: "4", + boost: 7.0 + } + } + } + ] + } + } + } + }, + { + # this should clause would generate the result set [1, 2, 3, 4] + standard: { + query: { + bool: { + should: [ + { + term: { + char_val: { + value: "A", + boost: 10.0 + } + } + }, + { + term: { + char_val: { + value: "B", + boost: 9.0 + } + } + }, + { + term: { + char_val: { + value: "C", + boost: 8.0 + } + } + }, + { + term: { + char_val: { + value: "D", + boost: 7.0 + } + } + } + ] + } + } + } + } + ], + rank_window_size: 3, + rank_constant: 10 + } + from : 2 + size : 2 + + - match: { hits.total.value : 4 } + - length: { hits.hits : 1 } + - match: { hits.hits.0._id: "3" } + # score for doc 3 is (1/13 + 1/13) + - close_to: {hits.hits.0._score: {value: 0.1538, error: 0.001}} + + +--- +"Pagination within interleaved results": + # perform two searches with different "from" parameter, ensuring that results are consistent + # rank_window_size covers the entire result set for both queries, so pagination should be consistent + # queryA has a result set of [1, 2, 3, 4] and + # queryB has a result set of [4, 3, 1, 2] + # so for rank_constant=10, the expected order is [1, 4, 3, 2] + - do: + search: + index: test + body: + track_total_hits: true + retriever: + rrf: + { + retrievers: [ + { + # this should clause would generate the result set [1, 2, 3, 4] + standard: { + query: { + bool: { + should: [ + { + term: { + number_val: { + value: "1", + boost: 10.0 + } + } + }, + { + term: { + number_val: { + value: "2", + boost: 9.0 + } + } + }, + { + term: { + number_val: { + value: "3", + boost: 8.0 + } + } + }, + { + term: { + number_val: { + value: "4", + boost: 7.0 + } + } + } + ] + } + } + } + }, + { + # this should clause would generate the result set [4, 3, 1, 2] + standard: { + query: { + bool: { + should: [ + { + term: { + char_val: { + value: "D", + boost: 10.0 + } + } + }, + { + term: { + char_val: { + value: "C", + boost: 9.0 + } + } + }, + { + term: { + char_val: { + value: "A", + boost: 8.0 + } + } + }, + { + term: { + char_val: { + value: "B", + boost: 7.0 + } + } + } + ] + } + } + } + } + ], + rank_window_size: 10, + rank_constant: 10 + } + from : 0 + size : 2 + + - match: { hits.total.value : 4 } + - length: { hits.hits : 2 } + - match: { hits.hits.0._id: "1" } + # score for doc 1 is (1/11 + 1/13) + - close_to: {hits.hits.0._score: {value: 0.1678, error: 0.001}} + - match: { hits.hits.1._id: "4" } + # score for doc 4 is (1/11 + 1/14) + - close_to: {hits.hits.1._score: {value: 0.1623, error: 0.001}} + + - do: + search: + index: test + body: + track_total_hits: true + retriever: + rrf: + { + retrievers: [ + { + # this should clause would generate the result set [1, 2, 3, 4] + standard: { + query: { + bool: { + should: [ + { + term: { + number_val: { + value: "1", + boost: 10.0 + } + } + }, + { + term: { + number_val: { + value: "2", + boost: 9.0 + } + } + }, + { + term: { + number_val: { + value: "3", + boost: 8.0 + } + } + }, + { + term: { + number_val: { + value: "4", + boost: 7.0 + } + } + } + ] + } + } + } + }, + { + # this should clause would generate the result set [4, 3, 1, 2] + standard: { + query: { + bool: { + should: [ + { + term: { + char_val: { + value: "D", + boost: 10.0 + } + } + }, + { + term: { + char_val: { + value: "C", + boost: 9.0 + } + } + }, + { + term: { + char_val: { + value: "A", + boost: 8.0 + } + } + }, + { + term: { + char_val: { + value: "B", + boost: 7.0 + } + } + } + ] + } + } + } + } + ], + rank_window_size: 10, + rank_constant: 10 + } + from : 2 + size : 2 + + - match: { hits.total.value : 4 } + - length: { hits.hits : 2 } + - match: { hits.hits.0._id: "3" } + # score for doc 3 is (1/12 + 1/13) + - close_to: {hits.hits.0._score: {value: 0.1602, error: 0.001}} + - match: { hits.hits.1._id: "2" } + # score for doc 2 is (1/12 + 1/14) + - close_to: {hits.hits.1._score: {value: 0.1547, error: 0.001}} + +--- +"Pagination within interleaved results, different result set sizes, rank_window_size covering all results": + # perform multiple searches with different "from" parameter, ensuring that results are consistent + # rank_window_size covers the entire result set for both queries, so pagination should be consistent + # queryA has a result set of [5, 1] and + # queryB has a result set of [4, 3, 1, 2] + # so for rank_constant=10, the expected order is [1, 4, 5, 3, 2] + - do: + search: + index: test + body: + track_total_hits: true + retriever: + rrf: + { + retrievers: [ + { + # this should clause would generate the result set [5, 1] + standard: { + query: { + bool: { + should: [ + { + term: { + number_val: { + value: "5", + boost: 10.0 + } + } + }, + { + term: { + number_val: { + value: "1", + boost: 9.0 + } + } + } + ] + } + } + } + }, + { + # this should clause would generate the result set [4, 3, 1, 2] + standard: { + query: { + bool: { + should: [ + { + term: { + char_val: { + value: "D", + boost: 10.0 + } + } + }, + { + term: { + char_val: { + value: "C", + boost: 9.0 + } + } + }, + { + term: { + char_val: { + value: "A", + boost: 8.0 + } + } + }, + { + term: { + char_val: { + value: "B", + boost: 7.0 + } + } + } + ] + } + } + } + } + ], + rank_window_size: 10, + rank_constant: 10 + } + from : 0 + size : 2 + + - match: { hits.total.value : 5 } + - length: { hits.hits : 2 } + - match: { hits.hits.0._id: "1" } + # score for doc 1 is (1/12 + 1/13) + - close_to: {hits.hits.0._score: {value: 0.1602, error: 0.001}} + - match: { hits.hits.1._id: "4" } + # score for doc 4 is (1/11) + - close_to: {hits.hits.1._score: {value: 0.0909, error: 0.001}} + + - do: + search: + index: test + body: + track_total_hits: true + retriever: + rrf: + { + retrievers: [ + { + # this should clause would generate the result set [5, 1] + standard: { + query: { + bool: { + should: [ + { + term: { + number_val: { + value: "5", + boost: 10.0 + } + } + }, + { + term: { + number_val: { + value: "1", + boost: 9.0 + } + } + } + ] + } + } + } + }, + { + # this should clause would generate the result set [4, 3, 1, 2] + standard: { + query: { + bool: { + should: [ + { + term: { + char_val: { + value: "D", + boost: 10.0 + } + } + }, + { + term: { + char_val: { + value: "C", + boost: 9.0 + } + } + }, + { + term: { + char_val: { + value: "A", + boost: 8.0 + } + } + }, + { + term: { + char_val: { + value: "B", + boost: 7.0 + } + } + } + ] + } + } + } + } + ], + rank_window_size: 10, + rank_constant: 10 + } + from : 2 + size : 2 + + - match: { hits.total.value : 5 } + - length: { hits.hits : 2 } + - match: { hits.hits.0._id: "5" } + # score for doc 5 is (1/11) + - close_to: {hits.hits.0._score: {value: 0.0909, error: 0.001}} + - match: { hits.hits.1._id: "3" } + # score for doc 3 is (1/12) + - close_to: {hits.hits.1._score: {value: 0.0833, error: 0.001}} + + - do: + search: + index: test + body: + track_total_hits: true + retriever: + rrf: + { + retrievers: [ + { + # this should clause would generate the result set [5, 1] + standard: { + query: { + bool: { + should: [ + { + term: { + number_val: { + value: "5", + boost: 10.0 + } + } + }, + { + term: { + number_val: { + value: "1", + boost: 9.0 + } + } + } + ] + } + } + } + }, + { + # this should clause would generate the result set [4, 3, 1, 2] + standard: { + query: { + bool: { + should: [ + { + term: { + char_val: { + value: "D", + boost: 10.0 + } + } + }, + { + term: { + char_val: { + value: "C", + boost: 9.0 + } + } + }, + { + term: { + char_val: { + value: "A", + boost: 8.0 + } + } + }, + { + term: { + char_val: { + value: "B", + boost: 7.0 + } + } + } + ] + } + } + } + } + ], + rank_window_size: 10, + rank_constant: 10 + } + from: 4 + size: 2 + + - match: { hits.total.value: 5 } + - length: { hits.hits: 1 } + - match: { hits.hits.0._id: "2" } + # score for doc 2 is (1/14) + - close_to: {hits.hits.0._score: {value: 0.0714, error: 0.001}} + + +--- +"Pagination within interleaved results, different result set sizes, rank_window_size not covering all results": + # perform multiple searches with different "from" parameter, ensuring that results are consistent + # rank_window_size does not cover the entire result set for both queries, so the results should be different + # from the test above. More specifically, we'd get to collect 2 results from each query, so we'd have: + # queryA has a result set of [5, 1] and + # queryB has a result set of [4, 3] + # so for rank_constant=10, the expected order is [4, 5, 1, 3], + # and the rank_window_size-sized result set that we'd paginate over is [4, 5] + - do: + search: + index: test + body: + track_total_hits: true + retriever: + rrf: + { + retrievers: [ + { + # this should clause would generate the result set [5, 1] + standard: { + query: { + bool: { + should: [ + { + term: { + number_val: { + value: "5", + boost: 10.0 + } + } + }, + { + term: { + number_val: { + value: "1", + boost: 9.0 + } + } + } + ] + } + } + } + }, + { + # this should clause would generate the result set [4, 3, 1, 2] + standard: { + query: { + bool: { + should: [ + { + term: { + char_val: { + value: "D", + boost: 10.0 + } + } + }, + { + term: { + char_val: { + value: "C", + boost: 9.0 + } + } + }, + { + term: { + char_val: { + value: "A", + boost: 8.0 + } + } + }, + { + term: { + char_val: { + value: "B", + boost: 7.0 + } + } + } + ] + } + } + } + } + ], + rank_window_size: 2, + rank_constant: 10 + } + from : 0 + size : 2 + + - match: { hits.total.value : 5 } + - length: { hits.hits : 2 } + - match: { hits.hits.0._id: "4" } + # score for doc 4 is (1/11) + - close_to: {hits.hits.0._score: {value: 0.0909, error: 0.001}} + - match: { hits.hits.1._id: "5" } + # score for doc 5 is (1/11) + - close_to: {hits.hits.1._score: {value: 0.0909, error: 0.001}} + + - do: + search: + index: test + body: + track_total_hits: true + retriever: + rrf: + { + retrievers: [ + { + # this should clause would generate the result set [5, 1] + standard: { + query: { + bool: { + should: [ + { + term: { + number_val: { + value: "5", + boost: 10.0 + } + } + }, + { + term: { + number_val: { + value: "1", + boost: 9.0 + } + } + } + ] + } + } + } + }, + { + # this should clause would generate the result set [4, 3, 1, 2] + standard: { + query: { + bool: { + should: [ + { + term: { + char_val: { + value: "D", + boost: 10.0 + } + } + }, + { + term: { + char_val: { + value: "C", + boost: 9.0 + } + } + }, + { + term: { + char_val: { + value: "A", + boost: 8.0 + } + } + }, + { + term: { + char_val: { + value: "B", + boost: 7.0 + } + } + } + ] + } + } + } + } + ], + rank_window_size: 2, + rank_constant: 10 + } + from : 2 + size : 2 + + - match: { hits.total.value : 5 } + - length: { hits.hits : 0 } diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/400_rrf_retriever_script.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/400_rrf_retriever_script.yml index 520389d51b737..bbc1087b05cc3 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/400_rrf_retriever_script.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/400_rrf_retriever_script.yml @@ -160,19 +160,10 @@ setup: ] - match: { hits.hits.0._id: "5" } - - match: { hits.hits.0._rank: 1 } - - match: { hits.hits.1._id: "6" } - - match: { hits.hits.1._rank: 2 } - - match: { hits.hits.2._id: "4" } - - match: { hits.hits.2._rank: 3 } - - match: { hits.hits.3._id: "7" } - - match: { hits.hits.3._rank: 4 } - - match: { hits.hits.4._id: "3" } - - match: { hits.hits.4._rank: 5 } - close_to: { aggregations.sums.value.asc_total: { value: 25.0, error: 0.001 }} - match: { aggregations.sums.value.text_total: 25 } @@ -228,13 +219,12 @@ setup: 'desc_total': states.stream().mapToDouble(v -> v['desc_total']).sum() ] - - match: { hits.total.value: 6 } + - match: { hits.total.value: 5 } - match: { hits.hits.0._id: "5" } - - match: { hits.hits.0._rank: 1 } - - close_to: { aggregations.sums.value.asc_total: { value: 33.0, error: 0.001 }} - - close_to: { aggregations.sums.value.desc_total: { value: 39.0, error: 0.001 }} + - close_to: { aggregations.sums.value.asc_total: { value: 25.0, error: 0.001 }} + - close_to: { aggregations.sums.value.desc_total: { value: 35.0, error: 0.001 }} --- "rrf retriever using multiple knn retrievers and a standard retriever with a scripted metric aggregation": @@ -333,19 +323,10 @@ setup: ] - match: { hits.hits.0._id: "6" } - - match: { hits.hits.0._rank: 1 } - - match: { hits.hits.1._id: "5" } - - match: { hits.hits.1._rank: 2 } - - match: { hits.hits.2._id: "7" } - - match: { hits.hits.2._rank: 3 } - - match: { hits.hits.3._id: "4" } - - match: { hits.hits.3._rank: 4 } - - match: { hits.hits.4._id: "8" } - - match: { hits.hits.4._rank: 5 } - close_to: { aggregations.sums.value.asc_total: { value: 30.0, error: 0.001 }} - close_to: { aggregations.sums.value.desc_total: { value: 30.0, error: 0.001 }} diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/500_rrf_retriever_explain.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/500_rrf_retriever_explain.yml index 8d74ecbccd328..a66d99a922ed0 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/500_rrf_retriever_explain.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/500_rrf_retriever_explain.yml @@ -117,7 +117,7 @@ setup: - match: {hits.hits.0._explanation.details.0.details.0.description: "/weight\\(text:term.*/" } - match: {hits.hits.0._explanation.details.1.value: 1} - match: {hits.hits.0._explanation.details.1.description: "/rrf.score:.\\[0.5\\].*/" } - - match: {hits.hits.0._explanation.details.1.details.0.description: "/within.top.*/" } + - match: {hits.hits.0._explanation.details.1.details.0.details.0.description: "/found.vector.with.calculated.similarity.*/" } - close_to: { hits.hits.1._explanation.value: { value: 0.5833334, error: 0.000001 } } - match: {hits.hits.1._explanation.description: "/rrf.score:.\\[0.5833334\\].*/" } @@ -126,7 +126,7 @@ setup: - match: {hits.hits.1._explanation.details.0.details.0.description: "/weight\\(text:term.*/" } - match: {hits.hits.1._explanation.details.1.value: 2} - match: {hits.hits.1._explanation.details.1.description: "/rrf.score:.\\[0.33333334\\].*/" } - - match: {hits.hits.1._explanation.details.1.details.0.description: "/within.top.*/" } + - match: {hits.hits.1._explanation.details.1.details.0.details.0.description: "/found.vector.with.calculated.similarity.*/" } - match: {hits.hits.2._explanation.value: 0.5} - match: {hits.hits.2._explanation.description: "/rrf.score:.\\[0.5\\].*/" } @@ -154,10 +154,10 @@ setup: term: { text: { value: "term", - _name: "my_query" } } - } + }, + _name: "my_query" } }, { @@ -186,7 +186,7 @@ setup: - match: {hits.hits.0._explanation.details.0.details.0.description: "/weight\\(text:term.*/" } - match: {hits.hits.0._explanation.details.1.value: 1} - match: {hits.hits.0._explanation.details.1.description: "/.*my_top_knn.*/" } - - match: {hits.hits.0._explanation.details.1.details.0.description: "/within.top.*/" } + - match: {hits.hits.0._explanation.details.1.details.0.details.0.description: "/found.vector.with.calculated.similarity.*/" } - close_to: { hits.hits.1._explanation.value: { value: 0.5833334, error: 0.000001 } } - match: {hits.hits.1._explanation.description: "/rrf.score:.\\[0.5833334\\].*/" } @@ -195,7 +195,7 @@ setup: - match: {hits.hits.1._explanation.details.0.details.0.description: "/weight\\(text:term.*/" } - match: {hits.hits.1._explanation.details.1.value: 2} - match: {hits.hits.1._explanation.details.1.description: "/.*my_top_knn.*/" } - - match: {hits.hits.1._explanation.details.1.details.0.description: "/within.top.*/" } + - match: {hits.hits.1._explanation.details.1.details.0.details.0.description: "/found.vector.with.calculated.similarity.*/" } - match: {hits.hits.2._explanation.value: 0.5} - match: {hits.hits.2._explanation.description: "/rrf.score:.\\[0.5\\].*/" } @@ -254,7 +254,7 @@ setup: - match: {hits.hits.0._explanation.details.0.details.0.description: "/weight\\(text:term.*/" } - match: {hits.hits.0._explanation.details.1.value: 1} - match: {hits.hits.0._explanation.details.1.description: "/.*my_top_knn.*/" } - - match: {hits.hits.0._explanation.details.1.details.0.description: "/within.top.*/" } + - match: {hits.hits.0._explanation.details.1.details.0.details.0.description: "/found.vector.with.calculated.similarity.*/" } - close_to: { hits.hits.1._explanation.value: { value: 0.5833334, error: 0.000001 } } - match: {hits.hits.1._explanation.description: "/rrf.score:.\\[0.5833334\\].*/" } @@ -263,7 +263,7 @@ setup: - match: {hits.hits.1._explanation.details.0.details.0.description: "/weight\\(text:term.*/" } - match: {hits.hits.1._explanation.details.1.value: 2} - match: {hits.hits.1._explanation.details.1.description: "/.*my_top_knn.*/" } - - match: {hits.hits.1._explanation.details.1.details.0.description: "/within.top.*/" } + - match: {hits.hits.1._explanation.details.1.details.0.details.0.description: "/found.vector.with.calculated.similarity.*/" } - match: {hits.hits.2._explanation.value: 0.5} - match: {hits.hits.2._explanation.description: "/rrf.score:.\\[0.5\\].*/" } diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/600_rrf_retriever_profile.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/600_rrf_retriever_profile.yml index 7308ce8947db7..e34885419c7f7 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/600_rrf_retriever_profile.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/600_rrf_retriever_profile.yml @@ -1,7 +1,7 @@ setup: - requires: - cluster_features: "gte_v8.15.0" - reason: 'profile for rrf was enabled in 8.15' + cluster_features: 'rrf_retriever_composition_supported' + reason: 'test requires rrf retriever composition support' test_runner_features: close_to - do: @@ -114,12 +114,13 @@ setup: - match: { hits.hits.2._id: "4" } - not_exists: profile.shards.0.dfs - - match: { profile.shards.0.searches.0.query.0.type: ConstantScoreQuery } - - length: { profile.shards.0.searches.0.query.0.children: 1 } - - match: { profile.shards.0.searches.0.query.0.children.0.type: BooleanQuery } - - length: { profile.shards.0.searches.0.query.0.children.0.children: 2 } - - match: { profile.shards.0.searches.0.query.0.children.0.children.0.type: TermQuery } - - match: { profile.shards.0.searches.0.query.0.children.0.children.1.type: DocAndScoreQuery } + - match: { profile.shards.0.searches.0.query.0.type: RankDocsQuery } + - length: { profile.shards.0.searches.0.query.0.children: 2 } + - match: { profile.shards.0.searches.0.query.0.children.0.type: TopQuery } + - match: { profile.shards.0.searches.0.query.0.children.1.type: BooleanQuery } + - length: { profile.shards.0.searches.0.query.0.children.1.children: 2 } + - match: { profile.shards.0.searches.0.query.0.children.1.children.0.type: TermQuery } + - match: { profile.shards.0.searches.0.query.0.children.1.children.1.type: DocAndScoreQuery } --- "profile standard and knn dfs retrievers": @@ -159,17 +160,14 @@ setup: - match: { hits.hits.1._id: "2" } - match: { hits.hits.2._id: "4" } - - exists: profile.shards.0.dfs - - length: { profile.shards.0.dfs.knn: 1 } - - length: { profile.shards.0.dfs.knn.0.query: 1 } - - match: { profile.shards.0.dfs.knn.0.query.0.type: DocAndScoreQuery } - - - match: { profile.shards.0.searches.0.query.0.type: ConstantScoreQuery } - - length: { profile.shards.0.searches.0.query.0.children: 1 } - - match: { profile.shards.0.searches.0.query.0.children.0.type: BooleanQuery } - - length: { profile.shards.0.searches.0.query.0.children.0.children: 2 } - - match: { profile.shards.0.searches.0.query.0.children.0.children.0.type: TermQuery } - - match: { profile.shards.0.searches.0.query.0.children.0.children.1.type: KnnScoreDocQuery } + - not_exists: profile.shards.0.dfs + - match: { profile.shards.0.searches.0.query.0.type: RankDocsQuery } + - length: { profile.shards.0.searches.0.query.0.children: 2 } + - match: { profile.shards.0.searches.0.query.0.children.0.type: TopQuery } + - match: { profile.shards.0.searches.0.query.0.children.1.type: BooleanQuery } + - length: { profile.shards.0.searches.0.query.0.children.1.children: 2 } + - match: { profile.shards.0.searches.0.query.0.children.1.children.0.type: TermQuery } + - match: { profile.shards.0.searches.0.query.0.children.1.children.1.type: TopQuery } --- "using query and dfs knn search": diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml new file mode 100644 index 0000000000000..1f7125377b892 --- /dev/null +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml @@ -0,0 +1,541 @@ +setup: + - skip: + features: close_to + + - requires: + cluster_features: 'rrf_retriever_composition_supported' + reason: 'test requires rrf retriever composition support' + + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 1 + mappings: + properties: + text: + type: text + text_to_highlight: + type: text + keyword: + type: keyword + integer: + type: integer + vector: + type: dense_vector + dims: 1 + index: true + similarity: l2_norm + index_options: + type: hnsw + ef_construction: 100 + m: 16 + nested: + type: nested + properties: + views: + type: long + + - do: + index: + index: test + id: "1" + body: + text: "term term term term term term term term term" + vector: [1.0] + + - do: + index: + index: test + id: "2" + body: + text: "term term term term term term term term" + text_to_highlight: "search for the truth" + keyword: "biology" + vector: [2.0] + + - do: + index: + index: test + id: "3" + body: + text: "term term term term term term term" + text_to_highlight: "nothing related but still a match" + keyword: "technology" + vector: [3.0] + + - do: + index: + index: test + id: "4" + body: + text: "term term term term term term" + vector: [4.0] + - do: + index: + index: test + id: "5" + body: + text: "term term term term term" + text_to_highlight: "You know, for Search!" + keyword: "technology" + integer: 5 + vector: [5.0] + - do: + index: + index: test + id: "6" + body: + text: "term term term term" + keyword: "biology" + integer: 6 + vector: [6.0] + - do: + index: + index: test + id: "7" + body: + text: "term term term" + keyword: "astronomy" + vector: [7.0] + nested: { views: 50} + - do: + index: + index: test + id: "8" + body: + text: "term term" + keyword: "technology" + vector: [8.0] + nested: { views: 100} + - do: + index: + index: test + id: "9" + body: + text: "term" + keyword: "technology" + vector: [9.0] + nested: { views: 10} + - do: + indices.refresh: {} + +--- +"rrf retriever with aggs": + + - do: + search: + index: test + body: + track_total_hits: false + retriever: + rrf: + retrievers: [ + { + knn: { + field: vector, + query_vector: [ 6.0 ], + k: 3, + num_candidates: 3 + } + }, + { + standard: { + query: { + term: { + text: term + } + } + } + } + ] + rank_window_size: 5 + rank_constant: 10 + size: 3 + aggs: + keyword_aggs: + terms: + field: keyword + + - match: { hits.hits.0._id: "5" } + - match: { hits.hits.1._id: "1" } + - match: { hits.hits.2._id: "6" } + + - match: { aggregations.keyword_aggs.buckets.0.key: "technology" } + - match: { aggregations.keyword_aggs.buckets.0.doc_count: 4 } + - match: { aggregations.keyword_aggs.buckets.1.key: "biology" } + - match: { aggregations.keyword_aggs.buckets.1.doc_count: 2 } + - match: { aggregations.keyword_aggs.buckets.2.key: "astronomy" } + - match: { aggregations.keyword_aggs.buckets.2.doc_count: 1 } + +--- +"rrf retriever with aggs - scripted metric using score": + + - do: + search: + index: test + body: + track_total_hits: false + retriever: + rrf: + retrievers: [ + { + knn: { + field: vector, + query_vector: [ 6.0 ], + k: 3, + num_candidates: 3 + } + }, + { + standard: { + query: { + term: { + text: term + } + } + } + } + ] + rank_window_size: 5 + rank_constant: 10 + size: 3 + aggs: + max_score: + max: + script: + lang: painless + source: "_score" + + + - match: { hits.hits.0._id: "5" } + - match: { hits.hits.1._id: "1" } + - match: { hits.hits.2._id: "6" } + + - close_to: { aggregations.max_score.value: { value: 0.15, error: 0.001 }} + +--- +"rrf retriever with top-level collapse": + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + track_total_hits: true + retriever: + rrf: + retrievers: [ + { + knn: { + field: vector, + query_vector: [ 6.0 ], + k: 3, + num_candidates: 3 + } + }, + { + standard: { + query: { + term: { + text: term + } + } + } + } + ] + rank_window_size: 5 + rank_constant: 10 + size: 3 + collapse: { field: keyword, inner_hits: { name: sub_hits, size: 2 } } + + - match: { hits.hits.0._id: "5" } + - match: { hits.hits.1._id: "1" } + - match: { hits.hits.2._id: "6" } + + - match: { hits.hits.0.inner_hits.sub_hits.hits.total : 4 } + - length: { hits.hits.0.inner_hits.sub_hits.hits.hits : 2 } + - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "5" } + - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.1._id: "3" } + + - length: { hits.hits.1.inner_hits.sub_hits.hits.hits : 2 } + - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "1" } + - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "4" } + + - length: { hits.hits.2.inner_hits.sub_hits.hits.hits: 2 } + - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "6" } + - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._id: "2" } + +--- +"rrf retriever with inner-level collapse": + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + track_total_hits: true + retriever: + rrf: + retrievers: [ + { + knn: { + field: vector, + query_vector: [ 6.0 ], + k: 3, + num_candidates: 10 + } + }, + { + standard: { + query: { + term: { + text: term + } + }, + collapse: { field: keyword, inner_hits: { name: sub_hits, size: 1 } } + } + } + ] + rank_window_size: 5 + rank_constant: 10 + size: 3 + + - match: { hits.hits.0._id: "7" } + - match: { hits.hits.1._id: "1" } + - match: { hits.hits.2._id: "6" } + +--- +"rrf retriever highlighting results": + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + track_total_hits: true + retriever: + rrf: + retrievers: [ + { + standard: { + query: { + match: { + text_to_highlight: "search" + } + } + } + }, + { + standard: { + query: { + term: { + keyword: technology + } + } + } + } + ] + rank_window_size: 5 + rank_constant: 10 + size: 3 + highlight: { + fields: { + "text_to_highlight": { + "fragment_size": 150, + "number_of_fragments": 3 + } + } + } + + - match: { hits.total : 5 } + + - match: { hits.hits.0._id: "5" } + - match: { hits.hits.0.highlight.text_to_highlight.0: "You know, for Search!" } + + - match: { hits.hits.1._id: "2" } + - match: { hits.hits.1.highlight.text_to_highlight.0: "search for the truth" } + + - match: { hits.hits.2._id: "3" } + - not_exists: hits.hits.2.highlight + +--- +"rrf retriever with custom nested sort": + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + track_total_hits: true + retriever: + rrf: + retrievers: [ + { + # this one retrievers docs 1, 2, 3, .., 9 + # but due to sorting, it will revert the order to 6, 5, .., 9 which due to + # rank_window_size: 2 will only return 6 and 5 + standard: { + query: { + term: { + text: term + } + }, + sort: [ + { + integer: { + order: desc + } + } + ] + } + }, + { + # this one retrieves doc 2 and 6 + standard: { + query: { + term: { + keyword: biology + } + } + } + } + ] + rank_window_size: 2 + rank_constant: 10 + size: 2 + + - match: { hits.total : 9 } + - length: {hits.hits: 2 } + + - match: { hits.hits.0._id: "6" } + - match: { hits.hits.1._id: "2" } + +--- +"rrf retriever with nested query": + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + track_total_hits: true + retriever: + rrf: + retrievers: [ + { + knn: { + field: vector, + query_vector: [ 7.0 ], + k: 1, + num_candidates: 3 + } + }, + { + standard: { + query: { + nested: { + path: nested, + query: { + range: { + nested.views: { + gte: 50 + } + } + } + } + } + } + } + ] + rank_window_size: 5 + rank_constant: 10 + size: 3 + + - match: { hits.total : 2 } + - match: { hits.hits.0._id: "7" } + - match: { hits.hits.1._id: "8" } + +--- +"rrf retriever with global min_score": + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + track_total_hits: true + retriever: + rrf: + retrievers: [ + { + # this one retrievers docs 1 and 2 + knn: { + field: vector, + query_vector: [ 1.0 ], + k: 2, + num_candidates: 10 + } + }, + { + # this one retrieves docs 2 and 6 + standard: { + query: { + term: { + keyword: biology + } + } + } + } + ] + rank_window_size: 5 + rank_constant: 10 + size: 3 + min_score: 0.1 + + - match: { hits.total : 1 } + + - match: { hits.hits.0._id: "2" } + +--- +"rrf retriever with retriever-level min_score": + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + track_total_hits: true + retriever: + rrf: + retrievers: [ + { + # this one retrieves doc 1 + knn: { + field: vector, + query_vector: [ 1.0 ], + k: 10, + num_candidates: 10, + similarity: 0.0 + } + }, + { + # this one retrieves no docs + standard: { + query: { + term: { + keyword: biology + } + }, + min_score: 100 + } + } + ] + rank_window_size: 10 + rank_constant: 10 + size: 10 + + - length: { hits.hits : 1 } + + - match: { hits.hits.0._id: "1" } From cd427198dc68764e9a36ee4b4b033700f256d736 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 3 Oct 2024 10:53:47 +0100 Subject: [PATCH 026/194] More verbose logging in `IndicesSegmentsRestCancellationIT` (#113844) Relates #88201 --- ...ockedSearcherRestCancellationTestCase.java | 34 +++++++++++++++++++ .../IndicesSegmentsRestCancellationIT.java | 11 ++++++ 2 files changed, 45 insertions(+) diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BlockedSearcherRestCancellationTestCase.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BlockedSearcherRestCancellationTestCase.java index a85ac9aefe694..c7e3a5b1c9a77 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BlockedSearcherRestCancellationTestCase.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/BlockedSearcherRestCancellationTestCase.java @@ -9,10 +9,12 @@ package org.elasticsearch.http; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.Cancellable; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; @@ -29,6 +31,8 @@ import org.elasticsearch.index.shard.IndexShardTestCase; import org.elasticsearch.index.translog.TranslogStats; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.plugins.EnginePlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.tasks.Task; @@ -141,6 +145,12 @@ public List> getSettings() { private static class SearcherBlockingEngine extends ReadOnlyEngine { + // using a specialized logger for this case because and "logger" means "Engine#logger" + // (relates investigation into https://github.com/elastic/elasticsearch/issues/88201) + private static final Logger blockedSearcherRestCancellationTestCaseLogger = LogManager.getLogger( + BlockedSearcherRestCancellationTestCase.class + ); + final Semaphore searcherBlock = new Semaphore(1); SearcherBlockingEngine(EngineConfig config) { @@ -149,12 +159,36 @@ private static class SearcherBlockingEngine extends ReadOnlyEngine { @Override public Searcher acquireSearcher(String source, SearcherScope scope, Function wrapper) throws EngineException { + if (blockedSearcherRestCancellationTestCaseLogger.isDebugEnabled()) { + blockedSearcherRestCancellationTestCaseLogger.debug( + Strings.format( + "in acquireSearcher for shard [%s] on thread [%s], availablePermits=%d", + config().getShardId(), + Thread.currentThread().getName(), + searcherBlock.availablePermits() + ), + new ElasticsearchException("stack trace") + ); + } + try { searcherBlock.acquire(); } catch (InterruptedException e) { throw new AssertionError(e); } searcherBlock.release(); + + if (blockedSearcherRestCancellationTestCaseLogger.isDebugEnabled()) { + blockedSearcherRestCancellationTestCaseLogger.debug( + Strings.format( + "continuing in acquireSearcher for shard [%s] on thread [%s], availablePermits=%d", + config().getShardId(), + Thread.currentThread().getName(), + searcherBlock.availablePermits() + ) + ); + } + return super.acquireSearcher(source, scope, wrapper); } } diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndicesSegmentsRestCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndicesSegmentsRestCancellationIT.java index a90b04d54649c..92fde6d7765cc 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndicesSegmentsRestCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/IndicesSegmentsRestCancellationIT.java @@ -12,12 +12,23 @@ import org.apache.http.client.methods.HttpGet; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsAction; import org.elasticsearch.client.Request; +import org.elasticsearch.test.junit.annotations.TestIssueLogging; public class IndicesSegmentsRestCancellationIT extends BlockedSearcherRestCancellationTestCase { + @TestIssueLogging( + issueUrl = "https://github.com/elastic/elasticsearch/issues/88201", + value = "org.elasticsearch.http.BlockedSearcherRestCancellationTestCase:DEBUG" + + ",org.elasticsearch.transport.TransportService:TRACE" + ) public void testIndicesSegmentsRestCancellation() throws Exception { runTest(new Request(HttpGet.METHOD_NAME, "/_segments"), IndicesSegmentsAction.NAME); } + @TestIssueLogging( + issueUrl = "https://github.com/elastic/elasticsearch/issues/88201", + value = "org.elasticsearch.http.BlockedSearcherRestCancellationTestCase:DEBUG" + + ",org.elasticsearch.transport.TransportService:TRACE" + ) public void testCatSegmentsRestCancellation() throws Exception { runTest(new Request(HttpGet.METHOD_NAME, "/_cat/segments"), IndicesSegmentsAction.NAME); } From 87deb99ef46da9406b2edf8043cad55662c51ab7 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 3 Oct 2024 11:11:14 +0100 Subject: [PATCH 027/194] Weaken `restoreFileFromSnapshot` assertion (#113768) It's possible to call `RecoveryTarget#restoreFileFromSnapshot` after the `RecoveryTarget` has been replaced with a new instance due to a retry, but before all its refs have been released. If the recovery was snapshot-based then in this situation we will have already transferred the permits to the new instance, so the assertion that this instance has permits will trip. This commit fixes the problem with a non-null placeholder value indicating that the recovery target was at least originally created while holding some snapshot recovery permits. Closes #96018 --- .../elasticsearch/indices/recovery/RecoveryTarget.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java index cf76404bcdf02..ea485a411143e 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java @@ -86,9 +86,12 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget private final AtomicInteger recoveryMonitorBlocks = new AtomicInteger(); - @Nullable // if we're not downloading files from snapshots in this recovery or we're retrying + @Nullable // if we're not downloading files from snapshots in this recovery private volatile Releasable snapshotFileDownloadsPermit; + // placeholder for snapshotFileDownloadsPermit for use when this RecoveryTarget has been replaced by a new one due to a retry + private static final Releasable SNAPSHOT_FILE_DOWNLOADS_PERMIT_PLACEHOLDER_FOR_RETRY = Releasables.wrap(); + // latch that can be used to blockingly wait for RecoveryTarget to be closed private final CountDownLatch closedLatch = new CountDownLatch(1); @@ -153,7 +156,9 @@ public RecoveryTarget retryCopy() { // If we're retrying we should remove the reference from this instance as the underlying resources // get released after the retry copy is created Releasable snapshotFileDownloadsPermitCopy = snapshotFileDownloadsPermit; - snapshotFileDownloadsPermit = null; + if (snapshotFileDownloadsPermitCopy != null) { + snapshotFileDownloadsPermit = SNAPSHOT_FILE_DOWNLOADS_PERMIT_PLACEHOLDER_FOR_RETRY; + } return new RecoveryTarget( indexShard, sourceNode, @@ -585,6 +590,7 @@ public void restoreFileFromSnapshot( BlobStoreIndexShardSnapshot.FileInfo fileInfo, ActionListener listener ) { + assert hasReferences(); assert hasPermitToDownloadSnapshotFiles(); try ( From b8b5a92de3a742dc060ba3b60b49dce155a97322 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Thu, 3 Oct 2024 12:22:15 +0200 Subject: [PATCH 028/194] ES|QL: fix EsqlQueryResponseTests.testEqualsAndHashcode (#113997) --- .../xpack/esql/action/EsqlQueryResponseTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index a344f8d46350d..7c0b6e6a2eaa3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -236,7 +236,8 @@ private Page randomPage(List columns) { protected EsqlQueryResponse mutateInstance(EsqlQueryResponse instance) { boolean allNull = true; for (ColumnInfoImpl info : instance.columns()) { - if (info.type() != DataType.NULL) { + // values inside NULL and UNSUPPORTED blocks cannot be mutated, because they are all null + if (info.type() != DataType.NULL && info.type() != DataType.UNSUPPORTED) { allNull = false; } } From 1a92975d2913e2a565d7f4ee4becf84c742457ca Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 3 Oct 2024 13:07:29 +0100 Subject: [PATCH 029/194] Remove obsolete tests in v9 (#113946) --- .../upgrades/FullClusterRestartIT.java | 96 ----- .../ReplicaShardAllocatorSyncIdIT.java | 327 ------------------ .../action/AutoFollowCoordinatorTests.java | 69 ---- .../SnapshotsRecoveryPlannerServiceTests.java | 41 --- 4 files changed, 533 deletions(-) delete mode 100644 server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 07d814c067bea..9255281e7e5da 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -42,11 +42,8 @@ import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.test.rest.RestTestLegacyFeatures; -import org.elasticsearch.test.rest.TestResponseParsers; -import org.elasticsearch.transport.Compression; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; import org.junit.Before; @@ -59,7 +56,6 @@ import java.util.ArrayList; import java.util.Base64; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -79,7 +75,6 @@ import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; -import static org.elasticsearch.transport.RemoteClusterService.REMOTE_CLUSTER_COMPRESS; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; @@ -1726,67 +1721,6 @@ public void testSystemIndexMetadataIsUpgraded() throws Exception { } } - /** - * This test ensures that soft deletes are enabled a when upgrading a pre-8 cluster to 8.0+ - */ - @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) // This test can be removed in v9 - public void testEnableSoftDeletesOnRestore() throws Exception { - var originalClusterDidNotEnforceSoftDeletes = oldClusterHasFeature(RestTestLegacyFeatures.SOFT_DELETES_ENFORCED) == false; - - assumeTrue("soft deletes must be enabled on 8.0+", originalClusterDidNotEnforceSoftDeletes); - final String snapshot = "snapshot-" + index; - if (isRunningAgainstOldCluster()) { - final Settings.Builder settings = indexSettings(1, 1); - settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false); - createIndex(index, settings.build()); - ensureGreen(index); - int numDocs = randomIntBetween(0, 100); - indexRandomDocuments( - numDocs, - true, - true, - randomBoolean(), - i -> jsonBuilder().startObject().field("field", "value").endObject() - ); - // create repo - client().performRequest(newXContentRequest(HttpMethod.PUT, "/_snapshot/repo", (repoConfig, params) -> { - repoConfig.field("type", "fs"); - repoConfig.startObject("settings"); - repoConfig.field("compress", randomBoolean()); - repoConfig.field("location", repoDirectory.getRoot().getPath()); - repoConfig.endObject(); - return repoConfig; - })); - // create snapshot - Request createSnapshot = newXContentRequest( - HttpMethod.PUT, - "/_snapshot/repo/" + snapshot, - (builder, params) -> builder.field("indices", index) - ); - createSnapshot.addParameter("wait_for_completion", "true"); - client().performRequest(createSnapshot); - } else { - String restoredIndex = "restored-" + index; - // Restore - Request restoreRequest = newXContentRequest( - HttpMethod.POST, - "/_snapshot/repo/" + snapshot + "/_restore", - (restoreCommand, params) -> { - restoreCommand.field("indices", index); - restoreCommand.field("rename_pattern", index); - restoreCommand.field("rename_replacement", restoredIndex); - restoreCommand.startObject("index_settings").field("index.soft_deletes.enabled", true).endObject(); - return restoreCommand; - } - ); - restoreRequest.addParameter("wait_for_completion", "true"); - client().performRequest(restoreRequest); - ensureGreen(restoredIndex); - int numDocs = countOfIndexedRandomDocuments(); - assertTotalHits(numDocs, entityAsMap(client().performRequest(new Request("GET", "/" + restoredIndex + "/_search")))); - } - } - public void testForbidDisableSoftDeletesOnRestore() throws Exception { final String snapshot = "snapshot-" + index; if (isRunningAgainstOldCluster()) { @@ -1837,36 +1771,6 @@ public void testForbidDisableSoftDeletesOnRestore() throws Exception { } } - /** - * In 7.14 the cluster.remote.*.transport.compress setting was changed from a boolean to an enum setting - * with true/false as options. This test ensures that the old boolean setting in cluster state is - * translated properly. This test can be removed in 9.0. - */ - @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) - public void testTransportCompressionSetting() throws IOException { - var originalClusterBooleanCompressSetting = oldClusterHasFeature(RestTestLegacyFeatures.NEW_TRANSPORT_COMPRESSED_SETTING) == false; - assumeTrue("the old transport.compress setting existed before 7.14", originalClusterBooleanCompressSetting); - if (isRunningAgainstOldCluster()) { - client().performRequest( - newXContentRequest( - HttpMethod.PUT, - "/_cluster/settings", - (builder, params) -> builder.startObject("persistent") - .field("cluster.remote.foo.seeds", Collections.singletonList("localhost:9200")) - .field("cluster.remote.foo.transport.compress", "true") - .endObject() - ) - ); - } else { - final Request getSettingsRequest = new Request("GET", "/_cluster/settings"); - final Response getSettingsResponse = client().performRequest(getSettingsRequest); - try (XContentParser parser = createParser(JsonXContent.jsonXContent, getSettingsResponse.getEntity().getContent())) { - final Settings settings = TestResponseParsers.parseClusterSettingsResponse(parser).getPersistentSettings(); - assertThat(REMOTE_CLUSTER_COMPRESS.getConcreteSettingForNamespace("foo").get(settings), equalTo(Compression.Enabled.TRUE)); - } - } - } - public static void assertNumHits(String index, int numHits, int totalShards) throws IOException { Map resp = entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))); assertNoFailures(resp); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java deleted file mode 100644 index 623933436a470..0000000000000 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/ReplicaShardAllocatorSyncIdIT.java +++ /dev/null @@ -1,327 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gateway; - -import org.apache.lucene.index.IndexWriter; -import org.elasticsearch.action.admin.indices.stats.ShardStats; -import org.elasticsearch.cluster.routing.UnassignedInfo; -import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; -import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.UpdateForV9; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.engine.EngineConfig; -import org.elasticsearch.index.engine.EngineFactory; -import org.elasticsearch.index.engine.InternalEngine; -import org.elasticsearch.index.seqno.SequenceNumbers; -import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.index.shard.IndexShardTestCase; -import org.elasticsearch.index.translog.Translog; -import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.indices.recovery.PeerRecoveryTargetService; -import org.elasticsearch.indices.recovery.RecoveryCleanFilesRequest; -import org.elasticsearch.indices.recovery.RecoveryState; -import org.elasticsearch.plugins.EnginePlugin; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; -import org.elasticsearch.test.transport.MockTransportService; -import org.junit.Before; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Semaphore; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.IntStream; - -import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; - -/** - * A legacy version of {@link ReplicaShardAllocatorIT#testPreferCopyCanPerformNoopRecovery()} verifying - * that the {@link ReplicaShardAllocator} prefers copies with matching sync_id. - */ -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) -@UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) // remove this test in v9 -public class ReplicaShardAllocatorSyncIdIT extends ESIntegTestCase { - - private static final AtomicBoolean allowFlush = new AtomicBoolean(); - - private static class SyncedFlushEngine extends InternalEngine { - private volatile IndexWriter indexWriter; - - SyncedFlushEngine(EngineConfig engineConfig) { - super(engineConfig); - } - - @Override - protected void commitIndexWriter(IndexWriter writer, Translog translog) throws IOException { - if (allowFlush.get() == false) { - throw new AssertionError( - "flush is not allowed:" - + "global checkpoint [" - + getLastSyncedGlobalCheckpoint() - + "] " - + "last commit [" - + getLastCommittedSegmentInfos().userData - + "]" - ); - } - indexWriter = writer; - super.commitIndexWriter(writer, translog); - } - - void syncFlush(String syncId) throws IOException { - // make sure that we have committed translog; otherwise, we can flush after relaying translog in store recovery - flush(true, true); - // make sure that background merges won't happen; otherwise, IndexWriter#hasUncommittedChanges can become true again - forceMerge(false, 1, false, UUIDs.randomBase64UUID()); - assertNotNull(indexWriter); - try (var ignored = acquireEnsureOpenRef()) { - assertThat(getTranslogStats().getUncommittedOperations(), equalTo(0)); - Map userData = new HashMap<>(getLastCommittedSegmentInfos().userData); - SequenceNumbers.CommitInfo commitInfo = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(userData.entrySet()); - assertThat(commitInfo.localCheckpoint(), equalTo(getLastSyncedGlobalCheckpoint())); - assertThat(commitInfo.maxSeqNo(), equalTo(getLastSyncedGlobalCheckpoint())); - userData.put(Engine.SYNC_COMMIT_ID, syncId); - indexWriter.setLiveCommitData(userData.entrySet()); - indexWriter.commit(); - } - } - } - - public static class SyncedFlushPlugin extends Plugin implements EnginePlugin { - @Override - public Optional getEngineFactory(IndexSettings indexSettings) { - return Optional.of(SyncedFlushEngine::new); - } - } - - @Override - protected boolean addMockInternalEngine() { - return false; - } - - @Override - protected Collection> nodePlugins() { - return Arrays.asList(MockTransportService.TestPlugin.class, InternalSettingsPlugin.class, SyncedFlushPlugin.class); - } - - @Before - public void allowFlush() { - allowFlush.set(true); - } - - private void syncFlush(String index) throws IOException { - String syncId = randomAlphaOfLength(10); - final Set nodes = internalCluster().nodesInclude(index); - for (String node : nodes) { - IndexService indexService = internalCluster().getInstance(IndicesService.class, node).indexServiceSafe(resolveIndex(index)); - for (IndexShard indexShard : indexService) { - SyncedFlushEngine engine = (SyncedFlushEngine) IndexShardTestCase.getEngine(indexShard); - engine.syncFlush(syncId); - } - } - // Once we have synced flushed, we do not allow regular flush as it will destroy the sync_id. - allowFlush.set(false); - } - - public void testPreferCopyCanPerformNoopRecovery() throws Exception { - String indexName = "test"; - String nodeWithPrimary = internalCluster().startNode(); - - updateClusterSettings( - Settings.builder().put(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE) - ); - assertAcked( - indicesAdmin().prepareCreate(indexName) - .setSettings( - indexSettings(1, 1) - // expire PRRLs quickly - .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING.getKey(), "1ms") - .put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "100ms") - .put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "100ms") - .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "1ms") - ) - ); - String nodeWithReplica = internalCluster().startDataOnlyNode(); - Settings nodeWithReplicaSettings = internalCluster().dataPathSettings(nodeWithReplica); - ensureGreen(indexName); - indexRandom( - randomBoolean(), - randomBoolean(), - randomBoolean(), - IntStream.range(0, between(100, 500)).mapToObj(n -> prepareIndex(indexName).setSource("f", "v")).toList() - ); - if (randomBoolean()) { - indicesAdmin().prepareFlush(indexName).get(); - } - ensureGlobalCheckpointAdvancedAndSynced(indexName); - syncFlush(indexName); - internalCluster().stopNode(nodeWithReplica); - // Wait until the peer recovery retention leases of the offline node are expired - assertBusy(() -> { - for (ShardStats shardStats : indicesAdmin().prepareStats(indexName).get().getShards()) { - assertThat(shardStats.getRetentionLeaseStats().retentionLeases().leases(), hasSize(1)); - } - }); - CountDownLatch blockRecovery = new CountDownLatch(1); - CountDownLatch recoveryStarted = new CountDownLatch(1); - final var transportServiceOnPrimary = MockTransportService.getInstance(nodeWithPrimary); - transportServiceOnPrimary.addSendBehavior((connection, requestId, action, request, options) -> { - if (PeerRecoveryTargetService.Actions.FILES_INFO.equals(action)) { - recoveryStarted.countDown(); - safeAwait(blockRecovery); - } - connection.sendRequest(requestId, action, request, options); - }); - internalCluster().startDataOnlyNode(); - recoveryStarted.await(); - nodeWithReplica = internalCluster().startDataOnlyNode(nodeWithReplicaSettings); - // AllocationService only calls GatewayAllocator if there are unassigned shards - assertAcked(indicesAdmin().prepareCreate("dummy-index").setWaitForActiveShards(0)); - ensureGreen(indexName); - assertThat(internalCluster().nodesInclude(indexName), containsInAnyOrder(nodeWithPrimary, nodeWithReplica)); - assertNoOpRecoveries(indexName); - blockRecovery.countDown(); - transportServiceOnPrimary.clearAllRules(); - } - - public void testFullClusterRestartPerformNoopRecovery() throws Exception { - int numOfReplicas = randomIntBetween(1, 2); - internalCluster().ensureAtLeastNumDataNodes(numOfReplicas + 2); - String indexName = "test"; - assertAcked( - indicesAdmin().prepareCreate(indexName) - .setSettings( - indexSettings(1, numOfReplicas).put( - IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), - randomIntBetween(10, 100) + "kb" - ) - .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING.getKey(), "1ms") // expire PRRLs quickly - .put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "100ms") - .put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "100ms") - ) - ); - ensureGreen(indexName); - indexRandom( - randomBoolean(), - randomBoolean(), - randomBoolean(), - IntStream.range(0, between(200, 500)).mapToObj(n -> prepareIndex(indexName).setSource("f", "v")).toList() - ); - if (randomBoolean()) { - indicesAdmin().prepareFlush(indexName).get(); - } - ensureGlobalCheckpointAdvancedAndSynced(indexName); - syncFlush(indexName); - updateClusterSettings(Settings.builder().put("cluster.routing.allocation.enable", "primaries")); - internalCluster().fullRestart(); - ensureYellow(indexName); - // Wait until the peer recovery retention leases of the offline node are expired - assertBusy(() -> { - for (ShardStats shardStats : indicesAdmin().prepareStats(indexName).get().getShards()) { - assertThat(shardStats.getRetentionLeaseStats().retentionLeases().leases(), hasSize(1)); - } - }); - updateClusterSettings(Settings.builder().putNull("cluster.routing.allocation.enable")); - ensureGreen(indexName); - assertNoOpRecoveries(indexName); - } - - /** - * If the recovery source is on an old node (before 
{@link org.elasticsearch.Version#V_7_2_0}
) then the recovery target - * won't have the safe commit after phase1 because the recovery source does not send the global checkpoint in the clean_files - * step. And if the recovery fails and retries, then the recovery stage might not transition properly. This test simulates - * this behavior by changing the global checkpoint in phase1 to unassigned. - */ - public void testSimulateRecoverySourceOnOldNode() throws Exception { - internalCluster().startMasterOnlyNode(); - String source = internalCluster().startDataOnlyNode(); - String indexName = "test"; - createIndex(indexName, 1, 0); - ensureGreen(indexName); - if (randomBoolean()) { - indexRandom( - randomBoolean(), - randomBoolean(), - randomBoolean(), - IntStream.range(0, between(200, 500)).mapToObj(n -> prepareIndex(indexName).setSource("f", "v")).toList() - ); - } - if (randomBoolean()) { - indicesAdmin().prepareFlush(indexName).get(); - } - if (randomBoolean()) { - syncFlush(indexName); - } - internalCluster().startDataOnlyNode(); - final var transportService = MockTransportService.getInstance(source); - Semaphore failRecovery = new Semaphore(1); - transportService.addSendBehavior((connection, requestId, action, request, options) -> { - if (action.equals(PeerRecoveryTargetService.Actions.CLEAN_FILES)) { - RecoveryCleanFilesRequest cleanFilesRequest = (RecoveryCleanFilesRequest) request; - request = new RecoveryCleanFilesRequest( - cleanFilesRequest.recoveryId(), - cleanFilesRequest.requestSeqNo(), - cleanFilesRequest.shardId(), - cleanFilesRequest.sourceMetaSnapshot(), - cleanFilesRequest.totalTranslogOps(), - SequenceNumbers.UNASSIGNED_SEQ_NO - ); - } - if (action.equals(PeerRecoveryTargetService.Actions.FINALIZE)) { - if (failRecovery.tryAcquire()) { - throw new IllegalStateException("simulated"); - } - } - connection.sendRequest(requestId, action, request, options); - }); - setReplicaCount(1, indexName); - ensureGreen(indexName); - transportService.clearAllRules(); - } - - private void assertNoOpRecoveries(String indexName) { - for (RecoveryState recovery : indicesAdmin().prepareRecoveries(indexName).get().shardRecoveryStates().get(indexName)) { - if (recovery.getPrimary() == false) { - assertThat(recovery.getIndex().fileDetails(), empty()); - assertThat(recovery.getTranslog().totalLocal(), equalTo(recovery.getTranslog().totalOperations())); - } - } - } - - private void ensureGlobalCheckpointAdvancedAndSynced(String indexName) throws Exception { - assertBusy(() -> { - Index index = resolveIndex(indexName); - for (String node : internalCluster().nodesInclude(indexName)) { - IndexService indexService = internalCluster().getInstance(IndicesService.class, node).indexService(index); - if (indexService != null) { - for (IndexShard shard : indexService) { - assertThat(shard.getLastSyncedGlobalCheckpoint(), equalTo(shard.seqNoStats().getMaxSeqNo())); - } - } - } - }); - } -} diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java index 41c1eded15ca7..40f8cb5ff1140 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; @@ -1813,74 +1812,6 @@ void updateAutoFollowMetadata(Function updateFunctio assertThat(counter.get(), equalTo(states.length)); } - @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) - @AwaitsFix(bugUrl = "ability to disable soft deletes was removed in 8.0 indexes so we can probably remove this test") - public void testAutoFollowerSoftDeletesDisabled() { - Client client = mock(Client.class); - when(client.getRemoteClusterClient(anyString(), any(), any())).thenReturn(new RedirectToLocalClusterRemoteClusterClient(client)); - - ClusterState remoteState = createRemoteClusterState("logs-20190101", false); - - AutoFollowPattern autoFollowPattern = createAutoFollowPattern("remote", "logs-*"); - Map patterns = new HashMap<>(); - patterns.put("remote", autoFollowPattern); - Map> followedLeaderIndexUUIDS = new HashMap<>(); - followedLeaderIndexUUIDS.put("remote", new ArrayList<>()); - Map> autoFollowHeaders = new HashMap<>(); - autoFollowHeaders.put("remote", Map.of("key", "val")); - AutoFollowMetadata autoFollowMetadata = new AutoFollowMetadata(patterns, followedLeaderIndexUUIDS, autoFollowHeaders); - - ClusterState currentState = ClusterState.builder(new ClusterName("name")) - .metadata(Metadata.builder().putCustom(AutoFollowMetadata.TYPE, autoFollowMetadata)) - .build(); - - List results = new ArrayList<>(); - Consumer> handler = results::addAll; - AutoFollower autoFollower = new AutoFollower("remote", handler, localClusterStateSupplier(currentState), () -> 1L, Runnable::run) { - @Override - void getRemoteClusterState(String remoteCluster, long metadataVersion, BiConsumer handler) { - assertThat(remoteCluster, equalTo("remote")); - handler.accept(new ClusterStateResponse(new ClusterName("name"), remoteState, false), null); - } - - @Override - void createAndFollow( - Map headers, - PutFollowAction.Request followRequest, - Runnable successHandler, - Consumer failureHandler - ) { - fail("soft deletes are disabled; index should not be followed"); - } - - @Override - void updateAutoFollowMetadata(Function updateFunction, Consumer handler) { - ClusterState resultCs = updateFunction.apply(currentState); - AutoFollowMetadata result = resultCs.metadata().custom(AutoFollowMetadata.TYPE); - assertThat(result.getFollowedLeaderIndexUUIDs().size(), equalTo(1)); - assertThat(result.getFollowedLeaderIndexUUIDs().get("remote").size(), equalTo(1)); - handler.accept(null); - } - - @Override - void cleanFollowedRemoteIndices(ClusterState remoteClusterState, List patterns) { - // Ignore, to avoid invoking updateAutoFollowMetadata(...) twice - } - }; - autoFollower.start(); - - assertThat(results.size(), equalTo(1)); - assertThat(results.get(0).clusterStateFetchException, nullValue()); - List> entries = new ArrayList<>(results.get(0).autoFollowExecutionResults.entrySet()); - assertThat(entries.size(), equalTo(1)); - assertThat(entries.get(0).getKey().getName(), equalTo("logs-20190101")); - assertThat(entries.get(0).getValue(), notNullValue()); - assertThat( - entries.get(0).getValue().getMessage(), - equalTo("index [logs-20190101] cannot be followed, " + "because soft deletes are not enabled") - ); - } - public void testAutoFollowerFollowerIndexAlreadyExists() { Client client = mock(Client.class); when(client.getRemoteClusterClient(anyString(), any(), any())).thenReturn(new RedirectToLocalClusterRemoteClusterClient(client)); diff --git a/x-pack/plugin/snapshot-based-recoveries/src/test/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/plan/SnapshotsRecoveryPlannerServiceTests.java b/x-pack/plugin/snapshot-based-recoveries/src/test/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/plan/SnapshotsRecoveryPlannerServiceTests.java index 485a482467f18..b082698254d17 100644 --- a/x-pack/plugin/snapshot-based-recoveries/src/test/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/plan/SnapshotsRecoveryPlannerServiceTests.java +++ b/x-pack/plugin/snapshot-based-recoveries/src/test/java/org/elasticsearch/xpack/snapshotbasedrecoveries/recovery/plan/SnapshotsRecoveryPlannerServiceTests.java @@ -390,47 +390,6 @@ public void fetchLatestSnapshotsForShard(ShardId shardId, ActionListener { - Store.MetadataSnapshot targetMetadataSnapshot = generateRandomTargetState(store); - - writeRandomDocs(store, randomIntBetween(10, 100)); - ShardSnapshot shardSnapshot = createShardSnapshotThatSharesSegmentFiles(store, "repo"); - - Store.MetadataSnapshot sourceMetadata = store.getMetadata(null); - - long startingSeqNo = randomNonNegativeLong(); - int translogOps = randomIntBetween(0, 100); - ShardRecoveryPlan shardRecoveryPlan = computeShardRecoveryPlan( - "shard-id", - sourceMetadata, - targetMetadataSnapshot, - startingSeqNo, - translogOps, - new ShardSnapshotsService(null, null, null, null) { - @Override - public void fetchLatestSnapshotsForShard(ShardId shardId, ActionListener> listener) { - listener.onResponse(Optional.of(shardSnapshot)); - } - }, - true, - IndexVersions.V_7_14_0, // Unsupported version, - randomBoolean() - ); - - assertPlanIsValid(shardRecoveryPlan, sourceMetadata); - assertAllSourceFilesAreAvailableInSource(shardRecoveryPlan, sourceMetadata); - assertAllIdenticalFilesAreAvailableInTarget(shardRecoveryPlan, targetMetadataSnapshot); - assertThat(shardRecoveryPlan.getSnapshotFilesToRecover(), is(equalTo(ShardRecoveryPlan.SnapshotFilesToRecover.EMPTY))); - assertThat(shardRecoveryPlan.canRecoverSnapshotFilesFromSourceNode(), is(equalTo(true))); - - assertThat(shardRecoveryPlan.getStartingSeqNo(), equalTo(startingSeqNo)); - assertThat(shardRecoveryPlan.getTranslogOps(), equalTo(translogOps)); - }); - } - private ShardRecoveryPlan computeShardRecoveryPlan( String shardIdentifier, Store.MetadataSnapshot sourceMetadataSnapshot, From 2337f49ca6a0c7b11085d25486546cae7e0598b2 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Thu, 3 Oct 2024 08:33:43 -0400 Subject: [PATCH 030/194] [ML] Check Search Inference ID Usage When Deleting An Inference Endpoint (#113895) --- .../ml/utils/SemanticTextInfoExtractor.java | 14 +++--- .../inference/InferenceBaseRestTest.java | 20 +++++++++ .../xpack/inference/InferenceCrudIT.java | 45 ++++++++++++++----- 3 files changed, 59 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/SemanticTextInfoExtractor.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/SemanticTextInfoExtractor.java index 544c1e344c91f..7683761737049 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/SemanticTextInfoExtractor.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/SemanticTextInfoExtractor.java @@ -33,15 +33,11 @@ public static Set extractIndexesReferencingInferenceEndpoints(Metadata m Map indices = metadata.indices(); indices.forEach((indexName, indexMetadata) -> { - if (indexMetadata.getInferenceFields() != null) { - Map inferenceFields = indexMetadata.getInferenceFields(); - if (inferenceFields.entrySet() - .stream() - .anyMatch( - entry -> entry.getValue().getInferenceId() != null && endpointIds.contains(entry.getValue().getInferenceId()) - )) { - referenceIndices.add(indexName); - } + Map inferenceFields = indexMetadata.getInferenceFields(); + if (inferenceFields.values() + .stream() + .anyMatch(im -> endpointIds.contains(im.getInferenceId()) || endpointIds.contains(im.getSearchInferenceId()))) { + referenceIndices.add(indexName); } }); diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java index 55a6292ecd165..3fa6159661b7e 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java @@ -171,6 +171,26 @@ protected void putSemanticText(String endpointId, String indexName) throws IOExc assertOkOrCreated(response); } + protected void putSemanticText(String endpointId, String searchEndpointId, String indexName) throws IOException { + var request = new Request("PUT", Strings.format("%s", indexName)); + String body = Strings.format(""" + { + "mappings": { + "properties": { + "inference_field": { + "type": "semantic_text", + "inference_id": "%s", + "search_inference_id": "%s" + } + } + } + } + """, endpointId, searchEndpointId); + request.setJsonEntity(body); + var response = client().performRequest(request); + assertOkOrCreated(response); + } + protected Map putModel(String modelId, String modelConfig, TaskType taskType) throws IOException { String endpoint = Strings.format("_inference/%s/%s", taskType, modelId); return putRequest(endpoint, modelConfig); diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java index 893d52435ec1c..92affbc043669 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java @@ -17,6 +17,7 @@ import java.io.IOException; import java.util.List; import java.util.Set; +import java.util.function.Function; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -154,27 +155,28 @@ public void testDeleteEndpointWhileReferencedByPipeline() throws IOException { } public void testDeleteEndpointWhileReferencedBySemanticText() throws IOException { - String endpointId = "endpoint_referenced_by_semantic_text"; + final String endpointId = "endpoint_referenced_by_semantic_text"; + final String searchEndpointId = "search_endpoint_referenced_by_semantic_text"; + final String indexName = randomAlphaOfLength(10).toLowerCase(); + final Function buildErrorString = endpointName -> " Inference endpoint " + + endpointName + + " is being used in the mapping for indexes: " + + Set.of(indexName) + + ". Ensure that no index mappings are using this inference endpoint, or use force to ignore this warning and delete the" + + " inference endpoint."; + putModel(endpointId, mockSparseServiceModelConfig(), TaskType.SPARSE_EMBEDDING); - String indexName = randomAlphaOfLength(10).toLowerCase(); putSemanticText(endpointId, indexName); { - - var errorString = new StringBuilder().append(" Inference endpoint ") - .append(endpointId) - .append(" is being used in the mapping for indexes: ") - .append(Set.of(indexName)) - .append(". ") - .append("Ensure that no index mappings are using this inference endpoint, ") - .append("or use force to ignore this warning and delete the inference endpoint."); var e = expectThrows(ResponseException.class, () -> deleteModel(endpointId)); - assertThat(e.getMessage(), containsString(errorString.toString())); + assertThat(e.getMessage(), containsString(buildErrorString.apply(endpointId))); } { var response = deleteModel(endpointId, "dry_run=true"); var entityString = EntityUtils.toString(response.getEntity()); assertThat(entityString, containsString("\"acknowledged\":false")); assertThat(entityString, containsString(indexName)); + assertThat(entityString, containsString(endpointId)); } { var response = deleteModel(endpointId, "force=true"); @@ -182,6 +184,26 @@ public void testDeleteEndpointWhileReferencedBySemanticText() throws IOException assertThat(entityString, containsString("\"acknowledged\":true")); } deleteIndex(indexName); + + putModel(searchEndpointId, mockSparseServiceModelConfig(), TaskType.SPARSE_EMBEDDING); + putSemanticText(endpointId, searchEndpointId, indexName); + { + var e = expectThrows(ResponseException.class, () -> deleteModel(searchEndpointId)); + assertThat(e.getMessage(), containsString(buildErrorString.apply(searchEndpointId))); + } + { + var response = deleteModel(searchEndpointId, "dry_run=true"); + var entityString = EntityUtils.toString(response.getEntity()); + assertThat(entityString, containsString("\"acknowledged\":false")); + assertThat(entityString, containsString(indexName)); + assertThat(entityString, containsString(searchEndpointId)); + } + { + var response = deleteModel(searchEndpointId, "force=true"); + var entityString = EntityUtils.toString(response.getEntity()); + assertThat(entityString, containsString("\"acknowledged\":true")); + } + deleteIndex(indexName); } public void testDeleteEndpointWhileReferencedBySemanticTextAndPipeline() throws IOException { @@ -217,6 +239,7 @@ public void testDeleteEndpointWhileReferencedBySemanticTextAndPipeline() throws assertThat(entityString, containsString("\"acknowledged\":false")); assertThat(entityString, containsString(indexName)); assertThat(entityString, containsString(pipelineId)); + assertThat(entityString, containsString(endpointId)); } { var response = deleteModel(endpointId, "force=true"); From a5ef13961616a7ae74dc35320c6c160638910e22 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 3 Oct 2024 22:35:10 +1000 Subject: [PATCH 031/194] Mute org.elasticsearch.test.rest.ClientYamlTestSuiteIT org.elasticsearch.test.rest.ClientYamlTestSuiteIT #114013 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 68422264221f2..2ec4faad550a6 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -347,6 +347,8 @@ tests: - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT method: testPutE5Small_withPlatformAgnosticVariant issue: https://github.com/elastic/elasticsearch/issues/113983 +- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT + issue: https://github.com/elastic/elasticsearch/issues/114013 # Examples: # From 1d8c94b7e4ff2f5030fe609abc5e671343a0b333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Cea=20Fontenla?= Date: Thu, 3 Oct 2024 14:53:50 +0200 Subject: [PATCH 032/194] Add CircuitBreaker to TDigest, Step 4: Take into account shallow classes size (#113613) Final (I wish) part of https://github.com/elastic/elasticsearch/issues/99815 Also, fixes https://github.com/elastic/elasticsearch/issues/113916 ## Steps 1. Migrate TDigest classes to use a custom Array implementation. Temporarily use a simple array wrapper (https://github.com/elastic/elasticsearch/pull/112810) 2. Implement CircuitBreaking in the `WrapperTDigestArrays` class. Add Releasable/AutoCloseable and ensure everything is closed (https://github.com/elastic/elasticsearch/pull/113105) 3. Pass the CircuitBreaker as a parameter to TDigestState from wherever it's being used (https://github.com/elastic/elasticsearch/pull/113387) - ESQL: Pass a real CB - Other aggs: Use the deprecated methods on `TDigestState`, that will use a No-op CB instead 4. Account remaining TDigest classes size ("SHALLOW_SIZE") (This PR) Every step should be safely mergeable to main: - The first and second steps should have no impact. - The third and fourth ones will start increasing the CB count partially. ## Remarks As TDigests are releasable now, I had to refactor all tests, adding try-with-resources or direct close() calls. That added a lot of changes, but most of them are trivial. Outside of it, in ESQL, TDigestStates are closed now. Old aggregations don't close them, as it's not trivial. However, as they are using the NoopCircuitBreaker, there's no problem with it. There's nothing to be closed. ## _Remarks 2_ I tried to follow the same pattern in how everything is accounted. On each TDigest class: - Static constant "SHALLOW_SIZE" with the object weight - Field `AtomicBoolean closed` to ensure indempotent `close()` - Static `create()` method that accounts the SHALLOW_SIZE, and returns a new isntance. And the important part: On exception, it discounts SHALLOW_SIZE again - A `ramBytesUsed()` (Accountable interface), barely used for anything really, but some assertions I believe - A constructor, that closes everything it created on exception (If it creates an array, and the next array surpasses the CB limit, the first one must be closed) - And a close() that will, well, close everything and discount SHALLOW_SIZE A lot of steps to make sure everything works well in this multi-level structure, but I believe the result was quite clean --- .../benchmark/tdigest/TDigestBench.java | 3 +- docs/changelog/113613.yaml | 7 + libs/tdigest/build.gradle | 1 + libs/tdigest/licenses/lucene-core-LICENSE.txt | 475 ++++++++++++++++ libs/tdigest/licenses/lucene-core-NOTICE.txt | 192 +++++++ libs/tdigest/src/main/java/module-info.java | 1 + .../elasticsearch/tdigest/AVLGroupTree.java | 133 +++-- .../elasticsearch/tdigest/AVLTreeDigest.java | 33 +- .../elasticsearch/tdigest/HybridDigest.java | 34 +- .../org/elasticsearch/tdigest/IntAVLTree.java | 127 ++++- .../elasticsearch/tdigest/MergingDigest.java | 69 ++- .../elasticsearch/tdigest/SortingDigest.java | 30 +- .../org/elasticsearch/tdigest/TDigest.java | 19 +- .../tdigest/arrays/TDigestArrays.java | 2 + .../tdigest/arrays/TDigestByteArray.java | 3 +- .../tdigest/arrays/TDigestDoubleArray.java | 3 +- .../tdigest/arrays/TDigestIntArray.java | 3 +- .../tdigest/arrays/TDigestLongArray.java | 3 +- .../tdigest/AVLGroupTreeTests.java | 112 ++-- .../tdigest/AVLTreeDigestTests.java | 2 +- .../tdigest/AlternativeMergeTests.java | 81 +-- .../elasticsearch/tdigest/BigCountTests.java | 13 +- .../BigCountTestsMergingDigestTests.java | 2 +- .../tdigest/BigCountTestsTreeDigestTests.java | 2 +- .../tdigest/ComparisonTests.java | 14 + .../tdigest/HybridDigestTests.java | 2 +- .../tdigest/IntAVLTreeTests.java | 77 +-- .../elasticsearch/tdigest/MedianTests.java | 52 +- .../tdigest/MergingDigestTests.java | 184 +++--- .../org/elasticsearch/tdigest/SortTests.java | 10 +- .../tdigest/SortingDigestTests.java | 2 +- .../tdigest/TDigestReleasingTests.java | 89 +++ .../tdigest/TDigestTestCase.java | 9 +- .../elasticsearch/tdigest/TDigestTests.java | 533 +++++++++--------- muted-tests.yml | 2 - .../InternalMedianAbsoluteDeviation.java | 1 - .../metrics/MemoryTrackingTDigestArrays.java | 15 +- .../aggregations/metrics/TDigestState.java | 69 ++- .../metrics/TDigestStateReleasingTests.java | 59 ++ .../compute/aggregation/QuantileStates.java | 6 +- 40 files changed, 1840 insertions(+), 634 deletions(-) create mode 100644 docs/changelog/113613.yaml create mode 100644 libs/tdigest/licenses/lucene-core-LICENSE.txt create mode 100644 libs/tdigest/licenses/lucene-core-NOTICE.txt create mode 100644 libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestReleasingTests.java create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestStateReleasingTests.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java index 36ffc34c482d7..7a5aaa0187533 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.breaker.NoopCircuitBreaker; import org.elasticsearch.search.aggregations.metrics.MemoryTrackingTDigestArrays; -import org.elasticsearch.tdigest.MergingDigest; import org.elasticsearch.tdigest.TDigest; import org.elasticsearch.tdigest.arrays.TDigestArrays; import org.openjdk.jmh.annotations.Benchmark; @@ -64,7 +63,7 @@ public enum TDigestFactory { MERGE { @Override TDigest create(double compression) { - return new MergingDigest(arrays, compression, (int) (10 * compression)); + return TDigest.createMergingDigest(arrays, compression); } }, AVL_TREE { diff --git a/docs/changelog/113613.yaml b/docs/changelog/113613.yaml new file mode 100644 index 0000000000000..4b020333aaa36 --- /dev/null +++ b/docs/changelog/113613.yaml @@ -0,0 +1,7 @@ +pr: 113613 +summary: "Add `CircuitBreaker` to TDigest, Step 4: Take into account shallow classes\ + \ size" +area: ES|QL +type: enhancement +issues: + - 113916 diff --git a/libs/tdigest/build.gradle b/libs/tdigest/build.gradle index df60862b27386..231eb845339aa 100644 --- a/libs/tdigest/build.gradle +++ b/libs/tdigest/build.gradle @@ -23,6 +23,7 @@ apply plugin: 'elasticsearch.publish' dependencies { api project(':libs:elasticsearch-core') + api "org.apache.lucene:lucene-core:${versions.lucene}" testImplementation(project(":test:framework")) { exclude group: 'org.elasticsearch', module: 'elasticsearch-tdigest' diff --git a/libs/tdigest/licenses/lucene-core-LICENSE.txt b/libs/tdigest/licenses/lucene-core-LICENSE.txt new file mode 100644 index 0000000000000..28b134f5f8e4d --- /dev/null +++ b/libs/tdigest/licenses/lucene-core-LICENSE.txt @@ -0,0 +1,475 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was +derived from unicode conversion examples available at +http://www.unicode.org/Public/PROGRAMS/CVTUTF. Here is the copyright +from those sources: + +/* + * Copyright 2001-2004 Unicode, Inc. + * + * Disclaimer + * + * This source code is provided as is by Unicode, Inc. No claims are + * made as to fitness for any particular purpose. No warranties of any + * kind are expressed or implied. The recipient agrees to determine + * applicability of information provided. If this file has been + * purchased on magnetic or optical media from Unicode, Inc., the + * sole remedy for any claim will be exchange of defective media + * within 90 days of receipt. + * + * Limitations on Rights to Redistribute This Code + * + * Unicode, Inc. hereby grants the right to freely use the information + * supplied in this file in the creation of products supporting the + * Unicode Standard, and to make copies of this file in any form + * for internal or external distribution as long as this notice + * remains attached. + */ + + +Some code in core/src/java/org/apache/lucene/util/ArrayUtil.java was +derived from Python 2.4.2 sources available at +http://www.python.org. Full license is here: + + http://www.python.org/download/releases/2.4.2/license/ + +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was +derived from Python 3.1.2 sources available at +http://www.python.org. Full license is here: + + http://www.python.org/download/releases/3.1.2/license/ + +Some code in core/src/java/org/apache/lucene/util/automaton was +derived from Brics automaton sources available at +www.brics.dk/automaton/. Here is the copyright from those sources: + +/* + * Copyright (c) 2001-2009 Anders Moeller + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +The levenshtein automata tables in core/src/java/org/apache/lucene/util/automaton +were automatically generated with the moman/finenight FSA package. +Here is the copyright for those sources: + +# Copyright (c) 2010, Jean-Philippe Barrette-LaPierre, +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was +derived from ICU (http://www.icu-project.org) +The full license is available here: + http://source.icu-project.org/repos/icu/icu/trunk/license.html + +/* + * Copyright (C) 1999-2010, International Business Machines + * Corporation and others. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * provided that the above copyright notice(s) and this permission notice appear + * in all copies of the Software and that both the above copyright notice(s) and + * this permission notice appear in supporting documentation. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE + * LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR + * ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER + * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT + * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall not + * be used in advertising or otherwise to promote the sale, use or other + * dealings in this Software without prior written authorization of the + * copyright holder. + */ + +The following license applies to the Snowball stemmers: + +Copyright (c) 2001, Dr Martin Porter +Copyright (c) 2002, Richard Boulton +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holders nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The following license applies to the KStemmer: + +Copyright © 2003, +Center for Intelligent Information Retrieval, +University of Massachusetts, Amherst. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. The names "Center for Intelligent Information Retrieval" and +"University of Massachusetts" must not be used to endorse or promote products +derived from this software without prior written permission. To obtain +permission, contact info@ciir.cs.umass.edu. + +THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF MASSACHUSETTS AND OTHER CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +The following license applies to the Morfologik project: + +Copyright (c) 2006 Dawid Weiss +Copyright (c) 2007-2011 Dawid Weiss, Marcin Miłkowski +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of Morfologik nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +The dictionary comes from Morfologik project. Morfologik uses data from +Polish ispell/myspell dictionary hosted at http://www.sjp.pl/slownik/en/ and +is licenced on the terms of (inter alia) LGPL and Creative Commons +ShareAlike. The part-of-speech tags were added in Morfologik project and +are not found in the data from sjp.pl. The tagset is similar to IPI PAN +tagset. + +--- + +The following license applies to the Morfeusz project, +used by org.apache.lucene.analysis.morfologik. + +BSD-licensed dictionary of Polish (SGJP) +http://sgjp.pl/morfeusz/ + +Copyright © 2011 Zygmunt Saloni, Włodzimierz Gruszczyński, + Marcin Woliński, Robert Wołosz + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDERS “AS IS” AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/libs/tdigest/licenses/lucene-core-NOTICE.txt b/libs/tdigest/licenses/lucene-core-NOTICE.txt new file mode 100644 index 0000000000000..1a1d51572432a --- /dev/null +++ b/libs/tdigest/licenses/lucene-core-NOTICE.txt @@ -0,0 +1,192 @@ +Apache Lucene +Copyright 2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Includes software from other Apache Software Foundation projects, +including, but not limited to: + - Apache Ant + - Apache Jakarta Regexp + - Apache Commons + - Apache Xerces + +ICU4J, (under analysis/icu) is licensed under an MIT styles license +and Copyright (c) 1995-2008 International Business Machines Corporation and others + +Some data files (under analysis/icu/src/data) are derived from Unicode data such +as the Unicode Character Database. See http://unicode.org/copyright.html for more +details. + +Brics Automaton (under core/src/java/org/apache/lucene/util/automaton) is +BSD-licensed, created by Anders Møller. See http://www.brics.dk/automaton/ + +The levenshtein automata tables (under core/src/java/org/apache/lucene/util/automaton) were +automatically generated with the moman/finenight FSA library, created by +Jean-Philippe Barrette-LaPierre. This library is available under an MIT license, +see http://sites.google.com/site/rrettesite/moman and +http://bitbucket.org/jpbarrette/moman/overview/ + +The class org.apache.lucene.util.WeakIdentityMap was derived from +the Apache CXF project and is Apache License 2.0. + +The Google Code Prettify is Apache License 2.0. +See http://code.google.com/p/google-code-prettify/ + +JUnit (junit-4.10) is licensed under the Common Public License v. 1.0 +See http://junit.sourceforge.net/cpl-v10.html + +This product includes code (JaspellTernarySearchTrie) from Java Spelling Checkin +g Package (jaspell): http://jaspell.sourceforge.net/ +License: The BSD License (http://www.opensource.org/licenses/bsd-license.php) + +The snowball stemmers in + analysis/common/src/java/net/sf/snowball +were developed by Martin Porter and Richard Boulton. +The snowball stopword lists in + analysis/common/src/resources/org/apache/lucene/analysis/snowball +were developed by Martin Porter and Richard Boulton. +The full snowball package is available from + http://snowball.tartarus.org/ + +The KStem stemmer in + analysis/common/src/org/apache/lucene/analysis/en +was developed by Bob Krovetz and Sergio Guzman-Lara (CIIR-UMass Amherst) +under the BSD-license. + +The Arabic,Persian,Romanian,Bulgarian, Hindi and Bengali analyzers (common) come with a default +stopword list that is BSD-licensed created by Jacques Savoy. These files reside in: +analysis/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/fa/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/ro/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/bg/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/bn/stopwords.txt +See http://members.unine.ch/jacques.savoy/clef/index.html. + +The German,Spanish,Finnish,French,Hungarian,Italian,Portuguese,Russian and Swedish light stemmers +(common) are based on BSD-licensed reference implementations created by Jacques Savoy and +Ljiljana Dolamic. These files reside in: +analysis/common/src/java/org/apache/lucene/analysis/de/GermanLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/de/GermanMinimalStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/es/SpanishLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchMinimalStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemmer.java + +The Stempel analyzer (stempel) includes BSD-licensed software developed +by the Egothor project http://egothor.sf.net/, created by Leo Galambos, Martin Kvapil, +and Edmond Nolan. + +The Polish analyzer (stempel) comes with a default +stopword list that is BSD-licensed created by the Carrot2 project. The file resides +in stempel/src/resources/org/apache/lucene/analysis/pl/stopwords.txt. +See http://project.carrot2.org/license.html. + +The SmartChineseAnalyzer source code (smartcn) was +provided by Xiaoping Gao and copyright 2009 by www.imdict.net. + +WordBreakTestUnicode_*.java (under modules/analysis/common/src/test/) +is derived from Unicode data such as the Unicode Character Database. +See http://unicode.org/copyright.html for more details. + +The Morfologik analyzer (morfologik) includes BSD-licensed software +developed by Dawid Weiss and Marcin Miłkowski (http://morfologik.blogspot.com/). + +Morfologik uses data from Polish ispell/myspell dictionary +(http://www.sjp.pl/slownik/en/) licenced on the terms of (inter alia) +LGPL and Creative Commons ShareAlike. + +Morfologic includes data from BSD-licensed dictionary of Polish (SGJP) +(http://sgjp.pl/morfeusz/) + +Servlet-api.jar and javax.servlet-*.jar are under the CDDL license, the original +source code for this can be found at http://www.eclipse.org/jetty/downloads.php + +=========================================================================== +Kuromoji Japanese Morphological Analyzer - Apache Lucene Integration +=========================================================================== + +This software includes a binary and/or source version of data from + + mecab-ipadic-2.7.0-20070801 + +which can be obtained from + + http://atilika.com/releases/mecab-ipadic/mecab-ipadic-2.7.0-20070801.tar.gz + +or + + http://jaist.dl.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/mecab-ipadic-2.7.0-20070801.tar.gz + +=========================================================================== +mecab-ipadic-2.7.0-20070801 Notice +=========================================================================== + +Nara Institute of Science and Technology (NAIST), +the copyright holders, disclaims all warranties with regard to this +software, including all implied warranties of merchantability and +fitness, in no event shall NAIST be liable for +any special, indirect or consequential damages or any damages +whatsoever resulting from loss of use, data or profits, whether in an +action of contract, negligence or other tortuous action, arising out +of or in connection with the use or performance of this software. + +A large portion of the dictionary entries +originate from ICOT Free Software. The following conditions for ICOT +Free Software applies to the current dictionary as well. + +Each User may also freely distribute the Program, whether in its +original form or modified, to any third party or parties, PROVIDED +that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear +on, or be attached to, the Program, which is distributed substantially +in the same form as set out herein and that such intended +distribution, if actually made, will neither violate or otherwise +contravene any of the laws and regulations of the countries having +jurisdiction over the User or the intended distribution itself. + +NO WARRANTY + +The program was produced on an experimental basis in the course of the +research and development conducted during the project and is provided +to users as so produced on an experimental basis. Accordingly, the +program is provided without any warranty whatsoever, whether express, +implied, statutory or otherwise. The term "warranty" used herein +includes, but is not limited to, any warranty of the quality, +performance, merchantability and fitness for a particular purpose of +the program and the nonexistence of any infringement or violation of +any right of any third party. + +Each user of the program will agree and understand, and be deemed to +have agreed and understood, that there is no warranty whatsoever for +the program and, accordingly, the entire risk arising from or +otherwise connected with the program is assumed by the user. + +Therefore, neither ICOT, the copyright holder, or any other +organization that participated in or was otherwise related to the +development of the program and their respective officials, directors, +officers and other employees shall be held liable for any and all +damages, including, without limitation, general, special, incidental +and consequential damages, arising out of or otherwise in connection +with the use or inability to use the program or any product, material +or result produced or otherwise obtained by using the program, +regardless of whether they have been advised of, or otherwise had +knowledge of, the possibility of such damages at any time during the +project or thereafter. Each user will be deemed to have agreed to the +foregoing by his or her commencement of use of the program. The term +"use" as used herein includes, but is not limited to, the use, +modification, copying and distribution of the program and the +production of secondary products from the program. + +In the case where the program, whether in its original form or +modified, was distributed or delivered to or received by a user from +any person, organization or entity other than ICOT, unless it makes or +grants independently of ICOT any specific warranty to the user in +writing, such person, organization or entity, will also be exempted +from and not be held liable to the user for any such damages as noted +above as far as the program is concerned. diff --git a/libs/tdigest/src/main/java/module-info.java b/libs/tdigest/src/main/java/module-info.java index cc7ff1810905f..79ddbe88ab3d3 100644 --- a/libs/tdigest/src/main/java/module-info.java +++ b/libs/tdigest/src/main/java/module-info.java @@ -19,6 +19,7 @@ module org.elasticsearch.tdigest { requires org.elasticsearch.base; + requires org.apache.lucene.core; exports org.elasticsearch.tdigest; exports org.elasticsearch.tdigest.arrays; diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLGroupTree.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLGroupTree.java index a1a65e1e71cde..66b4acc68db84 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLGroupTree.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLGroupTree.java @@ -21,6 +21,8 @@ package org.elasticsearch.tdigest; +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; @@ -33,7 +35,12 @@ /** * A tree of t-digest centroids. */ -final class AVLGroupTree extends AbstractCollection implements Releasable { +final class AVLGroupTree extends AbstractCollection implements Releasable, Accountable { + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(AVLGroupTree.class); + + private final TDigestArrays arrays; + private boolean closed = false; + /* For insertions into the tree */ private double centroid; private long count; @@ -42,49 +49,95 @@ final class AVLGroupTree extends AbstractCollection implements Releasa private final TDigestLongArray aggregatedCounts; private final IntAVLTree tree; - AVLGroupTree(TDigestArrays arrays) { - tree = new IntAVLTree(arrays) { + static AVLGroupTree create(TDigestArrays arrays) { + arrays.adjustBreaker(SHALLOW_SIZE); + try { + return new AVLGroupTree(arrays); + } catch (Exception e) { + arrays.adjustBreaker(-SHALLOW_SIZE); + throw e; + } + } - @Override - protected void resize(int newCapacity) { - super.resize(newCapacity); - centroids.resize(newCapacity); - counts.resize(newCapacity); - aggregatedCounts.resize(newCapacity); - } + private AVLGroupTree(TDigestArrays arrays) { + this.arrays = arrays; + + IntAVLTree tree = null; + TDigestDoubleArray centroids = null; + TDigestLongArray counts = null; + TDigestLongArray aggregatedCounts = null; + + try { + this.tree = tree = createIntAvlTree(arrays); + this.centroids = centroids = arrays.newDoubleArray(tree.capacity()); + this.counts = counts = arrays.newLongArray(tree.capacity()); + this.aggregatedCounts = aggregatedCounts = arrays.newLongArray(tree.capacity()); + + tree = null; + centroids = null; + counts = null; + aggregatedCounts = null; + } finally { + Releasables.close(tree, centroids, counts, aggregatedCounts); + } + } - @Override - protected void merge(int node) { - // two nodes are never considered equal - throw new UnsupportedOperationException(); - } + private IntAVLTree createIntAvlTree(TDigestArrays arrays) { + arrays.adjustBreaker(IntAVLTree.SHALLOW_SIZE); + try { + return new InternalIntAvlTree(arrays); + } catch (Exception e) { + arrays.adjustBreaker(-IntAVLTree.SHALLOW_SIZE); + throw e; + } + } - @Override - protected void copy(int node) { - centroids.set(node, centroid); - counts.set(node, count); - } + private class InternalIntAvlTree extends IntAVLTree { + private InternalIntAvlTree(TDigestArrays arrays) { + super(arrays); + } - @Override - protected int compare(int node) { - if (centroid < centroids.get(node)) { - return -1; - } else { - // upon equality, the newly added node is considered greater - return 1; - } - } + @Override + protected void resize(int newCapacity) { + super.resize(newCapacity); + centroids.resize(newCapacity); + counts.resize(newCapacity); + aggregatedCounts.resize(newCapacity); + } - @Override - protected void fixAggregates(int node) { - super.fixAggregates(node); - aggregatedCounts.set(node, counts.get(node) + aggregatedCounts.get(left(node)) + aggregatedCounts.get(right(node))); + @Override + protected void merge(int node) { + // two nodes are never considered equal + throw new UnsupportedOperationException(); + } + + @Override + protected void copy(int node) { + centroids.set(node, centroid); + counts.set(node, count); + } + + @Override + protected int compare(int node) { + if (centroid < centroids.get(node)) { + return -1; + } else { + // upon equality, the newly added node is considered greater + return 1; } + } + + @Override + protected void fixAggregates(int node) { + super.fixAggregates(node); + aggregatedCounts.set(node, counts.get(node) + aggregatedCounts.get(left(node)) + aggregatedCounts.get(right(node))); + } - }; - centroids = arrays.newDoubleArray(tree.capacity()); - counts = arrays.newLongArray(tree.capacity()); - aggregatedCounts = arrays.newLongArray(tree.capacity()); + } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + centroids.ramBytesUsed() + counts.ramBytesUsed() + aggregatedCounts.ramBytesUsed() + tree.ramBytesUsed(); } /** @@ -271,6 +324,10 @@ private void checkAggregates(int node) { @Override public void close() { - Releasables.close(centroids, counts, aggregatedCounts, tree); + if (closed == false) { + closed = true; + arrays.adjustBreaker(-SHALLOW_SIZE); + Releasables.close(centroids, counts, aggregatedCounts, tree); + } } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLTreeDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLTreeDigest.java index f6b027edb1e9c..8350f17d255cd 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLTreeDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/AVLTreeDigest.java @@ -21,6 +21,7 @@ package org.elasticsearch.tdigest; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; @@ -32,7 +33,10 @@ import static org.elasticsearch.tdigest.IntAVLTree.NIL; public class AVLTreeDigest extends AbstractTDigest { + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(AVLTreeDigest.class); + private final TDigestArrays arrays; + private boolean closed = false; final Random gen = new Random(); private final double compression; @@ -43,6 +47,16 @@ public class AVLTreeDigest extends AbstractTDigest { // Indicates if a sample has been added after the last compression. private boolean needsCompression; + static AVLTreeDigest create(TDigestArrays arrays, double compression) { + arrays.adjustBreaker(SHALLOW_SIZE); + try { + return new AVLTreeDigest(arrays, compression); + } catch (Exception e) { + arrays.adjustBreaker(-SHALLOW_SIZE); + throw e; + } + } + /** * A histogram structure that will record a sketch of a distribution. * @@ -51,15 +65,20 @@ public class AVLTreeDigest extends AbstractTDigest { * quantiles. Conversely, you should expect to track about 5 N centroids for this * accuracy. */ - AVLTreeDigest(TDigestArrays arrays, double compression) { + private AVLTreeDigest(TDigestArrays arrays, double compression) { this.arrays = arrays; this.compression = compression; - summary = new AVLGroupTree(arrays); + summary = AVLGroupTree.create(arrays); + } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + summary.ramBytesUsed(); } /** * Sets the seed for the RNG. - * In cases where a predicatable tree should be created, this function may be used to make the + * In cases where a predictable tree should be created, this function may be used to make the * randomness in this AVLTree become more deterministic. * * @param seed The random seed to use for RNG purposes @@ -155,7 +174,7 @@ public void compress() { needsCompression = false; try (AVLGroupTree centroids = summary) { - this.summary = new AVLGroupTree(arrays); + this.summary = AVLGroupTree.create(arrays); final int[] nodes = new int[centroids.size()]; nodes[0] = centroids.first(); @@ -361,6 +380,10 @@ public int byteSize() { @Override public void close() { - Releasables.close(summary); + if (closed == false) { + closed = true; + arrays.adjustBreaker(-SHALLOW_SIZE); + Releasables.close(summary); + } } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/HybridDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/HybridDigest.java index 8d03ce4e303a6..cf743db49acdd 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/HybridDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/HybridDigest.java @@ -19,6 +19,7 @@ package org.elasticsearch.tdigest; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; @@ -34,8 +35,10 @@ * bounded memory allocation and acceptable speed and accuracy for larger ones. */ public class HybridDigest extends AbstractTDigest { + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(HybridDigest.class); private final TDigestArrays arrays; + private boolean closed = false; // See MergingDigest's compression param. private final double compression; @@ -49,6 +52,16 @@ public class HybridDigest extends AbstractTDigest { // This gets initialized when the implementation switches to MergingDigest. private MergingDigest mergingDigest; + static HybridDigest create(TDigestArrays arrays, double compression) { + arrays.adjustBreaker(SHALLOW_SIZE); + try { + return new HybridDigest(arrays, compression); + } catch (Exception e) { + arrays.adjustBreaker(-SHALLOW_SIZE); + throw e; + } + } + /** * Creates a hybrid digest that uses a {@link SortingDigest} for up to {@param maxSortingSize} samples, * then switches to a {@link MergingDigest}. @@ -56,11 +69,11 @@ public class HybridDigest extends AbstractTDigest { * @param compression The compression factor for the MergingDigest * @param maxSortingSize The sample size limit for switching from a {@link SortingDigest} to a {@link MergingDigest} implementation */ - HybridDigest(TDigestArrays arrays, double compression, long maxSortingSize) { + private HybridDigest(TDigestArrays arrays, double compression, long maxSortingSize) { this.arrays = arrays; this.compression = compression; this.maxSortingSize = maxSortingSize; - this.sortingDigest = new SortingDigest(arrays); + this.sortingDigest = TDigest.createSortingDigest(arrays); } /** @@ -69,13 +82,20 @@ public class HybridDigest extends AbstractTDigest { * * @param compression The compression factor for the MergingDigest */ - HybridDigest(TDigestArrays arrays, double compression) { + private HybridDigest(TDigestArrays arrays, double compression) { // The default maxSortingSize is calculated so that the SortingDigest will have comparable size with the MergingDigest // at the point where implementations switch, e.g. for default compression 100 SortingDigest allocates ~16kB and MergingDigest // allocates ~15kB. this(arrays, compression, Math.round(compression) * 20); } + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + (sortingDigest != null ? sortingDigest.ramBytesUsed() : 0) + (mergingDigest != null + ? mergingDigest.ramBytesUsed() + : 0); + } + @Override public void add(double x, long w) { reserve(w); @@ -105,7 +125,7 @@ public void reserve(long size) { // Check if we need to switch implementations. assert sortingDigest != null; if (sortingDigest.size() + size >= maxSortingSize) { - mergingDigest = new MergingDigest(arrays, compression); + mergingDigest = TDigest.createMergingDigest(arrays, compression); for (int i = 0; i < sortingDigest.values.size(); i++) { mergingDigest.add(sortingDigest.values.get(i)); } @@ -201,6 +221,10 @@ public int byteSize() { @Override public void close() { - Releasables.close(sortingDigest, mergingDigest); + if (closed == false) { + closed = true; + arrays.adjustBreaker(-SHALLOW_SIZE); + Releasables.close(sortingDigest, mergingDigest); + } } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/IntAVLTree.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/IntAVLTree.java index b4a82257693d8..c86642e757caa 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/IntAVLTree.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/IntAVLTree.java @@ -21,21 +21,22 @@ package org.elasticsearch.tdigest; +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; import org.elasticsearch.tdigest.arrays.TDigestByteArray; import org.elasticsearch.tdigest.arrays.TDigestIntArray; -import java.util.Arrays; - /** * An AVL-tree structure stored in parallel arrays. * This class only stores the tree structure, so you need to extend it if you * want to add data to the nodes, typically by using arrays and node * identifiers as indices. */ -abstract class IntAVLTree implements Releasable { +abstract class IntAVLTree implements Releasable, Accountable { + static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(IntAVLTree.class); /** * We use 0 instead of -1 so that left(NIL) works without * condition. @@ -47,6 +48,9 @@ static int oversize(int size) { return size + (size >>> 3); } + private final TDigestArrays arrays; + private boolean closed = false; + private final NodeAllocator nodeAllocator; private int root; private final TDigestIntArray parent; @@ -55,18 +59,42 @@ static int oversize(int size) { private final TDigestByteArray depth; IntAVLTree(TDigestArrays arrays, int initialCapacity) { - nodeAllocator = new NodeAllocator(); + this.arrays = arrays; root = NIL; - parent = arrays.newIntArray(initialCapacity); - left = arrays.newIntArray(initialCapacity); - right = arrays.newIntArray(initialCapacity); - depth = arrays.newByteArray(initialCapacity); + + NodeAllocator nodeAllocator = null; + TDigestIntArray parent = null; + TDigestIntArray left = null; + TDigestIntArray right = null; + TDigestByteArray depth = null; + + try { + this.nodeAllocator = nodeAllocator = NodeAllocator.create(arrays); + this.parent = parent = arrays.newIntArray(initialCapacity); + this.left = left = arrays.newIntArray(initialCapacity); + this.right = right = arrays.newIntArray(initialCapacity); + this.depth = depth = arrays.newByteArray(initialCapacity); + + nodeAllocator = null; + parent = null; + left = null; + right = null; + depth = null; + } finally { + Releasables.close(nodeAllocator, parent, left, right, depth); + } } IntAVLTree(TDigestArrays arrays) { this(arrays, 16); } + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + nodeAllocator.ramBytesUsed() + parent.ramBytesUsed() + left.ramBytesUsed() + right.ramBytesUsed() + depth + .ramBytesUsed(); + } + /** * Return the current root of the tree. */ @@ -531,42 +559,85 @@ void checkBalance(int node) { /** * A stack of int values. */ - private static class IntStack { + private static class IntStack implements Releasable, Accountable { + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(IntStack.class); + + private final TDigestArrays arrays; + private boolean closed = false; - private int[] stack; + private final TDigestIntArray stack; private int size; - IntStack() { - stack = new int[0]; + IntStack(TDigestArrays arrays) { + this.arrays = arrays; + stack = arrays.newIntArray(0); size = 0; } + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + stack.ramBytesUsed(); + } + int size() { return size; } int pop() { - return stack[--size]; + int value = stack.get(--size); + stack.resize(size); + return value; } void push(int v) { - if (size >= stack.length) { - final int newLength = oversize(size + 1); - stack = Arrays.copyOf(stack, newLength); - } - stack[size++] = v; + stack.resize(++size); + stack.set(size - 1, v); } + @Override + public void close() { + if (closed == false) { + closed = true; + arrays.adjustBreaker(-SHALLOW_SIZE); + stack.close(); + } + } } - private static class NodeAllocator { + private static class NodeAllocator implements Releasable, Accountable { + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(NodeAllocator.class); + + private final TDigestArrays arrays; + private boolean closed = false; private int nextNode; private final IntStack releasedNodes; - NodeAllocator() { + static NodeAllocator create(TDigestArrays arrays) { + arrays.adjustBreaker(SHALLOW_SIZE); + try { + return new NodeAllocator(arrays); + } catch (Exception e) { + arrays.adjustBreaker(-SHALLOW_SIZE); + throw e; + } + } + + private NodeAllocator(TDigestArrays arrays) { + this.arrays = arrays; nextNode = NIL + 1; - releasedNodes = new IntStack(); + arrays.adjustBreaker(IntStack.SHALLOW_SIZE); + try { + releasedNodes = new IntStack(arrays); + } catch (Exception e) { + arrays.adjustBreaker(-IntStack.SHALLOW_SIZE); + throw e; + } + } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + releasedNodes.ramBytesUsed(); } int newNode() { @@ -586,10 +657,22 @@ int size() { return nextNode - releasedNodes.size() - 1; } + @Override + public void close() { + if (closed == false) { + closed = true; + arrays.adjustBreaker(-SHALLOW_SIZE); + releasedNodes.close(); + } + } } @Override public void close() { - Releasables.close(parent, left, right, depth); + if (closed == false) { + closed = true; + arrays.adjustBreaker(-SHALLOW_SIZE); + Releasables.close(nodeAllocator, parent, left, right, depth); + } } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java index f2ccfc33aa2a9..06724b049f821 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java @@ -21,6 +21,7 @@ package org.elasticsearch.tdigest; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; @@ -67,6 +68,11 @@ * what the AVLTreeDigest uses and no dynamic allocation is required at all. */ public class MergingDigest extends AbstractTDigest { + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(MergingDigest.class); + + private final TDigestArrays arrays; + private boolean closed = false; + private int mergeCount = 0; private final double publicCompression; @@ -107,6 +113,26 @@ public class MergingDigest extends AbstractTDigest { // weight limits. public static boolean useWeightLimit = true; + static MergingDigest create(TDigestArrays arrays, double compression) { + arrays.adjustBreaker(SHALLOW_SIZE); + try { + return new MergingDigest(arrays, compression); + } catch (Exception e) { + arrays.adjustBreaker(-SHALLOW_SIZE); + throw e; + } + } + + static MergingDigest create(TDigestArrays arrays, double compression, int bufferSize, int size) { + arrays.adjustBreaker(SHALLOW_SIZE); + try { + return new MergingDigest(arrays, compression, bufferSize, size); + } catch (Exception e) { + arrays.adjustBreaker(-SHALLOW_SIZE); + throw e; + } + } + /** * Allocates a buffer merging t-digest. This is the normally used constructor that * allocates default sized internal arrays. Other versions are available, but should @@ -114,7 +140,7 @@ public class MergingDigest extends AbstractTDigest { * * @param compression The compression factor */ - public MergingDigest(TDigestArrays arrays, double compression) { + private MergingDigest(TDigestArrays arrays, double compression) { this(arrays, compression, -1); } @@ -124,7 +150,7 @@ public MergingDigest(TDigestArrays arrays, double compression) { * @param compression Compression factor for t-digest. Same as 1/\delta in the paper. * @param bufferSize How many samples to retain before merging. */ - public MergingDigest(TDigestArrays arrays, double compression, int bufferSize) { + private MergingDigest(TDigestArrays arrays, double compression, int bufferSize) { // we can guarantee that we only need ceiling(compression). this(arrays, compression, bufferSize, -1); } @@ -136,7 +162,9 @@ public MergingDigest(TDigestArrays arrays, double compression, int bufferSize) { * @param bufferSize Number of temporary centroids * @param size Size of main buffer */ - public MergingDigest(TDigestArrays arrays, double compression, int bufferSize, int size) { + private MergingDigest(TDigestArrays arrays, double compression, int bufferSize, int size) { + this.arrays = arrays; + // ensure compression >= 10 // default size = 2 * ceil(compression) // default bufferSize = 5 * size @@ -210,16 +238,33 @@ public MergingDigest(TDigestArrays arrays, double compression, int bufferSize, i bufferSize = 2 * size; } - weight = arrays.newDoubleArray(size); - mean = arrays.newDoubleArray(size); - - tempWeight = arrays.newDoubleArray(bufferSize); - tempMean = arrays.newDoubleArray(bufferSize); - order = arrays.newIntArray(bufferSize); + TDigestDoubleArray weight = null; + TDigestDoubleArray mean = null; + TDigestDoubleArray tempWeight = null; + TDigestDoubleArray tempMean = null; + TDigestIntArray order = null; + + try { + this.weight = weight = arrays.newDoubleArray(size); + this.mean = mean = arrays.newDoubleArray(size); + + this.tempWeight = tempWeight = arrays.newDoubleArray(bufferSize); + this.tempMean = tempMean = arrays.newDoubleArray(bufferSize); + this.order = order = arrays.newIntArray(bufferSize); + } catch (Exception e) { + Releasables.close(weight, mean, tempWeight, tempMean, order); + throw e; + } lastUsedCell = 0; } + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + weight.ramBytesUsed() + mean.ramBytesUsed() + tempWeight.ramBytesUsed() + tempMean.ramBytesUsed() + order + .ramBytesUsed(); + } + @Override public void add(double x, long w) { checkValue(x); @@ -578,6 +623,10 @@ public String toString() { @Override public void close() { - Releasables.close(weight, mean, tempWeight, tempMean, order); + if (closed == false) { + closed = true; + arrays.adjustBreaker(-SHALLOW_SIZE); + Releasables.close(weight, mean, tempWeight, tempMean, order); + } } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/SortingDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/SortingDigest.java index f063ca9a511c6..6912982ae6f8f 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/SortingDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/SortingDigest.java @@ -19,6 +19,7 @@ package org.elasticsearch.tdigest; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.core.Releasables; import org.elasticsearch.tdigest.arrays.TDigestArrays; import org.elasticsearch.tdigest.arrays.TDigestDoubleArray; @@ -33,16 +34,37 @@ * samples, at the expense of allocating much more memory. */ public class SortingDigest extends AbstractTDigest { + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(SortingDigest.class); + + private final TDigestArrays arrays; + private boolean closed = false; + // Tracks all samples. Gets sorted on quantile and cdf calls. final TDigestDoubleArray values; // Indicates if all values have been sorted. private boolean isSorted = true; - public SortingDigest(TDigestArrays arrays) { + static SortingDigest create(TDigestArrays arrays) { + arrays.adjustBreaker(SHALLOW_SIZE); + try { + return new SortingDigest(arrays); + } catch (Exception e) { + arrays.adjustBreaker(-SHALLOW_SIZE); + throw e; + } + } + + private SortingDigest(TDigestArrays arrays) { + this.arrays = arrays; values = arrays.newDoubleArray(0); } + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + values.ramBytesUsed(); + } + @Override public void add(double x, long w) { checkValue(x); @@ -141,6 +163,10 @@ public int byteSize() { @Override public void close() { - Releasables.close(values); + if (closed == false) { + closed = true; + arrays.adjustBreaker(-SHALLOW_SIZE); + Releasables.close(values); + } } } diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/TDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/TDigest.java index e578a688738cb..cf232b4a23a87 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/TDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/TDigest.java @@ -21,6 +21,7 @@ package org.elasticsearch.tdigest; +import org.apache.lucene.util.Accountable; import org.elasticsearch.core.Releasable; import org.elasticsearch.tdigest.arrays.TDigestArrays; @@ -38,7 +39,7 @@ * - test coverage roughly at 90% * - easy to adapt for use with map-reduce */ -public abstract class TDigest implements Releasable { +public abstract class TDigest implements Releasable, Accountable { protected ScaleFunction scale = ScaleFunction.K_2; double min = Double.POSITIVE_INFINITY; double max = Double.NEGATIVE_INFINITY; @@ -51,8 +52,8 @@ public abstract class TDigest implements Releasable { * The number of centroids retained will be a smallish (usually less than 10) multiple of this number. * @return the MergingDigest */ - public static TDigest createMergingDigest(TDigestArrays arrays, double compression) { - return new MergingDigest(arrays, compression); + public static MergingDigest createMergingDigest(TDigestArrays arrays, double compression) { + return MergingDigest.create(arrays, compression); } /** @@ -64,8 +65,8 @@ public static TDigest createMergingDigest(TDigestArrays arrays, double compressi * The number of centroids retained will be a smallish (usually less than 10) multiple of this number. * @return the AvlTreeDigest */ - public static TDigest createAvlTreeDigest(TDigestArrays arrays, double compression) { - return new AVLTreeDigest(arrays, compression); + public static AVLTreeDigest createAvlTreeDigest(TDigestArrays arrays, double compression) { + return AVLTreeDigest.create(arrays, compression); } /** @@ -74,8 +75,8 @@ public static TDigest createAvlTreeDigest(TDigestArrays arrays, double compressi * * @return the SortingDigest */ - public static TDigest createSortingDigest(TDigestArrays arrays) { - return new SortingDigest(arrays); + public static SortingDigest createSortingDigest(TDigestArrays arrays) { + return SortingDigest.create(arrays); } /** @@ -87,8 +88,8 @@ public static TDigest createSortingDigest(TDigestArrays arrays) { * The number of centroids retained will be a smallish (usually less than 10) multiple of this number. * @return the HybridDigest */ - public static TDigest createHybridDigest(TDigestArrays arrays, double compression) { - return new HybridDigest(arrays, compression); + public static HybridDigest createHybridDigest(TDigestArrays arrays, double compression) { + return HybridDigest.create(arrays, compression); } /** diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestArrays.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestArrays.java index 5e15c4c82f796..e444eeda458e9 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestArrays.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestArrays.java @@ -25,6 +25,8 @@ * Minimal interface for BigArrays-like classes used within TDigest. */ public interface TDigestArrays { + void adjustBreaker(long size); + TDigestDoubleArray newDoubleArray(int initialSize); TDigestIntArray newIntArray(int initialSize); diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestByteArray.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestByteArray.java index ae8e84800b433..3416ace3bd095 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestByteArray.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestByteArray.java @@ -21,12 +21,13 @@ package org.elasticsearch.tdigest.arrays; +import org.apache.lucene.util.Accountable; import org.elasticsearch.core.Releasable; /** * Minimal interface for ByteArray-like classes used within TDigest. */ -public interface TDigestByteArray extends Releasable { +public interface TDigestByteArray extends Releasable, Accountable { int size(); byte get(int index); diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestDoubleArray.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestDoubleArray.java index 1699dbd9beaf1..707b983358e36 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestDoubleArray.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestDoubleArray.java @@ -21,12 +21,13 @@ package org.elasticsearch.tdigest.arrays; +import org.apache.lucene.util.Accountable; import org.elasticsearch.core.Releasable; /** * Minimal interface for DoubleArray-like classes used within TDigest. */ -public interface TDigestDoubleArray extends Releasable { +public interface TDigestDoubleArray extends Releasable, Accountable { int size(); double get(int index); diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestIntArray.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestIntArray.java index 44e366aacd173..b131e194e8be4 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestIntArray.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestIntArray.java @@ -21,12 +21,13 @@ package org.elasticsearch.tdigest.arrays; +import org.apache.lucene.util.Accountable; import org.elasticsearch.core.Releasable; /** * Minimal interface for IntArray-like classes used within TDigest. */ -public interface TDigestIntArray extends Releasable { +public interface TDigestIntArray extends Releasable, Accountable { int size(); int get(int index); diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestLongArray.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestLongArray.java index 5deea6b28b1ed..a41742dabb205 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestLongArray.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/arrays/TDigestLongArray.java @@ -21,12 +21,13 @@ package org.elasticsearch.tdigest.arrays; +import org.apache.lucene.util.Accountable; import org.elasticsearch.core.Releasable; /** * Minimal interface for LongArray-like classes used within TDigest. */ -public interface TDigestLongArray extends Releasable { +public interface TDigestLongArray extends Releasable, Accountable { int size(); long get(int index); diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLGroupTreeTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLGroupTreeTests.java index 7ac55afd87808..bbaa829fd37e0 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLGroupTreeTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLGroupTreeTests.java @@ -24,81 +24,85 @@ public class AVLGroupTreeTests extends TDigestTestCase { public void testSimpleAdds() { - AVLGroupTree x = new AVLGroupTree(arrays()); - assertEquals(IntAVLTree.NIL, x.floor(34)); - assertEquals(IntAVLTree.NIL, x.first()); - assertEquals(IntAVLTree.NIL, x.last()); - assertEquals(0, x.size()); - assertEquals(0, x.sum()); + try (AVLGroupTree x = AVLGroupTree.create(arrays())) { + assertEquals(IntAVLTree.NIL, x.floor(34)); + assertEquals(IntAVLTree.NIL, x.first()); + assertEquals(IntAVLTree.NIL, x.last()); + assertEquals(0, x.size()); + assertEquals(0, x.sum()); - x.add(new Centroid(1)); - assertEquals(1, x.sum()); - Centroid centroid = new Centroid(2); - centroid.add(3, 1); - centroid.add(4, 1); - x.add(centroid); + x.add(new Centroid(1)); + assertEquals(1, x.sum()); + Centroid centroid = new Centroid(2); + centroid.add(3, 1); + centroid.add(4, 1); + x.add(centroid); - assertEquals(2, x.size()); - assertEquals(4, x.sum()); + assertEquals(2, x.size()); + assertEquals(4, x.sum()); + } } public void testBalancing() { - AVLGroupTree x = new AVLGroupTree(arrays()); - for (int i = 0; i < 101; i++) { - x.add(new Centroid(i)); - } + try (AVLGroupTree x = AVLGroupTree.create(arrays())) { + for (int i = 0; i < 101; i++) { + x.add(new Centroid(i)); + } - assertEquals(101, x.size()); - assertEquals(101, x.sum()); + assertEquals(101, x.size()); + assertEquals(101, x.sum()); - x.checkBalance(); - x.checkAggregates(); + x.checkBalance(); + x.checkAggregates(); + } } public void testFloor() { // mostly tested in other tests - AVLGroupTree x = new AVLGroupTree(arrays()); - for (int i = 0; i < 101; i++) { - x.add(new Centroid(i / 2)); - } + try (AVLGroupTree x = AVLGroupTree.create(arrays())) { + for (int i = 0; i < 101; i++) { + x.add(new Centroid(i / 2)); + } - assertEquals(IntAVLTree.NIL, x.floor(-30)); + assertEquals(IntAVLTree.NIL, x.floor(-30)); - for (Centroid centroid : x) { - assertEquals(centroid.mean(), x.mean(x.floor(centroid.mean() + 0.1)), 0); + for (Centroid centroid : x) { + assertEquals(centroid.mean(), x.mean(x.floor(centroid.mean() + 0.1)), 0); + } } } public void testHeadSum() { - AVLGroupTree x = new AVLGroupTree(arrays()); - for (int i = 0; i < 1000; ++i) { - x.add(randomDouble(), randomIntBetween(1, 10)); + try (AVLGroupTree x = AVLGroupTree.create(arrays())) { + for (int i = 0; i < 1000; ++i) { + x.add(randomDouble(), randomIntBetween(1, 10)); + } + long sum = 0; + long last = -1; + for (int node = x.first(); node != IntAVLTree.NIL; node = x.next(node)) { + assertEquals(sum, x.headSum(node)); + sum += x.count(node); + last = x.count(node); + } + assertEquals(last, x.count(x.last())); } - long sum = 0; - long last = -1; - for (int node = x.first(); node != IntAVLTree.NIL; node = x.next(node)) { - assertEquals(sum, x.headSum(node)); - sum += x.count(node); - last = x.count(node); - } - assertEquals(last, x.count(x.last())); } public void testFloorSum() { - AVLGroupTree x = new AVLGroupTree(arrays()); - int total = 0; - for (int i = 0; i < 1000; ++i) { - int count = randomIntBetween(1, 10); - x.add(randomDouble(), count); - total += count; - } - assertEquals(IntAVLTree.NIL, x.floorSum(-1)); - for (long i = 0; i < total + 10; ++i) { - final int floorNode = x.floorSum(i); - assertTrue(x.headSum(floorNode) <= i); - final int next = x.next(floorNode); - assertTrue(next == IntAVLTree.NIL || x.headSum(next) > i); + try (AVLGroupTree x = AVLGroupTree.create(arrays())) { + int total = 0; + for (int i = 0; i < 1000; ++i) { + int count = randomIntBetween(1, 10); + x.add(randomDouble(), count); + total += count; + } + assertEquals(IntAVLTree.NIL, x.floorSum(-1)); + for (long i = 0; i < total + 10; ++i) { + final int floorNode = x.floorSum(i); + assertTrue(x.headSum(floorNode) <= i); + final int next = x.next(floorNode); + assertTrue(next == IntAVLTree.NIL || x.headSum(next) > i); + } } } - } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLTreeDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLTreeDigestTests.java index f6dde4e168291..e7cac9fbef725 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLTreeDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AVLTreeDigestTests.java @@ -25,7 +25,7 @@ public class AVLTreeDigestTests extends TDigestTests { protected DigestFactory factory(final double compression) { return () -> { - AVLTreeDigest digest = new AVLTreeDigest(arrays(), compression); + AVLTreeDigest digest = AVLTreeDigest.create(arrays(), compression); digest.setRandomSeed(randomLong()); return digest; }; diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AlternativeMergeTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AlternativeMergeTests.java index 0d095ec37fa45..0639c8a0acbd1 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AlternativeMergeTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/AlternativeMergeTests.java @@ -34,48 +34,51 @@ public class AlternativeMergeTests extends TDigestTestCase { public void testMerges() { for (int n : new int[] { 100, 1000, 10000, 100000 }) { for (double compression : new double[] { 50, 100, 200, 400 }) { - MergingDigest mergingDigest = new MergingDigest(arrays(), compression); - AVLTreeDigest treeDigest = new AVLTreeDigest(arrays(), compression); - List data = new ArrayList<>(); - Random gen = random(); - for (int i = 0; i < n; i++) { - double x = gen.nextDouble(); - data.add(x); - mergingDigest.add(x); - treeDigest.add(x); - } - Collections.sort(data); - List counts = new ArrayList<>(); - double soFar = 0; - double current = 0; - for (Double x : data) { - double q = (soFar + (current + 1.0) / 2) / n; - if (current == 0 || current + 1 < n * Math.PI / compression * Math.sqrt(q * (1 - q))) { - current += 1; - } else { + try ( + MergingDigest mergingDigest = TDigest.createMergingDigest(arrays(), compression); + AVLTreeDigest treeDigest = TDigest.createAvlTreeDigest(arrays(), compression); + ) { + List data = new ArrayList<>(); + Random gen = random(); + for (int i = 0; i < n; i++) { + double x = gen.nextDouble(); + data.add(x); + mergingDigest.add(x); + treeDigest.add(x); + } + Collections.sort(data); + List counts = new ArrayList<>(); + double soFar = 0; + double current = 0; + for (Double x : data) { + double q = (soFar + (current + 1.0) / 2) / n; + if (current == 0 || current + 1 < n * Math.PI / compression * Math.sqrt(q * (1 - q))) { + current += 1; + } else { + counts.add(current); + soFar += current; + current = 1; + } + } + if (current > 0) { counts.add(current); - soFar += current; - current = 1; } + soFar = 0; + for (Double count : counts) { + soFar += count; + } + assertEquals(n, soFar, 0); + soFar = 0; + for (Centroid c : mergingDigest.centroids()) { + soFar += c.count(); + } + assertEquals(n, soFar, 0); + soFar = 0; + for (Centroid c : treeDigest.centroids()) { + soFar += c.count(); + } + assertEquals(n, soFar, 0); } - if (current > 0) { - counts.add(current); - } - soFar = 0; - for (Double count : counts) { - soFar += count; - } - assertEquals(n, soFar, 0); - soFar = 0; - for (Centroid c : mergingDigest.centroids()) { - soFar += c.count(); - } - assertEquals(n, soFar, 0); - soFar = 0; - for (Centroid c : treeDigest.centroids()) { - soFar += c.count(); - } - assertEquals(n, soFar, 0); } } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTests.java index 7520d76172ef9..ac39bf0f7e8b5 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTests.java @@ -24,11 +24,14 @@ public abstract class BigCountTests extends TDigestTestCase { public void testBigMerge() { - TDigest digest = createDigest(); - for (int i = 0; i < 5; i++) { - digest.add(getDigest()); - double actual = digest.quantile(0.5); - assertEquals("Count = " + digest.size(), 3000, actual, 0.001); + try (TDigest digest = createDigest()) { + for (int i = 0; i < 5; i++) { + try (TDigest digestToMerge = getDigest()) { + digest.add(digestToMerge); + } + double actual = digest.quantile(0.5); + assertEquals("Count = " + digest.size(), 3000, actual, 0.001); + } } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsMergingDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsMergingDigestTests.java index ab28628200cce..7a7094691fb95 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsMergingDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsMergingDigestTests.java @@ -24,6 +24,6 @@ public class BigCountTestsMergingDigestTests extends BigCountTests { @Override public TDigest createDigest() { - return new MergingDigest(arrays(), 100); + return TDigest.createMergingDigest(arrays(), 100); } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsTreeDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsTreeDigestTests.java index a9af82164c2ba..2978e1c98bcdb 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsTreeDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/BigCountTestsTreeDigestTests.java @@ -24,6 +24,6 @@ public class BigCountTestsTreeDigestTests extends BigCountTests { @Override public TDigest createDigest() { - return new AVLTreeDigest(arrays(), 100); + return TDigest.createAvlTreeDigest(arrays(), 100); } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/ComparisonTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/ComparisonTests.java index 82620459891ec..b9b22bf1f8480 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/ComparisonTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/ComparisonTests.java @@ -21,6 +21,8 @@ package org.elasticsearch.tdigest; +import org.elasticsearch.core.Releasables; + import java.util.Arrays; import java.util.function.Supplier; @@ -53,6 +55,10 @@ private void loadData(Supplier sampleGenerator) { Arrays.sort(samples); } + private void releaseData() { + Releasables.close(avlTreeDigest, mergingDigest, sortingDigest, hybridDigest); + } + public void testRandomDenseDistribution() { loadData(() -> random().nextDouble()); @@ -65,6 +71,8 @@ public void testRandomDenseDistribution() { assertEquals(String.valueOf(percentile), expected, mergingDigest.quantile(q), accuracy); assertEquals(String.valueOf(percentile), expected, hybridDigest.quantile(q), accuracy); } + + releaseData(); } public void testRandomSparseDistribution() { @@ -79,6 +87,8 @@ public void testRandomSparseDistribution() { assertEquals(String.valueOf(percentile), expected, mergingDigest.quantile(q), accuracy); assertEquals(String.valueOf(percentile), expected, hybridDigest.quantile(q), accuracy); } + + releaseData(); } public void testDenseGaussianDistribution() { @@ -99,6 +109,8 @@ public void testDenseGaussianDistribution() { assertEquals(expectedMedian, avlTreeDigest.quantile(0.5), 0.01); assertEquals(expectedMedian, mergingDigest.quantile(0.5), 0.01); assertEquals(expectedMedian, hybridDigest.quantile(0.5), 0.01); + + releaseData(); } public void testSparseGaussianDistribution() { @@ -120,5 +132,7 @@ public void testSparseGaussianDistribution() { assertEquals(expectedMedian, avlTreeDigest.quantile(0.5), 5000); assertEquals(expectedMedian, mergingDigest.quantile(0.5), 5000); assertEquals(expectedMedian, hybridDigest.quantile(0.5), 5000); + + releaseData(); } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/HybridDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/HybridDigestTests.java index 01b3dc8f5da2a..96adb7b13203d 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/HybridDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/HybridDigestTests.java @@ -24,6 +24,6 @@ public class HybridDigestTests extends TDigestTests { protected DigestFactory factory(final double compression) { - return () -> new HybridDigest(arrays(), compression); + return () -> HybridDigest.create(arrays(), compression); } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/IntAVLTreeTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/IntAVLTreeTests.java index 5178701e96c2c..53c4664cbcc0d 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/IntAVLTreeTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/IntAVLTreeTests.java @@ -39,6 +39,8 @@ static class IntegerBag extends IntAVLTree { IntegerBag(TDigestArrays arrays) { super(arrays); + // We adjust the breaker after creation as this is just a test class + arrays.adjustBreaker(IntAVLTree.SHALLOW_SIZE); values = new int[capacity()]; counts = new int[capacity()]; } @@ -88,53 +90,54 @@ protected void merge(int node) { public void testDualAdd() { Random r = random(); TreeMap map = new TreeMap<>(); - IntegerBag bag = new IntegerBag(arrays()); - for (int i = 0; i < 100000; ++i) { - final int v = r.nextInt(100000); - if (map.containsKey(v)) { - map.put(v, map.get(v) + 1); - assertFalse(bag.addValue(v)); - } else { - map.put(v, 1); - assertTrue(bag.addValue(v)); + try (IntegerBag bag = new IntegerBag(arrays())) { + for (int i = 0; i < 100000; ++i) { + final int v = r.nextInt(100000); + if (map.containsKey(v)) { + map.put(v, map.get(v) + 1); + assertFalse(bag.addValue(v)); + } else { + map.put(v, 1); + assertTrue(bag.addValue(v)); + } } + Iterator> it = map.entrySet().iterator(); + for (int node = bag.first(bag.root()); node != IntAVLTree.NIL; node = bag.next(node)) { + final Map.Entry next = it.next(); + assertEquals(next.getKey().intValue(), bag.values[node]); + assertEquals(next.getValue().intValue(), bag.counts[node]); + } + assertFalse(it.hasNext()); } - Iterator> it = map.entrySet().iterator(); - for (int node = bag.first(bag.root()); node != IntAVLTree.NIL; node = bag.next(node)) { - final Map.Entry next = it.next(); - assertEquals(next.getKey().intValue(), bag.values[node]); - assertEquals(next.getValue().intValue(), bag.counts[node]); - } - assertFalse(it.hasNext()); } public void testDualAddRemove() { Random r = random(); TreeMap map = new TreeMap<>(); - IntegerBag bag = new IntegerBag(arrays()); - for (int i = 0; i < 100000; ++i) { - final int v = r.nextInt(1000); - if (r.nextBoolean()) { - // add - if (map.containsKey(v)) { - map.put(v, map.get(v) + 1); - assertFalse(bag.addValue(v)); + try (IntegerBag bag = new IntegerBag(arrays())) { + for (int i = 0; i < 100000; ++i) { + final int v = r.nextInt(1000); + if (r.nextBoolean()) { + // add + if (map.containsKey(v)) { + map.put(v, map.get(v) + 1); + assertFalse(bag.addValue(v)); + } else { + map.put(v, 1); + assertTrue(bag.addValue(v)); + } } else { - map.put(v, 1); - assertTrue(bag.addValue(v)); + // remove + assertEquals(map.remove(v) != null, bag.removeValue(v)); } - } else { - // remove - assertEquals(map.remove(v) != null, bag.removeValue(v)); } + Iterator> it = map.entrySet().iterator(); + for (int node = bag.first(bag.root()); node != IntAVLTree.NIL; node = bag.next(node)) { + final Map.Entry next = it.next(); + assertEquals(next.getKey().intValue(), bag.values[node]); + assertEquals(next.getValue().intValue(), bag.counts[node]); + } + assertFalse(it.hasNext()); } - Iterator> it = map.entrySet().iterator(); - for (int node = bag.first(bag.root()); node != IntAVLTree.NIL; node = bag.next(node)) { - final Map.Entry next = it.next(); - assertEquals(next.getKey().intValue(), bag.values[node]); - assertEquals(next.getValue().intValue(), bag.counts[node]); - } - assertFalse(it.hasNext()); } - } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MedianTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MedianTests.java index c8acec935c040..524c1df2a8c92 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MedianTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MedianTests.java @@ -25,46 +25,50 @@ public class MedianTests extends TDigestTestCase { public void testAVL() { double[] data = new double[] { 7, 15, 36, 39, 40, 41 }; - TDigest digest = new AVLTreeDigest(arrays(), 100); - for (double value : data) { - digest.add(value); - } + try (TDigest digest = TDigest.createAvlTreeDigest(arrays(), 100)) { + for (double value : data) { + digest.add(value); + } - assertEquals(37.5, digest.quantile(0.5), 0); - assertEquals(0.5, digest.cdf(37.5), 0); + assertEquals(37.5, digest.quantile(0.5), 0); + assertEquals(0.5, digest.cdf(37.5), 0); + } } public void testMergingDigest() { double[] data = new double[] { 7, 15, 36, 39, 40, 41 }; - TDigest digest = new MergingDigest(arrays(), 100); - for (double value : data) { - digest.add(value); - } + try (TDigest digest = TDigest.createMergingDigest(arrays(), 100)) { + for (double value : data) { + digest.add(value); + } - assertEquals(37.5, digest.quantile(0.5), 0); - assertEquals(0.5, digest.cdf(37.5), 0); + assertEquals(37.5, digest.quantile(0.5), 0); + assertEquals(0.5, digest.cdf(37.5), 0); + } } public void testSortingDigest() { double[] data = new double[] { 7, 15, 36, 39, 40, 41 }; - TDigest digest = new SortingDigest(arrays()); - for (double value : data) { - digest.add(value); - } + try (TDigest digest = TDigest.createSortingDigest(arrays())) { + for (double value : data) { + digest.add(value); + } - assertEquals(37.5, digest.quantile(0.5), 0); - assertEquals(0.5, digest.cdf(37.5), 0); + assertEquals(37.5, digest.quantile(0.5), 0); + assertEquals(0.5, digest.cdf(37.5), 0); + } } public void testHybridDigest() { double[] data = new double[] { 7, 15, 36, 39, 40, 41 }; - TDigest digest = new HybridDigest(arrays(), 100); - for (double value : data) { - digest.add(value); - } + try (TDigest digest = TDigest.createHybridDigest(arrays(), 100)) { + for (double value : data) { + digest.add(value); + } - assertEquals(37.5, digest.quantile(0.5), 0); - assertEquals(0.5, digest.cdf(37.5), 0); + assertEquals(37.5, digest.quantile(0.5), 0); + assertEquals(0.5, digest.cdf(37.5), 0); + } } public void testReferenceWikipedia() { diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MergingDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MergingDigestTests.java index 263d0fe920208..18ef7984242ff 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MergingDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/MergingDigestTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.tdigest; +import org.elasticsearch.core.Releasables; import org.junit.Assert; import java.util.ArrayList; @@ -33,46 +34,47 @@ public class MergingDigestTests extends TDigestTests { protected DigestFactory factory(final double compression) { - - return () -> new MergingDigest(arrays(), compression); + return () -> MergingDigest.create(arrays(), compression); } public void testNanDueToBadInitialization() { int compression = 100; int factor = 5; - MergingDigest md = new MergingDigest(arrays(), compression, (factor + 1) * compression, compression); + try (MergingDigest md = MergingDigest.create(arrays(), compression, (factor + 1) * compression, compression)) { - final int M = 10; - List mds = new ArrayList<>(); - for (int i = 0; i < M; ++i) { - mds.add(new MergingDigest(arrays(), compression, (factor + 1) * compression, compression)); - } + final int M = 10; + List mds = new ArrayList<>(); + for (int i = 0; i < M; ++i) { + mds.add(MergingDigest.create(arrays(), compression, (factor + 1) * compression, compression)); + } - // Fill all digests with values (0,10,20,...,80). - List raw = new ArrayList<>(); - for (int i = 0; i < 9; ++i) { - double x = 10 * i; - md.add(x); - raw.add(x); - for (int j = 0; j < M; ++j) { - mds.get(j).add(x); + // Fill all digests with values (0,10,20,...,80). + List raw = new ArrayList<>(); + for (int i = 0; i < 9; ++i) { + double x = 10 * i; + md.add(x); raw.add(x); + for (int j = 0; j < M; ++j) { + mds.get(j).add(x); + raw.add(x); + } } - } - Collections.sort(raw); + Collections.sort(raw); - // Merge all mds one at a time into md. - for (int i = 0; i < M; ++i) { - md.add(mds.get(i)); - } - Assert.assertFalse(Double.isNaN(md.quantile(0.01))); - - for (double q : new double[] { 0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.90, 0.95, 0.99 }) { - double est = md.quantile(q); - double actual = Dist.quantile(q, raw); - double qx = md.cdf(actual); - Assert.assertEquals(q, qx, 0.5); - Assert.assertEquals(est, actual, 3.8); + // Merge all mds one at a time into md. + for (int i = 0; i < M; ++i) { + md.add(mds.get(i)); + } + Assert.assertFalse(Double.isNaN(md.quantile(0.01))); + + for (double q : new double[] { 0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.90, 0.95, 0.99 }) { + double est = md.quantile(q); + double actual = Dist.quantile(q, raw); + double qx = md.cdf(actual); + Assert.assertEquals(q, qx, 0.5); + Assert.assertEquals(est, actual, 3.8); + } + Releasables.close(mds); } } @@ -80,86 +82,90 @@ public void testNanDueToBadInitialization() { * Verifies interpolation between a singleton and a larger centroid. */ public void testSingleMultiRange() { - TDigest digest = factory(100).create(); - digest.setScaleFunction(ScaleFunction.K_0); - for (int i = 0; i < 100; i++) { - digest.add(1); - digest.add(2); - digest.add(3); + try (TDigest digest = factory(100).create()) { + digest.setScaleFunction(ScaleFunction.K_0); + for (int i = 0; i < 100; i++) { + digest.add(1); + digest.add(2); + digest.add(3); + } + // this check is, of course true, but it also forces merging before we change scale + assertTrue(digest.centroidCount() < 300); + digest.add(0); + // we now have a digest with a singleton first, then a heavier centroid next + Iterator ix = digest.centroids().iterator(); + Centroid first = ix.next(); + Centroid second = ix.next(); + assertEquals(1, first.count()); + assertEquals(0, first.mean(), 0); + // assertTrue(second.count() > 1); + assertEquals(1.0, second.mean(), 0); + + assertEquals(0.00166, digest.cdf(0), 1e-5); + assertEquals(0.00166, digest.cdf(1e-10), 1e-5); + assertEquals(0.0025, digest.cdf(0.25), 1e-5); } - // this check is, of course true, but it also forces merging before we change scale - assertTrue(digest.centroidCount() < 300); - digest.add(0); - // we now have a digest with a singleton first, then a heavier centroid next - Iterator ix = digest.centroids().iterator(); - Centroid first = ix.next(); - Centroid second = ix.next(); - assertEquals(1, first.count()); - assertEquals(0, first.mean(), 0); - // assertTrue(second.count() > 1); - assertEquals(1.0, second.mean(), 0); - - assertEquals(0.00166, digest.cdf(0), 1e-5); - assertEquals(0.00166, digest.cdf(1e-10), 1e-5); - assertEquals(0.0025, digest.cdf(0.25), 1e-5); } /** * Make sure that the first and last centroids have unit weight */ public void testSingletonsAtEnds() { - TDigest d = new MergingDigest(arrays(), 50); - Random gen = random(); - double[] data = new double[100]; - for (int i = 0; i < data.length; i++) { - data[i] = Math.floor(gen.nextGaussian() * 3); - } - for (int i = 0; i < 100; i++) { - for (double x : data) { - d.add(x); + try (TDigest d = MergingDigest.create(arrays(), 50)) { + Random gen = random(); + double[] data = new double[100]; + for (int i = 0; i < data.length; i++) { + data[i] = Math.floor(gen.nextGaussian() * 3); } - } - long last = 0; - for (Centroid centroid : d.centroids()) { - if (last == 0) { - assertEquals(1, centroid.count()); + for (int i = 0; i < 100; i++) { + for (double x : data) { + d.add(x); + } } - last = centroid.count(); + long last = 0; + for (Centroid centroid : d.centroids()) { + if (last == 0) { + assertEquals(1, centroid.count()); + } + last = centroid.count(); + } + assertEquals(1, last); } - assertEquals(1, last); } /** * Verify centroid sizes. */ public void testFill() { - MergingDigest x = new MergingDigest(arrays(), 300); - Random gen = random(); - ScaleFunction scale = x.getScaleFunction(); - double compression = x.compression(); - for (int i = 0; i < 1000000; i++) { - x.add(gen.nextGaussian()); - } - double q0 = 0; - int i = 0; - for (Centroid centroid : x.centroids()) { - double q1 = q0 + (double) centroid.count() / x.size(); - double dk = scale.k(q1, compression, x.size()) - scale.k(q0, compression, x.size()); - if (centroid.count() > 1) { - assertTrue(String.format(Locale.ROOT, "K-size for centroid %d at %.3f is %.3f", i, centroid.mean(), dk), dk <= 1); + try (MergingDigest x = MergingDigest.create(arrays(), 300)) { + Random gen = random(); + ScaleFunction scale = x.getScaleFunction(); + double compression = x.compression(); + for (int i = 0; i < 1000000; i++) { + x.add(gen.nextGaussian()); + } + double q0 = 0; + int i = 0; + for (Centroid centroid : x.centroids()) { + double q1 = q0 + (double) centroid.count() / x.size(); + double dk = scale.k(q1, compression, x.size()) - scale.k(q0, compression, x.size()); + if (centroid.count() > 1) { + assertTrue(String.format(Locale.ROOT, "K-size for centroid %d at %.3f is %.3f", i, centroid.mean(), dk), dk <= 1); + } + q0 = q1; + i++; } - q0 = q1; - i++; } } public void testLargeInputSmallCompression() { - MergingDigest td = new MergingDigest(arrays(), 10); - for (int i = 0; i < 10_000_000; i++) { - td.add(between(0, 3_600_000)); + try (MergingDigest td = MergingDigest.create(arrays(), 10)) { + for (int i = 0; i < 10_000_000; i++) { + td.add(between(0, 3_600_000)); + } + assertTrue(td.centroidCount() < 100); + assertTrue(td.quantile(0.00001) < 100_000); + assertTrue(td.quantile(0.99999) > 3_000_000); } - assertTrue(td.centroidCount() < 100); - assertTrue(td.quantile(0.00001) < 100_000); - assertTrue(td.quantile(0.99999) > 3_000_000); } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortTests.java index 425e4d1497eda..f12004d3d6d02 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortTests.java @@ -35,6 +35,7 @@ public void testReverse() { Sort.reverse(x, 0, x.size()); // reverse stuff! + x.close(); x = arrays().newIntArray(new int[] { 1, 2, 3, 4, 5 }); Sort.reverse(x, 0, x.size()); for (int i = 0; i < 5; i++) { @@ -57,11 +58,13 @@ public void testReverse() { assertEquals(4, x.get(3)); assertEquals(1, x.get(4)); + x.close(); x = arrays().newIntArray(new int[] { 1, 2, 3, 4, 5, 6 }); Sort.reverse(x, 0, x.size()); for (int i = 0; i < 6; i++) { assertEquals(6 - i, x.get(i)); } + x.close(); } public void testEmpty() { @@ -227,9 +230,8 @@ private void checkOrder(int[] order, double[] values) { } private void sort(int[] order, double[] values, int n) { - var wrappedOrder = arrays().newIntArray(order); - var wrappedValues = arrays().newDoubleArray(values); - - Sort.stableSort(wrappedOrder, wrappedValues, n); + try (var wrappedOrder = arrays().newIntArray(order); var wrappedValues = arrays().newDoubleArray(values);) { + Sort.stableSort(wrappedOrder, wrappedValues, n); + } } } diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortingDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortingDigestTests.java index 2478e85421f07..ea38959019f0b 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortingDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/SortingDigestTests.java @@ -24,7 +24,7 @@ public class SortingDigestTests extends TDigestTests { protected DigestFactory factory(final double compression) { - return () -> new SortingDigest(arrays()); + return () -> SortingDigest.create(arrays()); } // Make this test a noop to avoid OOMs. diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestReleasingTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestReleasingTests.java new file mode 100644 index 0000000000000..fbd423e3d0a37 --- /dev/null +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestReleasingTests.java @@ -0,0 +1,89 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * This project is based on a modification of https://github.com/tdunning/t-digest which is licensed under the Apache 2.0 License. + */ + +package org.elasticsearch.tdigest; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.search.aggregations.metrics.MemoryTrackingTDigestArrays; +import org.elasticsearch.tdigest.arrays.TDigestArrays; +import org.elasticsearch.test.ESTestCase; + +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +public class TDigestReleasingTests extends ESTestCase { + @ParametersFactory + public static Iterable parameters() { + return List.of( + makeTDigestParams("Hybrid", (arrays) -> TDigest.createHybridDigest(arrays, 100)), + makeTDigestParams("Merging", (arrays) -> TDigest.createMergingDigest(arrays, 100)), + makeTDigestParams("Sorting", TDigest::createSortingDigest), + makeTDigestParams("AvlTree", (arrays) -> TDigest.createAvlTreeDigest(arrays, 100)) + ); + } + + public record TestCase(String name, CircuitBreaker breaker, Supplier tDigestSupplier) { + @Override + public String toString() { + return name; + } + } + + private static Object[] makeTDigestParams(String name, Function tDigestSupplier) { + var breaker = newLimitedBreaker(ByteSizeValue.ofMb(100)); + return new Object[] { new TestCase(name, breaker, () -> tDigestSupplier.apply(new MemoryTrackingTDigestArrays(breaker))) }; + } + + private final TestCase testCase; + + public TDigestReleasingTests(TestCase testCase) { + this.testCase = testCase; + } + + public void testRelease() { + var breaker = testCase.breaker; + assertThat(breaker.getUsed(), equalTo(0L)); + + var tDigest = testCase.tDigestSupplier.get(); + assertThat(breaker.getUsed(), greaterThan(0L)); + assertThat(breaker.getUsed(), equalTo(tDigest.ramBytesUsed())); + + for (int i = 0; i < 10_000; i++) { + tDigest.add(randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true)); + } + assertThat(breaker.getUsed(), greaterThan(0L)); + assertThat(breaker.getUsed(), equalTo(tDigest.ramBytesUsed())); + + tDigest.close(); + assertThat("close() must release all memory", breaker.getUsed(), equalTo(0L)); + + tDigest.close(); + assertThat("close() must be idempotent", breaker.getUsed(), equalTo(0L)); + } + +} diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTestCase.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTestCase.java index 76db01d5dd0bf..d4240a8e633d1 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTestCase.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTestCase.java @@ -53,8 +53,8 @@ public abstract class TDigestTestCase extends ESTestCase { * The arrays created by this method will be automatically released after the test. *

*/ - protected DelegatingTDigestArrays arrays() { - return new DelegatingTDigestArrays(); + protected MemoryTrackingTDigestArrays arrays() { + return new MemoryTrackingTDigestArrays(newLimitedBreaker(ByteSizeValue.ofMb(100))); } /** @@ -82,6 +82,11 @@ public TDigestDoubleArray newDoubleArray(double[] data) { return register(delegate.newDoubleArray(data)); } + @Override + public void adjustBreaker(long size) { + delegate.adjustBreaker(size); + } + @Override public TDigestDoubleArray newDoubleArray(int size) { return register(delegate.newDoubleArray(size)); diff --git a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java index 89a0c037dc864..24dac8242bee9 100644 --- a/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java +++ b/libs/tdigest/src/test/java/org/elasticsearch/tdigest/TDigestTests.java @@ -59,6 +59,7 @@ public void testBigJump() { assertEquals(0.95, digest.cdf(500_000), 1e-5); assertEquals(0.975, digest.cdf(1_000_000), 1e-5); + digest.close(); digest = factory(80).create(); digest.setScaleFunction(ScaleFunction.K_0); @@ -72,21 +73,23 @@ public void testBigJump() { assertEquals(19.0, digest.quantile(0.915), 0.1); assertEquals(19.0, digest.quantile(0.935), 0.1); assertEquals(1_000_000.0, digest.quantile(0.965), 0.1); + digest.close(); } public void testSmallCountQuantile() { List data = List.of(15.0, 20.0, 32.0, 60.0); - TDigest td = factory(200).create(); - for (Double datum : data) { - td.add(datum); + try (TDigest td = factory(200).create()) { + for (Double datum : data) { + td.add(datum); + } + assertEquals(15.0, td.quantile(0.00), 1e-5); + assertEquals(16.0, td.quantile(0.10), 1.0); + assertEquals(18.0, td.quantile(0.25), 1.0); + assertEquals(26.0, td.quantile(0.50), 1e-5); + assertEquals(42.0, td.quantile(0.75), 4.0); + assertEquals(55.0, td.quantile(0.90), 5.0); + assertEquals(60.0, td.quantile(1.00), 1e-5); } - assertEquals(15.0, td.quantile(0.00), 1e-5); - assertEquals(16.0, td.quantile(0.10), 1.0); - assertEquals(18.0, td.quantile(0.25), 1.0); - assertEquals(26.0, td.quantile(0.50), 1e-5); - assertEquals(42.0, td.quantile(0.75), 4.0); - assertEquals(55.0, td.quantile(0.90), 5.0); - assertEquals(60.0, td.quantile(1.00), 1e-5); } public void testExplicitSkewedData() { @@ -123,35 +126,37 @@ public void testExplicitSkewedData() { 51242, 54241 }; - TDigest digest = factory().create(); - for (double x : data) { - digest.add(x); - } + try (TDigest digest = factory().create()) { + for (double x : data) { + digest.add(x); + } - assertEquals(Dist.quantile(0.5, data), digest.quantile(0.5), 0); + assertEquals(Dist.quantile(0.5, data), digest.quantile(0.5), 0); + } } public void testQuantile() { double[] samples = new double[] { 1.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 3.0, 3.0, 3.0, 3.0, 4.0, 5.0, 6.0, 7.0 }; - TDigest hist1 = factory().create(); - List data = new ArrayList<>(); + try (TDigest hist1 = factory().create(); TDigest hist2 = factory().create()) { + List data = new ArrayList<>(); - for (int j = 0; j < 100; j++) { - for (double x : samples) { - data.add(x); - hist1.add(x); + for (int j = 0; j < 100; j++) { + for (double x : samples) { + data.add(x); + hist1.add(x); + } } + + hist1.compress(); + hist2.add(hist1); + Collections.sort(data); + hist2.compress(); + double x1 = hist1.quantile(0.5); + double x2 = hist2.quantile(0.5); + assertEquals(Dist.quantile(0.5, data), x1, 0.25); + assertEquals(x1, x2, 0.01); } - TDigest hist2 = factory().create(); - hist1.compress(); - hist2.add(hist1); - Collections.sort(data); - hist2.compress(); - double x1 = hist1.quantile(0.5); - double x2 = hist2.quantile(0.5); - assertEquals(Dist.quantile(0.5, data), x1, 0.25); - assertEquals(x1, x2, 0.01); } /** @@ -159,43 +164,45 @@ public void testQuantile() { */ public void testSingletonQuantiles() { double[] data = new double[11]; - TDigest digest = factory().create(); - for (int i = 0; i < data.length; i++) { - digest.add(i); - data[i] = i; - } + try (TDigest digest = factory().create()) { + for (int i = 0; i < data.length; i++) { + digest.add(i); + data[i] = i; + } - for (double x = digest.getMin() - 0.1; x <= digest.getMax() + 0.1; x += 1e-3) { - assertEquals(String.valueOf(x), Dist.cdf(x, data), digest.cdf(x), 0.1); - } + for (double x = digest.getMin() - 0.1; x <= digest.getMax() + 0.1; x += 1e-3) { + assertEquals(String.valueOf(x), Dist.cdf(x, data), digest.cdf(x), 0.1); + } - for (int i = 0; i <= 1000; i++) { - double q = 0.001 * i; - double dist = Dist.quantile(q, data); - double td = digest.quantile(q); - assertEquals(String.valueOf(q), dist, td, 0.5); + for (int i = 0; i <= 1000; i++) { + double q = 0.001 * i; + double dist = Dist.quantile(q, data); + double td = digest.quantile(q); + assertEquals(String.valueOf(q), dist, td, 0.5); + } } } public void testCentroidsWithIncreasingWeights() { ArrayList data = new ArrayList<>(); - TDigest digest = factory().create(); - for (int i = 1; i <= 10; i++) { - digest.add(i, i); - for (int j = 0; j < i; j++) { - data.add((double) i); + try (TDigest digest = factory().create()) { + for (int i = 1; i <= 10; i++) { + digest.add(i, i); + for (int j = 0; j < i; j++) { + data.add((double) i); + } } - } - for (double x = digest.getMin() - 0.1; x <= digest.getMax() + 0.1; x += 1e-3) { - assertEquals(String.valueOf(x), Dist.cdf(x, data), digest.cdf(x), 0.5); - } + for (double x = digest.getMin() - 0.1; x <= digest.getMax() + 0.1; x += 1e-3) { + assertEquals(String.valueOf(x), Dist.cdf(x, data), digest.cdf(x), 0.5); + } - for (int i = 0; i <= 1000; i++) { - double q = 0.001 * i; - double dist = Dist.quantile(q, data); - double td = digest.quantile(q); - assertEquals(String.valueOf(q), dist, td, 0.75); + for (int i = 0; i <= 1000; i++) { + double q = 0.001 * i; + double dist = Dist.quantile(q, data); + double td = digest.quantile(q); + assertEquals(String.valueOf(q), dist, td, 0.75); + } } } @@ -203,15 +210,16 @@ public void testCentroidsWithIncreasingWeights() { * Verifies behavior involving interpolation between singleton centroids. */ public void testSingleSingleRange() { - TDigest digest = factory().create(); - digest.add(1); - digest.add(2); - digest.add(3); + try (TDigest digest = factory().create()) { + digest.add(1); + digest.add(2); + digest.add(3); - // verify the cdf is a step between singletons - assertEquals(0.5 / 3.0, digest.cdf(1), 0); - assertEquals(1.5 / 3.0, digest.cdf(2), 0); - assertEquals(2.5 / 3.0, digest.cdf(3), 0); + // verify the cdf is a step between singletons + assertEquals(0.5 / 3.0, digest.cdf(1), 0); + assertEquals(1.5 / 3.0, digest.cdf(2), 0); + assertEquals(2.5 / 3.0, digest.cdf(3), 0); + } } /** @@ -240,6 +248,7 @@ public void testSingletonAtEnd() { // normally min == mean[0] because weight[0] == 1 // we can force this not to be true for testing + digest.close(); digest = factory().create(); digest.setScaleFunction(ScaleFunction.K_0); for (int i = 0; i < 100; i++) { @@ -278,219 +287,229 @@ public void testSingletonAtEnd() { assertEquals(4, digest.quantile(1), 0); assertEquals(last.mean(), 4, 0); + digest.close(); } public void testFewRepeatedValues() { - TDigest d = factory().create(); - for (int i = 0; i < 2; ++i) { - d.add(9000); - } - for (int i = 0; i < 11; ++i) { - d.add(3000); - } - for (int i = 0; i < 26; ++i) { - d.add(1000); - } + try (TDigest d = factory().create()) { + for (int i = 0; i < 2; ++i) { + d.add(9000); + } + for (int i = 0; i < 11; ++i) { + d.add(3000); + } + for (int i = 0; i < 26; ++i) { + d.add(1000); + } - assertEquals(3000.0, d.quantile(0.90), 1e-5); - assertEquals(4500.0, d.quantile(0.95), 2000); - assertEquals(8500.0, d.quantile(0.97), 500); - assertEquals(9000.0, d.quantile(0.98), 1e-5); - assertEquals(9000.0, d.quantile(1.00), 1e-5); + assertEquals(3000.0, d.quantile(0.90), 1e-5); + assertEquals(4500.0, d.quantile(0.95), 2000); + assertEquals(8500.0, d.quantile(0.97), 500); + assertEquals(9000.0, d.quantile(0.98), 1e-5); + assertEquals(9000.0, d.quantile(1.00), 1e-5); + } } public void testSingleValue() { Random rand = random(); - final TDigest digest = factory().create(); - final double value = rand.nextDouble() * 1000; - digest.add(value); + try (TDigest digest = factory().create()) { + final double value = rand.nextDouble() * 1000; + digest.add(value); - assertEquals(value, digest.quantile(0.0), 0); - assertEquals(value, digest.quantile(1.0), 0); - assertEquals(value, digest.quantile(rand.nextDouble()), 0); + assertEquals(value, digest.quantile(0.0), 0); + assertEquals(value, digest.quantile(1.0), 0); + assertEquals(value, digest.quantile(rand.nextDouble()), 0); - assertEquals(0.0, digest.cdf(value - 1e-5), 0.0); - assertEquals(1.0, digest.cdf(value + 1e5), 0.0); - assertEquals(0.5, digest.cdf(value), 0.0); + assertEquals(0.0, digest.cdf(value - 1e-5), 0.0); + assertEquals(1.0, digest.cdf(value + 1e5), 0.0); + assertEquals(0.5, digest.cdf(value), 0.0); + } } public void testFewValues() { // When there are few values in the tree, quantiles should be exact - final TDigest digest = factory().create(); - final Random r = random(); - final int length = r.nextInt(10); - final List values = new ArrayList<>(); - for (int i = 0; i < length; ++i) { - final double value; - if (i == 0 || r.nextBoolean()) { - value = r.nextDouble() * 100; - } else { - // introduce duplicates - value = values.get(i - 1); + try (TDigest digest = factory().create()) { + final Random r = random(); + final int length = r.nextInt(10); + final List values = new ArrayList<>(); + for (int i = 0; i < length; ++i) { + final double value; + if (i == 0 || r.nextBoolean()) { + value = r.nextDouble() * 100; + } else { + // introduce duplicates + value = values.get(i - 1); + } + digest.add(value); + values.add(value); + } + Collections.sort(values); + + // for this value of the compression, the tree shouldn't have merged any node + assertEquals(digest.centroids().size(), values.size()); + for (double q : new double[] { 0, 1e-10, 0.5, 1 - 1e-10, 1 }) { + double q1 = Dist.quantile(q, values); + double q2 = digest.quantile(q); + assertEquals(String.valueOf(q), q1, q2, q1); } - digest.add(value); - values.add(value); - } - Collections.sort(values); - - // for this value of the compression, the tree shouldn't have merged any node - assertEquals(digest.centroids().size(), values.size()); - for (double q : new double[] { 0, 1e-10, 0.5, 1 - 1e-10, 1 }) { - double q1 = Dist.quantile(q, values); - double q2 = digest.quantile(q); - assertEquals(String.valueOf(q), q1, q2, q1); } } public void testEmptyDigest() { - TDigest digest = factory().create(); - assertEquals(0, digest.centroids().size()); - assertEquals(0, digest.size()); - assertTrue(Double.isNaN(digest.quantile(random().nextDouble()))); - assertTrue(Double.isNaN(digest.cdf(0))); + try (TDigest digest = factory().create()) { + assertEquals(0, digest.centroids().size()); + assertEquals(0, digest.size()); + assertTrue(Double.isNaN(digest.quantile(random().nextDouble()))); + assertTrue(Double.isNaN(digest.cdf(0))); + } } public void testMoreThan2BValues() { - final TDigest digest = factory().create(); - // carefully build a t-digest that is as if we added 3 uniform values from [0,1] - double n = 3e9; - double q0 = 0; - for (int i = 0; i < 200 && q0 < 1 - 1e-10; ++i) { - double k0 = digest.scale.k(q0, digest.compression(), n); - double q = digest.scale.q(k0 + 1, digest.compression(), n); - int m = (int) Math.max(1, n * (q - q0)); - digest.add((q + q0) / 2, m); - q0 = q0 + m / n; - } - digest.compress(); - assertEquals(3_000_000_000L, digest.size()); - assertTrue(digest.size() > Integer.MAX_VALUE); - final double[] quantiles = new double[] { 0, 0.1, 0.5, 0.9, 1 }; - double prev = Double.NEGATIVE_INFINITY; - for (double q : quantiles) { - final double v = digest.quantile(q); - assertTrue(String.format(Locale.ROOT, "q=%.1f, v=%.4f, pref=%.4f", q, v, prev), v >= prev); - prev = v; + try (TDigest digest = factory().create()) { + // carefully build a t-digest that is as if we added 3 uniform values from [0,1] + double n = 3e9; + double q0 = 0; + for (int i = 0; i < 200 && q0 < 1 - 1e-10; ++i) { + double k0 = digest.scale.k(q0, digest.compression(), n); + double q = digest.scale.q(k0 + 1, digest.compression(), n); + int m = (int) Math.max(1, n * (q - q0)); + digest.add((q + q0) / 2, m); + q0 = q0 + m / n; + } + digest.compress(); + assertEquals(3_000_000_000L, digest.size()); + assertTrue(digest.size() > Integer.MAX_VALUE); + final double[] quantiles = new double[] { 0, 0.1, 0.5, 0.9, 1 }; + double prev = Double.NEGATIVE_INFINITY; + for (double q : quantiles) { + final double v = digest.quantile(q); + assertTrue(String.format(Locale.ROOT, "q=%.1f, v=%.4f, pref=%.4f", q, v, prev), v >= prev); + prev = v; + } } } public void testSorted() { - final TDigest digest = factory().create(); - Random gen = random(); - for (int i = 0; i < 10000; ++i) { - int w = 1 + gen.nextInt(10); - double x = gen.nextDouble(); - for (int j = 0; j < w; j++) { - digest.add(x); + try (TDigest digest = factory().create()) { + Random gen = random(); + for (int i = 0; i < 10000; ++i) { + int w = 1 + gen.nextInt(10); + double x = gen.nextDouble(); + for (int j = 0; j < w; j++) { + digest.add(x); + } } - } - Centroid previous = null; - for (Centroid centroid : digest.centroids()) { - if (previous != null) { - if (previous.mean() <= centroid.mean()) { - assertTrue(Double.compare(previous.mean(), centroid.mean()) <= 0); + Centroid previous = null; + for (Centroid centroid : digest.centroids()) { + if (previous != null) { + if (previous.mean() <= centroid.mean()) { + assertTrue(Double.compare(previous.mean(), centroid.mean()) <= 0); + } } + previous = centroid; } - previous = centroid; } } public void testNaN() { - final TDigest digest = factory().create(); - Random gen = random(); - final int iters = gen.nextInt(100); - for (int i = 0; i < iters; ++i) { - digest.add(gen.nextDouble(), 1 + gen.nextInt(10)); - } - try { - // both versions should fail - if (gen.nextBoolean()) { - digest.add(Double.NaN); - } else { - digest.add(Double.NaN, 1); + try (TDigest digest = factory().create()) { + Random gen = random(); + final int iters = gen.nextInt(100); + for (int i = 0; i < iters; ++i) { + digest.add(gen.nextDouble(), 1 + gen.nextInt(10)); + } + try { + // both versions should fail + if (gen.nextBoolean()) { + digest.add(Double.NaN); + } else { + digest.add(Double.NaN, 1); + } + fail("NaN should be an illegal argument"); + } catch (IllegalArgumentException e) { + // expected } - fail("NaN should be an illegal argument"); - } catch (IllegalArgumentException e) { - // expected } } public void testMidPointRule() { - TDigest dist = factory(200).create(); - dist.add(1); - dist.add(2); - - for (int i = 0; i < 1000; i++) { + try (TDigest dist = factory(200).create()) { dist.add(1); dist.add(2); - if (i % 8 == 0) { - String message = String.format(Locale.ROOT, "i = %d", i); - assertEquals(message, 0, dist.cdf(1 - 1e-9), 0); - assertEquals(message, 0.3, dist.cdf(1), 0.2); - assertEquals(message, 0.8, dist.cdf(2), 0.2); - assertEquals(message, 1, dist.cdf(2 + 1e-9), 0); - - assertEquals(1.0, dist.quantile(0.0), 1e-5); - assertEquals(1.0, dist.quantile(0.1), 1e-5); - assertEquals(1.0, dist.quantile(0.2), 1e-5); - - assertTrue(dist.quantile(0.5) > 1.0); - assertTrue(dist.quantile(0.5) < 2.0); - - assertEquals(2.0, dist.quantile(0.7), 1e-5); - assertEquals(2.0, dist.quantile(0.8), 1e-5); - assertEquals(2.0, dist.quantile(0.9), 1e-5); - assertEquals(2.0, dist.quantile(1.0), 1e-5); + + for (int i = 0; i < 1000; i++) { + dist.add(1); + dist.add(2); + if (i % 8 == 0) { + String message = String.format(Locale.ROOT, "i = %d", i); + assertEquals(message, 0, dist.cdf(1 - 1e-9), 0); + assertEquals(message, 0.3, dist.cdf(1), 0.2); + assertEquals(message, 0.8, dist.cdf(2), 0.2); + assertEquals(message, 1, dist.cdf(2 + 1e-9), 0); + + assertEquals(1.0, dist.quantile(0.0), 1e-5); + assertEquals(1.0, dist.quantile(0.1), 1e-5); + assertEquals(1.0, dist.quantile(0.2), 1e-5); + + assertTrue(dist.quantile(0.5) > 1.0); + assertTrue(dist.quantile(0.5) < 2.0); + + assertEquals(2.0, dist.quantile(0.7), 1e-5); + assertEquals(2.0, dist.quantile(0.8), 1e-5); + assertEquals(2.0, dist.quantile(0.9), 1e-5); + assertEquals(2.0, dist.quantile(1.0), 1e-5); + } } } - } public void testThreePointExample() { - TDigest tdigest = factory().create(); - double x0 = 0.18615591526031494; - double x1 = 0.4241943657398224; - double x2 = 0.8813006281852722; - - tdigest.add(x0); - tdigest.add(x1); - tdigest.add(x2); - - double p10 = tdigest.quantile(0.1); - double p50 = tdigest.quantile(0.5); - double p90 = tdigest.quantile(0.9); - double p95 = tdigest.quantile(0.95); - double p99 = tdigest.quantile(0.99); - - assertTrue(Double.compare(p10, p50) <= 0); - assertTrue(Double.compare(p50, p90) <= 0); - assertTrue(Double.compare(p90, p95) <= 0); - assertTrue(Double.compare(p95, p99) <= 0); - - assertEquals(x0, tdigest.quantile(0.0), 0); - assertEquals(x2, tdigest.quantile(1.0), 0); - - assertTrue(String.valueOf(p10), Double.compare(x0, p10) <= 0); - assertTrue(String.valueOf(p10), Double.compare(x1, p10) >= 0); - assertTrue(String.valueOf(p99), Double.compare(x1, p99) <= 0); - assertTrue(String.valueOf(p99), Double.compare(x2, p99) >= 0); + try (TDigest tdigest = factory().create()) { + double x0 = 0.18615591526031494; + double x1 = 0.4241943657398224; + double x2 = 0.8813006281852722; + + tdigest.add(x0); + tdigest.add(x1); + tdigest.add(x2); + + double p10 = tdigest.quantile(0.1); + double p50 = tdigest.quantile(0.5); + double p90 = tdigest.quantile(0.9); + double p95 = tdigest.quantile(0.95); + double p99 = tdigest.quantile(0.99); + + assertTrue(Double.compare(p10, p50) <= 0); + assertTrue(Double.compare(p50, p90) <= 0); + assertTrue(Double.compare(p90, p95) <= 0); + assertTrue(Double.compare(p95, p99) <= 0); + + assertEquals(x0, tdigest.quantile(0.0), 0); + assertEquals(x2, tdigest.quantile(1.0), 0); + + assertTrue(String.valueOf(p10), Double.compare(x0, p10) <= 0); + assertTrue(String.valueOf(p10), Double.compare(x1, p10) >= 0); + assertTrue(String.valueOf(p99), Double.compare(x1, p99) <= 0); + assertTrue(String.valueOf(p99), Double.compare(x2, p99) >= 0); + } } public void testSingletonInACrowd() { - TDigest dist = factory().create(); - for (int i = 0; i < 10000; i++) { - dist.add(10); + try (TDigest dist = factory().create()) { + for (int i = 0; i < 10000; i++) { + dist.add(10); + } + dist.add(20); + dist.compress(); + + // The actual numbers depend on how the digest get constructed. + // A singleton on the right boundary yields much better accuracy, e.g. q(0.9999) == 10. + // Otherwise, quantiles above 0.9 use interpolation between 10 and 20, thus returning higher values. + assertEquals(10.0, dist.quantile(0), 0); + assertEquals(10.0, dist.quantile(0.9), 0); + assertEquals(19.0, dist.quantile(0.99999), 1); + assertEquals(20.0, dist.quantile(1), 0); } - dist.add(20); - dist.compress(); - - // The actual numbers depend on how the digest get constructed. - // A singleton on the right boundary yields much better accuracy, e.g. q(0.9999) == 10. - // Otherwise, quantiles above 0.9 use interpolation between 10 and 20, thus returning higher values. - assertEquals(10.0, dist.quantile(0), 0); - assertEquals(10.0, dist.quantile(0.9), 0); - assertEquals(19.0, dist.quantile(0.99999), 1); - assertEquals(20.0, dist.quantile(1), 0); } public void testScaling() { @@ -503,41 +522,43 @@ public void testScaling() { Collections.sort(data); for (double compression : new double[] { 10, 20, 50, 100, 200, 500, 1000 }) { - TDigest dist = factory(compression).create(); - for (Double x : data) { - dist.add(x); - } - dist.compress(); - - for (double q : new double[] { 0.001, 0.01, 0.1, 0.5 }) { - double estimate = dist.quantile(q); - double actual = data.get((int) (q * data.size())); - if (Double.compare(estimate, 0) != 0) { - assertTrue(Double.compare(Math.abs(actual - estimate) / estimate, 1) < 0); - } else { - assertEquals(Double.compare(estimate, 0), 0); + try (TDigest dist = factory(compression).create()) { + for (Double x : data) { + dist.add(x); + } + dist.compress(); + + for (double q : new double[] { 0.001, 0.01, 0.1, 0.5 }) { + double estimate = dist.quantile(q); + double actual = data.get((int) (q * data.size())); + if (Double.compare(estimate, 0) != 0) { + assertTrue(Double.compare(Math.abs(actual - estimate) / estimate, 1) < 0); + } else { + assertEquals(Double.compare(estimate, 0), 0); + } } } } } public void testMonotonicity() { - TDigest digest = factory().create(); - final Random gen = random(); - for (int i = 0; i < 100000; i++) { - digest.add(gen.nextDouble()); - } + try (TDigest digest = factory().create()) { + final Random gen = random(); + for (int i = 0; i < 100000; i++) { + digest.add(gen.nextDouble()); + } - double lastQuantile = -1; - double lastX = -1; - for (double z = 0; z <= 1; z += 1e-4) { - double x = digest.quantile(z); - assertTrue("q: " + z + " x: " + x + " last: " + lastX, Double.compare(x, lastX) >= 0); - lastX = x; + double lastQuantile = -1; + double lastX = -1; + for (double z = 0; z <= 1; z += 1e-4) { + double x = digest.quantile(z); + assertTrue("q: " + z + " x: " + x + " last: " + lastX, Double.compare(x, lastX) >= 0); + lastX = x; - double q = digest.cdf(z); - assertTrue("Q: " + z, Double.compare(q, lastQuantile) >= 0); - lastQuantile = q; + double q = digest.cdf(z); + assertTrue("Q: " + z, Double.compare(q, lastQuantile) >= 0); + lastQuantile = q; + } } } } diff --git a/muted-tests.yml b/muted-tests.yml index 2ec4faad550a6..94dc2baaab681 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -339,8 +339,6 @@ tests: - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=range/20_synthetic_source/Date range} issue: https://github.com/elastic/elasticsearch/issues/113874 -- class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT - issue: https://github.com/elastic/elasticsearch/issues/113916 - class: org.elasticsearch.kibana.KibanaThreadPoolIT method: testBlockedThreadPoolsRejectUserRequests issue: https://github.com/elastic/elasticsearch/issues/113939 diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMedianAbsoluteDeviation.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMedianAbsoluteDeviation.java index d5a2492b2503e..da0aec23a56b6 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMedianAbsoluteDeviation.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMedianAbsoluteDeviation.java @@ -25,7 +25,6 @@ public class InternalMedianAbsoluteDeviation extends InternalNumericMetricsAggregation.SingleValue implements MedianAbsoluteDeviation { public static double computeMedianAbsoluteDeviation(TDigestState valuesSketch) { - if (valuesSketch.size() == 0) { return Double.NaN; } else { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArrays.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArrays.java index 52e77ddfa9c3b..d9e53973996e6 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArrays.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/MemoryTrackingTDigestArrays.java @@ -21,7 +21,6 @@ import org.elasticsearch.tdigest.arrays.TDigestLongArray; import java.util.Arrays; -import java.util.concurrent.atomic.AtomicBoolean; /** * TDigestArrays with raw arrays and circuit breaking. @@ -34,6 +33,15 @@ public MemoryTrackingTDigestArrays(CircuitBreaker breaker) { this.breaker = breaker; } + @Override + public void adjustBreaker(long size) { + if (size > 0) { + breaker.addEstimateBytesAndMaybeBreak(size, "tdigest-adjust-breaker"); + } else { + breaker.addWithoutBreaking(size); + } + } + @Override public MemoryTrackingTDigestDoubleArray newDoubleArray(int initialSize) { breaker.addEstimateBytesAndMaybeBreak( @@ -80,7 +88,7 @@ private static long estimatedArraySize(long arrayLength, long bytesPerElement) { private abstract static class AbstractMemoryTrackingArray implements Releasable, Accountable { protected final CircuitBreaker breaker; - private final AtomicBoolean closed = new AtomicBoolean(false); + private boolean closed = false; AbstractMemoryTrackingArray(CircuitBreaker breaker) { this.breaker = breaker; @@ -88,7 +96,8 @@ private abstract static class AbstractMemoryTrackingArray implements Releasable, @Override public final void close() { - if (closed.compareAndSet(false, true)) { + if (closed == false) { + closed = true; breaker.addWithoutBreaking(-ramBytesUsed()); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestState.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestState.java index c8791c1d95a19..79455484580ca 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestState.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestState.java @@ -8,6 +8,8 @@ */ package org.elasticsearch.search.aggregations.metrics; +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.NoopCircuitBreaker; @@ -27,11 +29,13 @@ * through factory method params, providing one optimized for performance (e.g. MergingDigest or HybridDigest) by default, or optionally one * that produces highly accurate results regardless of input size but its construction over the sample population takes 2x-10x longer. */ -public class TDigestState implements Releasable { +public class TDigestState implements Releasable, Accountable { + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(TDigestState.class); - protected static final CircuitBreaker DEFAULT_NOOP_BREAKER = new NoopCircuitBreaker("default-tdigest-state-noop-breaker"); + private static final CircuitBreaker DEFAULT_NOOP_BREAKER = new NoopCircuitBreaker("default-tdigest-state-noop-breaker"); private final CircuitBreaker breaker; + private boolean closed = false; private final double compression; @@ -71,7 +75,23 @@ public static TDigestState createWithoutCircuitBreaking(double compression) { * @return a TDigestState object that's optimized for performance */ public static TDigestState create(CircuitBreaker breaker, double compression) { - return new TDigestState(breaker, Type.defaultValue(), compression); + breaker.addEstimateBytesAndMaybeBreak(SHALLOW_SIZE, "tdigest-state-create"); + try { + return new TDigestState(breaker, Type.defaultValue(), compression); + } catch (Exception e) { + breaker.addWithoutBreaking(-SHALLOW_SIZE); + throw e; + } + } + + static TDigestState create(CircuitBreaker breaker, Type type, double compression) { + breaker.addEstimateBytesAndMaybeBreak(SHALLOW_SIZE, "tdigest-state-create-with-type"); + try { + return new TDigestState(breaker, type, compression); + } catch (Exception e) { + breaker.addWithoutBreaking(-SHALLOW_SIZE); + throw e; + } } /** @@ -80,7 +100,13 @@ public static TDigestState create(CircuitBreaker breaker, double compression) { * @return a TDigestState object that's optimized for performance */ static TDigestState createOptimizedForAccuracy(CircuitBreaker breaker, double compression) { - return new TDigestState(breaker, Type.valueForHighAccuracy(), compression); + breaker.addEstimateBytesAndMaybeBreak(SHALLOW_SIZE, "tdigest-state-create-optimized-for-accuracy"); + try { + return new TDigestState(breaker, Type.valueForHighAccuracy(), compression); + } catch (Exception e) { + breaker.addWithoutBreaking(-SHALLOW_SIZE); + throw e; + } } /** @@ -114,7 +140,13 @@ public static TDigestState create(CircuitBreaker breaker, double compression, TD * @return a TDigestState object */ public static TDigestState createUsingParamsFrom(TDigestState state) { - return new TDigestState(state.breaker, state.type, state.compression); + state.breaker.addEstimateBytesAndMaybeBreak(SHALLOW_SIZE, "tdigest-state-create-using-params-from"); + try { + return new TDigestState(state.breaker, state.type, state.compression); + } catch (Exception e) { + state.breaker.addWithoutBreaking(-SHALLOW_SIZE); + throw e; + } } protected TDigestState(CircuitBreaker breaker, Type type, double compression) { @@ -130,6 +162,11 @@ protected TDigestState(CircuitBreaker breaker, Type type, double compression) { this.compression = compression; } + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + tdigest.ramBytesUsed(); + } + public final double compression() { return compression; } @@ -161,11 +198,17 @@ public static TDigestState read(CircuitBreaker breaker, StreamInput in) throws I double compression = in.readDouble(); TDigestState state; long size = 0; - if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { - state = new TDigestState(breaker, Type.valueOf(in.readString()), compression); - size = in.readVLong(); - } else { - state = new TDigestState(breaker, Type.valueForHighAccuracy(), compression); + breaker.addEstimateBytesAndMaybeBreak(SHALLOW_SIZE, "tdigest-state-read"); + try { + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { + state = new TDigestState(breaker, Type.valueOf(in.readString()), compression); + size = in.readVLong(); + } else { + state = new TDigestState(breaker, Type.valueForHighAccuracy(), compression); + } + } catch (Exception e) { + breaker.addWithoutBreaking(-SHALLOW_SIZE); + throw e; } int n = in.readVInt(); if (size > 0) { @@ -281,6 +324,10 @@ public final double getMax() { @Override public void close() { - Releasables.close(tdigest); + if (closed == false) { + closed = true; + breaker.addWithoutBreaking(-SHALLOW_SIZE); + Releasables.close(tdigest); + } } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestStateReleasingTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestStateReleasingTests.java new file mode 100644 index 0000000000000..176028b7bf3fd --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestStateReleasingTests.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.search.aggregations.metrics; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.test.ESTestCase; + +import java.util.Arrays; + +import static org.hamcrest.Matchers.equalTo; + +public class TDigestStateReleasingTests extends ESTestCase { + @ParametersFactory + public static Iterable parameters() { + return Arrays.stream(TDigestState.Type.values()).map(type -> new Object[] { type }).toList(); + } + + private final TDigestState.Type digestType; + + public TDigestStateReleasingTests(TDigestState.Type digestType) { + this.digestType = digestType; + } + + /** + * Tests that a circuit breaker trip leaves no unreleased memory. + */ + public void testCircuitBreakerTrip() { + for (int bytes = randomIntBetween(0, 16); bytes < 50_000; bytes += 17) { + CircuitBreaker breaker = newLimitedBreaker(ByteSizeValue.ofBytes(bytes)); + + try (TDigestState state = TDigestState.create(breaker, digestType, 100)) { + // Add some data to make it trip. It won't work in all digest types + for (int i = 0; i < 100; i++) { + state.add(randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true)); + } + + // Testing with more memory shouldn't change anything, we finished the test + return; + } catch (CircuitBreakingException e) { + // Expected + } finally { + assertThat("unreleased bytes with a " + bytes + " bytes limit", breaker.getUsed(), equalTo(0L)); + } + } + + fail("Test case didn't reach a non-tripping breaker limit"); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/QuantileStates.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/QuantileStates.java index b6b79bafb08a8..329e798dcb3f0 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/QuantileStates.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/QuantileStates.java @@ -18,7 +18,6 @@ import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.search.aggregations.metrics.InternalMedianAbsoluteDeviation; import org.elasticsearch.search.aggregations.metrics.TDigestState; @@ -235,10 +234,7 @@ Block evaluatePercentile(IntVector selected, DriverContext driverContext) { @Override public void close() { - Releasables.close( - Releasables.wrap(LongStream.range(0, digests.size()).mapToObj(i -> (Releasable) digests.get(i)).toList()), - digests - ); + Releasables.close(Releasables.wrap(LongStream.range(0, digests.size()).mapToObj(i -> digests.get(i)).toList()), digests); } } } From 7d19a16c12291bd3dd58b4808fada57bfa8c5a4c Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 3 Oct 2024 23:04:36 +1000 Subject: [PATCH 033/194] Mute org.elasticsearch.xpack.rank.rrf.RRFRankClientYamlTestSuiteIT test {yaml=rrf/700_rrf_retriever_search_api_compatibility/rrf retriever with top-level collapse} #114019 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 94dc2baaab681..625d210ced883 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -347,6 +347,9 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/113983 - class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT issue: https://github.com/elastic/elasticsearch/issues/114013 +- class: org.elasticsearch.xpack.rank.rrf.RRFRankClientYamlTestSuiteIT + method: test {yaml=rrf/700_rrf_retriever_search_api_compatibility/rrf retriever with top-level collapse} + issue: https://github.com/elastic/elasticsearch/issues/114019 # Examples: # From 7157c4bb483f7df391ee6e1f2834710507472fd6 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Thu, 3 Oct 2024 23:50:30 +1000 Subject: [PATCH 034/194] Mute org.elasticsearch.xpack.inference.TextEmbeddingCrudIT testPutE5WithTrainedModelAndInference #114023 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 625d210ced883..22ecb333a1908 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -350,6 +350,9 @@ tests: - class: org.elasticsearch.xpack.rank.rrf.RRFRankClientYamlTestSuiteIT method: test {yaml=rrf/700_rrf_retriever_search_api_compatibility/rrf retriever with top-level collapse} issue: https://github.com/elastic/elasticsearch/issues/114019 +- class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT + method: testPutE5WithTrainedModelAndInference + issue: https://github.com/elastic/elasticsearch/issues/114023 # Examples: # From 08fb7c0945332a5a2917f7bb9d89e1ea122bc764 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Thu, 3 Oct 2024 10:12:27 -0400 Subject: [PATCH 035/194] Rollback Semantic Query Inner Hits (#113986) --- docs/changelog/111834.yaml | 5 - .../mapper/SemanticTextFieldMapper.java | 14 +- .../queries/SemanticQueryBuilder.java | 50 +-- .../queries/SemanticQueryInnerHitBuilder.java | 132 ------ .../queries/SemanticQueryBuilderTests.java | 42 +- .../test/inference/40_semantic_text_query.yml | 415 ------------------ 6 files changed, 6 insertions(+), 652 deletions(-) delete mode 100644 docs/changelog/111834.yaml delete mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryInnerHitBuilder.java diff --git a/docs/changelog/111834.yaml b/docs/changelog/111834.yaml deleted file mode 100644 index 4548dee5f91e5..0000000000000 --- a/docs/changelog/111834.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111834 -summary: Add inner hits support to semantic query -area: Search -type: enhancement -issues: [] diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index e0ad044f597ab..0483296cd2c6a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -40,7 +40,6 @@ import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; -import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.MatchNoneQueryBuilder; import org.elasticsearch.index.query.NestedQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -55,7 +54,6 @@ import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; -import org.elasticsearch.xpack.inference.queries.SemanticQueryInnerHitBuilder; import java.io.IOException; import java.util.ArrayList; @@ -470,12 +468,7 @@ public boolean fieldHasValue(FieldInfos fieldInfos) { return fieldInfos.fieldInfo(getEmbeddingsFieldName(name())) != null; } - public QueryBuilder semanticQuery( - InferenceResults inferenceResults, - float boost, - String queryName, - SemanticQueryInnerHitBuilder semanticInnerHitBuilder - ) { + public QueryBuilder semanticQuery(InferenceResults inferenceResults, float boost, String queryName) { String nestedFieldPath = getChunksFieldName(name()); String inferenceResultsFieldName = getEmbeddingsFieldName(name()); QueryBuilder childQueryBuilder; @@ -531,10 +524,7 @@ public QueryBuilder semanticQuery( }; } - InnerHitBuilder innerHitBuilder = semanticInnerHitBuilder != null ? semanticInnerHitBuilder.toInnerHitBuilder() : null; - return new NestedQueryBuilder(nestedFieldPath, childQueryBuilder, ScoreMode.Max).boost(boost) - .queryName(queryName) - .innerHit(innerHitBuilder); + return new NestedQueryBuilder(nestedFieldPath, childQueryBuilder, ScoreMode.Max).boost(boost).queryName(queryName); } private String generateQueryInferenceResultsTypeMismatchMessage(InferenceResults inferenceResults, String expectedResultsType) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java index 901de30145f7d..00d44b8e7a0e2 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java @@ -16,7 +16,6 @@ import org.elasticsearch.cluster.metadata.InferenceFieldMetadata; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.Nullable; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.AbstractQueryBuilder; @@ -46,9 +45,7 @@ import java.util.Map; import java.util.Objects; -import static org.elasticsearch.TransportVersions.SEMANTIC_QUERY_INNER_HITS; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; -import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; @@ -59,33 +56,26 @@ public class SemanticQueryBuilder extends AbstractQueryBuilder PARSER = new ConstructingObjectParser<>( NAME, false, - args -> new SemanticQueryBuilder((String) args[0], (String) args[1], (SemanticQueryInnerHitBuilder) args[2]) + args -> new SemanticQueryBuilder((String) args[0], (String) args[1]) ); static { PARSER.declareString(constructorArg(), FIELD_FIELD); PARSER.declareString(constructorArg(), QUERY_FIELD); - PARSER.declareObject(optionalConstructorArg(), (p, c) -> SemanticQueryInnerHitBuilder.fromXContent(p), INNER_HITS_FIELD); declareStandardFields(PARSER); } private final String fieldName; private final String query; - private final SemanticQueryInnerHitBuilder innerHitBuilder; private final SetOnce inferenceResultsSupplier; private final InferenceResults inferenceResults; private final boolean noInferenceResults; public SemanticQueryBuilder(String fieldName, String query) { - this(fieldName, query, null); - } - - public SemanticQueryBuilder(String fieldName, String query, @Nullable SemanticQueryInnerHitBuilder innerHitBuilder) { if (fieldName == null) { throw new IllegalArgumentException("[" + NAME + "] requires a " + FIELD_FIELD.getPreferredName() + " value"); } @@ -94,25 +84,15 @@ public SemanticQueryBuilder(String fieldName, String query, @Nullable SemanticQu } this.fieldName = fieldName; this.query = query; - this.innerHitBuilder = innerHitBuilder; this.inferenceResults = null; this.inferenceResultsSupplier = null; this.noInferenceResults = false; - - if (this.innerHitBuilder != null) { - this.innerHitBuilder.setFieldName(fieldName); - } } public SemanticQueryBuilder(StreamInput in) throws IOException { super(in); this.fieldName = in.readString(); this.query = in.readString(); - if (in.getTransportVersion().onOrAfter(SEMANTIC_QUERY_INNER_HITS)) { - this.innerHitBuilder = in.readOptionalWriteable(SemanticQueryInnerHitBuilder::new); - } else { - this.innerHitBuilder = null; - } this.inferenceResults = in.readOptionalNamedWriteable(InferenceResults.class); this.noInferenceResults = in.readBoolean(); this.inferenceResultsSupplier = null; @@ -125,21 +105,6 @@ protected void doWriteTo(StreamOutput out) throws IOException { } out.writeString(fieldName); out.writeString(query); - if (out.getTransportVersion().onOrAfter(SEMANTIC_QUERY_INNER_HITS)) { - out.writeOptionalWriteable(innerHitBuilder); - } else if (innerHitBuilder != null) { - throw new IllegalStateException( - "Transport version must be at least [" - + SEMANTIC_QUERY_INNER_HITS.toReleaseVersion() - + "] to use [ " - + INNER_HITS_FIELD.getPreferredName() - + "] in [" - + NAME - + "], current transport version is [" - + out.getTransportVersion().toReleaseVersion() - + "]. Are you running a mixed-version cluster?" - ); - } out.writeOptionalNamedWriteable(inferenceResults); out.writeBoolean(noInferenceResults); } @@ -152,7 +117,6 @@ private SemanticQueryBuilder( ) { this.fieldName = other.fieldName; this.query = other.query; - this.innerHitBuilder = other.innerHitBuilder; this.boost = other.boost; this.queryName = other.queryName; this.inferenceResultsSupplier = inferenceResultsSupplier; @@ -160,10 +124,6 @@ private SemanticQueryBuilder( this.noInferenceResults = noInferenceResults; } - public SemanticQueryInnerHitBuilder innerHit() { - return innerHitBuilder; - } - @Override public String getWriteableName() { return NAME; @@ -183,9 +143,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep builder.startObject(NAME); builder.field(FIELD_FIELD.getPreferredName(), fieldName); builder.field(QUERY_FIELD.getPreferredName(), query); - if (innerHitBuilder != null) { - builder.field(INNER_HITS_FIELD.getPreferredName(), innerHitBuilder); - } boostAndQueryNameToXContent(builder); builder.endObject(); } @@ -212,7 +169,7 @@ private QueryBuilder doRewriteBuildSemanticQuery(SearchExecutionContext searchEx ); } - return semanticTextFieldType.semanticQuery(inferenceResults, boost(), queryName(), innerHitBuilder); + return semanticTextFieldType.semanticQuery(inferenceResults, boost(), queryName()); } else { throw new IllegalArgumentException( "Field [" + fieldName + "] of type [" + fieldType.typeName() + "] does not support " + NAME + " queries" @@ -347,12 +304,11 @@ private static String getInferenceIdForForField(Collection indexM protected boolean doEquals(SemanticQueryBuilder other) { return Objects.equals(fieldName, other.fieldName) && Objects.equals(query, other.query) - && Objects.equals(innerHitBuilder, other.innerHitBuilder) && Objects.equals(inferenceResults, other.inferenceResults); } @Override protected int doHashCode() { - return Objects.hash(fieldName, query, innerHitBuilder, inferenceResults); + return Objects.hash(fieldName, query, inferenceResults); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryInnerHitBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryInnerHitBuilder.java deleted file mode 100644 index 776ce990665ac..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryInnerHitBuilder.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.queries; - -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.index.query.InnerHitBuilder; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.FetchSourceContext; -import org.elasticsearch.xcontent.ObjectParser; -import org.elasticsearch.xcontent.ToXContentObject; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.inference.mapper.SemanticTextField; - -import java.io.IOException; -import java.util.Objects; - -import static org.elasticsearch.index.query.InnerHitBuilder.DEFAULT_FROM; -import static org.elasticsearch.index.query.InnerHitBuilder.DEFAULT_SIZE; - -public class SemanticQueryInnerHitBuilder implements Writeable, ToXContentObject { - private static final ObjectParser PARSER = new ObjectParser<>( - "semantic_query_inner_hits", - SemanticQueryInnerHitBuilder::new - ); - - static { - PARSER.declareInt(SemanticQueryInnerHitBuilder::setFrom, SearchSourceBuilder.FROM_FIELD); - PARSER.declareInt(SemanticQueryInnerHitBuilder::setSize, SearchSourceBuilder.SIZE_FIELD); - } - - private String fieldName; - private int from = DEFAULT_FROM; - private int size = DEFAULT_SIZE; - - public SemanticQueryInnerHitBuilder() { - this.fieldName = null; - } - - public SemanticQueryInnerHitBuilder(StreamInput in) throws IOException { - fieldName = in.readOptionalString(); - from = in.readVInt(); - size = in.readVInt(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeOptionalString(fieldName); - out.writeVInt(from); - out.writeVInt(size); - } - - public String getFieldName() { - return fieldName; - } - - public void setFieldName(String fieldName) { - this.fieldName = fieldName; - } - - public int getFrom() { - return from; - } - - public SemanticQueryInnerHitBuilder setFrom(int from) { - this.from = from; - return this; - } - - public int getSize() { - return size; - } - - public SemanticQueryInnerHitBuilder setSize(int size) { - this.size = size; - return this; - } - - public InnerHitBuilder toInnerHitBuilder() { - if (fieldName == null) { - throw new IllegalStateException("fieldName must have a value"); - } - - return new InnerHitBuilder(fieldName).setFrom(from) - .setSize(size) - .setFetchSourceContext(FetchSourceContext.of(true, null, new String[] { SemanticTextField.getEmbeddingsFieldName(fieldName) })); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - // Don't include name in XContent because it is hard-coded - builder.startObject(); - if (from != DEFAULT_FROM) { - builder.field(SearchSourceBuilder.FROM_FIELD.getPreferredName(), from); - } - if (size != DEFAULT_SIZE) { - builder.field(SearchSourceBuilder.SIZE_FIELD.getPreferredName(), size); - } - builder.endObject(); - return builder; - } - - public static SemanticQueryInnerHitBuilder fromXContent(XContentParser parser) throws IOException { - return PARSER.parse(parser, new SemanticQueryInnerHitBuilder(), null); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - SemanticQueryInnerHitBuilder that = (SemanticQueryInnerHitBuilder) o; - return from == that.from && size == that.size && Objects.equals(fieldName, that.fieldName); - } - - @Override - public int hashCode() { - return Objects.hash(fieldName, from, size); - } - - @Override - public String toString() { - return Strings.toString(this, true, true); - } -} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java index 47ac33a5cf9ab..f54ce89183079 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java @@ -31,9 +31,7 @@ import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; -import org.elasticsearch.index.query.InnerHitContextBuilder; import org.elasticsearch.index.query.MatchNoneQueryBuilder; -import org.elasticsearch.index.query.NestedQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.SearchExecutionContext; @@ -64,9 +62,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.HashMap; import java.util.List; -import java.util.Map; import static org.apache.lucene.search.BooleanClause.Occur.FILTER; import static org.apache.lucene.search.BooleanClause.Occur.MUST; @@ -169,14 +165,7 @@ protected SemanticQueryBuilder doCreateTestQueryBuilder() { queryTokens.add(randomAlphaOfLength(QUERY_TOKEN_LENGTH)); } - SemanticQueryInnerHitBuilder innerHitBuilder = null; - if (randomBoolean()) { - innerHitBuilder = new SemanticQueryInnerHitBuilder(); - innerHitBuilder.setFrom(randomIntBetween(0, 100)); - innerHitBuilder.setSize(randomIntBetween(0, 100)); - } - - SemanticQueryBuilder builder = new SemanticQueryBuilder(SEMANTIC_TEXT_FIELD, String.join(" ", queryTokens), innerHitBuilder); + SemanticQueryBuilder builder = new SemanticQueryBuilder(SEMANTIC_TEXT_FIELD, String.join(" ", queryTokens)); if (randomBoolean()) { builder.boost((float) randomDoubleBetween(0.1, 10.0, true)); } @@ -201,21 +190,6 @@ protected void doAssertLuceneQuery(SemanticQueryBuilder queryBuilder, Query quer case SPARSE_EMBEDDING -> assertSparseEmbeddingLuceneQuery(nestedQuery.getChildQuery()); case TEXT_EMBEDDING -> assertTextEmbeddingLuceneQuery(nestedQuery.getChildQuery()); } - - if (queryBuilder.innerHit() != null) { - // Rewrite to a nested query - QueryBuilder rewrittenQueryBuilder = rewriteQuery(queryBuilder, createQueryRewriteContext(), createSearchExecutionContext()); - assertThat(rewrittenQueryBuilder, instanceOf(NestedQueryBuilder.class)); - - NestedQueryBuilder nestedQueryBuilder = (NestedQueryBuilder) rewrittenQueryBuilder; - Map innerHitInternals = new HashMap<>(); - InnerHitContextBuilder.extractInnerHits(nestedQueryBuilder, innerHitInternals); - assertThat(innerHitInternals.size(), equalTo(1)); - - InnerHitContextBuilder innerHits = innerHitInternals.get(queryBuilder.innerHit().getFieldName()); - assertNotNull(innerHits); - assertThat(innerHits.innerHitBuilder(), equalTo(queryBuilder.innerHit().toInnerHitBuilder())); - } } private void assertSparseEmbeddingLuceneQuery(Query query) { @@ -338,20 +312,6 @@ public void testToXContent() throws IOException { "query": "bar" } }""", queryBuilder); - - SemanticQueryInnerHitBuilder innerHitBuilder = new SemanticQueryInnerHitBuilder().setFrom(1).setSize(2); - queryBuilder = new SemanticQueryBuilder("foo", "bar", innerHitBuilder); - checkGeneratedJson(""" - { - "semantic": { - "field": "foo", - "query": "bar", - "inner_hits": { - "from": 1, - "size": 2 - } - } - }""", queryBuilder); } public void testSerializingQueryWhenNoInferenceId() throws IOException { diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/40_semantic_text_query.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/40_semantic_text_query.yml index 4d90d8faeb3f3..2070b3752791a 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/40_semantic_text_query.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/40_semantic_text_query.yml @@ -122,147 +122,6 @@ setup: - close_to: { hits.hits.0._score: { value: 3.7837332e17, error: 1e10 } } - length: { hits.hits.0._source.inference_field.inference.chunks: 2 } ---- -"Query using a sparse embedding model and inner hits": - - requires: - cluster_features: "semantic_text.inner_hits" - reason: semantic_text inner hits support added in 8.16.0 - - - skip: - features: [ "headers", "close_to" ] - - - do: - index: - index: test-sparse-index - id: doc_1 - body: - inference_field: ["inference test", "another inference test", "yet another inference test"] - non_inference_field: "non inference test" - refresh: true - - - do: - headers: - # Force JSON content type so that we use a parser that interprets the floating-point score as a double - Content-Type: application/json - search: - index: test-sparse-index - body: - query: - semantic: - field: "inference_field" - query: "inference test" - inner_hits: {} - - - match: { hits.total.value: 1 } - - match: { hits.hits.0._id: "doc_1" } - - close_to: { hits.hits.0._score: { value: 3.7837332e17, error: 1e10 } } - - length: { hits.hits.0._source.inference_field.inference.chunks: 3 } - - match: { hits.hits.0.inner_hits.inference_field.hits.total.value: 3 } - - length: { hits.hits.0.inner_hits.inference_field.hits.hits: 3 } - - match: { hits.hits.0.inner_hits.inference_field.hits.hits.0._source.text: "another inference test" } - - not_exists: hits.hits.0.inner_hits.inference_field.hits.hits.0._source.embeddings - - match: { hits.hits.0.inner_hits.inference_field.hits.hits.1._source.text: "yet another inference test" } - - not_exists: hits.hits.0.inner_hits.inference_field.hits.hits.1._source.embeddings - - match: { hits.hits.0.inner_hits.inference_field.hits.hits.2._source.text: "inference test" } - - not_exists: hits.hits.0.inner_hits.inference_field.hits.hits.2._source.embeddings - - - do: - headers: - # Force JSON content type so that we use a parser that interprets the floating-point score as a double - Content-Type: application/json - search: - index: test-sparse-index - body: - query: - semantic: - field: "inference_field" - query: "inference test" - inner_hits: { - "size": 1 - } - - - match: { hits.total.value: 1 } - - match: { hits.hits.0._id: "doc_1" } - - close_to: { hits.hits.0._score: { value: 3.7837332e17, error: 1e10 } } - - length: { hits.hits.0._source.inference_field.inference.chunks: 3 } - - match: { hits.hits.0.inner_hits.inference_field.hits.total.value: 3 } - - length: { hits.hits.0.inner_hits.inference_field.hits.hits: 1 } - - match: { hits.hits.0.inner_hits.inference_field.hits.hits.0._source.text: "another inference test" } - - not_exists: hits.hits.0.inner_hits.inference_field.hits.hits.0._source.embeddings - - - do: - headers: - # Force JSON content type so that we use a parser that interprets the floating-point score as a double - Content-Type: application/json - search: - index: test-sparse-index - body: - query: - semantic: - field: "inference_field" - query: "inference test" - inner_hits: { - "from": 1 - } - - - match: { hits.total.value: 1 } - - match: { hits.hits.0._id: "doc_1" } - - close_to: { hits.hits.0._score: { value: 3.7837332e17, error: 1e10 } } - - length: { hits.hits.0._source.inference_field.inference.chunks: 3 } - - match: { hits.hits.0.inner_hits.inference_field.hits.total.value: 3 } - - length: { hits.hits.0.inner_hits.inference_field.hits.hits: 2 } - - match: { hits.hits.0.inner_hits.inference_field.hits.hits.0._source.text: "yet another inference test" } - - not_exists: hits.hits.0.inner_hits.inference_field.hits.hits.0._source.embeddings - - match: { hits.hits.0.inner_hits.inference_field.hits.hits.1._source.text: "inference test" } - - not_exists: hits.hits.0.inner_hits.inference_field.hits.hits.1._source.embeddings - - - do: - headers: - # Force JSON content type so that we use a parser that interprets the floating-point score as a double - Content-Type: application/json - search: - index: test-sparse-index - body: - query: - semantic: - field: "inference_field" - query: "inference test" - inner_hits: { - "from": 1, - "size": 1 - } - - - match: { hits.total.value: 1 } - - match: { hits.hits.0._id: "doc_1" } - - close_to: { hits.hits.0._score: { value: 3.7837332e17, error: 1e10 } } - - length: { hits.hits.0._source.inference_field.inference.chunks: 3 } - - match: { hits.hits.0.inner_hits.inference_field.hits.total.value: 3 } - - length: { hits.hits.0.inner_hits.inference_field.hits.hits: 1 } - - match: { hits.hits.0.inner_hits.inference_field.hits.hits.0._source.text: "yet another inference test" } - - not_exists: hits.hits.0.inner_hits.inference_field.hits.hits.0._source.embeddings - - - do: - headers: - # Force JSON content type so that we use a parser that interprets the floating-point score as a double - Content-Type: application/json - search: - index: test-sparse-index - body: - query: - semantic: - field: "inference_field" - query: "inference test" - inner_hits: { - "from": 3 - } - - - match: { hits.total.value: 1 } - - match: { hits.hits.0._id: "doc_1" } - - close_to: { hits.hits.0._score: { value: 3.7837332e17, error: 1e10 } } - - length: { hits.hits.0._source.inference_field.inference.chunks: 3 } - - match: { hits.hits.0.inner_hits.inference_field.hits.total.value: 0 } # Hits total drops to zero when you page off the end - - length: { hits.hits.0.inner_hits.inference_field.hits.hits: 0 } - --- "Numeric query using a sparse embedding model": - skip: @@ -391,147 +250,6 @@ setup: - close_to: { hits.hits.0._score: { value: 1.0, error: 0.0001 } } - length: { hits.hits.0._source.inference_field.inference.chunks: 2 } ---- -"Query using a dense embedding model and inner hits": - - requires: - cluster_features: "semantic_text.inner_hits" - reason: semantic_text inner hits support added in 8.16.0 - - - skip: - features: [ "headers", "close_to" ] - - - do: - index: - index: test-dense-index - id: doc_1 - body: - inference_field: ["inference test", "another inference test", "yet another inference test"] - non_inference_field: "non inference test" - refresh: true - - - do: - headers: - # Force JSON content type so that we use a parser that interprets the floating-point score as a double - Content-Type: application/json - search: - index: test-dense-index - body: - query: - semantic: - field: "inference_field" - query: "inference test" - inner_hits: {} - - - match: { hits.total.value: 1 } - - match: { hits.hits.0._id: "doc_1" } - - close_to: { hits.hits.0._score: { value: 1.0, error: 0.0001 } } - - length: { hits.hits.0._source.inference_field.inference.chunks: 3 } - - match: { hits.hits.0.inner_hits.inference_field.hits.total.value: 3 } - - length: { hits.hits.0.inner_hits.inference_field.hits.hits: 3 } - - match: { hits.hits.0.inner_hits.inference_field.hits.hits.0._source.text: "inference test" } - - not_exists: hits.hits.0.inner_hits.inference_field.hits.hits.0._source.embeddings - - match: { hits.hits.0.inner_hits.inference_field.hits.hits.1._source.text: "yet another inference test" } - - not_exists: hits.hits.0.inner_hits.inference_field.hits.hits.1._source.embeddings - - match: { hits.hits.0.inner_hits.inference_field.hits.hits.2._source.text: "another inference test" } - - not_exists: hits.hits.0.inner_hits.inference_field.hits.hits.2._source.embeddings - - - do: - headers: - # Force JSON content type so that we use a parser that interprets the floating-point score as a double - Content-Type: application/json - search: - index: test-dense-index - body: - query: - semantic: - field: "inference_field" - query: "inference test" - inner_hits: { - "size": 1 - } - - - match: { hits.total.value: 1 } - - match: { hits.hits.0._id: "doc_1" } - - close_to: { hits.hits.0._score: { value: 1.0, error: 0.0001 } } - - length: { hits.hits.0._source.inference_field.inference.chunks: 3 } - - match: { hits.hits.0.inner_hits.inference_field.hits.total.value: 3 } - - length: { hits.hits.0.inner_hits.inference_field.hits.hits: 1 } - - match: { hits.hits.0.inner_hits.inference_field.hits.hits.0._source.text: "inference test" } - - not_exists: hits.hits.0.inner_hits.inference_field.hits.hits.0._source.embeddings - - - do: - headers: - # Force JSON content type so that we use a parser that interprets the floating-point score as a double - Content-Type: application/json - search: - index: test-dense-index - body: - query: - semantic: - field: "inference_field" - query: "inference test" - inner_hits: { - "from": 1 - } - - - match: { hits.total.value: 1 } - - match: { hits.hits.0._id: "doc_1" } - - close_to: { hits.hits.0._score: { value: 1.0, error: 0.0001 } } - - length: { hits.hits.0._source.inference_field.inference.chunks: 3 } - - match: { hits.hits.0.inner_hits.inference_field.hits.total.value: 3 } - - length: { hits.hits.0.inner_hits.inference_field.hits.hits: 2 } - - match: { hits.hits.0.inner_hits.inference_field.hits.hits.0._source.text: "yet another inference test" } - - not_exists: hits.hits.0.inner_hits.inference_field.hits.hits.0._source.embeddings - - match: { hits.hits.0.inner_hits.inference_field.hits.hits.1._source.text: "another inference test" } - - not_exists: hits.hits.0.inner_hits.inference_field.hits.hits.1._source.embeddings - - - do: - headers: - # Force JSON content type so that we use a parser that interprets the floating-point score as a double - Content-Type: application/json - search: - index: test-dense-index - body: - query: - semantic: - field: "inference_field" - query: "inference test" - inner_hits: { - "from": 1, - "size": 1 - } - - - match: { hits.total.value: 1 } - - match: { hits.hits.0._id: "doc_1" } - - close_to: { hits.hits.0._score: { value: 1.0, error: 0.0001 } } - - length: { hits.hits.0._source.inference_field.inference.chunks: 3 } - - match: { hits.hits.0.inner_hits.inference_field.hits.total.value: 3 } - - length: { hits.hits.0.inner_hits.inference_field.hits.hits: 1 } - - match: { hits.hits.0.inner_hits.inference_field.hits.hits.0._source.text: "yet another inference test" } - - not_exists: hits.hits.0.inner_hits.inference_field.hits.hits.0._source.embeddings - - - do: - headers: - # Force JSON content type so that we use a parser that interprets the floating-point score as a double - Content-Type: application/json - search: - index: test-dense-index - body: - query: - semantic: - field: "inference_field" - query: "inference test" - inner_hits: { - "from": 3 - } - - - match: { hits.total.value: 1 } - - match: { hits.hits.0._id: "doc_1" } - - close_to: { hits.hits.0._score: { value: 1.0, error: 0.0001 } } - - length: { hits.hits.0._source.inference_field.inference.chunks: 3 } - - match: { hits.hits.0.inner_hits.inference_field.hits.total.value: 0 } # Hits total drops to zero when you page off the end - - length: { hits.hits.0.inner_hits.inference_field.hits.hits: 0 } - --- "Numeric query using a dense embedding model": - skip: @@ -760,101 +478,6 @@ setup: - close_to: { hits.hits.0._score: { value: 3.7837332e17, error: 1e10 } } - length: { hits.hits.0._source.inference_field.inference.chunks: 2 } ---- -"Query multiple semantic text fields with inner hits": - - requires: - cluster_features: "semantic_text.inner_hits" - reason: semantic_text inner hits support added in 8.16.0 - - - do: - indices.create: - index: test-multi-semantic-text-field-index - body: - mappings: - properties: - inference_field_1: - type: semantic_text - inference_id: sparse-inference-id - inference_field_2: - type: semantic_text - inference_id: sparse-inference-id - - - do: - index: - index: test-multi-semantic-text-field-index - id: doc_1 - body: - inference_field_1: [ "inference test 1", "another inference test 1" ] - inference_field_2: [ "inference test 2", "another inference test 2", "yet another inference test 2" ] - refresh: true - - - do: - search: - index: test-multi-semantic-text-field-index - body: - query: - bool: - must: - - semantic: - field: "inference_field_1" - query: "inference test" - inner_hits: { } - - semantic: - field: "inference_field_2" - query: "inference test" - inner_hits: { } - - - match: { hits.total.value: 1 } - - match: { hits.hits.0._id: "doc_1" } - - length: { hits.hits.0._source.inference_field_1.inference.chunks: 2 } - - length: { hits.hits.0._source.inference_field_2.inference.chunks: 3 } - - match: { hits.hits.0.inner_hits.inference_field_1.hits.total.value: 2 } - - length: { hits.hits.0.inner_hits.inference_field_1.hits.hits: 2 } - - match: { hits.hits.0.inner_hits.inference_field_2.hits.total.value: 3 } - - length: { hits.hits.0.inner_hits.inference_field_2.hits.hits: 3 } - ---- -"Query semantic text field in object with inner hits": - - requires: - cluster_features: "semantic_text.inner_hits" - reason: semantic_text inner hits support added in 8.16.0 - - - do: - indices.create: - index: test-semantic-text-in-object-index - body: - mappings: - properties: - container: - properties: - inference_field: - type: semantic_text - inference_id: sparse-inference-id - - - do: - index: - index: test-semantic-text-in-object-index - id: doc_1 - body: - container.inference_field: ["inference test", "another inference test", "yet another inference test"] - refresh: true - - - do: - search: - index: test-semantic-text-in-object-index - body: - query: - semantic: - field: "container.inference_field" - query: "inference test" - inner_hits: {} - - - match: { hits.total.value: 1 } - - match: { hits.hits.0._id: "doc_1" } - - exists: hits.hits.0.inner_hits.container\.inference_field - - match: { hits.hits.0.inner_hits.container\.inference_field.hits.total.value: 3 } - - length: { hits.hits.0.inner_hits.container\.inference_field.hits.hits: 3 } - --- "Query the wrong field type": - do: @@ -1216,41 +839,3 @@ setup: - match: { error.type: "resource_not_found_exception" } - match: { error.reason: "Inference endpoint not found [invalid-inference-id]" } - ---- -"Query using inner hits with invalid args": - - requires: - cluster_features: "semantic_text.inner_hits" - reason: semantic_text inner hits support added in 8.16.0 - - - do: - catch: bad_request - search: - index: test-sparse-index - body: - query: - semantic: - field: "inference_field" - query: "inference test" - inner_hits: { - "from": -1 - } - - - match: { error.root_cause.0.type: "illegal_argument_exception" } - - match: { error.root_cause.0.reason: "illegal from value, at least 0 or higher" } - - - do: - catch: bad_request - search: - index: test-sparse-index - body: - query: - semantic: - field: "inference_field" - query: "inference test" - inner_hits: { - "size": -1 - } - - - match: { error.root_cause.0.type: "illegal_argument_exception" } - - match: { error.root_cause.0.reason: "illegal size value, at least 0 or higher" } From 7fd0a666cc2e1dbf23e1c303ae04a9c0b9cb695c Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Thu, 3 Oct 2024 15:17:25 +0100 Subject: [PATCH 036/194] Copy 8.16 CLDR migration notes to main (#113954) Copy from the 8.x branch to main --- .../reference/migration/migrate_8_16.asciidoc | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/reference/migration/migrate_8_16.asciidoc b/docs/reference/migration/migrate_8_16.asciidoc index aea6322f292bf..950b8f7ec3964 100644 --- a/docs/reference/migration/migrate_8_16.asciidoc +++ b/docs/reference/migration/migrate_8_16.asciidoc @@ -16,5 +16,22 @@ coming::[8.16.0] [[breaking-changes-8.16]] === Breaking changes -There are no breaking changes in {es} 8.16. +The following changes in {es} 8.16 might affect your applications +and prevent them from operating normally. +Before upgrading to 8.16, review these changes and take the described steps +to mitigate the impact. +[discrete] +[[breaking_816_locale_change]] +==== JDK locale database change + +{es} 8.16 changes the version of the JDK that is included from version 22 to version 23. This changes +the locale database that is used by Elasticsearch from the _COMPAT_ database to the _CLDR_ database. +This can result in significant changes to custom textual date field formats, +and calculations for custom week-date date fields. + +For more information see <>. + +If you run {es} 8.16 on JDK version 22 or below, it will use the _COMPAT_ locale database +to match the behavior of 8.15. However, please note that starting with {es} 9.0, +{es} will use the _CLDR_ database regardless of JDK version it is run on. From fa06764bad714f73989b05a1deb6cdf6c55cd040 Mon Sep 17 00:00:00 2001 From: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com> Date: Thu, 3 Oct 2024 10:36:25 -0400 Subject: [PATCH 037/194] [Inference API] Deprecate elser service (#113216) * merging * copy elser service files into elasticsearch service * Add deprecation log message for elser service * improve deprecation warning * change elasticsearch internal service elser case to use elser model * switch elasticsearch elser tests to use elasticsearch elser * Update docs/changelog/113216.yaml * alias elser service to elasticsearch * delete elser service package now that elasticsearch service supports it and has aliased it * Add deprecation warning to infer API for elser * Fix accidentally introduced NPE and retain BWC support for null model ID (with deprecation message) * change "area" to "REST API" because "Machine Learning" isn't an option for deprecation * change elser literals to static variable * change Put and Elasticsearch Internal service to pass the service name if it is elser or elasticsearch this will allow the elasticsearch service to maintain BWC for null model IDs if the service was elser. * fix up tests to match new elasticsearch service semantics regarding elser. * Move passing of service name * add persistence for elser models in elasticsearch * copy elser service files into elasticsearch service * Add deprecation log message for elser service * Add deprecation warning to infer API for elser * fix merge conflicts * fix merge --- docs/changelog/113216.yaml | 10 + .../inference/InferenceServiceRegistry.java | 8 +- .../integration/ModelRegistryIT.java | 27 +- .../InferenceNamedWriteablesProvider.java | 4 +- .../xpack/inference/InferencePlugin.java | 2 - .../action/TransportInferenceAction.java | 2 + .../TransportPutInferenceModelAction.java | 7 + ...InferenceServiceSparseEmbeddingsModel.java | 2 +- ...erviceSparseEmbeddingsServiceSettings.java | 2 +- .../BaseElasticsearchInternalService.java | 1 - .../ElasticsearchInternalService.java | 126 +++- .../ElasticsearchInternalServiceSettings.java | 2 +- .../ElserInternalModel.java | 3 +- .../ElserInternalServiceSettings.java | 5 +- .../ElserMlNodeTaskSettings.java | 2 +- .../{elser => elasticsearch}/ElserModels.java | 4 +- .../services/elser/ElserInternalService.java | 300 ---------- .../inference/ModelConfigurationsTests.java | 4 +- ...enceServiceSparseEmbeddingsModelTests.java | 2 +- ...eSparseEmbeddingsServiceSettingsTests.java | 4 +- .../elastic/ElasticInferenceServiceTests.java | 2 +- ...ticsearchInternalServiceSettingsTests.java | 1 - .../ElasticsearchInternalServiceTests.java | 160 ++++- .../ElserInternalServiceSettingsTests.java | 6 +- .../ElserMlNodeTaskSettingsTests.java | 2 +- .../elasticsearch/ElserModelsTests.java | 39 ++ .../elser/ElserInternalServiceTests.java | 548 ------------------ .../services/elser/ElserModelsTests.java | 33 -- 28 files changed, 371 insertions(+), 937 deletions(-) create mode 100644 docs/changelog/113216.yaml rename x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/{elser => elasticsearch}/ElserInternalModel.java (93%) rename x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/{elser => elasticsearch}/ElserInternalServiceSettings.java (89%) rename x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/{elser => elasticsearch}/ElserMlNodeTaskSettings.java (96%) rename x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/{elser => elasticsearch}/ElserModels.java (87%) delete mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java rename x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/{elser => elasticsearch}/ElserInternalServiceSettingsTests.java (89%) rename x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/{elser => elasticsearch}/ElserMlNodeTaskSettingsTests.java (93%) create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserModelsTests.java delete mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java delete mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserModelsTests.java diff --git a/docs/changelog/113216.yaml b/docs/changelog/113216.yaml new file mode 100644 index 0000000000000..dec0b991fdacf --- /dev/null +++ b/docs/changelog/113216.yaml @@ -0,0 +1,10 @@ +pr: 113216 +summary: "[Inference API] Deprecate elser service" +area: Machine Learning +type: deprecation +issues: [] +deprecation: + title: "[Inference API] Deprecate elser service" + area: REST API + details: The `elser` service of the inference API will be removed in an upcoming release. Please use the elasticsearch service instead. + impact: In the current version there is no impact. In a future version, users of the `elser` service will no longer be able to use it, and will be required to use the `elasticsearch` service to access elser through the inference API. diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceServiceRegistry.java b/server/src/main/java/org/elasticsearch/inference/InferenceServiceRegistry.java index 40b4e37f36509..f1ce94173a550 100644 --- a/server/src/main/java/org/elasticsearch/inference/InferenceServiceRegistry.java +++ b/server/src/main/java/org/elasticsearch/inference/InferenceServiceRegistry.java @@ -46,7 +46,13 @@ public Map getServices() { } public Optional getService(String serviceName) { - return Optional.ofNullable(services.get(serviceName)); + + if ("elser".equals(serviceName)) { // ElserService.NAME before removal + // here we are aliasing the elser service to use the elasticsearch service instead + return Optional.ofNullable(services.get("elasticsearch")); // ElasticsearchInternalService.NAME + } else { + return Optional.ofNullable(services.get(serviceName)); + } } public List getNamedWriteables() { diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java index 524cd5014c19e..ea8b32f36f54c 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java @@ -28,11 +28,10 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.inference.InferencePlugin; import org.elasticsearch.xpack.inference.registry.ModelRegistry; -import org.elasticsearch.xpack.inference.services.elser.ElserInternalModel; -import org.elasticsearch.xpack.inference.services.elser.ElserInternalService; -import org.elasticsearch.xpack.inference.services.elser.ElserInternalServiceSettingsTests; -import org.elasticsearch.xpack.inference.services.elser.ElserInternalServiceTests; -import org.elasticsearch.xpack.inference.services.elser.ElserMlNodeTaskSettingsTests; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalModel; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElserInternalServiceSettingsTests; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElserMlNodeTaskSettingsTests; import org.junit.Before; import java.io.IOException; @@ -118,10 +117,10 @@ public void testGetModel() throws Exception { assertEquals(model.getConfigurations().getService(), modelHolder.get().service()); - var elserService = new ElserInternalService( + var elserService = new ElasticsearchInternalService( new InferenceServiceExtension.InferenceServiceFactoryContext(mock(Client.class), mock(ThreadPool.class)) ); - ElserInternalModel roundTripModel = elserService.parsePersistedConfigWithSecrets( + ElasticsearchInternalModel roundTripModel = (ElasticsearchInternalModel) elserService.parsePersistedConfigWithSecrets( modelHolder.get().inferenceEntityId(), modelHolder.get().taskType(), modelHolder.get().settings(), @@ -277,7 +276,17 @@ public void testGetModelWithSecrets() throws InterruptedException { } private Model buildElserModelConfig(String inferenceEntityId, TaskType taskType) { - return ElserInternalServiceTests.randomModelConfig(inferenceEntityId, taskType); + return switch (taskType) { + case SPARSE_EMBEDDING -> new org.elasticsearch.xpack.inference.services.elasticsearch.ElserInternalModel( + inferenceEntityId, + taskType, + ElasticsearchInternalService.NAME, + ElserInternalServiceSettingsTests.createRandom(), + ElserMlNodeTaskSettingsTests.createRandom() + ); + default -> throw new IllegalArgumentException("task type " + taskType + " is not supported"); + }; + } protected void blockingCall(Consumer> function, AtomicReference response, AtomicReference error) @@ -300,7 +309,7 @@ private static Model buildModelWithUnknownField(String inferenceEntityId) { new ModelWithUnknownField( inferenceEntityId, TaskType.SPARSE_EMBEDDING, - ElserInternalService.NAME, + ElasticsearchInternalService.NAME, ElserInternalServiceSettingsTests.createRandom(), ElserMlNodeTaskSettingsTests.createRandom() ) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java index 336626cd1db20..02bddb6076d69 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceNamedWriteablesProvider.java @@ -64,9 +64,9 @@ import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandInternalTextEmbeddingServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.CustomElandRerankTaskSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElserInternalServiceSettings; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElserMlNodeTaskSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.MultilingualE5SmallInternalServiceSettings; -import org.elasticsearch.xpack.inference.services.elser.ElserInternalServiceSettings; -import org.elasticsearch.xpack.inference.services.elser.ElserMlNodeTaskSettings; import org.elasticsearch.xpack.inference.services.googleaistudio.completion.GoogleAiStudioCompletionServiceSettings; import org.elasticsearch.xpack.inference.services.googleaistudio.embeddings.GoogleAiStudioEmbeddingsServiceSettings; import org.elasticsearch.xpack.inference.services.googlevertexai.GoogleVertexAiSecretSettings; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index f2f019490444e..0ab395f4bfa39 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -86,7 +86,6 @@ import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceFeature; import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceServiceSettings; import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService; -import org.elasticsearch.xpack.inference.services.elser.ElserInternalService; import org.elasticsearch.xpack.inference.services.googleaistudio.GoogleAiStudioService; import org.elasticsearch.xpack.inference.services.googlevertexai.GoogleVertexAiService; import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceService; @@ -229,7 +228,6 @@ public void loadExtensions(ExtensionLoader loader) { public List getInferenceServiceFactories() { return List.of( - ElserInternalService::new, context -> new HuggingFaceElserService(httpFactory.get(), serviceComponents.get()), context -> new HuggingFaceService(httpFactory.get(), serviceComponents.get()), context -> new OpenAiService(httpFactory.get(), serviceComponents.get()), diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java index 4186b281a35b5..d2a73b7df77c1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.inference.InferenceService; @@ -42,6 +43,7 @@ public class TransportInferenceAction extends HandledTransportAction serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); Map taskSettingsMap = removeFromMap(config, ModelConfigurations.TASK_SETTINGS); + String serviceName = (String) config.remove(ModelConfigurations.SERVICE); // required for elser service in elasticsearch service throwIfNotEmptyMap(config, name()); String modelId = (String) serviceSettingsMap.get(ElasticsearchInternalServiceSettings.MODEL_ID); if (modelId == null) { - throw new ValidationException().addValidationError("Error parsing request config, model id is missing"); - } - if (MULTILINGUAL_E5_SMALL_VALID_IDS.contains(modelId)) { + if (OLD_ELSER_SERVICE_NAME.equals(serviceName)) { + // TODO complete deprecation of null model ID + // throw new ValidationException().addValidationError("Error parsing request config, model id is missing"); + DEPRECATION_LOGGER.critical( + DeprecationCategory.API, + "inference_api_null_model_id_in_elasticsearch_service", + "Putting elasticsearch service inference endpoints (including elser service) without a model_id field is" + + " deprecated and will be removed in a future release. Please specify a model_id field." + ); + platformArch.accept( + modelListener.delegateFailureAndWrap( + (delegate, arch) -> elserCase(inferenceEntityId, taskType, config, arch, serviceSettingsMap, modelListener) + ) + ); + } else { + throw new IllegalArgumentException("Error parsing service settings, model_id must be provided"); + } + } else if (MULTILINGUAL_E5_SMALL_VALID_IDS.contains(modelId)) { platformArch.accept( modelListener.delegateFailureAndWrap( (delegate, arch) -> e5Case(inferenceEntityId, taskType, config, arch, serviceSettingsMap, modelListener) ) ); + } else if (ElserModels.isValidModel(modelId)) { + platformArch.accept( + modelListener.delegateFailureAndWrap( + (delegate, arch) -> elserCase(inferenceEntityId, taskType, config, arch, serviceSettingsMap, modelListener) + ) + ); } else { customElandCase(inferenceEntityId, taskType, serviceSettingsMap, taskSettingsMap, modelListener); } @@ -239,7 +270,86 @@ static boolean modelVariantValidForArchitecture(Set platformArchitecture // platform agnostic model is always compatible return true; } + return modelId.equals( + selectDefaultModelVariantBasedOnClusterArchitecture( + platformArchitectures, + MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86, + MULTILINGUAL_E5_SMALL_MODEL_ID + ) + ); + } + private void elserCase( + String inferenceEntityId, + TaskType taskType, + Map config, + Set platformArchitectures, + Map serviceSettingsMap, + ActionListener modelListener + ) { + var esServiceSettingsBuilder = ElasticsearchInternalServiceSettings.fromRequestMap(serviceSettingsMap); + final String defaultModelId = selectDefaultModelVariantBasedOnClusterArchitecture( + platformArchitectures, + ELSER_V2_MODEL_LINUX_X86, + ELSER_V2_MODEL + ); + if (false == defaultModelId.equals(esServiceSettingsBuilder.getModelId())) { + + if (esServiceSettingsBuilder.getModelId() == null) { + // TODO remove this case once we remove the option to not pass model ID + esServiceSettingsBuilder.setModelId(defaultModelId); + } else if (esServiceSettingsBuilder.getModelId().equals(ELSER_V2_MODEL)) { + logger.warn( + "The platform agnostic model [{}] was requested on Linux x86_64. " + + "It is recommended to use the optimized model instead [{}]", + ELSER_V2_MODEL, + ELSER_V2_MODEL_LINUX_X86 + ); + } else { + throw new IllegalArgumentException( + "Error parsing request config, model id does not match any models available on this platform. Was [" + + esServiceSettingsBuilder.getModelId() + + "]. You may need to use a platform agnostic model." + ); + } + } + + DEPRECATION_LOGGER.warn( + DeprecationCategory.API, + "inference_api_elser_service", + "The [{}] service is deprecated and will be removed in a future release. Use the [{}] service instead, with" + + " [model_id] set to [{}] in the [service_settings]", + OLD_ELSER_SERVICE_NAME, + ElasticsearchInternalService.NAME, + defaultModelId + ); + + if (modelVariantDoesNotMatchArchitecturesAndIsNotPlatformAgnostic(platformArchitectures, esServiceSettingsBuilder.getModelId())) { + throw new IllegalArgumentException( + "Error parsing request config, model id does not match any models available on this platform. Was [" + + esServiceSettingsBuilder.getModelId() + + "]" + ); + } + + throwIfNotEmptyMap(config, name()); + throwIfNotEmptyMap(serviceSettingsMap, name()); + + modelListener.onResponse( + new ElserInternalModel( + inferenceEntityId, + taskType, + NAME, + new ElserInternalServiceSettings(esServiceSettingsBuilder.build()), + ElserMlNodeTaskSettings.DEFAULT + ) + ); + } + + private static boolean modelVariantDoesNotMatchArchitecturesAndIsNotPlatformAgnostic( + Set platformArchitectures, + String modelId + ) { return modelId.equals( selectDefaultModelVariantBasedOnClusterArchitecture( platformArchitectures, @@ -276,6 +386,14 @@ public Model parsePersistedConfig(String inferenceEntityId, TaskType taskType, M NAME, new MultilingualE5SmallInternalServiceSettings(ElasticsearchInternalServiceSettings.fromPersistedMap(serviceSettingsMap)) ); + } else if (ElserModels.isValidModel(modelId)) { + return new ElserInternalModel( + inferenceEntityId, + taskType, + NAME, + new ElserInternalServiceSettings(ElasticsearchInternalServiceSettings.fromPersistedMap(serviceSettingsMap)), + ElserMlNodeTaskSettings.DEFAULT + ); } else { return createCustomElandModel( inferenceEntityId, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java index 1acf19c5373b7..f8b5837ef387e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java @@ -83,7 +83,7 @@ protected static ElasticsearchInternalServiceSettings.Builder fromMap( validationException ); - // model id is optional as the ELSER and E5 service will default it + // model id is optional as the ELSER service will default it. TODO make this a required field once the elser service is removed String modelId = extractOptionalString(map, MODEL_ID, ModelConfigurations.SERVICE_SETTINGS, validationException); if (numAllocations == null && adaptiveAllocationsSettings == null) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalModel.java similarity index 93% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalModel.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalModel.java index bb668c314649d..827eb178f7633 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalModel.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.inference.services.elser; +package org.elasticsearch.xpack.inference.services.elasticsearch; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; @@ -13,7 +13,6 @@ import org.elasticsearch.inference.TaskType; import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalModel; public class ElserInternalModel extends ElasticsearchInternalModel { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalServiceSettings.java similarity index 89% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettings.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalServiceSettings.java index fcbabd5a88fc6..f7bcd95c8bd28 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalServiceSettings.java @@ -5,14 +5,13 @@ * 2.0. */ -package org.elasticsearch.xpack.inference.services.elser; +package org.elasticsearch.xpack.inference.services.elasticsearch; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; -import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings; import java.io.IOException; import java.util.Arrays; @@ -22,7 +21,7 @@ public class ElserInternalServiceSettings extends ElasticsearchInternalServiceSe public static final String NAME = "elser_mlnode_service_settings"; - public static ElasticsearchInternalServiceSettings.Builder fromRequestMap(Map map) { + public static Builder fromRequestMap(Map map) { ValidationException validationException = new ValidationException(); var baseSettings = ElasticsearchInternalServiceSettings.fromMap(map, validationException); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserMlNodeTaskSettings.java similarity index 96% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeTaskSettings.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserMlNodeTaskSettings.java index 9b9f6e41113e5..934edaa96a15c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserMlNodeTaskSettings.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.inference.services.elser; +package org.elasticsearch.xpack.inference.services.elasticsearch; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserModels.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserModels.java similarity index 87% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserModels.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserModels.java index af94d2813dd2c..37f528ea3a750 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserModels.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserModels.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.inference.services.elser; +package org.elasticsearch.xpack.inference.services.elasticsearch; import java.util.Set; @@ -23,7 +23,7 @@ public class ElserModels { ); public static boolean isValidModel(String model) { - return VALID_ELSER_MODEL_IDS.contains(model); + return model != null && VALID_ELSER_MODEL_IDS.contains(model); } public static boolean isValidEisModel(String model) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java deleted file mode 100644 index d36b8eca7661e..0000000000000 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java +++ /dev/null @@ -1,300 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - * - * this file has been contributed to by a Generative AI - */ - -package org.elasticsearch.xpack.inference.services.elser; - -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.inference.ChunkedInferenceServiceResults; -import org.elasticsearch.inference.ChunkingOptions; -import org.elasticsearch.inference.InferenceResults; -import org.elasticsearch.inference.InferenceServiceExtension; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.inference.InputType; -import org.elasticsearch.inference.Model; -import org.elasticsearch.inference.ModelConfigurations; -import org.elasticsearch.inference.TaskType; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; -import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; -import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; -import org.elasticsearch.xpack.core.ml.action.InferModelAction; -import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; -import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults; -import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextExpansionConfigUpdate; -import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate; -import org.elasticsearch.xpack.inference.services.ServiceUtils; -import org.elasticsearch.xpack.inference.services.elasticsearch.BaseElasticsearchInternalService; - -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Consumer; - -import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; -import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; -import static org.elasticsearch.xpack.inference.services.elser.ElserModels.ELSER_V2_MODEL; -import static org.elasticsearch.xpack.inference.services.elser.ElserModels.ELSER_V2_MODEL_LINUX_X86; - -public class ElserInternalService extends BaseElasticsearchInternalService { - - public static final String NAME = "elser"; - - private static final String OLD_MODEL_ID_FIELD_NAME = "model_version"; - - public ElserInternalService(InferenceServiceExtension.InferenceServiceFactoryContext context) { - super(context); - } - - // for testing - ElserInternalService( - InferenceServiceExtension.InferenceServiceFactoryContext context, - Consumer>> platformArch - ) { - super(context, platformArch); - } - - @Override - protected EnumSet supportedTaskTypes() { - return EnumSet.of(TaskType.SPARSE_EMBEDDING); - } - - @Override - public void parseRequestConfig( - String inferenceEntityId, - TaskType taskType, - Map config, - ActionListener parsedModelListener - ) { - try { - Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); - var serviceSettingsBuilder = ElserInternalServiceSettings.fromRequestMap(serviceSettingsMap); - - Map taskSettingsMap; - // task settings are optional - if (config.containsKey(ModelConfigurations.TASK_SETTINGS)) { - taskSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.TASK_SETTINGS); - } else { - taskSettingsMap = Map.of(); - } - - var taskSettings = taskSettingsFromMap(taskType, taskSettingsMap); - - throwIfNotEmptyMap(config, NAME); - throwIfNotEmptyMap(serviceSettingsMap, NAME); - throwIfNotEmptyMap(taskSettingsMap, NAME); - - if (serviceSettingsBuilder.getModelId() == null) { - platformArch.accept(parsedModelListener.delegateFailureAndWrap((delegate, arch) -> { - serviceSettingsBuilder.setModelId( - selectDefaultModelVariantBasedOnClusterArchitecture(arch, ELSER_V2_MODEL_LINUX_X86, ELSER_V2_MODEL) - ); - parsedModelListener.onResponse( - new ElserInternalModel( - inferenceEntityId, - taskType, - NAME, - new ElserInternalServiceSettings(serviceSettingsBuilder.build()), - taskSettings - ) - ); - })); - } else { - parsedModelListener.onResponse( - new ElserInternalModel( - inferenceEntityId, - taskType, - NAME, - new ElserInternalServiceSettings(serviceSettingsBuilder.build()), - taskSettings - ) - ); - } - } catch (Exception e) { - parsedModelListener.onFailure(e); - } - } - - @Override - public ElserInternalModel parsePersistedConfigWithSecrets( - String inferenceEntityId, - TaskType taskType, - Map config, - Map secrets - ) { - return parsePersistedConfig(inferenceEntityId, taskType, config); - } - - @Override - public ElserInternalModel parsePersistedConfig(String inferenceEntityId, TaskType taskType, Map config) { - Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); - - // Change from old model_version field name to new model_id field name as of - // TransportVersions.ML_TEXT_EMBEDDING_INFERENCE_SERVICE_ADDED - if (serviceSettingsMap.containsKey(OLD_MODEL_ID_FIELD_NAME)) { - String modelId = ServiceUtils.removeAsType(serviceSettingsMap, OLD_MODEL_ID_FIELD_NAME, String.class); - serviceSettingsMap.put(ElserInternalServiceSettings.MODEL_ID, modelId); - } - - var serviceSettings = ElserInternalServiceSettings.fromPersistedMap(serviceSettingsMap); - - Map taskSettingsMap; - // task settings are optional - if (config.containsKey(ModelConfigurations.TASK_SETTINGS)) { - taskSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.TASK_SETTINGS); - } else { - taskSettingsMap = Map.of(); - } - - var taskSettings = taskSettingsFromMap(taskType, taskSettingsMap); - - return new ElserInternalModel(inferenceEntityId, taskType, NAME, new ElserInternalServiceSettings(serviceSettings), taskSettings); - } - - @Override - public void infer( - Model model, - @Nullable String query, - List inputs, - boolean stream, - Map taskSettings, - InputType inputType, - TimeValue timeout, - ActionListener listener - ) { - // No task settings to override with requestTaskSettings - - try { - checkCompatibleTaskType(model.getConfigurations().getTaskType()); - } catch (Exception e) { - listener.onFailure(e); - return; - } - - var request = buildInferenceRequest( - model.getConfigurations().getInferenceEntityId(), - TextExpansionConfigUpdate.EMPTY_UPDATE, - inputs, - inputType, - timeout, - false // chunk - ); - - client.execute( - InferModelAction.INSTANCE, - request, - listener.delegateFailureAndWrap( - (l, inferenceResult) -> l.onResponse(SparseEmbeddingResults.of(inferenceResult.getInferenceResults())) - ) - ); - } - - public void chunkedInfer( - Model model, - List input, - Map taskSettings, - InputType inputType, - @Nullable ChunkingOptions chunkingOptions, - TimeValue timeout, - ActionListener> listener - ) { - chunkedInfer(model, null, input, taskSettings, inputType, chunkingOptions, timeout, listener); - } - - @Override - public void chunkedInfer( - Model model, - @Nullable String query, - List inputs, - Map taskSettings, - InputType inputType, - @Nullable ChunkingOptions chunkingOptions, - TimeValue timeout, - ActionListener> listener - ) { - try { - checkCompatibleTaskType(model.getConfigurations().getTaskType()); - } catch (Exception e) { - listener.onFailure(e); - return; - } - - var configUpdate = chunkingOptions != null - ? new TokenizationConfigUpdate(chunkingOptions.windowSize(), chunkingOptions.span()) - : new TokenizationConfigUpdate(null, null); - - var request = buildInferenceRequest( - model.getConfigurations().getInferenceEntityId(), - configUpdate, - inputs, - inputType, - timeout, - true // chunk - ); - - client.execute( - InferModelAction.INSTANCE, - request, - listener.delegateFailureAndWrap( - (l, inferenceResult) -> l.onResponse(translateChunkedResults(inferenceResult.getInferenceResults())) - ) - ); - } - - private void checkCompatibleTaskType(TaskType taskType) { - if (TaskType.SPARSE_EMBEDDING.isAnyOrSame(taskType) == false) { - throw new ElasticsearchStatusException(TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), RestStatus.BAD_REQUEST); - } - } - - private static ElserMlNodeTaskSettings taskSettingsFromMap(TaskType taskType, Map config) { - if (taskType != TaskType.SPARSE_EMBEDDING) { - throw new ElasticsearchStatusException(TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), RestStatus.BAD_REQUEST); - } - - // no config options yet - return ElserMlNodeTaskSettings.DEFAULT; - } - - private List translateChunkedResults(List inferenceResults) { - var translated = new ArrayList(); - - for (var inferenceResult : inferenceResults) { - if (inferenceResult instanceof MlChunkedTextExpansionResults mlChunkedResult) { - translated.add(InferenceChunkedSparseEmbeddingResults.ofMlResult(mlChunkedResult)); - } else if (inferenceResult instanceof ErrorInferenceResults error) { - translated.add(new ErrorChunkedInferenceResults(error.getException())); - } else { - throw new ElasticsearchStatusException( - "Expected a chunked inference [{}] received [{}]", - RestStatus.INTERNAL_SERVER_ERROR, - MlChunkedTextExpansionResults.NAME, - inferenceResult.getWriteableName() - ); - } - } - return translated; - } - - @Override - public String name() { - return NAME; - } - - @Override - public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_8_12_0; - } -} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelConfigurationsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelConfigurationsTests.java index 5a1922fd200f5..03613901c7816 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelConfigurationsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelConfigurationsTests.java @@ -16,8 +16,8 @@ import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xpack.core.inference.ChunkingSettingsFeatureFlag; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests; -import org.elasticsearch.xpack.inference.services.elser.ElserInternalServiceSettingsTests; -import org.elasticsearch.xpack.inference.services.elser.ElserMlNodeTaskSettings; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElserInternalServiceSettingsTests; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElserMlNodeTaskSettings; public class ModelConfigurationsTests extends AbstractWireSerializingTestCase { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java index af13ce7944685..c9f4234331221 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsModelTests.java @@ -11,7 +11,7 @@ import org.elasticsearch.inference.EmptyTaskSettings; import org.elasticsearch.inference.TaskType; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.inference.services.elser.ElserModels; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels; public class ElasticInferenceServiceSparseEmbeddingsModelTests extends ESTestCase { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests.java index a2b36cf9abdd5..1751e1c3be5e8 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceSparseEmbeddingsServiceSettingsTests.java @@ -16,13 +16,13 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.ServiceFields; -import org.elasticsearch.xpack.inference.services.elser.ElserModels; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels; import java.io.IOException; import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.xpack.inference.services.elser.ElserModelsTests.randomElserModel; +import static org.elasticsearch.xpack.inference.services.elasticsearch.ElserModelsTests.randomElserModel; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java index ab85e112418f5..d10c70c6f0f5e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceServiceTests.java @@ -36,7 +36,7 @@ import org.elasticsearch.xpack.inference.logging.ThrottlerManager; import org.elasticsearch.xpack.inference.results.SparseEmbeddingResultsTests; import org.elasticsearch.xpack.inference.services.ServiceFields; -import org.elasticsearch.xpack.inference.services.elser.ElserModels; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels; import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; import org.junit.After; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettingsTests.java index 41afef88d22c6..419db748d793d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettingsTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xpack.core.ml.inference.assignment.AdaptiveAllocationsSettings; -import org.elasticsearch.xpack.inference.services.elser.ElserInternalServiceSettings; import java.io.IOException; import java.util.HashMap; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java index de9298f1b08dd..cd6da4c0ad8d8 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java @@ -9,10 +9,12 @@ package org.elasticsearch.xpack.inference.services.elasticsearch; +import org.apache.logging.log4j.Level; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; @@ -69,6 +71,8 @@ import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID; import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86; +import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.NAME; +import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.OLD_ELSER_SERVICE_NAME; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; @@ -97,9 +101,11 @@ public void shutdownThreadPool() { } public void testParseRequestConfig() { + // Null model variant var service = createService(mock(Client.class)); - var settings = new HashMap(); - settings.put( + var config = new HashMap(); + config.put(ModelConfigurations.SERVICE, ElasticsearchInternalService.NAME); + config.put( ModelConfigurations.SERVICE_SETTINGS, new HashMap<>( Map.of(ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS, 1, ElasticsearchInternalServiceSettings.NUM_THREADS, 4) @@ -112,15 +118,16 @@ public void testParseRequestConfig() { ); var taskType = randomFrom(TaskType.TEXT_EMBEDDING, TaskType.RERANK, TaskType.SPARSE_EMBEDDING); - service.parseRequestConfig(randomInferenceEntityId, taskType, settings, modelListener); + service.parseRequestConfig(randomInferenceEntityId, taskType, config, modelListener); } public void testParseRequestConfig_Misconfigured() { - // Null model variant + // Non-existent model variant { var service = createService(mock(Client.class)); - var settings = new HashMap(); - settings.put( + var config = new HashMap(); + config.put(ModelConfigurations.SERVICE, ElasticsearchInternalService.NAME); + config.put( ModelConfigurations.SERVICE_SETTINGS, new HashMap<>( Map.of(ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS, 1, ElasticsearchInternalServiceSettings.NUM_THREADS, 4) @@ -133,20 +140,21 @@ public void testParseRequestConfig_Misconfigured() { ); var taskType = randomFrom(TaskType.TEXT_EMBEDDING, TaskType.RERANK, TaskType.SPARSE_EMBEDDING); - service.parseRequestConfig(randomInferenceEntityId, taskType, settings, modelListener); + service.parseRequestConfig(randomInferenceEntityId, taskType, config, modelListener); } // Invalid config map { var service = createService(mock(Client.class)); - var settings = new HashMap(); - settings.put( + var config = new HashMap(); + config.put(ModelConfigurations.SERVICE, ElasticsearchInternalService.NAME); + config.put( ModelConfigurations.SERVICE_SETTINGS, new HashMap<>( Map.of(ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS, 1, ElasticsearchInternalServiceSettings.NUM_THREADS, 4) ) ); - settings.put("not_a_valid_config_setting", randomAlphaOfLength(10)); + config.put("not_a_valid_config_setting", randomAlphaOfLength(10)); ActionListener modelListener = ActionListener.wrap( model -> fail("Model parsing should have failed"), @@ -154,7 +162,7 @@ public void testParseRequestConfig_Misconfigured() { ); var taskType = randomFrom(TaskType.TEXT_EMBEDDING, TaskType.RERANK, TaskType.SPARSE_EMBEDDING); - service.parseRequestConfig(randomInferenceEntityId, taskType, settings, modelListener); + service.parseRequestConfig(randomInferenceEntityId, taskType, config, modelListener); } } @@ -182,7 +190,7 @@ public void testParseRequestConfig_E5() { randomInferenceEntityId, TaskType.TEXT_EMBEDDING, settings, - getModelVerificationActionListener(e5ServiceSettings) + getE5ModelVerificationActionListener(e5ServiceSettings) ); } @@ -214,7 +222,7 @@ public void testParseRequestConfig_E5() { randomInferenceEntityId, TaskType.TEXT_EMBEDDING, settings, - getModelVerificationActionListener(e5ServiceSettings) + getE5ModelVerificationActionListener(e5ServiceSettings) ); } @@ -247,6 +255,106 @@ public void testParseRequestConfig_E5() { } } + public void testParseRequestConfig_elser() { + // General happy case + { + Client mockClient = mock(Client.class); + when(mockClient.threadPool()).thenReturn(threadPool); + var service = createService(mockClient); + var config = new HashMap(); + config.put(ModelConfigurations.SERVICE, OLD_ELSER_SERVICE_NAME); + config.put( + ModelConfigurations.SERVICE_SETTINGS, + new HashMap<>( + Map.of( + ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS, + 1, + ElasticsearchInternalServiceSettings.NUM_THREADS, + 4, + ElasticsearchInternalServiceSettings.MODEL_ID, + ElserModels.ELSER_V2_MODEL + ) + ) + ); + + var elserServiceSettings = new ElserInternalServiceSettings(1, 4, ElserModels.ELSER_V2_MODEL, null); + + service.parseRequestConfig( + randomInferenceEntityId, + TaskType.SPARSE_EMBEDDING, + config, + getElserModelVerificationActionListener( + elserServiceSettings, + null, + "The [elser] service is deprecated and will be removed in a future release. Use the [elasticsearch] service " + + "instead, with [model_id] set to [.elser_model_2] in the [service_settings]" + ) + ); + } + + // null model ID returns elser model for the provided platform (not linux) + { + Client mockClient = mock(Client.class); + when(mockClient.threadPool()).thenReturn(threadPool); + var service = createService(mockClient); + var config = new HashMap(); + config.put(ModelConfigurations.SERVICE, OLD_ELSER_SERVICE_NAME); + config.put( + ModelConfigurations.SERVICE_SETTINGS, + new HashMap<>( + Map.of(ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS, 1, ElasticsearchInternalServiceSettings.NUM_THREADS, 4) + ) + ); + + var elserServiceSettings = new ElserInternalServiceSettings(1, 4, ElserModels.ELSER_V2_MODEL, null); + + String criticalWarning = + "Putting elasticsearch service inference endpoints (including elser service) without a model_id field is" + + " deprecated and will be removed in a future release. Please specify a model_id field."; + String warnWarning = + "The [elser] service is deprecated and will be removed in a future release. Use the [elasticsearch] service " + + "instead, with [model_id] set to [.elser_model_2] in the [service_settings]"; + service.parseRequestConfig( + randomInferenceEntityId, + TaskType.SPARSE_EMBEDDING, + config, + getElserModelVerificationActionListener(elserServiceSettings, criticalWarning, warnWarning) + ); + assertWarnings(true, new DeprecationWarning(DeprecationLogger.CRITICAL, criticalWarning)); + } + + // Invalid service settings + { + Client mockClient = mock(Client.class); + when(mockClient.threadPool()).thenReturn(threadPool); + var service = createService(mockClient); + var config = new HashMap(); + config.put(ModelConfigurations.SERVICE, OLD_ELSER_SERVICE_NAME); + config.put( + ModelConfigurations.SERVICE_SETTINGS, + new HashMap<>( + Map.of( + ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS, + 1, + ElasticsearchInternalServiceSettings.NUM_THREADS, + 4, + ElasticsearchInternalServiceSettings.MODEL_ID, + ElserModels.ELSER_V2_MODEL, + "not_a_valid_service_setting", + randomAlphaOfLength(10) + ) + ) + ); + + ActionListener modelListener = ActionListener.wrap( + model -> fail("Model parsing should have failed"), + e -> assertThat(e, instanceOf(ElasticsearchStatusException.class)) + ); + + service.parseRequestConfig(randomInferenceEntityId, TaskType.SPARSE_EMBEDDING, config, modelListener); + } + } + @SuppressWarnings("unchecked") public void testParseRequestConfig_Rerank() { // with task settings @@ -374,7 +482,7 @@ public void testParseRequestConfig_SparseEmbedding() { service.parseRequestConfig(randomInferenceEntityId, TaskType.SPARSE_EMBEDDING, settings, modelListener); } - private ActionListener getModelVerificationActionListener(MultilingualE5SmallInternalServiceSettings e5ServiceSettings) { + private ActionListener getE5ModelVerificationActionListener(MultilingualE5SmallInternalServiceSettings e5ServiceSettings) { return ActionListener.wrap(model -> { assertEquals( new MultilingualE5SmallModel( @@ -388,6 +496,30 @@ private ActionListener getModelVerificationActionListener(MultilingualE5S }, e -> { fail("Model parsing failed " + e.getMessage()); }); } + private ActionListener getElserModelVerificationActionListener( + ElserInternalServiceSettings elserServiceSettings, + String criticalWarning, + String warnWarning + ) { + return ActionListener.wrap(model -> { + assertWarnings( + true, + new DeprecationWarning(DeprecationLogger.CRITICAL, criticalWarning), + new DeprecationWarning(Level.WARN, warnWarning) + ); + assertEquals( + new ElserInternalModel( + randomInferenceEntityId, + TaskType.SPARSE_EMBEDDING, + NAME, + elserServiceSettings, + ElserMlNodeTaskSettings.DEFAULT + ), + model + ); + }, e -> { fail("Model parsing failed " + e.getMessage()); }); + } + public void testParsePersistedConfig() { // Null model variant diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalServiceSettingsTests.java similarity index 89% rename from x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettingsTests.java rename to x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalServiceSettingsTests.java index ffbdf1a5a6178..f4e97b2c2e5e0 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserInternalServiceSettingsTests.java @@ -5,18 +5,16 @@ * 2.0. */ -package org.elasticsearch.xpack.inference.services.elser; +package org.elasticsearch.xpack.inference.services.elasticsearch; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings; -import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettingsTests; import java.io.IOException; import java.util.HashSet; -import static org.elasticsearch.xpack.inference.services.elser.ElserModelsTests.randomElserModel; +import static org.elasticsearch.xpack.inference.services.elasticsearch.ElserModelsTests.randomElserModel; public class ElserInternalServiceSettingsTests extends AbstractWireSerializingTestCase { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeTaskSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserMlNodeTaskSettingsTests.java similarity index 93% rename from x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeTaskSettingsTests.java rename to x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserMlNodeTaskSettingsTests.java index d55065a5f9b27..a7de3fe8b8fdc 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserMlNodeTaskSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserMlNodeTaskSettingsTests.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.inference.services.elser; +package org.elasticsearch.xpack.inference.services.elasticsearch; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserModelsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserModelsTests.java new file mode 100644 index 0000000000000..fa0148ac69df5 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElserModelsTests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.elasticsearch; + +import org.elasticsearch.test.ESTestCase; + +public class ElserModelsTests extends ESTestCase { + + public static String randomElserModel() { + return randomFrom(org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels.VALID_ELSER_MODEL_IDS); + } + + public void testIsValidModel() { + assertTrue(org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels.isValidModel(randomElserModel())); + } + + public void testIsValidEisModel() { + assertTrue( + org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels.isValidEisModel( + org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels.ELSER_V2_MODEL + ) + ); + } + + public void testIsInvalidModel() { + assertFalse(org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels.isValidModel("invalid")); + } + + public void testIsInvalidEisModel() { + assertFalse( + org.elasticsearch.xpack.inference.services.elasticsearch.ElserModels.isValidEisModel(ElserModels.ELSER_V2_MODEL_LINUX_X86) + ); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java deleted file mode 100644 index 09abeb9b9b389..0000000000000 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java +++ /dev/null @@ -1,548 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - * - * this file was contributed to by a generative AI - */ - -package org.elasticsearch.xpack.inference.services.elser; - -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.inference.ChunkedInferenceServiceResults; -import org.elasticsearch.inference.ChunkingOptions; -import org.elasticsearch.inference.InferenceResults; -import org.elasticsearch.inference.InferenceServiceExtension; -import org.elasticsearch.inference.InputType; -import org.elasticsearch.inference.Model; -import org.elasticsearch.inference.ModelConfigurations; -import org.elasticsearch.inference.TaskType; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.threadpool.TestThreadPool; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.inference.action.InferenceAction; -import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; -import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; -import org.elasticsearch.xpack.core.ml.action.InferModelAction; -import org.elasticsearch.xpack.core.ml.action.InferTrainedModelDeploymentAction; -import org.elasticsearch.xpack.core.ml.action.PutTrainedModelAction; -import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig; -import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; -import org.elasticsearch.xpack.core.ml.inference.results.InferenceChunkedTextExpansionResultsTests; -import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults; -import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate; -import org.elasticsearch.xpack.inference.InferencePlugin; -import org.junit.After; -import org.junit.Before; -import org.mockito.ArgumentCaptor; -import org.mockito.Mockito; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.instanceOf; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.same; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class ElserInternalServiceTests extends ESTestCase { - - private static ThreadPool threadPool; - - @Before - public void setUpThreadPool() { - threadPool = createThreadPool(InferencePlugin.inferenceUtilityExecutor(Settings.EMPTY)); - } - - @After - public void shutdownThreadPool() { - TestThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); - } - - public static Model randomModelConfig(String inferenceEntityId, TaskType taskType) { - return switch (taskType) { - case SPARSE_EMBEDDING -> new ElserInternalModel( - inferenceEntityId, - taskType, - ElserInternalService.NAME, - ElserInternalServiceSettingsTests.createRandom(), - ElserMlNodeTaskSettingsTests.createRandom() - ); - default -> throw new IllegalArgumentException("task type " + taskType + " is not supported"); - }; - } - - public void testParseConfigStrict() { - var service = createService(mock(Client.class)); - - var settings = new HashMap(); - settings.put( - ModelConfigurations.SERVICE_SETTINGS, - new HashMap<>( - Map.of( - ElserInternalServiceSettings.NUM_ALLOCATIONS, - 1, - ElserInternalServiceSettings.NUM_THREADS, - 4, - "model_id", - ".elser_model_1" - ) - ) - ); - settings.put(ModelConfigurations.TASK_SETTINGS, Map.of()); - - var expectedModel = new ElserInternalModel( - "foo", - TaskType.SPARSE_EMBEDDING, - ElserInternalService.NAME, - new ElserInternalServiceSettings(1, 4, ".elser_model_1", null), - ElserMlNodeTaskSettings.DEFAULT - ); - - var modelVerificationListener = getModelVerificationListener(expectedModel); - - service.parseRequestConfig("foo", TaskType.SPARSE_EMBEDDING, settings, modelVerificationListener); - - } - - public void testParseConfigWithoutModelId() { - Client mockClient = mock(Client.class); - when(mockClient.threadPool()).thenReturn(threadPool); - var service = createService(mockClient); - - var settings = new HashMap(); - settings.put( - ModelConfigurations.SERVICE_SETTINGS, - new HashMap<>(Map.of(ElserInternalServiceSettings.NUM_ALLOCATIONS, 1, ElserInternalServiceSettings.NUM_THREADS, 4)) - ); - - var expectedModel = new ElserInternalModel( - "foo", - TaskType.SPARSE_EMBEDDING, - ElserInternalService.NAME, - new ElserInternalServiceSettings(1, 4, ".elser_model_2", null), - ElserMlNodeTaskSettings.DEFAULT - ); - - var modelVerificationListener = getModelVerificationListener(expectedModel); - - service.parseRequestConfig("foo", TaskType.SPARSE_EMBEDDING, settings, modelVerificationListener); - - } - - public void testParseConfigLooseWithOldModelId() { - var service = createService(mock(Client.class)); - - var settings = new HashMap(); - settings.put( - ModelConfigurations.SERVICE_SETTINGS, - new HashMap<>( - Map.of( - ElserInternalServiceSettings.NUM_ALLOCATIONS, - 1, - ElserInternalServiceSettings.NUM_THREADS, - 4, - "model_version", - ".elser_model_1" - ) - ) - ); - settings.put(ModelConfigurations.TASK_SETTINGS, Map.of()); - - var expectedModel = new ElserInternalModel( - "foo", - TaskType.SPARSE_EMBEDDING, - ElserInternalService.NAME, - new ElserInternalServiceSettings(1, 4, ".elser_model_1", null), - ElserMlNodeTaskSettings.DEFAULT - ); - - var realModel = service.parsePersistedConfig("foo", TaskType.SPARSE_EMBEDDING, settings); - - assertEquals(expectedModel, realModel); - - } - - private static ActionListener getModelVerificationListener(ElserInternalModel expectedModel) { - return ActionListener.wrap( - (model) -> { assertEquals(expectedModel, model); }, - (e) -> fail("Model verification should not fail " + e.getMessage()) - ); - } - - public void testParseConfigStrictWithNoTaskSettings() { - var service = createService(mock(Client.class), Set.of("Aarch64")); - - var settings = new HashMap(); - settings.put( - ModelConfigurations.SERVICE_SETTINGS, - new HashMap<>(Map.of(ElserInternalServiceSettings.NUM_ALLOCATIONS, 1, ElserInternalServiceSettings.NUM_THREADS, 4)) - ); - - var expectedModel = new ElserInternalModel( - "foo", - TaskType.SPARSE_EMBEDDING, - ElserInternalService.NAME, - new ElserInternalServiceSettings(1, 4, ElserModels.ELSER_V2_MODEL, null), - ElserMlNodeTaskSettings.DEFAULT - ); - - var modelVerificationListener = getModelVerificationListener(expectedModel); - - service.parseRequestConfig("foo", TaskType.SPARSE_EMBEDDING, settings, modelVerificationListener); - } - - public void testParseConfigStrictWithUnknownSettings() { - - var service = createService(mock(Client.class)); - - for (boolean throwOnUnknown : new boolean[] { true, false }) { - { - var settings = new HashMap(); - settings.put( - ModelConfigurations.SERVICE_SETTINGS, - new HashMap<>( - Map.of( - ElserInternalServiceSettings.NUM_ALLOCATIONS, - 1, - ElserInternalServiceSettings.NUM_THREADS, - 4, - ElserInternalServiceSettings.MODEL_ID, - ".elser_model_2" - ) - ) - ); - settings.put(ModelConfigurations.TASK_SETTINGS, Map.of()); - settings.put("foo", "bar"); - - ActionListener errorVerificationListener = ActionListener.wrap((model) -> { - if (throwOnUnknown) { - fail("Model verification should fail when throwOnUnknown is true"); - } - }, (e) -> { - if (throwOnUnknown) { - assertThat( - e.getMessage(), - containsString("Model configuration contains settings [{foo=bar}] unknown to the [elser] service") - ); - } else { - fail("Model verification should not fail when throwOnUnknown is false"); - } - }); - - if (throwOnUnknown == false) { - var parsed = service.parsePersistedConfigWithSecrets( - "foo", - TaskType.SPARSE_EMBEDDING, - settings, - Collections.emptyMap() - ); - } else { - - service.parseRequestConfig("foo", TaskType.SPARSE_EMBEDDING, settings, errorVerificationListener); - } - } - - { - var settings = new HashMap(); - settings.put( - ModelConfigurations.SERVICE_SETTINGS, - new HashMap<>( - Map.of( - ElserInternalServiceSettings.NUM_ALLOCATIONS, - 1, - ElserInternalServiceSettings.NUM_THREADS, - 4, - ElserInternalServiceSettings.MODEL_ID, - ".elser_model_2" - ) - ) - ); - settings.put(ModelConfigurations.TASK_SETTINGS, Map.of("foo", "bar")); - - ActionListener errorVerificationListener = ActionListener.wrap((model) -> { - if (throwOnUnknown) { - fail("Model verification should fail when throwOnUnknown is true"); - } - }, (e) -> { - if (throwOnUnknown) { - assertThat( - e.getMessage(), - containsString("Model configuration contains settings [{foo=bar}] unknown to the [elser] service") - ); - } else { - fail("Model verification should not fail when throwOnUnknown is false"); - } - }); - if (throwOnUnknown == false) { - var parsed = service.parsePersistedConfigWithSecrets( - "foo", - TaskType.SPARSE_EMBEDDING, - settings, - Collections.emptyMap() - ); - } else { - service.parseRequestConfig("foo", TaskType.SPARSE_EMBEDDING, settings, errorVerificationListener); - } - } - - { - var settings = new HashMap(); - settings.put( - ModelConfigurations.SERVICE_SETTINGS, - new HashMap<>( - Map.of( - ElserInternalServiceSettings.NUM_ALLOCATIONS, - 1, - ElserInternalServiceSettings.NUM_THREADS, - 4, - ElserInternalServiceSettings.MODEL_ID, - ".elser_model_2", - "foo", - "bar" - ) - ) - ); - settings.put(ModelConfigurations.TASK_SETTINGS, Map.of("foo", "bar")); - - ActionListener errorVerificationListener = ActionListener.wrap((model) -> { - if (throwOnUnknown) { - fail("Model verification should fail when throwOnUnknown is true"); - } - }, (e) -> { - if (throwOnUnknown) { - assertThat( - e.getMessage(), - containsString("Model configuration contains settings [{foo=bar}] unknown to the [elser] service") - ); - } else { - fail("Model verification should not fail when throwOnUnknown is false"); - } - }); - if (throwOnUnknown == false) { - var parsed = service.parsePersistedConfigWithSecrets( - "foo", - TaskType.SPARSE_EMBEDDING, - settings, - Collections.emptyMap() - ); - } else { - service.parseRequestConfig("foo", TaskType.SPARSE_EMBEDDING, settings, errorVerificationListener); - } - } - } - } - - public void testParseRequestConfig_DefaultModel() { - { - var service = createService(mock(Client.class), Set.of()); - var settings = new HashMap(); - settings.put( - ModelConfigurations.SERVICE_SETTINGS, - new HashMap<>(Map.of(ElserInternalServiceSettings.NUM_ALLOCATIONS, 1, ElserInternalServiceSettings.NUM_THREADS, 4)) - ); - - ActionListener modelActionListener = ActionListener.wrap((model) -> { - assertEquals(".elser_model_2", ((ElserInternalModel) model).getServiceSettings().modelId()); - }, (e) -> { fail(e, "Model verification should not fail"); }); - - service.parseRequestConfig("foo", TaskType.SPARSE_EMBEDDING, settings, modelActionListener); - } - { - var service = createService(mock(Client.class), Set.of("linux-x86_64")); - var settings = new HashMap(); - settings.put( - ModelConfigurations.SERVICE_SETTINGS, - new HashMap<>(Map.of(ElserInternalServiceSettings.NUM_ALLOCATIONS, 1, ElserInternalServiceSettings.NUM_THREADS, 4)) - ); - - ActionListener modelActionListener = ActionListener.wrap((model) -> { - assertEquals(".elser_model_2_linux-x86_64", ((ElserInternalModel) model).getServiceSettings().modelId()); - }, (e) -> { fail(e, "Model verification should not fail"); }); - - service.parseRequestConfig("foo", TaskType.SPARSE_EMBEDDING, settings, modelActionListener); - } - } - - @SuppressWarnings("unchecked") - public void testChunkInfer() { - var mlTrainedModelResults = new ArrayList(); - mlTrainedModelResults.add(InferenceChunkedTextExpansionResultsTests.createRandomResults()); - mlTrainedModelResults.add(InferenceChunkedTextExpansionResultsTests.createRandomResults()); - mlTrainedModelResults.add(new ErrorInferenceResults(new RuntimeException("boom"))); - var response = new InferModelAction.Response(mlTrainedModelResults, "foo", true); - - ThreadPool threadpool = new TestThreadPool("test"); - Client client = mock(Client.class); - when(client.threadPool()).thenReturn(threadpool); - doAnswer(invocationOnMock -> { - var listener = (ActionListener) invocationOnMock.getArguments()[2]; - listener.onResponse(response); - return null; - }).when(client).execute(same(InferModelAction.INSTANCE), any(InferModelAction.Request.class), any(ActionListener.class)); - - var model = new ElserInternalModel( - "foo", - TaskType.SPARSE_EMBEDDING, - "elser", - new ElserInternalServiceSettings(1, 1, "elser", null), - new ElserMlNodeTaskSettings() - ); - var service = createService(client); - - var gotResults = new AtomicBoolean(); - var resultsListener = ActionListener.>wrap(chunkedResponse -> { - assertThat(chunkedResponse, hasSize(3)); - assertThat(chunkedResponse.get(0), instanceOf(InferenceChunkedSparseEmbeddingResults.class)); - var result1 = (InferenceChunkedSparseEmbeddingResults) chunkedResponse.get(0); - assertEquals(((MlChunkedTextExpansionResults) mlTrainedModelResults.get(0)).getChunks(), result1.getChunkedResults()); - assertThat(chunkedResponse.get(1), instanceOf(InferenceChunkedSparseEmbeddingResults.class)); - var result2 = (InferenceChunkedSparseEmbeddingResults) chunkedResponse.get(1); - assertEquals(((MlChunkedTextExpansionResults) mlTrainedModelResults.get(1)).getChunks(), result2.getChunkedResults()); - var result3 = (ErrorChunkedInferenceResults) chunkedResponse.get(2); - assertThat(result3.getException(), instanceOf(RuntimeException.class)); - assertThat(result3.getException().getMessage(), containsString("boom")); - gotResults.set(true); - }, ESTestCase::fail); - - service.chunkedInfer( - model, - null, - List.of("foo", "bar"), - Map.of(), - InputType.SEARCH, - new ChunkingOptions(null, null), - InferenceAction.Request.DEFAULT_TIMEOUT, - ActionListener.runAfter(resultsListener, () -> terminate(threadpool)) - ); - - if (gotResults.get() == false) { - terminate(threadpool); - } - assertTrue("Listener not called", gotResults.get()); - } - - @SuppressWarnings("unchecked") - public void testChunkInferSetsTokenization() { - var expectedSpan = new AtomicInteger(); - var expectedWindowSize = new AtomicReference(); - - ThreadPool threadpool = new TestThreadPool("test"); - Client client = mock(Client.class); - try { - when(client.threadPool()).thenReturn(threadpool); - doAnswer(invocationOnMock -> { - var request = (InferTrainedModelDeploymentAction.Request) invocationOnMock.getArguments()[1]; - assertThat(request.getUpdate(), instanceOf(TokenizationConfigUpdate.class)); - var update = (TokenizationConfigUpdate) request.getUpdate(); - assertEquals(update.getSpanSettings().span(), expectedSpan.get()); - assertEquals(update.getSpanSettings().maxSequenceLength(), expectedWindowSize.get()); - return null; - }).when(client) - .execute( - same(InferTrainedModelDeploymentAction.INSTANCE), - any(InferTrainedModelDeploymentAction.Request.class), - any(ActionListener.class) - ); - - var model = new ElserInternalModel( - "foo", - TaskType.SPARSE_EMBEDDING, - "elser", - new ElserInternalServiceSettings(1, 1, "elser", null), - new ElserMlNodeTaskSettings() - ); - var service = createService(client); - - expectedSpan.set(-1); - expectedWindowSize.set(null); - service.chunkedInfer( - model, - List.of("foo", "bar"), - Map.of(), - InputType.SEARCH, - null, - InferenceAction.Request.DEFAULT_TIMEOUT, - ActionListener.wrap(r -> fail("unexpected result"), e -> fail(e.getMessage())) - ); - - expectedSpan.set(-1); - expectedWindowSize.set(256); - service.chunkedInfer( - model, - List.of("foo", "bar"), - Map.of(), - InputType.SEARCH, - new ChunkingOptions(256, null), - InferenceAction.Request.DEFAULT_TIMEOUT, - ActionListener.wrap(r -> fail("unexpected result"), e -> fail(e.getMessage())) - ); - } finally { - terminate(threadpool); - } - } - - @SuppressWarnings("unchecked") - public void testPutModel() { - var client = mock(Client.class); - ArgumentCaptor argument = ArgumentCaptor.forClass(PutTrainedModelAction.Request.class); - - doAnswer(invocation -> { - var listener = (ActionListener) invocation.getArguments()[2]; - listener.onResponse(new PutTrainedModelAction.Response(mock(TrainedModelConfig.class))); - return null; - }).when(client).execute(Mockito.same(PutTrainedModelAction.INSTANCE), argument.capture(), any()); - - when(client.threadPool()).thenReturn(threadPool); - - var service = createService(client); - - var model = new ElserInternalModel( - "my-elser", - TaskType.SPARSE_EMBEDDING, - "elser", - new ElserInternalServiceSettings(1, 1, ".elser_model_2", null), - ElserMlNodeTaskSettings.DEFAULT - ); - - service.putModel(model, new ActionListener<>() { - @Override - public void onResponse(Boolean success) { - assertTrue(success); - } - - @Override - public void onFailure(Exception e) { - fail(e); - } - }); - - var putConfig = argument.getValue().getTrainedModelConfig(); - assertEquals("text_field", putConfig.getInput().getFieldNames().get(0)); - } - - private ElserInternalService createService(Client client) { - var context = new InferenceServiceExtension.InferenceServiceFactoryContext(client, threadPool); - return new ElserInternalService(context); - } - - private ElserInternalService createService(Client client, Set architectures) { - var context = new InferenceServiceExtension.InferenceServiceFactoryContext(client, threadPool); - return new ElserInternalService(context, (l) -> l.onResponse(architectures)); - } -} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserModelsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserModelsTests.java deleted file mode 100644 index f56e941dcc8c0..0000000000000 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserModelsTests.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.inference.services.elser; - -import org.elasticsearch.test.ESTestCase; - -public class ElserModelsTests extends ESTestCase { - - public static String randomElserModel() { - return randomFrom(ElserModels.VALID_ELSER_MODEL_IDS); - } - - public void testIsValidModel() { - assertTrue(ElserModels.isValidModel(randomElserModel())); - } - - public void testIsValidEisModel() { - assertTrue(ElserModels.isValidEisModel(ElserModels.ELSER_V2_MODEL)); - } - - public void testIsInvalidModel() { - assertFalse(ElserModels.isValidModel("invalid")); - } - - public void testIsInvalidEisModel() { - assertFalse(ElserModels.isValidEisModel(ElserModels.ELSER_V2_MODEL_LINUX_X86)); - } -} From fe6f8d4b376b11f5bbea99d0c110a66b9a62a1a1 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 3 Oct 2024 10:37:34 -0400 Subject: [PATCH 038/194] ESQL: Mention EXPLAIN ANALYZE in profile docs (#114025) This mentions EXPLAIN ANALYZE and EXPLAIN PLAN in the docs for ESQL's `profile` option. Those are things that folks from PostgreSQL and Oracle are used to and might search for. And `profile` is the closest thing we have to them. EXPLAIN PLAN doesn't run the query - it just tells you what the plan is. ESQL's `profile` always runs the query. So that's different. But it's close! EXPLAIN ANALYZE *does* run the query. It's pretty much the same. --- docs/reference/esql/esql-query-api.asciidoc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/reference/esql/esql-query-api.asciidoc b/docs/reference/esql/esql-query-api.asciidoc index c8c735b73d2a4..3c8d1f89496ff 100644 --- a/docs/reference/esql/esql-query-api.asciidoc +++ b/docs/reference/esql/esql-query-api.asciidoc @@ -79,6 +79,8 @@ For syntax, refer to <>. (Optional, boolean) If provided and `true` the response will include an extra `profile` object with information about how the query was executed. It provides insight into the performance of each part of the query. This is for human debugging as the object's format might change at any time. +Think of this like https://www.postgresql.org/docs/current/sql-explain.html[EXPLAIN ANALYZE] or +https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/EXPLAIN-PLAN.html[EXPLAIN PLAN]. `query`:: (Required, string) {esql} query to run. For syntax, refer to <>. @@ -109,4 +111,6 @@ Values for the search results. `profile`:: (object) Profile describing the execution of the query. Only returned if `profile` was sent in the body. -The object itself is for human debugging and can change at any time. +The object itself is for human debugging and can change at any time. Think of this like +https://www.postgresql.org/docs/current/sql-explain.html[EXPLAIN ANALYZE] or +https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/EXPLAIN-PLAN.html[EXPLAIN PLAN]. From ff53cb7158fbac2125419c6eafd559c7d41faf30 Mon Sep 17 00:00:00 2001 From: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com> Date: Thu, 3 Oct 2024 11:00:25 -0400 Subject: [PATCH 039/194] [ML] Add DeBERTa-V2/V3 tokenizer (#111852) * start to add deberta-v2 tokenizer classes * continue to add basic tokenizer stuff * Finish adding DeBERTa-2 tokenizer Still need to review & test * Complete test setup and linting * Update docs/changelog/111852.yaml * Add serialization of deberta tokenization * fix request buillder to match model * debugging * add balanced truncation * remove full vocabulary and use tiny vocab for tests * Remove TODO * precommit * Add named writables and known tokenizers * Add deberta to list of known tokenizers in test * Add tests for balanced tokenizer and fix errors in tokenizer logic * fix order of parameters passed to deberta * Add support for byte_fallback which is enabled for DeBERTa byte_fallback decomposes unknown tokens into multiple tokens each of one byte if those bytes are in the vocabulary. * precommit * update tests to account for byte decomposition * remove sysout * fix tests for byteFallback, for real this time * Move defaultSpanForChunking into super class to avoid repitition * simplify decomposeBytePieces --------- Co-authored-by: Elastic Machine --- docs/changelog/111852.yaml | 5 + .../MlInferenceNamedXContentProvider.java | 20 ++ .../trainedmodel/DebertaV2Tokenization.java | 83 +++++ .../DebertaV2TokenizationUpdate.java | 90 ++++++ .../trainedmodel/NlpConfigUpdate.java | 4 +- .../inference/trainedmodel/Tokenization.java | 3 +- .../trainedmodel/NlpConfigUpdateTests.java | 4 +- .../nlp/tokenizers/BertTokenizer.java | 5 - .../tokenizers/DebertaTokenizationResult.java | 143 +++++++++ .../nlp/tokenizers/DebertaV2Tokenizer.java | 301 ++++++++++++++++++ .../nlp/tokenizers/NlpTokenizer.java | 38 ++- .../nlp/tokenizers/RobertaTokenizer.java | 5 - .../nlp/tokenizers/UnigramTokenizer.java | 71 ++++- .../nlp/tokenizers/XLMRobertaTokenizer.java | 7 +- .../nlp/tokenizers/BertTokenizerTests.java | 113 +++++++ .../tokenizers/DebertaV2TokenizerTests.java | 206 ++++++++++++ .../nlp/tokenizers/NlpTokenizerTests.java | 11 + .../nlp/tokenizers/UnigramTokenizerTests.java | 47 ++- 18 files changed, 1126 insertions(+), 30 deletions(-) create mode 100644 docs/changelog/111852.yaml create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/DebertaV2Tokenization.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/DebertaV2TokenizationUpdate.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/DebertaTokenizationResult.java create mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/DebertaV2Tokenizer.java create mode 100644 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/DebertaV2TokenizerTests.java diff --git a/docs/changelog/111852.yaml b/docs/changelog/111852.yaml new file mode 100644 index 0000000000000..c043cab43ebbd --- /dev/null +++ b/docs/changelog/111852.yaml @@ -0,0 +1,5 @@ +pr: 111852 +summary: Add DeBERTa-V2/V3 tokenizer +area: Machine Learning +type: enhancement +issues: [] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java index 65e30072d9870..667d7bf63efc9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java @@ -40,6 +40,8 @@ import org.elasticsearch.xpack.core.ml.inference.trainedmodel.BertTokenizationUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ClassificationConfig; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ClassificationConfigUpdate; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.DebertaV2Tokenization; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.DebertaV2TokenizationUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.EmptyConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.FillMaskConfig; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.FillMaskConfigUpdate; @@ -547,6 +549,13 @@ public List getNamedXContentParsers() { (p, c) -> XLMRobertaTokenization.fromXContent(p, (boolean) c) ) ); + namedXContent.add( + new NamedXContentRegistry.Entry( + Tokenization.class, + new ParseField(DebertaV2Tokenization.NAME), + (p, c) -> DebertaV2Tokenization.fromXContent(p, (boolean) c) + ) + ); namedXContent.add( new NamedXContentRegistry.Entry( @@ -583,6 +592,13 @@ public List getNamedXContentParsers() { (p, c) -> XLMRobertaTokenizationUpdate.fromXContent(p) ) ); + namedXContent.add( + new NamedXContentRegistry.Entry( + TokenizationUpdate.class, + DebertaV2TokenizationUpdate.NAME, + (p, c) -> DebertaV2TokenizationUpdate.fromXContent(p) + ) + ); return namedXContent; } @@ -791,6 +807,7 @@ public List getNamedWriteables() { ); namedWriteables.add(new NamedWriteableRegistry.Entry(Tokenization.class, RobertaTokenization.NAME, RobertaTokenization::new)); namedWriteables.add(new NamedWriteableRegistry.Entry(Tokenization.class, XLMRobertaTokenization.NAME, XLMRobertaTokenization::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(Tokenization.class, DebertaV2Tokenization.NAME, DebertaV2Tokenization::new)); namedWriteables.add( new NamedWriteableRegistry.Entry( @@ -827,6 +844,9 @@ public List getNamedWriteables() { XLMRobertaTokenizationUpdate::new ) ); + namedWriteables.add( + new NamedWriteableRegistry.Entry(TokenizationUpdate.class, DebertaV2Tokenization.NAME, DebertaV2TokenizationUpdate::new) + ); return namedWriteables; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/DebertaV2Tokenization.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/DebertaV2Tokenization.java new file mode 100644 index 0000000000000..ce5464832b6d5 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/DebertaV2Tokenization.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ml.inference.trainedmodel; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; + +import java.io.IOException; + +public class DebertaV2Tokenization extends Tokenization { + + public static final String NAME = "deberta_v2"; + public static final String MASK_TOKEN = "[MASK]"; + + public static ConstructingObjectParser createParser(boolean ignoreUnknownFields) { + ConstructingObjectParser parser = new ConstructingObjectParser<>( + NAME, + ignoreUnknownFields, + a -> new DebertaV2Tokenization( + (Boolean) a[0], + (Boolean) a[1], + (Integer) a[2], + a[3] == null ? null : Truncate.fromString((String) a[3]), + (Integer) a[4] + ) + ); + declareCommonFields(parser); + return parser; + } + + private static final ConstructingObjectParser LENIENT_PARSER = createParser(true); + private static final ConstructingObjectParser STRICT_PARSER = createParser(false); + + public static DebertaV2Tokenization fromXContent(XContentParser parser, boolean lenient) { + return lenient ? LENIENT_PARSER.apply(parser, null) : STRICT_PARSER.apply(parser, null); + } + + public DebertaV2Tokenization( + Boolean doLowerCase, + Boolean withSpecialTokens, + Integer maxSequenceLength, + Truncate truncate, + Integer span + ) { + super(doLowerCase, withSpecialTokens, maxSequenceLength, truncate, span); + } + + public DebertaV2Tokenization(StreamInput in) throws IOException { + super(in); + } + + @Override + Tokenization buildWindowingTokenization(int updatedMaxSeqLength, int updatedSpan) { + return new DebertaV2Tokenization(doLowerCase, withSpecialTokens, updatedMaxSeqLength, truncate, updatedSpan); + } + + @Override + public String getMaskToken() { + return MASK_TOKEN; + } + + @Override + XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public String getName() { + return NAME; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/DebertaV2TokenizationUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/DebertaV2TokenizationUpdate.java new file mode 100644 index 0000000000000..683b27793402d --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/DebertaV2TokenizationUpdate.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.ml.inference.trainedmodel; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Optional; + +public class DebertaV2TokenizationUpdate extends AbstractTokenizationUpdate { + public static final ParseField NAME = new ParseField(DebertaV2Tokenization.NAME); + + public static ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "deberta_v2_tokenization_update", + a -> new DebertaV2TokenizationUpdate(a[0] == null ? null : Tokenization.Truncate.fromString((String) a[0]), (Integer) a[1]) + ); + + static { + declareCommonParserFields(PARSER); + } + + public static DebertaV2TokenizationUpdate fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + public DebertaV2TokenizationUpdate(@Nullable Tokenization.Truncate truncate, @Nullable Integer span) { + super(truncate, span); + } + + public DebertaV2TokenizationUpdate(StreamInput in) throws IOException { + super(in); + } + + @Override + public Tokenization apply(Tokenization originalConfig) { + if (originalConfig instanceof DebertaV2Tokenization debertaV2Tokenization) { + if (isNoop()) { + return debertaV2Tokenization; + } + + Tokenization.validateSpanAndTruncate(getTruncate(), getSpan()); + + if (getTruncate() != null && getTruncate().isInCompatibleWithSpan() == false) { + // When truncate value is incompatible with span wipe out + // the existing span setting to avoid an invalid combination of settings. + // This avoids the user have to set span to the special unset value + return new DebertaV2Tokenization( + debertaV2Tokenization.doLowerCase(), + debertaV2Tokenization.withSpecialTokens(), + debertaV2Tokenization.maxSequenceLength(), + getTruncate(), + null + ); + } + + return new DebertaV2Tokenization( + debertaV2Tokenization.doLowerCase(), + debertaV2Tokenization.withSpecialTokens(), + debertaV2Tokenization.maxSequenceLength(), + Optional.ofNullable(this.getTruncate()).orElse(originalConfig.getTruncate()), + Optional.ofNullable(this.getSpan()).orElse(originalConfig.getSpan()) + ); + } + throw ExceptionsHelper.badRequestException( + "Tokenization config of type [{}] can not be updated with a request of type [{}]", + originalConfig.getName(), + getName() + ); + } + + @Override + public String getWriteableName() { + return NAME.getPreferredName(); + } + + @Override + public String getName() { + return NAME.getPreferredName(); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NlpConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NlpConfigUpdate.java index 92e44edcd1259..328c851d63be6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NlpConfigUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NlpConfigUpdate.java @@ -42,7 +42,9 @@ public static TokenizationUpdate tokenizationFromMap(Map map) { RobertaTokenizationUpdate.NAME.getPreferredName(), RobertaTokenizationUpdate::new, XLMRobertaTokenizationUpdate.NAME.getPreferredName(), - XLMRobertaTokenizationUpdate::new + XLMRobertaTokenizationUpdate::new, + DebertaV2Tokenization.NAME, + DebertaV2TokenizationUpdate::new ); Map tokenizationConfig = null; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/Tokenization.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/Tokenization.java index 4fec726b9fa5d..a7c46a68538c0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/Tokenization.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/Tokenization.java @@ -36,7 +36,8 @@ public enum Truncate { public boolean isInCompatibleWithSpan() { return false; } - }; + }, + BALANCED; public boolean isInCompatibleWithSpan() { return true; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NlpConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NlpConfigUpdateTests.java index 8bc3a339ab0ee..83dc0b2a06376 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NlpConfigUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/NlpConfigUpdateTests.java @@ -73,7 +73,9 @@ public void testTokenizationFromMap() { ); assertThat( e.getMessage(), - containsString("unknown tokenization type expecting one of [bert, bert_ja, mpnet, roberta, xlm_roberta] got [not_bert]") + containsString( + "unknown tokenization type expecting one of [bert, bert_ja, deberta_v2, mpnet, roberta, xlm_roberta] got [not_bert]" + ) ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizer.java index 464c8eac8c9dd..1b53a7642abf3 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizer.java @@ -169,11 +169,6 @@ boolean isWithSpecialTokens() { return withSpecialTokens; } - @Override - int defaultSpanForChunking(int maxWindowSize) { - return (maxWindowSize - numExtraTokensForSingleSequence()) / 2; - } - @Override int getNumExtraTokensForSeqPair() { return 3; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/DebertaTokenizationResult.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/DebertaTokenizationResult.java new file mode 100644 index 0000000000000..2a50172fcc722 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/DebertaTokenizationResult.java @@ -0,0 +1,143 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + * + * this file was contributed to by a Generative AI model + */ + +package org.elasticsearch.xpack.ml.inference.nlp.tokenizers; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.Tokenization; +import org.elasticsearch.xpack.ml.inference.nlp.NlpTask; + +import java.io.IOException; +import java.util.List; +import java.util.function.Function; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +public class DebertaTokenizationResult extends TokenizationResult { + static final String REQUEST_ID = "request_id"; + static final String TOKENS = "tokens"; + static final String ARG1 = "arg_1"; + static final String ARG2 = "arg_2"; + + private static final Logger logger = LogManager.getLogger(DebertaTokenizationResult.class); + + protected DebertaTokenizationResult(List vocab, List tokenizations, int padTokenId) { + super(vocab, tokenizations, padTokenId); + } + + @Override + public NlpTask.Request buildRequest(String requestId, Tokenization.Truncate t) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.field(REQUEST_ID, requestId); + writePaddedTokens(TOKENS, builder); + writeAttentionMask(ARG1, builder); + writeTokenTypeIds(ARG2, builder); + builder.endObject(); + + // BytesReference.bytes closes the builder + BytesReference jsonRequest = BytesReference.bytes(builder); + return new NlpTask.Request(this, jsonRequest); + } + + static class DebertaTokensBuilder implements TokenizationResult.TokensBuilder { + private final int clsTokenId; + private final int sepTokenId; + private final boolean withSpecialTokens; + protected final Stream.Builder tokenIds; + protected final Stream.Builder tokenMap; + protected int seqPairOffset = 0; + + DebertaTokensBuilder(int clsTokenId, int sepTokenId, boolean withSpecialTokens) { + this.clsTokenId = clsTokenId; + this.sepTokenId = sepTokenId; + this.withSpecialTokens = withSpecialTokens; + this.tokenIds = Stream.builder(); + this.tokenMap = Stream.builder(); + } + + @Override + public TokensBuilder addSequence(List tokenIds, List tokenMap) { + // DeBERTa-v2 single sequence: [CLS] X [SEP] + if (withSpecialTokens) { + this.tokenIds.add(IntStream.of(clsTokenId)); + this.tokenMap.add(IntStream.of(SPECIAL_TOKEN_POSITION)); + } + this.tokenIds.add(tokenIds.stream().mapToInt(Integer::valueOf)); + this.tokenMap.add(tokenMap.stream().mapToInt(Integer::valueOf)); + if (withSpecialTokens) { + this.tokenIds.add(IntStream.of(sepTokenId)); + this.tokenMap.add(IntStream.of(SPECIAL_TOKEN_POSITION)); + } + return this; + } + + @Override + public TokensBuilder addSequencePair( + List tokenId1s, + List tokenMap1, + List tokenId2s, + List tokenMap2 + ) { + if (tokenId1s.isEmpty() || tokenId2s.isEmpty()) { + throw new IllegalArgumentException("Both sequences must have at least one token"); + } + + // DeBERTa-v2 pair of sequences: [CLS] A [SEP] B [SEP] + if (withSpecialTokens) { + tokenIds.add(IntStream.of(clsTokenId)); + tokenMap.add(IntStream.of(SPECIAL_TOKEN_POSITION)); + } + tokenIds.add(tokenId1s.stream().mapToInt(Integer::valueOf)); + tokenMap.add(tokenMap1.stream().mapToInt(Integer::valueOf)); + int previouslyFinalMap = tokenMap1.get(tokenMap1.size() - 1); + if (withSpecialTokens) { + tokenIds.add(IntStream.of(sepTokenId)); + tokenMap.add(IntStream.of(SPECIAL_TOKEN_POSITION)); + } + tokenIds.add(tokenId2s.stream().mapToInt(Integer::valueOf)); + tokenMap.add(tokenMap2.stream().mapToInt(i -> i + previouslyFinalMap)); + if (withSpecialTokens) { + tokenIds.add(IntStream.of(sepTokenId)); + tokenMap.add(IntStream.of(SPECIAL_TOKEN_POSITION)); + } + seqPairOffset = withSpecialTokens ? tokenId1s.size() + 2 : tokenId1s.size(); + return this; + } + + @Override + public Tokens build( + List input, + boolean truncated, + List> allTokens, + int spanPrev, + int seqId + ) { + return new Tokens( + input, + allTokens, + truncated, + tokenIds.build().flatMapToInt(Function.identity()).toArray(), + tokenMap.build().flatMapToInt(Function.identity()).toArray(), + spanPrev, + seqId, + seqPairOffset + ); + } + + @Override + public Tokens build(String input, boolean truncated, List allTokens, int spanPrev, int seqId) { + return TokensBuilder.super.build(input, truncated, allTokens, spanPrev, seqId); + } + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/DebertaV2Tokenizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/DebertaV2Tokenizer.java new file mode 100644 index 0000000000000..3f7094bcce29d --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/DebertaV2Tokenizer.java @@ -0,0 +1,301 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + * + * This Java port DeBERTa-V2 tokenizer, was derived from + * Microsoft's DeBERTa-V2 project at https://github.com/microsoft/DeBERTa + * and + * Huggingface's DeBERTa-V2 transformers + * project at https://github.com/huggingface/transformers/blob/main/src/transformers/models/deberta_v2/tokenization_deberta_v2.py + */ + +package org.elasticsearch.xpack.ml.inference.nlp.tokenizers; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.DebertaV2Tokenization; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.inference.nlp.NlpTask; + +import java.io.IOException; +import java.io.Reader; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.OptionalInt; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class DebertaV2Tokenizer extends NlpTokenizer { + + public static final String UNKNOWN_TOKEN = "[UNK]"; + public static final String SEPARATOR_TOKEN = "[SEP]"; + public static final String PAD_TOKEN = "[PAD]"; + public static final String CLASS_TOKEN = "[CLS]"; + public static final String MASK_TOKEN = "[MASK]"; + + private static final Set NEVER_SPLIT = Set.of(UNKNOWN_TOKEN, SEPARATOR_TOKEN, PAD_TOKEN, CLASS_TOKEN, MASK_TOKEN); + + private final DebertaAnalyzer debertaAnalyzer; + protected final List originalVocab; + private final SortedMap vocab; + protected final boolean withSpecialTokens; + protected final int sepTokenId; + private final int clsTokenId; + protected final int padTokenId; + private final int maxSequenceLength; + + protected DebertaV2Tokenizer( + List originalVocab, + SortedMap vocab, + List scores, + boolean withSpecialTokens, + int maxSequenceLength, + Set neverSplit + ) throws IOException { + this.originalVocab = originalVocab; + this.debertaAnalyzer = new DebertaAnalyzer( + originalVocab, + scores, + new ArrayList<>(Sets.union(NEVER_SPLIT, neverSplit)), + UNKNOWN_TOKEN + ); + this.vocab = vocab; + this.withSpecialTokens = withSpecialTokens; + this.maxSequenceLength = maxSequenceLength; + if (vocab.containsKey(UNKNOWN_TOKEN) == false) { + throw ExceptionsHelper.conflictStatusException("stored vocabulary is missing required [{}] token", UNKNOWN_TOKEN); + } + if (vocab.containsKey(PAD_TOKEN) == false) { + throw ExceptionsHelper.conflictStatusException("stored vocabulary is missing required [{}] token", PAD_TOKEN); + } + this.padTokenId = vocab.get(PAD_TOKEN); + if (withSpecialTokens) { + Set missingSpecialTokens = Sets.difference(Set.of(SEPARATOR_TOKEN, CLASS_TOKEN), vocab.keySet()); + if (missingSpecialTokens.isEmpty() == false) { + throw ExceptionsHelper.conflictStatusException("stored vocabulary is missing required {} token(s)", missingSpecialTokens); + } + this.sepTokenId = vocab.get(SEPARATOR_TOKEN); + this.clsTokenId = vocab.get(CLASS_TOKEN); + } else { + this.sepTokenId = -1; + this.clsTokenId = -1; + } + } + + @Override + int clsTokenId() { + return clsTokenId; + } + + @Override + int sepTokenId() { + return sepTokenId; + } + + @Override + int maxSequenceLength() { + return maxSequenceLength; + } + + @Override + boolean isWithSpecialTokens() { + return withSpecialTokens; + } + + @Override + int numExtraTokensForSingleSequence() { + // https://github.com/huggingface/transformers/blob/v4.44.0/src/transformers/models/deberta_v2/tokenization_deberta_v2.py#L164 + // single sequence: [CLS] X [SEP] + return 2; + } + + @Override + int getNumExtraTokensForSeqPair() { + // https://github.com/huggingface/transformers/blob/v4.44.0/src/transformers/models/deberta_v2/tokenization_deberta_v2.py#L165 + // pair of sequences: [CLS] A [SEP] B [SEP] + return 3; + } + + @Override + public TokenizationResult buildTokenizationResult(List tokenizations) { + return new DebertaTokenizationResult(originalVocab, tokenizations, padTokenId); + } + + @Override + public NlpTask.RequestBuilder requestBuilder() { + return (inputs, requestId, truncate, span, windowSize) -> buildTokenizationResult( + IntStream.range(0, inputs.size()) + .boxed() + .flatMap(seqId -> tokenize(inputs.get(seqId), truncate, span, seqId, windowSize).stream()) + .collect(Collectors.toList()) + ).buildRequest(requestId, truncate); + } + + @Override + public OptionalInt getPadTokenId() { + return OptionalInt.of(padTokenId); + } + + @Override + public String getPadToken() { + return PAD_TOKEN; + } + + @Override + public OptionalInt getMaskTokenId() { + Integer maskId = vocab.get(MASK_TOKEN); + if (maskId == null) { + return OptionalInt.empty(); + } + return OptionalInt.of(maskId); + } + + @Override + public String getMaskToken() { + return MASK_TOKEN; + } + + @Override + public List getVocabulary() { + return originalVocab; + } + + @Override + TokenizationResult.TokensBuilder createTokensBuilder(int clsTokenId, int sepTokenId, boolean withSpecialTokens) { + return new DebertaTokenizationResult.DebertaTokensBuilder(clsTokenId, sepTokenId, withSpecialTokens); + } + + public static DebertaV2Tokenizer.Builder builder(List vocab, List scores, DebertaV2Tokenization tokenization) { + return new DebertaV2Tokenizer.Builder(vocab, scores, tokenization); + } + + public static class Builder { + + protected final List originalVocab; + protected final List scores; + protected final SortedMap vocab; + protected boolean withSpecialTokens; + protected int maxSequenceLength; + protected Set neverSplit; + + protected Builder(List vocab, List scores, DebertaV2Tokenization tokenization) { + this.originalVocab = vocab; + this.vocab = buildSortedVocab(vocab); + this.scores = scores; + this.withSpecialTokens = tokenization.withSpecialTokens(); + this.maxSequenceLength = tokenization.maxSequenceLength(); + } + + private static SortedMap buildSortedVocab(List vocab) { + SortedMap sortedVocab = new TreeMap<>(); + for (int i = 0; i < vocab.size(); i++) { + sortedVocab.put(vocab.get(i), i); + } + return sortedVocab; + } + + public DebertaV2Tokenizer.Builder setNeverSplit(Set neverSplit) { + this.neverSplit = neverSplit; + return this; + } + + public DebertaV2Tokenizer.Builder setMaxSequenceLength(int maxSequenceLength) { + this.maxSequenceLength = maxSequenceLength; + return this; + } + + /** + * Include CLS and SEP tokens + * @param withSpecialTokens if true include CLS and SEP tokens + * @return this + */ + public DebertaV2Tokenizer.Builder setWithSpecialTokens(boolean withSpecialTokens) { + this.withSpecialTokens = withSpecialTokens; + return this; + } + + public DebertaV2Tokenizer build() throws IOException { + if (neverSplit == null) { + neverSplit = Collections.emptySet(); + } + + return new DebertaV2Tokenizer(originalVocab, vocab, scores, withSpecialTokens, maxSequenceLength, neverSplit); + } + } + + @Override + public InnerTokenization innerTokenize(String seq) { + List tokenPositionMap = new ArrayList<>(); + try (TokenStream ts = debertaAnalyzer.tokenStream("input", seq)) { + ts.reset(); + PositionIncrementAttribute tokenPos = ts.addAttribute(PositionIncrementAttribute.class); + int currPos = -1; // the PositionIncrement starts at one, so this aligns the first token at position 0 + while (ts.incrementToken()) { + currPos += tokenPos.getPositionIncrement(); + tokenPositionMap.add(currPos); + } + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } + return new InnerTokenization(new ArrayList<>(debertaAnalyzer.getTokens()), tokenPositionMap); + } + + @Override + public void close() { + this.debertaAnalyzer.close(); + } + + static class DebertaAnalyzer extends Analyzer { + private final List vocabulary; + private final List neverSplit; + private final double[] scores; + private UnigramTokenizer innerTokenizer; + private final String unknownToken; + private final PrecompiledCharMapNormalizer.Config normalizer; + + DebertaAnalyzer(List vocabulary, List scores, List neverSplit, String unknownToken) throws IOException { + this.vocabulary = vocabulary; + this.neverSplit = neverSplit; + this.unknownToken = unknownToken; + this.scores = new double[scores.size()]; + int i = 0; + for (Double s : scores) { + this.scores[i++] = s; + } + normalizer = PrecompiledCharMapNormalizer.fromBase64EncodedResource( + "/org/elasticsearch/xpack/ml/inference.nlp.tokenizers/spm_precompiled_normalizer.txt" + ); + } + + @Override + protected Reader initReader(String fieldName, Reader reader) { + if (normalizer.offsets().length > 0) { + return new PrecompiledCharMapNormalizer(normalizer.offsets(), normalizer.utf8str(), reader); + } + return reader; + } + + @Override + protected TokenStreamComponents createComponents(String fieldName) { + this.innerTokenizer = UnigramTokenizer.build(neverSplit, vocabulary, scores, unknownToken, true); + return new TokenStreamComponents(this.innerTokenizer); + } + + public List getTokens() { + if (innerTokenizer != null) { + return innerTokenizer.getTokenizedValues(); + } else { + return List.of(); + } + } + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/NlpTokenizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/NlpTokenizer.java index 5014eb269b081..0b4a5b651d8d4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/NlpTokenizer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/NlpTokenizer.java @@ -11,6 +11,7 @@ import org.elasticsearch.core.Strings; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.BertJapaneseTokenization; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.BertTokenization; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.DebertaV2Tokenization; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.MPNetTokenization; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RobertaTokenization; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.Tokenization; @@ -26,6 +27,7 @@ import java.util.OptionalInt; import java.util.stream.Collectors; +import static java.lang.Math.min; import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.NlpConfig.TOKENIZATION; import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.NlpConfig.VOCABULARY; @@ -48,7 +50,9 @@ public abstract class NlpTokenizer implements Releasable { abstract int getNumExtraTokensForSeqPair(); - abstract int defaultSpanForChunking(int maxWindowSize); + int defaultSpanForChunking(int maxWindowSize) { + return (maxWindowSize - numExtraTokensForSingleSequence()) / 2; + } public abstract TokenizationResult buildTokenizationResult(List tokenizations); @@ -85,7 +89,7 @@ public final List tokenize( if (numTokens > windowSize) { switch (truncate) { - case FIRST, SECOND -> { + case FIRST, SECOND, BALANCED -> { // only one sequence exists in this case isTruncated = true; tokenIds = tokenIds.subList(0, isWithSpecialTokens() ? windowSize - numExtraTokensForSingleSequence() : windowSize); tokenPositionMap = tokenPositionMap.subList( @@ -123,7 +127,7 @@ public final List tokenize( int splitStartPos = 0; int spanPrev = -1; while (splitEndPos < tokenIds.size()) { - splitEndPos = Math.min( + splitEndPos = min( splitStartPos + (isWithSpecialTokens() ? windowSize - numExtraTokensForSingleSequence() : windowSize), tokenIds.size() ); @@ -232,6 +236,29 @@ public TokenizationResult.Tokens tokenize( tokenIdsSeq2 = tokenIdsSeq2.subList(0, maxSequenceLength() - extraTokens - tokenIdsSeq1.size()); tokenPositionMapSeq2 = tokenPositionMapSeq2.subList(0, maxSequenceLength() - extraTokens - tokenIdsSeq1.size()); } + case BALANCED -> { + isTruncated = true; + int firstSequenceLength = 0; + + if (tokenIdsSeq2.size() > (maxSequenceLength() - getNumExtraTokensForSeqPair()) / 2) { + firstSequenceLength = min(tokenIdsSeq1.size(), (maxSequenceLength() - getNumExtraTokensForSeqPair()) / 2); + } else { + firstSequenceLength = min( + tokenIdsSeq1.size(), + maxSequenceLength() - tokenIdsSeq2.size() - getNumExtraTokensForSeqPair() + ); + } + int secondSequenceLength = min( + tokenIdsSeq2.size(), + maxSequenceLength() - firstSequenceLength - getNumExtraTokensForSeqPair() + ); + + tokenIdsSeq1 = tokenIdsSeq1.subList(0, firstSequenceLength); + tokenPositionMapSeq1 = tokenPositionMapSeq1.subList(0, firstSequenceLength); + + tokenIdsSeq2 = tokenIdsSeq2.subList(0, secondSequenceLength); + tokenPositionMapSeq2 = tokenPositionMapSeq2.subList(0, secondSequenceLength); + } case NONE -> throw ExceptionsHelper.badRequestException( "Input too large. The tokenized input length [{}] exceeds the maximum sequence length [{}]", numTokens, @@ -355,7 +382,7 @@ public List tokenize(String seq1, String seq2, Tokeni } while (splitEndPos < tokenIdsSeq2.size()) { - splitEndPos = Math.min(splitStartPos + trueMaxSeqLength, tokenIdsSeq2.size()); + splitEndPos = min(splitStartPos + trueMaxSeqLength, tokenIdsSeq2.size()); // Make sure we do not end on a word if (splitEndPos != tokenIdsSeq2.size()) { while (splitEndPos > splitStartPos + 1 @@ -447,6 +474,9 @@ public static NlpTokenizer build(Vocabulary vocabulary, Tokenization params) thr if (params instanceof XLMRobertaTokenization xlmRobertaTokenization) { return XLMRobertaTokenizer.builder(vocabulary.get(), vocabulary.scores(), xlmRobertaTokenization).build(); } + if (params instanceof DebertaV2Tokenization debertaV2Tokenization) { + return DebertaV2Tokenizer.builder(vocabulary.get(), vocabulary.scores(), debertaV2Tokenization).build(); + } throw new IllegalArgumentException("unknown tokenization type [" + params.getName() + "]"); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/RobertaTokenizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/RobertaTokenizer.java index e884e84faa85d..6d58d2e2dc2cf 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/RobertaTokenizer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/RobertaTokenizer.java @@ -106,11 +106,6 @@ int getNumExtraTokensForSeqPair() { return 4; } - @Override - int defaultSpanForChunking(int maxWindowSize) { - return (maxWindowSize - numExtraTokensForSingleSequence()) / 2; - } - @Override int numExtraTokensForSingleSequence() { return 2; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/UnigramTokenizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/UnigramTokenizer.java index acb1f6c038ef9..31deac066cba2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/UnigramTokenizer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/UnigramTokenizer.java @@ -14,6 +14,7 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.UnicodeUtil; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Nullable; @@ -49,7 +50,13 @@ public final class UnigramTokenizer extends Tokenizer { private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); - static UnigramTokenizer build(List neverSplit, List dictionary, double[] scores, String unknownToken) { + static UnigramTokenizer build( + List neverSplit, + List dictionary, + double[] scores, + String unknownToken, + boolean byteFallback + ) { if (dictionary.isEmpty()) { throw new IllegalArgumentException("vocab empty"); } @@ -84,7 +91,8 @@ static UnigramTokenizer build(List neverSplit, List dictionary, Optional.ofNullable(tokenToId.get(new BytesRef(unknownToken))) .orElseThrow( () -> new IllegalArgumentException("provided vocabulary does not contain the unknown token of [" + unknownToken + "]") - ) + ), + byteFallback ); } @@ -94,7 +102,7 @@ static UnigramTokenizer build(List neverSplit, List dictionary, private final double minScore; // This may be configurable in the future - private final boolean fuseUnk = true; + private boolean fuseUnk = true; private final double[] vocabScores; private final CharTrie neverSplit; private final CharArraySet neverSplitHash; @@ -104,6 +112,7 @@ static UnigramTokenizer build(List neverSplit, List dictionary, // This is a buffer that is reused per token for decoding the normalized char-sequence into utf-8 bytes // It's usage is NOT thread safe private byte[] normalizedByteBuffer = new byte[128]; + private boolean byteFallback = false; // If true, decompose unknown pieces into UTF-8 byte pieces public UnigramTokenizer( double minScore, @@ -127,6 +136,31 @@ public UnigramTokenizer( this.whitespaceTokenizer = new SimpleWhitespaceTokenizer(); } + public UnigramTokenizer( + double minScore, + double[] vocabScores, + CharTrie neverSplit, + CharArraySet neverSplitHash, + Map vocabToId, + BytesTrie vocabTrie, + int unknownTokenId, + boolean byteFallback + ) { + super(); + this.tokens = new LinkedList<>(); + this.tokenizedValues = new ArrayList<>(); + this.minScore = minScore; + this.neverSplit = neverSplit; + this.neverSplitHash = neverSplitHash; + this.vocabToId = vocabToId; + this.vocabTrie = vocabTrie; + this.unknownTokenId = unknownTokenId; + this.vocabScores = vocabScores; + this.whitespaceTokenizer = new SimpleWhitespaceTokenizer(); + this.byteFallback = byteFallback; + this.fuseUnk = byteFallback == false; + } + List getTokenizedValues() { return tokenizedValues; } @@ -231,6 +265,21 @@ public boolean incrementToken() throws IOException { return false; } + private int[] decomposeBytePieces(byte[] bytes) { + assert this.byteFallback; + + int[] pieces = new int[bytes.length]; + for (int i = 0; i < bytes.length; i++) { + BytesRef decomposedToken = new BytesRef(Strings.format("<0x%02X>", bytes[i])); + Integer piece = vocabToId.get(decomposedToken); + if (piece == null) { + piece = unknownTokenId; + } + pieces[i] = piece; + } + return pieces; + } + /** * This algorithm does the following: * @@ -309,7 +358,21 @@ List tokenize(CharSequence inputSequence, IntToIntFuncti while (endsAtBytes > 0) { BestPathNode node = bestPathNodes[endsAtBytes]; int startsAtBytes = node.startsAtBytePos; - if (node.id == unknownTokenId && fuseUnk) { + if (node.id == unknownTokenId && byteFallback) { + CharSequence multiByteSequence = inputSequence.subSequence(node.startsAtCharPos, endsAtChars); + byte[] bytes = multiByteSequence.toString().getBytes(StandardCharsets.UTF_8); + int[] pieces = decomposeBytePieces(bytes); + for (int i = pieces.length - 1; i >= 0; i--) { + results.add( + new DelimitedToken.Encoded( + Strings.format("<0x%02X>", bytes[i]), + pieces[i], + offsetCorrection.apply(node.startsAtCharPos), + offsetCorrection.apply(startsAtBytes + i) + ) + ); + } + } else if (node.id == unknownTokenId && fuseUnk) { unknownTokens.add( new DelimitedToken.Encoded( new String(normalizedByteBuffer, startsAtBytes, endsAtBytes - startsAtBytes, StandardCharsets.UTF_8), diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/XLMRobertaTokenizer.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/XLMRobertaTokenizer.java index 7a856d8e4735a..0e8793eb374ca 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/XLMRobertaTokenizer.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/XLMRobertaTokenizer.java @@ -101,11 +101,6 @@ int getNumExtraTokensForSeqPair() { return 4; } - @Override - int defaultSpanForChunking(int maxWindowSize) { - return (maxWindowSize - numExtraTokensForSingleSequence()) / 2; - } - @Override int numExtraTokensForSingleSequence() { return 2; @@ -284,7 +279,7 @@ protected Reader initReader(String fieldName, Reader reader) { @Override protected TokenStreamComponents createComponents(String fieldName) { - this.innerTokenizer = UnigramTokenizer.build(neverSplit, vocabulary, scores, unknownToken); + this.innerTokenizer = UnigramTokenizer.build(neverSplit, vocabulary, scores, unknownToken, false); return new TokenStreamComponents(this.innerTokenizer); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizerTests.java index 901fea45d9de9..ccebe3bf0ca98 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/BertTokenizerTests.java @@ -760,6 +760,119 @@ public void testTokenizeLargeInputMultiSequenceTruncation() { } + public void testTokenizeLargeInputMultiSequenceBalancedTruncation() { + try ( + BertTokenizer tokenizer = BertTokenizer.builder( + TEST_CASED_VOCAB, + new BertTokenization(null, true, 10, Tokenization.Truncate.BALANCED, -1) + ).build() + ) { + + { // both sequences are truncated + TokenizationResult.Tokens tokenization = tokenizer.tokenize( + "Elasticsearch is fun", + "Godzilla my little red car", + Tokenization.Truncate.BALANCED, + 0 + ); + + var tokenStream = Arrays.stream(tokenization.tokenIds()).mapToObj(TEST_CASED_VOCAB::get).collect(Collectors.toList()); + assertThat( + tokenStream, + contains( + BertTokenizer.CLASS_TOKEN, + "Elastic", + "##search", + "is", + BertTokenizer.SEPARATOR_TOKEN, + "God", + "##zilla", + "my", + "little", + BertTokenizer.SEPARATOR_TOKEN + ) + ); + } + + { // first sequence is too short to be truncated + TokenizationResult.Tokens tokenization = tokenizer.tokenize( + "Elasticsearch", + "Godzilla my little red car", + Tokenization.Truncate.BALANCED, + 0 + ); + + var tokenStream = Arrays.stream(tokenization.tokenIds()).mapToObj(TEST_CASED_VOCAB::get).collect(Collectors.toList()); + assertThat( + tokenStream, + contains( + BertTokenizer.CLASS_TOKEN, + "Elastic", + "##search", + BertTokenizer.SEPARATOR_TOKEN, + "God", + "##zilla", + "my", + "little", + "red", + BertTokenizer.SEPARATOR_TOKEN + ) + ); + } + + { // second sequence is too short to be truncated + TokenizationResult.Tokens tokenization = tokenizer.tokenize( + "Elasticsearch is my little red fun", + "Godzilla", + Tokenization.Truncate.BALANCED, + 0 + ); + + var tokenStream = Arrays.stream(tokenization.tokenIds()).mapToObj(TEST_CASED_VOCAB::get).collect(Collectors.toList()); + assertThat( + tokenStream, + contains( + BertTokenizer.CLASS_TOKEN, + "Elastic", + "##search", + "is", + "my", + "little", + BertTokenizer.SEPARATOR_TOKEN, + "God", + "##zilla", + BertTokenizer.SEPARATOR_TOKEN + ) + ); + } + + { // both sequences are too short to be truncated + TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch", "Godzilla", Tokenization.Truncate.BALANCED, 0); + + var tokenStream = Arrays.stream(tokenization.tokenIds()).mapToObj(TEST_CASED_VOCAB::get).collect(Collectors.toList()); + assertThat( + tokenStream, + contains( + BertTokenizer.CLASS_TOKEN, + "Elastic", + "##search", + BertTokenizer.SEPARATOR_TOKEN, + "God", + "##zilla", + BertTokenizer.SEPARATOR_TOKEN + ) + ); + } + + expectThrows( + ElasticsearchStatusException.class, + () -> BertTokenizer.builder(TEST_CASED_VOCAB, new BertTokenization(null, true, 8, Tokenization.Truncate.NONE, -1)) + .build() + .tokenize("Elasticsearch is fun", "Godzilla my little red car", Tokenization.Truncate.NONE, 0) + ); + } + } + public void testMultiSeqRequiresSpecialTokens() { try ( BertTokenizer tokenizer = BertTokenizer.builder( diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/DebertaV2TokenizerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/DebertaV2TokenizerTests.java new file mode 100644 index 0000000000000..bbe509da67452 --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/DebertaV2TokenizerTests.java @@ -0,0 +1,206 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ml.inference.nlp.tokenizers; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.DebertaV2Tokenization; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.Tokenization; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.ml.inference.nlp.tokenizers.DebertaV2Tokenizer.MASK_TOKEN; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + +public class DebertaV2TokenizerTests extends ESTestCase { + + private static final List TEST_CASE_VOCAB = List.of( + DebertaV2Tokenizer.CLASS_TOKEN, + DebertaV2Tokenizer.PAD_TOKEN, + DebertaV2Tokenizer.SEPARATOR_TOKEN, + DebertaV2Tokenizer.UNKNOWN_TOKEN, + "▁Ela", + "stic", + "search", + "▁is", + "▁fun", + "▁God", + "z", + "illa", + "▁my", + "▁little", + "▁red", + "▁car", + "▁😀", + "▁🇸🇴", + MASK_TOKEN, + ".", + "<0xC2>", + "<0xAD>", + "▁" + ); + private static final List TEST_CASE_SCORES = List.of( + 0.0, + 0.0, + 0.0, + 0.0, + -12.535264015197754, + -12.300995826721191, + -13.255199432373047, + -7.402246475219727, + -11.201482772827148, + -10.576351165771484, + -7.898513317108154, + -10.230172157287598, + -9.18289566040039, + -11.451579093933105, + -10.858806610107422, + -10.214239120483398, + -10.230172157287598, + -9.451579093933105, + 0.0, + -3.0, + 1.0, + 2.0, + -7.97025 + ); + + private List tokenStrings(List tokens) { + return tokens.stream().map(DelimitedToken::toString).collect(Collectors.toList()); + } + + public void testTokenize() throws IOException { + try ( + DebertaV2Tokenizer tokenizer = DebertaV2Tokenizer.builder( + TEST_CASE_VOCAB, + TEST_CASE_SCORES, + new DebertaV2Tokenization(false, false, null, Tokenization.Truncate.NONE, -1) + ).build() + ) { + TokenizationResult.Tokens tokenization = tokenizer.tokenize("Elasticsearch fun", Tokenization.Truncate.NONE, -1, 0, null) + .get(0); + assertThat(tokenStrings(tokenization.tokens().get(0)), contains("▁Ela", "stic", "search", "▁fun")); + assertArrayEquals(new int[] { 4, 5, 6, 8 }, tokenization.tokenIds()); + assertArrayEquals(new int[] { 0, 1, 2, 3 }, tokenization.tokenMap()); + } + } + + public void testSurrogatePair() throws IOException { + try ( + DebertaV2Tokenizer tokenizer = DebertaV2Tokenizer.builder( + TEST_CASE_VOCAB, + TEST_CASE_SCORES, + new DebertaV2Tokenization(false, false, null, Tokenization.Truncate.NONE, -1) + ).build() + ) { + TokenizationResult.Tokens tokenization = tokenizer.tokenize( + "Elastic" + "\u00AD" + "search 😀" + "\u00AD" + " fun", + Tokenization.Truncate.NONE, + -1, + 0, + null + ).get(0); + assertArrayEquals(new int[] { 4, 5, 20, 21, 6, 16, 20, 21, 8 }, tokenization.tokenIds()); + assertThat( + tokenStrings(tokenization.tokens().get(0)), + contains("▁Ela", "stic", "<0xC2>", "<0xAD>", "search", "▁\uD83D\uDE00", "<0xC2>", "<0xAD>", "▁fun") + ); + + tokenization = tokenizer.tokenize("😀", Tokenization.Truncate.NONE, -1, 0, null).get(0); + assertThat(tokenStrings(tokenization.tokens().get(0)), contains("▁\uD83D\uDE00")); + + tokenization = tokenizer.tokenize("Elasticsearch 😀", Tokenization.Truncate.NONE, -1, 0, null).get(0); + assertThat(tokenStrings(tokenization.tokens().get(0)), contains("▁Ela", "stic", "search", "▁\uD83D\uDE00")); + + tokenization = tokenizer.tokenize("Elasticsearch 😀 fun", Tokenization.Truncate.NONE, -1, 0, null).get(0); + assertThat(tokenStrings(tokenization.tokens().get(0)), contains("▁Ela", "stic", "search", "▁\uD83D\uDE00", "▁fun")); + + } + } + + public void testMultiByteEmoji() throws IOException { + try ( + DebertaV2Tokenizer tokenizer = DebertaV2Tokenizer.builder( + TEST_CASE_VOCAB, + TEST_CASE_SCORES, + new DebertaV2Tokenization(false, false, null, Tokenization.Truncate.NONE, -1) + ).build() + ) { + TokenizationResult.Tokens tokenization = tokenizer.tokenize("🇸🇴", Tokenization.Truncate.NONE, -1, 0, null).get(0); + assertThat(tokenStrings(tokenization.tokens().get(0)), contains("▁🇸🇴")); + assertThat(tokenization.tokenIds()[0], not(equalTo(3))); // not the unknown token + + tokenization = tokenizer.tokenize("🏁", Tokenization.Truncate.NONE, -1, 0, null).get(0); + assertThat(tokenStrings(tokenization.tokens().get(0)), contains("▁", "<0xF0>", "<0x9F>", "<0x8F>", "<0x81>")); + // contains the 4-byte sequence representing the emoji which is not in the vocab, due to byteFallback enabled + } + } + + public void testTokenizeWithNeverSplit() throws IOException { + try ( + DebertaV2Tokenizer tokenizer = DebertaV2Tokenizer.builder( + TEST_CASE_VOCAB, + TEST_CASE_SCORES, + new DebertaV2Tokenization(false, true, null, Tokenization.Truncate.NONE, -1) + ).build() + ) { + TokenizationResult.Tokens tokenization = tokenizer.tokenize( + "Elasticsearch ." + MASK_TOKEN + ".", + Tokenization.Truncate.NONE, + -1, + 0, + null + ).get(0); + assertThat(tokenStrings(tokenization.tokens().get(0)), contains("▁Ela", "stic", "search", "▁", ".", MASK_TOKEN, "▁", ".")); + } + } + + public void testMultiSeqTokenization() throws IOException { + try ( + DebertaV2Tokenizer tokenizer = DebertaV2Tokenizer.builder( + TEST_CASE_VOCAB, + TEST_CASE_SCORES, + new DebertaV2Tokenization(false, false, null, Tokenization.Truncate.NONE, -1) + ).setWithSpecialTokens(true).build() + ) { + TokenizationResult.Tokens tokenization = tokenizer.tokenize( + "Elasticsearch is fun", + "Godzilla my little red car", + Tokenization.Truncate.NONE, + 0 + ); + + var tokenStream = Arrays.stream(tokenization.tokenIds()).mapToObj(TEST_CASE_VOCAB::get).collect(Collectors.toList()); + assertThat( + tokenStream, + contains( + DebertaV2Tokenizer.CLASS_TOKEN, + "▁Ela", + "stic", + "search", + "▁is", + "▁fun", + DebertaV2Tokenizer.SEPARATOR_TOKEN, + "▁God", + "z", + "illa", + "▁my", + "▁little", + "▁red", + "▁car", + DebertaV2Tokenizer.SEPARATOR_TOKEN + ) + ); + } + } + +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/NlpTokenizerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/NlpTokenizerTests.java index fc2a31a06e187..ad6f44e77aafc 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/NlpTokenizerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/NlpTokenizerTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.BertJapaneseTokenization; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.BertTokenization; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.DebertaV2Tokenization; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.MPNetTokenization; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.RobertaTokenization; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.Tokenization; @@ -44,6 +45,13 @@ public class NlpTokenizerTests extends ESTestCase { RobertaTokenizer.CLASS_TOKEN, RobertaTokenizer.MASK_TOKEN ); + public static final List DEBERTA_REQUIRED_VOCAB = List.of( + DebertaV2Tokenizer.UNKNOWN_TOKEN, + DebertaV2Tokenizer.SEPARATOR_TOKEN, + DebertaV2Tokenizer.PAD_TOKEN, + DebertaV2Tokenizer.CLASS_TOKEN, + DebertaV2Tokenizer.MASK_TOKEN + ); void validateBuilder(List vocab, Tokenization tokenization, Class expectedClass) throws IOException { Vocabulary vocabulary = new Vocabulary(vocab, "model-name", null, null); @@ -66,5 +74,8 @@ public void testBuildTokenizer() throws IOException { Tokenization xlmRoberta = new XLMRobertaTokenization(null, null, Tokenization.Truncate.NONE, -1); validateBuilder(ROBERTA_REQUIRED_VOCAB, xlmRoberta, XLMRobertaTokenizer.class); + + Tokenization debertaV2 = new DebertaV2Tokenization(false, null, null, Tokenization.Truncate.NONE, -1); + validateBuilder(DEBERTA_REQUIRED_VOCAB, debertaV2, DebertaV2Tokenizer.class); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/UnigramTokenizerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/UnigramTokenizerTests.java index f97055b29ca7b..d1ce2fea9d1dc 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/UnigramTokenizerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/tokenizers/UnigramTokenizerTests.java @@ -39,8 +39,24 @@ public void testSimpleTokenization() throws IOException { public void testLessSimpleTokenization() throws IOException { TestNLPAnalyzer analyzer = new TestNLPAnalyzer( - List.of(UNKNOWN_TOKEN, PREFIX + "ab", "cd", PREFIX + "abc", "a", "b", "c", "ABC", "abcdabcd", "q", "r", "qr", ""), - List.of(0.0, 0.0, -0.1, -0.2, -0.3, -0.4, -0.5, -0.5, 20.0, 20.5, 20.5, -0.5, 0.0), + List.of( + UNKNOWN_TOKEN, + PREFIX + "ab", + "cd", + PREFIX + "abc", + "a", + "b", + "c", + "ABC", + "abcdabcd", + "q", + "r", + "qr", + "", + "aa", + "aaaa" + ), + List.of(0.0, 0.0, -0.1, -0.2, -0.3, -0.4, -0.5, -0.5, 20.0, 20.5, 20.5, -0.5, 0.0, -13.5467, -14.9644), UNKNOWN_TOKEN, new PrecompiledCharMapNormalizer.Config(new int[0], "") ); @@ -53,6 +69,31 @@ public void testLessSimpleTokenization() throws IOException { assertAnalyzesToNoCharFilter(analyzer, " \nabcd \n\n abcc \n", new String[] { PREFIX + "ab", "cd", PREFIX + "abc", "c" }); } + public void testLessSimpleTokenizationForRepeatingCharacters() throws IOException { + TestNLPAnalyzer analyzer = new TestNLPAnalyzer( + List.of(UNKNOWN_TOKEN, "HH", "HHHH", PREFIX + "H", "HHH", PREFIX + "HH", PREFIX, PREFIX + "HHH"), + List.of(0.0, -13.5467, -14.9644, -9.17478, -15.1165, -13.201, -7.97025, -15.602), + UNKNOWN_TOKEN, + PrecompiledCharMapNormalizer.fromBase64EncodedResource( + "/org/elasticsearch/xpack/ml/inference.nlp.tokenizers/spm_precompiled_normalizer.txt" + ) + ); + + assertAnalyzesToNoCharFilter(analyzer, "HHHHHHHHHHHH", new String[] { PREFIX, "HHHH", "HHHH", "HHHH" }); + assertAnalyzesToNoCharFilter(analyzer, "HHHHHHHHHHH", new String[] { PREFIX + "HHH", "HHHH", "HHHH" }); + assertAnalyzesToNoCharFilter(analyzer, "HHHHHHHHHH", new String[] { PREFIX + "HH", "HHHH", "HHHH" }); + assertAnalyzesToNoCharFilter(analyzer, "HHHHHHHHH", new String[] { PREFIX + "H", "HHHH", "HHHH" }); + assertAnalyzesToNoCharFilter(analyzer, "HHHHHHHH", new String[] { PREFIX, "HHHH", "HHHH" }); + assertAnalyzesToNoCharFilter(analyzer, "HHHHHHH", new String[] { PREFIX + "HHH", "HHHH" }); + assertAnalyzesToNoCharFilter(analyzer, "HHHHHH", new String[] { PREFIX + "HH", "HHHH" }); + assertAnalyzesToNoCharFilter(analyzer, "HHHHH", new String[] { PREFIX + "H", "HHHH" }); + assertAnalyzesToNoCharFilter(analyzer, "HHHH", new String[] { PREFIX, "HHHH" }); + assertAnalyzesToNoCharFilter(analyzer, "HHH", new String[] { PREFIX + "HHH" }); + assertAnalyzesToNoCharFilter(analyzer, "HH", new String[] { PREFIX + "HH" }); + assertAnalyzesToNoCharFilter(analyzer, "H", new String[] { PREFIX + "H" }); + + } + public void testLessSimpleTokenizationWithNeverSplit() throws IOException { TestNLPAnalyzer analyzer = new TestNLPAnalyzer( List.of( @@ -153,7 +194,7 @@ protected Reader initReader(String fieldName, Reader reader) { @Override protected TokenStreamComponents createComponents(String fieldName) { - UnigramTokenizer tokenizer = UnigramTokenizer.build(NEVER_SPLIT, dictionary, scores, unknownToken); + UnigramTokenizer tokenizer = UnigramTokenizer.build(NEVER_SPLIT, dictionary, scores, unknownToken, false); return new TokenStreamComponents(tokenizer); } } From c45707012a2ff7f6136d5de1935c19df73a7c282 Mon Sep 17 00:00:00 2001 From: Pat Whelan Date: Thu, 3 Oct 2024 11:08:09 -0400 Subject: [PATCH 040/194] [ML] Enable OpenAI Streaming (#113911) Enables end-to-end streaming for OpenAI's chat completion API. Additional changes: - InferenceServiceResults now returns a wildcard, allowing for tests to work with classes that extend ChunkedToXContent. - StreamingChatCompletionResults now defines the Results structure so that future providers can reuse the same structure. - DelegatingProcessor now cancels the upstream if `next` throws an exception and forwards the exception downstream. - Moved the streaming handler from OpenAiChatCompletionResponseHandler to OpenAiResponseHandler so that Azure Open AI can reuse it. - OpenAiStreamingProcessor now iterates over the returned choices array, handling both OpenAI and Azure response formats. - SenderService declares a helper `Set` for implementations to reuse when enabling streaming. - Added an InferenceEventsAssertion test helper to fluidly assert responses from the mock webserver. --- docs/changelog/113911.yaml | 5 + .../inference/InferenceServiceResults.java | 2 +- .../StreamingChatCompletionResults.java | 33 +++- .../inference/common/DelegatingProcessor.java | 10 +- .../AzureOpenAiResponseHandler.java | 2 +- .../OpenAiCompletionRequestManager.java | 7 +- .../OpenAiEmbeddingsRequestManager.java | 2 +- .../OpenAiChatCompletionResponseHandler.java | 23 +-- .../openai/OpenAiResponseHandler.java | 26 ++- .../openai/OpenAiStreamingProcessor.java | 92 ++++----- .../openai/OpenAiChatCompletionRequest.java | 11 +- .../OpenAiChatCompletionRequestEntity.java | 9 +- .../inference/services/SenderService.java | 3 + .../services/openai/OpenAiService.java | 6 + .../common/DelegatingProcessorTests.java | 13 +- .../openai/OpenAiResponseHandlerTests.java | 2 +- .../openai/OpenAiStreamingProcessorTests.java | 8 +- ...penAiChatCompletionRequestEntityTests.java | 8 +- .../OpenAiChatCompletionRequestTests.java | 25 ++- .../services/InferenceEventsAssertion.java | 186 ++++++++++++++++++ .../services/openai/OpenAiServiceTests.java | 72 +++++++ 21 files changed, 444 insertions(+), 101 deletions(-) create mode 100644 docs/changelog/113911.yaml create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/InferenceEventsAssertion.java diff --git a/docs/changelog/113911.yaml b/docs/changelog/113911.yaml new file mode 100644 index 0000000000000..5c2f93a6ea76a --- /dev/null +++ b/docs/changelog/113911.yaml @@ -0,0 +1,5 @@ +pr: 113911 +summary: Enable OpenAI Streaming +area: Machine Learning +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceServiceResults.java b/server/src/main/java/org/elasticsearch/inference/InferenceServiceResults.java index 5724a738209e2..34c8ffcb82d09 100644 --- a/server/src/main/java/org/elasticsearch/inference/InferenceServiceResults.java +++ b/server/src/main/java/org/elasticsearch/inference/InferenceServiceResults.java @@ -52,7 +52,7 @@ default boolean isStreaming() { * When {@link #isStreaming()} is {@code true}, the InferenceAction.Results will subscribe to this publisher. * Implementations should follow the {@link java.util.concurrent.Flow.Publisher} spec to stream the chunks. */ - default Flow.Publisher publisher() { + default Flow.Publisher publisher() { assert isStreaming() == false : "This must be implemented when isStreaming() == true"; throw new UnsupportedOperationException("This must be implemented when isStreaming() == true"); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/StreamingChatCompletionResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/StreamingChatCompletionResults.java index 672ad74419c8f..05a181d3fc5b6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/StreamingChatCompletionResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/StreamingChatCompletionResults.java @@ -7,22 +7,27 @@ package org.elasticsearch.xpack.core.inference.results; +import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ChunkedToXContent; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.inference.InferenceResults; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.xcontent.ToXContent; import java.io.IOException; +import java.util.Deque; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.Flow; +import static org.elasticsearch.xpack.core.inference.results.ChatCompletionResults.COMPLETION; + /** * Chat Completion results that only contain a Flow.Publisher. */ -public record StreamingChatCompletionResults(Flow.Publisher publisher) implements InferenceServiceResults { +public record StreamingChatCompletionResults(Flow.Publisher publisher) implements InferenceServiceResults { @Override public boolean isStreaming() { @@ -58,4 +63,30 @@ public void writeTo(StreamOutput out) throws IOException { public Iterator toXContentChunked(ToXContent.Params params) { throw new UnsupportedOperationException("Not implemented"); } + + public record Results(Deque results) implements ChunkedToXContent { + @Override + public Iterator toXContentChunked(ToXContent.Params params) { + return Iterators.concat( + ChunkedToXContentHelper.startObject(), + ChunkedToXContentHelper.startArray(COMPLETION), + Iterators.flatMap(results.iterator(), d -> d.toXContentChunked(params)), + ChunkedToXContentHelper.endArray(), + ChunkedToXContentHelper.endObject() + ); + } + } + + public record Result(String delta) implements ChunkedToXContent { + private static final String RESULT = "delta"; + + @Override + public Iterator toXContentChunked(ToXContent.Params params) { + return Iterators.concat( + ChunkedToXContentHelper.startObject(), + ChunkedToXContentHelper.field(RESULT, delta), + ChunkedToXContentHelper.endObject() + ); + } + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/DelegatingProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/DelegatingProcessor.java index 2273150d70ad6..9af5668ecf75b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/DelegatingProcessor.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/DelegatingProcessor.java @@ -89,7 +89,12 @@ public void onNext(T item) { if (isClosed.get()) { upstream.cancel(); } else { - next(item); + try { + next(item); + } catch (Exception e) { + upstream().cancel(); + onError(e); + } } } @@ -97,8 +102,9 @@ public void onNext(T item) { * An {@link #onNext(Object)} that is only called when the stream is still open. * Implementations can pass the resulting R object to the downstream subscriber via {@link #downstream()}, or the upstream can be * accessed via {@link #upstream()}. + * Any Exceptions thrown by this method will cancel the upstream and be sent to the downstream {@link #onError(Throwable)}. */ - protected abstract void next(T item); + protected abstract void next(T item) throws Exception; @Override public void onError(Throwable throwable) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/azureopenai/AzureOpenAiResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/azureopenai/AzureOpenAiResponseHandler.java index 2f72088327468..4b2168a42e3ac 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/azureopenai/AzureOpenAiResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/azureopenai/AzureOpenAiResponseHandler.java @@ -32,7 +32,7 @@ public class AzureOpenAiResponseHandler extends OpenAiResponseHandler { static final String REMAINING_TOKENS = "x-ratelimit-remaining-tokens"; public AzureOpenAiResponseHandler(String requestType, ResponseParser parseFunction) { - super(requestType, parseFunction); + super(requestType, parseFunction, false); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiCompletionRequestManager.java index 65f25c0baf8dc..cea89332e5bf0 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiCompletionRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiCompletionRequestManager.java @@ -19,7 +19,6 @@ import org.elasticsearch.xpack.inference.external.response.openai.OpenAiChatCompletionResponseEntity; import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModel; -import java.util.List; import java.util.Objects; import java.util.function.Supplier; @@ -47,8 +46,10 @@ public void execute( Supplier hasRequestCompletedFunction, ActionListener listener ) { - List docsInput = DocumentsOnlyInput.of(inferenceInputs).getInputs(); - OpenAiChatCompletionRequest request = new OpenAiChatCompletionRequest(docsInput, model); + var docsOnly = DocumentsOnlyInput.of(inferenceInputs); + var docsInput = docsOnly.getInputs(); + var stream = docsOnly.stream(); + OpenAiChatCompletionRequest request = new OpenAiChatCompletionRequest(docsInput, model, stream); execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener)); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiEmbeddingsRequestManager.java index 5c164f2eb9644..49fa15e5bc843 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiEmbeddingsRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiEmbeddingsRequestManager.java @@ -33,7 +33,7 @@ public class OpenAiEmbeddingsRequestManager extends OpenAiRequestManager { private static final ResponseHandler HANDLER = createEmbeddingsHandler(); private static ResponseHandler createEmbeddingsHandler() { - return new OpenAiResponseHandler("openai text embedding", OpenAiEmbeddingsResponseEntity::fromResponse); + return new OpenAiResponseHandler("openai text embedding", OpenAiEmbeddingsResponseEntity::fromResponse, false); } public static OpenAiEmbeddingsRequestManager of(OpenAiEmbeddingsModel model, Truncator truncator, ThreadPool threadPool) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiChatCompletionResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiChatCompletionResponseHandler.java index 08426dc4bbc4a..7607e5e4ed3a2 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiChatCompletionResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiChatCompletionResponseHandler.java @@ -7,20 +7,14 @@ package org.elasticsearch.xpack.inference.external.openai; -import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser; import org.elasticsearch.xpack.inference.external.http.retry.RetryException; import org.elasticsearch.xpack.inference.external.request.Request; -import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventParser; -import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventProcessor; - -import java.util.concurrent.Flow; public class OpenAiChatCompletionResponseHandler extends OpenAiResponseHandler { public OpenAiChatCompletionResponseHandler(String requestType, ResponseParser parseFunction) { - super(requestType, parseFunction); + super(requestType, parseFunction, true); } @Override @@ -28,19 +22,4 @@ protected RetryException buildExceptionHandling429(Request request, HttpResult r // We don't retry, if the chat completion input is too large return new RetryException(false, buildError(RATE_LIMIT, request, result)); } - - @Override - public boolean canHandleStreamingResponses() { - return true; - } - - @Override - public InferenceServiceResults parseResult(Request request, Flow.Publisher flow) { - var serverSentEventProcessor = new ServerSentEventProcessor(new ServerSentEventParser()); - var openAiProcessor = new OpenAiStreamingProcessor(); - - flow.subscribe(serverSentEventProcessor); - serverSentEventProcessor.subscribe(openAiProcessor); - return new StreamingChatCompletionResults(openAiProcessor); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandler.java index c23b94351c187..c193280e1978b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandler.java @@ -9,6 +9,8 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.common.Strings; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler; import org.elasticsearch.xpack.inference.external.http.retry.ContentTooLargeException; @@ -16,8 +18,12 @@ import org.elasticsearch.xpack.inference.external.http.retry.RetryException; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.external.response.openai.OpenAiErrorResponseEntity; +import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventParser; +import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventProcessor; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import java.util.concurrent.Flow; + import static org.elasticsearch.xpack.inference.external.http.HttpUtils.checkForEmptyBody; import static org.elasticsearch.xpack.inference.external.http.retry.ResponseHandlerUtils.getFirstHeaderOrUnknown; @@ -38,8 +44,11 @@ public class OpenAiResponseHandler extends BaseResponseHandler { static final String OPENAI_SERVER_BUSY = "Received a server busy error status code"; - public OpenAiResponseHandler(String requestType, ResponseParser parseFunction) { + private final boolean canHandleStreamingResponses; + + public OpenAiResponseHandler(String requestType, ResponseParser parseFunction, boolean canHandleStreamingResponses) { super(requestType, parseFunction, OpenAiErrorResponseEntity::fromResponse); + this.canHandleStreamingResponses = canHandleStreamingResponses; } @Override @@ -120,4 +129,19 @@ static String buildRateLimitErrorMessage(HttpResult result) { return RATE_LIMIT + ". " + usageMessage; } + + @Override + public boolean canHandleStreamingResponses() { + return canHandleStreamingResponses; + } + + @Override + public InferenceServiceResults parseResult(Request request, Flow.Publisher flow) { + var serverSentEventProcessor = new ServerSentEventProcessor(new ServerSentEventParser()); + var openAiProcessor = new OpenAiStreamingProcessor(); + + flow.subscribe(serverSentEventProcessor); + serverSentEventProcessor.subscribe(openAiProcessor); + return new StreamingChatCompletionResults(openAiProcessor); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessor.java index dcda832091e05..803bae40b33ed 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessor.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessor.java @@ -9,27 +9,28 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.xcontent.ChunkedToXContent; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; import org.elasticsearch.xpack.inference.common.DelegatingProcessor; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEvent; import org.elasticsearch.xpack.inference.external.response.streaming.ServerSentEventField; import java.io.IOException; import java.util.ArrayDeque; +import java.util.Collections; import java.util.Deque; import java.util.Iterator; -import java.util.Optional; +import java.util.Objects; +import java.util.function.Predicate; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; -import static org.elasticsearch.core.Strings.format; -import static org.elasticsearch.xpack.core.inference.results.ChatCompletionResults.COMPLETION; +import static org.elasticsearch.common.xcontent.XContentParserUtils.parseList; +import static org.elasticsearch.xpack.inference.external.response.XContentUtils.consumeUntilObjectEnd; import static org.elasticsearch.xpack.inference.external.response.XContentUtils.moveToFirstToken; import static org.elasticsearch.xpack.inference.external.response.XContentUtils.positionParserAtTokenAfterField; @@ -105,7 +106,6 @@ public class OpenAiStreamingProcessor extends DelegatingProcessor, ChunkedToXContent> { private static final Logger log = LogManager.getLogger(OpenAiStreamingProcessor.class); private static final String FAILED_TO_FIND_FIELD_TEMPLATE = "Failed to find required field [%s] in OpenAI chat completions response"; - private static final String RESULT = "delta"; private static final String CHOICES_FIELD = "choices"; private static final String DELTA_FIELD = "delta"; @@ -115,19 +115,18 @@ public class OpenAiStreamingProcessor extends DelegatingProcessor item) { + protected void next(Deque item) throws Exception { var parserConfig = XContentParserConfiguration.EMPTY.withDeprecationHandler(LoggingDeprecationHandler.INSTANCE); - var results = new ArrayDeque(item.size()); + var results = new ArrayDeque(item.size()); for (ServerSentEvent event : item) { if (ServerSentEventField.DATA == event.name() && event.hasValue()) { try { var delta = parse(parserConfig, event); - delta.map(this::deltaChunk).ifPresent(results::offer); + delta.forEachRemaining(results::offer); } catch (Exception e) { log.warn("Failed to parse event from inference provider: {}", event); - onError(new IOException("Failed to parse event from inference provider.", e)); - return; + throw e; } } } @@ -135,13 +134,14 @@ protected void next(Deque item) { if (results.isEmpty()) { upstream().request(1); } else { - downstream().onNext(completionChunk(results.iterator())); + downstream().onNext(new StreamingChatCompletionResults.Results(results)); } } - private Optional parse(XContentParserConfiguration parserConfig, ServerSentEvent event) throws IOException { + private Iterator parse(XContentParserConfiguration parserConfig, ServerSentEvent event) + throws IOException { if (DONE_MESSAGE.equalsIgnoreCase(event.value())) { - return Optional.empty(); + return Collections.emptyIterator(); } try (XContentParser jsonParser = XContentFactory.xContent(XContentType.JSON).createParser(parserConfig, event.value())) { @@ -150,54 +150,38 @@ private Optional parse(XContentParserConfiguration parserConfig, ServerS XContentParser.Token token = jsonParser.currentToken(); ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser); - // choices is an array, but since we don't send 'n' in the request then we only get one value in the result positionParserAtTokenAfterField(jsonParser, CHOICES_FIELD, FAILED_TO_FIND_FIELD_TEMPLATE); - jsonParser.nextToken(); - ensureExpectedToken(XContentParser.Token.START_OBJECT, jsonParser.currentToken(), jsonParser); + return parseList(jsonParser, parser -> { + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); - positionParserAtTokenAfterField(jsonParser, DELTA_FIELD, FAILED_TO_FIND_FIELD_TEMPLATE); + positionParserAtTokenAfterField(parser, DELTA_FIELD, FAILED_TO_FIND_FIELD_TEMPLATE); - token = jsonParser.currentToken(); + var currentToken = parser.currentToken(); - ensureExpectedToken(XContentParser.Token.START_OBJECT, token, jsonParser); + ensureExpectedToken(XContentParser.Token.START_OBJECT, currentToken, parser); - while (token != null) { - if (token == XContentParser.Token.FIELD_NAME && jsonParser.currentName().equals(CONTENT_FIELD)) { - jsonParser.nextToken(); - var contentToken = jsonParser.currentToken(); - ensureExpectedToken(XContentParser.Token.VALUE_STRING, contentToken, jsonParser); - return Optional.ofNullable(jsonParser.text()); - } else if (token == XContentParser.Token.FIELD_NAME && jsonParser.currentName().equals(FINISH_REASON_FIELD)) { - jsonParser.nextToken(); - var contentToken = jsonParser.currentToken(); - ensureExpectedToken(XContentParser.Token.VALUE_STRING, contentToken, jsonParser); - if (STOP_MESSAGE.equalsIgnoreCase(jsonParser.text())) { - return Optional.empty(); - } + currentToken = parser.nextToken(); + if (currentToken == XContentParser.Token.END_OBJECT) { + consumeUntilObjectEnd(parser); // end choices + return ""; // stopped } - token = jsonParser.nextToken(); - } - throw new IllegalStateException(format(FAILED_TO_FIND_FIELD_TEMPLATE, CONTENT_FIELD)); + if (currentToken == XContentParser.Token.FIELD_NAME && parser.currentName().equals(CONTENT_FIELD)) { + parser.nextToken(); + } else { + positionParserAtTokenAfterField(parser, CONTENT_FIELD, FAILED_TO_FIND_FIELD_TEMPLATE); + } + ensureExpectedToken(XContentParser.Token.VALUE_STRING, parser.currentToken(), parser); + var content = parser.text(); + consumeUntilObjectEnd(parser); // end delta + consumeUntilObjectEnd(parser); // end choices + return content; + }).stream() + .filter(Objects::nonNull) + .filter(Predicate.not(String::isEmpty)) + .map(StreamingChatCompletionResults.Result::new) + .iterator(); } } - - private ChunkedToXContent deltaChunk(String delta) { - return params -> Iterators.concat( - ChunkedToXContentHelper.startObject(), - ChunkedToXContentHelper.field(RESULT, delta), - ChunkedToXContentHelper.endObject() - ); - } - - private ChunkedToXContent completionChunk(Iterator delta) { - return params -> Iterators.concat( - ChunkedToXContentHelper.startObject(), - ChunkedToXContentHelper.startArray(COMPLETION), - Iterators.flatMap(delta, d -> d.toXContentChunked(params)), - ChunkedToXContentHelper.endArray(), - ChunkedToXContentHelper.endObject() - ); - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequest.java index 9fa6533161745..99a025e70d003 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequest.java @@ -32,11 +32,13 @@ public class OpenAiChatCompletionRequest implements OpenAiRequest { private final OpenAiAccount account; private final List input; private final OpenAiChatCompletionModel model; + private final boolean stream; - public OpenAiChatCompletionRequest(List input, OpenAiChatCompletionModel model) { + public OpenAiChatCompletionRequest(List input, OpenAiChatCompletionModel model, boolean stream) { this.account = OpenAiAccount.of(model, OpenAiChatCompletionRequest::buildDefaultUri); this.input = Objects.requireNonNull(input); this.model = Objects.requireNonNull(model); + this.stream = stream; } @Override @@ -45,7 +47,7 @@ public HttpRequest createHttpRequest() { ByteArrayEntity byteEntity = new ByteArrayEntity( Strings.toString( - new OpenAiChatCompletionRequestEntity(input, model.getServiceSettings().modelId(), model.getTaskSettings().user()) + new OpenAiChatCompletionRequestEntity(input, model.getServiceSettings().modelId(), model.getTaskSettings().user(), stream) ).getBytes(StandardCharsets.UTF_8) ); httpPost.setEntity(byteEntity); @@ -83,6 +85,11 @@ public String getInferenceEntityId() { return model.getInferenceEntityId(); } + @Override + public boolean isStreaming() { + return stream; + } + public static URI buildDefaultUri() throws URISyntaxException { return new URIBuilder().setScheme("https") .setHost(OpenAiUtils.HOST) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestEntity.java index c9aa225c77941..867a7ca80cbcb 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestEntity.java @@ -25,19 +25,22 @@ public class OpenAiChatCompletionRequestEntity implements ToXContentObject { private static final String ROLE_FIELD = "role"; private static final String USER_FIELD = "user"; private static final String CONTENT_FIELD = "content"; + private static final String STREAM_FIELD = "stream"; private final List messages; private final String model; private final String user; + private final boolean stream; - public OpenAiChatCompletionRequestEntity(List messages, String model, String user) { + public OpenAiChatCompletionRequestEntity(List messages, String model, String user, boolean stream) { Objects.requireNonNull(messages); Objects.requireNonNull(model); this.messages = messages; this.model = model; this.user = user; + this.stream = stream; } @Override @@ -65,6 +68,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(USER_FIELD, user); } + if (stream) { + builder.field(STREAM_FIELD, true); + } + builder.endObject(); return builder; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java index 21b2df6af1ab6..71b38d7a0785a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/SenderService.java @@ -17,6 +17,7 @@ import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.TaskType; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; @@ -27,8 +28,10 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; public abstract class SenderService implements InferenceService { + protected static final Set COMPLETION_ONLY = Set.of(TaskType.COMPLETION); private final Sender sender; private final ServiceComponents serviceComponents; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java index bba8721c48c88..fd5f7197475aa 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java @@ -42,6 +42,7 @@ import java.util.List; import java.util.Map; +import java.util.Set; import static org.elasticsearch.xpack.inference.services.ServiceFields.MODEL_ID; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; @@ -314,6 +315,11 @@ public TransportVersion getMinimalSupportedVersion() { return TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED; } + @Override + public Set supportedStreamingTasks() { + return COMPLETION_ONLY; + } + /** * Model was originally defined in task settings, but it should * have been part of the service settings. diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/DelegatingProcessorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/DelegatingProcessorTests.java index 826eaf6cd6860..e86b909af91aa 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/DelegatingProcessorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/DelegatingProcessorTests.java @@ -14,6 +14,7 @@ import java.util.concurrent.atomic.AtomicReference; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; @@ -28,17 +29,25 @@ public class DelegatingProcessorTests extends ESTestCase { public static R onNext(DelegatingProcessor processor, T item) { var response = new AtomicReference(); + var error = new AtomicReference(); processor.onSubscribe(mock()); Flow.Subscriber downstream = mock(); + doAnswer(ans -> { response.set(ans.getArgument(0)); return null; }).when(downstream).onNext(any()); + doAnswer(ans -> { + error.set(ans.getArgument(0)); + return null; + }).when(downstream).onError(any()); + processor.subscribe(downstream); processor.onNext(item); + assertThat("onError should not be called", error.get(), nullValue()); assertThat("Response from processor was null", response.get(), notNullValue()); return response.get(); } @@ -46,7 +55,8 @@ public static R onNext(DelegatingProcessor processor, T item) { public static Throwable onError(DelegatingProcessor processor, T item) { var response = new AtomicReference(); - processor.onSubscribe(mock()); + Flow.Subscription upstream = mock(); + processor.onSubscribe(upstream); Flow.Subscriber downstream = mock(); doAnswer(ans -> { @@ -57,6 +67,7 @@ public static Throwable onError(DelegatingProcessor processor, T it processor.onNext(item); assertThat("Error from processor was null", response.get(), notNullValue()); + verify(upstream, times(1)).cancel(); return response.get(); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandlerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandlerTests.java index bda709017f266..de5b2416cc766 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandlerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandlerTests.java @@ -42,7 +42,7 @@ public void testCheckForFailureStatusCode() { var mockRequest = RequestTests.mockRequest("id"); var httpResult = new HttpResult(httpResponse, new byte[] {}); - var handler = new OpenAiResponseHandler("", (request, result) -> null); + var handler = new OpenAiResponseHandler("", (request, result) -> null, false); // 200 ok when(statusLine.getStatusCode()).thenReturn(200); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessorTests.java index 992990a476a0c..a57e7c1b64c07 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/openai/OpenAiStreamingProcessorTests.java @@ -111,12 +111,10 @@ public void testParseErrorCallsOnError() { item.offer(new ServerSentEvent(ServerSentEventField.DATA, "this isn't json")); var exception = onError(new OpenAiStreamingProcessor(), item); - assertThat(exception, instanceOf(IOException.class)); - assertThat(exception.getMessage(), equalTo("Failed to parse event from inference provider.")); - assertThat(exception.getCause(), instanceOf(XContentParseException.class)); + assertThat(exception, instanceOf(XContentParseException.class)); } - public void testEmptyResultsRequestsMoreData() { + public void testEmptyResultsRequestsMoreData() throws Exception { var emptyDeque = new ArrayDeque(); var processor = new OpenAiStreamingProcessor(); @@ -133,7 +131,7 @@ public void testEmptyResultsRequestsMoreData() { verify(downstream, times(0)).onNext(any()); } - public void testDoneMessageIsIgnored() { + public void testDoneMessageIsIgnored() throws Exception { var item = new ArrayDeque(); item.offer(new ServerSentEvent(ServerSentEventField.DATA, "[DONE]")); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestEntityTests.java index 0b61bf060fc5f..9d5492f9e9516 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestEntityTests.java @@ -21,7 +21,7 @@ public class OpenAiChatCompletionRequestEntityTests extends ESTestCase { public void testXContent_WritesUserWhenDefined() throws IOException { - var entity = new OpenAiChatCompletionRequestEntity(List.of("abc"), "model", "user"); + var entity = new OpenAiChatCompletionRequestEntity(List.of("abc"), "model", "user", false); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); entity.toXContent(builder, null); @@ -33,7 +33,7 @@ public void testXContent_WritesUserWhenDefined() throws IOException { } public void testXContent_DoesNotWriteUserWhenItIsNull() throws IOException { - var entity = new OpenAiChatCompletionRequestEntity(List.of("abc"), "model", null); + var entity = new OpenAiChatCompletionRequestEntity(List.of("abc"), "model", null, false); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); entity.toXContent(builder, null); @@ -44,10 +44,10 @@ public void testXContent_DoesNotWriteUserWhenItIsNull() throws IOException { } public void testXContent_ThrowsIfModelIsNull() { - assertThrows(NullPointerException.class, () -> new OpenAiChatCompletionRequestEntity(List.of("abc"), null, "user")); + assertThrows(NullPointerException.class, () -> new OpenAiChatCompletionRequestEntity(List.of("abc"), null, "user", false)); } public void testXContent_ThrowsIfMessagesAreNull() { - assertThrows(NullPointerException.class, () -> new OpenAiChatCompletionRequestEntity(null, "model", "user")); + assertThrows(NullPointerException.class, () -> new OpenAiChatCompletionRequestEntity(null, "model", "user", false)); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestTests.java index b71508021eddd..b6ebfd02941f3 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestTests.java @@ -87,6 +87,17 @@ public void testCreateRequest_WithDefaultUrlAndWithoutUserOrganization() throws assertThat(requestMap.get("n"), is(1)); } + public void testCreateRequest_WithStreaming() throws URISyntaxException, IOException { + var request = createRequest(null, null, "secret", "abc", "model", null, true); + var httpRequest = request.createHttpRequest(); + + assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); + var httpPost = (HttpPost) httpRequest.httpRequestBase(); + + var requestMap = entityAsMap(httpPost.getEntity().getContent()); + assertThat(requestMap.get("stream"), is(true)); + } + public void testTruncate_DoesNotReduceInputTextSize() throws URISyntaxException, IOException { var request = createRequest(null, null, "secret", "abcd", "model", null); var truncatedRequest = request.truncate(); @@ -117,9 +128,21 @@ public static OpenAiChatCompletionRequest createRequest( String input, String model, @Nullable String user + ) { + return createRequest(url, org, apiKey, input, model, user, false); + } + + public static OpenAiChatCompletionRequest createRequest( + @Nullable String url, + @Nullable String org, + String apiKey, + String input, + String model, + @Nullable String user, + boolean stream ) { var chatCompletionModel = OpenAiChatCompletionModelTests.createChatCompletionModel(url, org, apiKey, model, user); - return new OpenAiChatCompletionRequest(List.of(input), chatCompletionModel); + return new OpenAiChatCompletionRequest(List.of(input), chatCompletionModel, stream); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/InferenceEventsAssertion.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/InferenceEventsAssertion.java new file mode 100644 index 0000000000000..f23ea2aa414b2 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/InferenceEventsAssertion.java @@ -0,0 +1,186 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ChunkedToXContent; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentFactory; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Flow; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; +import static org.hamcrest.CoreMatchers.is; + +/** + * Helper to chain together assertions for streaming {@link InferenceServiceResults}. + */ +public record InferenceEventsAssertion(Iterator events, Throwable error, boolean isComplete, int iterations) { + + public static InferenceEventsAssertion assertThat(InferenceServiceResults results) throws Exception { + return TestSubscriber.subscribeAndWait(results.publisher()).toAssertion(); + } + + public InferenceEventsAssertion hasFinishedStream() { + MatcherAssert.assertThat( + "Expected publisher to eventually call onComplete, but it stopped after [" + iterations + "] iterations.", + isComplete + ); + return this; + } + + public InferenceEventsAssertion hasNoErrors() { + MatcherAssert.assertThat("Expected no errors from stream.", error, Matchers.nullValue()); + return this; + } + + public InferenceEventsAssertion hasError() { + MatcherAssert.assertThat("Expected error from stream.", error, Matchers.notNullValue()); + return this; + } + + public InferenceEventsAssertion hasErrorWithStatusCode(int statusCode) { + hasError(); + Throwable t = error; + while (t != null) { + if (t instanceof ElasticsearchStatusException statusException) { + MatcherAssert.assertThat(statusException.status().getStatus(), Matchers.equalTo(statusCode)); + return this; + } + t = t.getCause(); + } + ESTestCase.fail(error, "Expected an underlying ElasticsearchStatusException."); + return this; + } + + public InferenceEventsAssertion hasErrorContaining(String message) { + hasError(); + Throwable t = error; + while (t != null) { + if (t.getMessage() != null && t.getMessage().contains(message)) { + return this; + } + t = t.getCause(); + } + ESTestCase.fail(error, "Expected exception to contain string: " + message); + return this; + } + + public InferenceEventsAssertion hasEvents(String... events) { + Arrays.stream(events).forEach(this::hasEvent); + return this; + } + + public InferenceEventsAssertion hasNoEvents() { + MatcherAssert.assertThat("Expected no items processed by the subscriber.", iterations, Matchers.is(0)); + return this; + } + + public InferenceEventsAssertion hasEvent(String event) { + MatcherAssert.assertThat( + "Subscriber returned [" + iterations + "] results, but we expect at least one more.", + this.events.hasNext(), + Matchers.is(true) + ); + MatcherAssert.assertThat(this.events.next(), Matchers.equalTo(event)); + return this; + } + + private static class TestSubscriber implements Flow.Subscriber { + private final CountDownLatch latch = new CountDownLatch(1); + private final AtomicInteger infiniteLoopCheck = new AtomicInteger(0); + private final Stream.Builder events = Stream.builder(); + private Throwable error; + private boolean isComplete; + private Flow.Subscription subscription; + + private static TestSubscriber subscribeAndWait(Flow.Publisher publisher) throws Exception { + var testSubscriber = new TestSubscriber(); + publisher.subscribe(testSubscriber); + // the subscriber will initiate response handling on another thread, so we need to wait for that thread to finish + try { + MatcherAssert.assertThat( + "Timed out waiting for publisher or mock web server to finish. Collected [" + + testSubscriber.infiniteLoopCheck.get() + + "] items.", + testSubscriber.latch.await(10, TimeUnit.SECONDS), + is(true) + ); + } catch (Exception e) { + // the test is about to fail, but stop the mock server from responding anyway + testSubscriber.subscription.cancel(); + throw e; + } + return testSubscriber; + } + + @Override + public void onSubscribe(Flow.Subscription subscription) { + this.subscription = subscription; + subscription.request(1); + } + + @Override + public void onNext(T item) { + if (infiniteLoopCheck.incrementAndGet() > 10) { + subscription.cancel(); + latch.countDown(); + return; + } + + try { + events.add(toJsonString(item)); + subscription.request(1); + } catch (IOException e) { + onError(e); + } + } + + @Override + public void onError(Throwable throwable) { + error = throwable; + isComplete = true; + latch.countDown(); + } + + @Override + public void onComplete() { + isComplete = true; + latch.countDown(); + } + + private String toJsonString(ChunkedToXContent chunkedToXContent) throws IOException { + try (var builder = XContentFactory.jsonBuilder()) { + chunkedToXContent.toXContentChunked(EMPTY_PARAMS).forEachRemaining(xContent -> { + try { + xContent.toXContent(builder, EMPTY_PARAMS); + } catch (IOException e) { + throw new IllegalStateException(e); + } + }); + return XContentHelper.convertToJson(BytesReference.bytes(builder), false, builder.contentType()); + } + } + + private InferenceEventsAssertion toAssertion() { + return new InferenceEventsAssertion(events.build().iterator(), error, isComplete, infiniteLoopCheck.get()); + } + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java index 508da45ac2fc2..cf1438b334478 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceTests.java @@ -38,8 +38,10 @@ import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import org.elasticsearch.xpack.inference.services.InferenceEventsAssertion; import org.elasticsearch.xpack.inference.services.ServiceFields; import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModel; +import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModelTests; import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsModel; import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsModelTests; import org.hamcrest.CoreMatchers; @@ -1005,6 +1007,76 @@ public void testInfer_SendsRequest() throws IOException { } } + public void testInfer_StreamRequest() throws Exception { + String responseJson = """ + data: {\ + "id":"12345",\ + "object":"chat.completion.chunk",\ + "created":123456789,\ + "model":"gpt-4o-mini",\ + "system_fingerprint": "123456789",\ + "choices":[\ + {\ + "index":0,\ + "delta":{\ + "content":"hello, world"\ + },\ + "logprobs":null,\ + "finish_reason":null\ + }\ + ]\ + } + + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var result = streamChatCompletion(); + + InferenceEventsAssertion.assertThat(result).hasFinishedStream().hasNoErrors().hasEvent(""" + {"completion":[{"delta":"hello, world"}]}"""); + } + + private InferenceServiceResults streamChatCompletion() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new OpenAiService(senderFactory, createWithEmptySettings(threadPool))) { + var model = OpenAiChatCompletionModelTests.createChatCompletionModel(getUrl(webServer), "org", "secret", "model", "user"); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + null, + List.of("abc"), + true, + new HashMap<>(), + InputType.INGEST, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + return listener.actionGet(TIMEOUT); + } + } + + public void testInfer_StreamRequest_ErrorResponse() throws Exception { + String responseJson = """ + { + "error": { + "message": "You didn't provide an API key...", + "type": "invalid_request_error", + "param": null, + "code": null + } + }"""; + webServer.enqueue(new MockResponse().setResponseCode(401).setBody(responseJson)); + + var result = streamChatCompletion(); + + InferenceEventsAssertion.assertThat(result) + .hasFinishedStream() + .hasNoEvents() + .hasErrorWithStatusCode(401) + .hasErrorContaining("You didn't provide an API key..."); + } + public void testCheckModelConfig_IncludesMaxTokens() throws IOException { var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); From fa01665ea961dc528fa11c3474089781b7332701 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 3 Oct 2024 11:27:54 -0400 Subject: [PATCH 041/194] ESQL: More tests for anyTrue and anyFalse (#113958) These were added in #113804 and missing the `false`/`false` test case. --- .../elasticsearch/compute/data/BasicBlockTests.java | 10 +++++++--- .../compute/data/BigArrayVectorTests.java | 5 ++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java index 1fd670f836df3..439ebe34c7d4a 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java @@ -830,16 +830,20 @@ public void testBooleanBlock() { randomBoolean() ? randomIntBetween(1, positionCount) : positionCount ); Boolean value = randomFrom(random(), null, true, false); - IntStream.range(0, positionCount).mapToObj(ii -> { + Boolean[] bools = IntStream.range(0, positionCount).mapToObj(ii -> { if (value == null) { return randomBoolean(); } return value; - }).forEach(vectorBuilder::appendBoolean); + }).toArray(Boolean[]::new); + Arrays.stream(bools).forEach(vectorBuilder::appendBoolean); BooleanVector vector = vectorBuilder.build(); assertSingleValueDenseBlock(vector.asBlock()); assertToMask(vector); - if (value != null) { + if (value == null) { + assertThat(vector.allTrue(), equalTo(Arrays.stream(bools).allMatch(v -> v))); + assertThat(vector.allFalse(), equalTo(Arrays.stream(bools).allMatch(v -> v == false))); + } else { if (value) { assertTrue(vector.allTrue()); assertFalse(vector.allFalse()); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayVectorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayVectorTests.java index 6225aa1a6f2a0..929e38dc31751 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayVectorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BigArrayVectorTests.java @@ -84,7 +84,10 @@ public void testBoolean() throws IOException { assertThat(mask.mask().getBoolean(p), equalTo(values[p])); } } - if (value != null) { + if (value == null) { + assertThat(vector.allTrue(), equalTo(Arrays.stream(values).allMatch(v -> v))); + assertThat(vector.allFalse(), equalTo(Arrays.stream(values).allMatch(v -> v == false))); + } else { if (value) { assertTrue(vector.allTrue()); assertFalse(vector.allFalse()); From f4cb32991a2a15e3aebecdcf1740e18045f4aa55 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 3 Oct 2024 11:30:03 -0400 Subject: [PATCH 042/194] ESQL: change link to profile explanation (#114032) Let's use a vendor neutral link. --- docs/reference/esql/esql-query-api.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/esql/esql-query-api.asciidoc b/docs/reference/esql/esql-query-api.asciidoc index 3c8d1f89496ff..d1db21043a5b5 100644 --- a/docs/reference/esql/esql-query-api.asciidoc +++ b/docs/reference/esql/esql-query-api.asciidoc @@ -80,7 +80,7 @@ For syntax, refer to <>. with information about how the query was executed. It provides insight into the performance of each part of the query. This is for human debugging as the object's format might change at any time. Think of this like https://www.postgresql.org/docs/current/sql-explain.html[EXPLAIN ANALYZE] or -https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/EXPLAIN-PLAN.html[EXPLAIN PLAN]. +https://en.wikipedia.org/wiki/Query_plan[EXPLAIN PLAN]. `query`:: (Required, string) {esql} query to run. For syntax, refer to <>. @@ -113,4 +113,4 @@ Values for the search results. Profile describing the execution of the query. Only returned if `profile` was sent in the body. The object itself is for human debugging and can change at any time. Think of this like https://www.postgresql.org/docs/current/sql-explain.html[EXPLAIN ANALYZE] or -https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/EXPLAIN-PLAN.html[EXPLAIN PLAN]. +https://en.wikipedia.org/wiki/Query_plan[EXPLAIN PLAN]. From 24260ce88a4c36a98944af7f5e5716c4e3a2944a Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 3 Oct 2024 11:36:07 -0400 Subject: [PATCH 043/194] [CI] Replace openjdk17 with openjdk21 in failing java-matrix tasks (#113899) --- .buildkite/pipelines/periodic.template.yml | 2 +- .buildkite/pipelines/periodic.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipelines/periodic.template.yml b/.buildkite/pipelines/periodic.template.yml index 5048916a9cac9..08648e2eedd3c 100644 --- a/.buildkite/pipelines/periodic.template.yml +++ b/.buildkite/pipelines/periodic.template.yml @@ -69,7 +69,7 @@ steps: matrix: setup: ES_RUNTIME_JAVA: - - openjdk17 + - openjdk21 BWC_VERSION: $BWC_LIST agents: provider: gcp diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index fa7e84fae160b..8e35d1561b6d7 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -430,7 +430,7 @@ steps: matrix: setup: ES_RUNTIME_JAVA: - - openjdk17 + - openjdk21 BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"] agents: provider: gcp From 5923d32221aa6a0fbb092e7d9b646f0287db66f7 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 3 Oct 2024 16:43:48 +0100 Subject: [PATCH 044/194] Revert `es.trigger_merge_after_recovery` sysprop name (#114026) Closes ES-9623 --- .../java/org/elasticsearch/indices/PostRecoveryMerger.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/PostRecoveryMerger.java b/server/src/main/java/org/elasticsearch/indices/PostRecoveryMerger.java index 44b5d857946cc..17382e7854e90 100644 --- a/server/src/main/java/org/elasticsearch/indices/PostRecoveryMerger.java +++ b/server/src/main/java/org/elasticsearch/indices/PostRecoveryMerger.java @@ -46,14 +46,14 @@ class PostRecoveryMerger { private static final boolean TRIGGER_MERGE_AFTER_RECOVERY; static { - final var propertyValue = System.getProperty("es.trigger_merge_after_recovery_8_515_00_0"); + final var propertyValue = System.getProperty("es.trigger_merge_after_recovery"); if (propertyValue == null) { TRIGGER_MERGE_AFTER_RECOVERY = true; } else if ("false".equals(propertyValue)) { TRIGGER_MERGE_AFTER_RECOVERY = false; } else { throw new IllegalStateException( - "system property [es.trigger_merge_after_recovery_8_515_00_0] may only be set to [false], but was [" + propertyValue + "]" + "system property [es.trigger_merge_after_recovery] may only be set to [false], but was [" + propertyValue + "]" ); } } From 5bee44daddaa52510258026b495d5d5a369406b0 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 3 Oct 2024 08:47:05 -0700 Subject: [PATCH 045/194] Track search and fetch failure stats (#113988) This PR tracks the total number of query and fetch failures, in addition to the existing metrics for each shard, and exposes them through the stats API. --- docs/changelog/113988.yaml | 5 + .../search/stats/SearchStatsIT.java | 118 +++++++++++++++++- .../org/elasticsearch/TransportVersions.java | 1 + .../index/search/stats/SearchStats.java | 76 ++++++++--- .../index/search/stats/ShardSearchStats.java | 22 +++- .../cluster/node/stats/NodeStatsTests.java | 2 + .../index/search/stats/SearchStatsTests.java | 6 +- .../indices/IndexStatsMonitoringDocTests.java | 2 +- .../IndicesStatsMonitoringDocTests.java | 2 +- .../node/NodeStatsMonitoringDocTests.java | 2 +- 10 files changed, 205 insertions(+), 31 deletions(-) create mode 100644 docs/changelog/113988.yaml diff --git a/docs/changelog/113988.yaml b/docs/changelog/113988.yaml new file mode 100644 index 0000000000000..d55e7eb2db326 --- /dev/null +++ b/docs/changelog/113988.yaml @@ -0,0 +1,5 @@ +pr: 113988 +summary: Track search and fetch failure stats +area: Stats +type: enhancement +issues: [] diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java index d0bae2b9ee47e..9f40b1928dce6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java @@ -13,23 +13,35 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.mapper.OnScriptError; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.search.stats.SearchStats.Stats; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.LongFieldScript; import org.elasticsearch.script.MockScriptPlugin; import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; @@ -53,7 +65,7 @@ public class SearchStatsIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Collections.singleton(CustomScriptPlugin.class); + return List.of(CustomScriptPlugin.class, FailingFieldPlugin.class); } public static class CustomScriptPlugin extends MockScriptPlugin { @@ -68,6 +80,50 @@ protected Map, Object>> pluginScripts() { } } + public static class FailingFieldPlugin extends Plugin implements ScriptPlugin { + + @Override + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { + return new ScriptEngine() { + @Override + public String getType() { + return "failing_field"; + } + + @Override + @SuppressWarnings("unchecked") + public FactoryType compile( + String name, + String code, + ScriptContext context, + Map params + ) { + return (FactoryType) new LongFieldScript.Factory() { + @Override + public LongFieldScript.LeafFactory newFactory( + String fieldName, + Map params, + SearchLookup searchLookup, + OnScriptError onScriptError + ) { + return ctx -> new LongFieldScript(fieldName, params, searchLookup, onScriptError, ctx) { + @Override + public void execute() { + throw new IllegalArgumentException("Accessing failing field"); + } + }; + } + }; + } + + @Override + public Set> getSupportedContexts() { + return Set.of(LongFieldScript.CONTEXT); + } + }; + } + } + @Override protected int numberOfReplicas() { return 0; @@ -244,4 +300,64 @@ protected int numAssignedShards(String... indices) { GroupShardsIterator allAssignedShardsGrouped = state.routingTable().allAssignedShardsGrouped(indices, true); return allAssignedShardsGrouped.size(); } + + public void testFailureStats() throws Exception { + String indexName = "test"; + XContentBuilder mapping = JsonXContent.contentBuilder().startObject(); + mapping.startObject("runtime"); + { + mapping.startObject("fail_me"); + { + mapping.field("type", "long"); + mapping.startObject("script").field("source", "").field("lang", "failing_field").endObject(); + } + mapping.endObject(); + } + mapping.endObject(); + mapping.endObject(); + int numOfShards = between(1, 5); + client().admin() + .indices() + .prepareCreate(indexName) + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards)) + .setMapping(mapping) + .get(); + int numDocs = between(20, 100); + for (int i = 1; i < numDocs; i++) { + index(indexName, Integer.toString(i), Map.of("position", i)); + } + refresh(indexName); + int numQueries = between(1, 10); + long failedQueries = 0; + for (int q = 0; q < numQueries; q++) { + expectThrows(Exception.class, () -> { + client().prepareSearch(indexName) + .setQuery(new RangeQueryBuilder("fail_me").gt(10)) + .setAllowPartialSearchResults(true) + .get(); + }); + failedQueries += numOfShards; + var stats = client().admin().indices().prepareStats(indexName).all().get().getTotal().search.getTotal(); + assertThat(stats.getQueryCount(), equalTo(0L)); + assertThat(stats.getQueryFailure(), equalTo(failedQueries)); + assertThat(stats.getFetchCount(), equalTo(0L)); + assertThat(stats.getFetchFailure(), equalTo(0L)); + } + int numFetches = between(1, 10); + for (int q = 0; q < numFetches; q++) { + expectThrows(Exception.class, () -> { + client().prepareSearch(indexName) + .setQuery(new RangeQueryBuilder("position").gt(0)) + .setFetchSource(false) + .addFetchField("fail_me") + .setSize(1000) + .get(); + }); + var stats = client().admin().indices().prepareStats(indexName).all().get().getTotal().search.getTotal(); + assertThat(stats.getQueryCount(), equalTo((q + 1L) * numOfShards)); + assertThat(stats.getQueryFailure(), equalTo(failedQueries)); + assertThat(stats.getFetchCount(), equalTo(0L)); + assertThat(stats.getFetchFailure(), equalTo((q + 1L) * numOfShards)); + } + } } diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 7ff0ed1bbe82c..b82afe2a22fa6 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -232,6 +232,7 @@ static TransportVersion def(int id) { public static final TransportVersion ESQL_CCS_EXECUTION_INFO = def(8_756_00_0); public static final TransportVersion REGEX_AND_RANGE_INTERVAL_QUERIES = def(8_757_00_0); public static final TransportVersion RRF_QUERY_REWRITE = def(8_758_00_0); + public static final TransportVersion SEARCH_FAILURE_STATS = def(8_759_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java b/server/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java index f6521960be290..ff514091979c3 100644 --- a/server/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java +++ b/server/src/main/java/org/elasticsearch/index/search/stats/SearchStats.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.search.stats; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -45,6 +46,9 @@ public static class Stats implements Writeable, ToXContentFragment { private long suggestTimeInMillis; private long suggestCurrent; + private long queryFailure; + private long fetchFailure; + private Stats() { // for internal use, initializes all counts to 0 } @@ -53,9 +57,11 @@ public Stats( long queryCount, long queryTimeInMillis, long queryCurrent, + long queryFailure, long fetchCount, long fetchTimeInMillis, long fetchCurrent, + long fetchFailure, long scrollCount, long scrollTimeInMillis, long scrollCurrent, @@ -66,10 +72,12 @@ public Stats( this.queryCount = queryCount; this.queryTimeInMillis = queryTimeInMillis; this.queryCurrent = queryCurrent; + this.queryFailure = queryFailure; this.fetchCount = fetchCount; this.fetchTimeInMillis = fetchTimeInMillis; this.fetchCurrent = fetchCurrent; + this.fetchFailure = fetchFailure; this.scrollCount = scrollCount; this.scrollTimeInMillis = scrollTimeInMillis; @@ -96,16 +104,47 @@ private Stats(StreamInput in) throws IOException { suggestCount = in.readVLong(); suggestTimeInMillis = in.readVLong(); suggestCurrent = in.readVLong(); + + if (in.getTransportVersion().onOrAfter(TransportVersions.SEARCH_FAILURE_STATS)) { + queryFailure = in.readVLong(); + fetchFailure = in.readVLong(); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(queryCount); + out.writeVLong(queryTimeInMillis); + out.writeVLong(queryCurrent); + + out.writeVLong(fetchCount); + out.writeVLong(fetchTimeInMillis); + out.writeVLong(fetchCurrent); + + out.writeVLong(scrollCount); + out.writeVLong(scrollTimeInMillis); + out.writeVLong(scrollCurrent); + + out.writeVLong(suggestCount); + out.writeVLong(suggestTimeInMillis); + out.writeVLong(suggestCurrent); + + if (out.getTransportVersion().onOrAfter(TransportVersions.SEARCH_FAILURE_STATS)) { + out.writeVLong(queryFailure); + out.writeVLong(fetchFailure); + } } public void add(Stats stats) { queryCount += stats.queryCount; queryTimeInMillis += stats.queryTimeInMillis; queryCurrent += stats.queryCurrent; + queryFailure += stats.queryFailure; fetchCount += stats.fetchCount; fetchTimeInMillis += stats.fetchTimeInMillis; fetchCurrent += stats.fetchCurrent; + fetchFailure += stats.fetchFailure; scrollCount += stats.scrollCount; scrollTimeInMillis += stats.scrollTimeInMillis; @@ -119,9 +158,11 @@ public void add(Stats stats) { public void addForClosingShard(Stats stats) { queryCount += stats.queryCount; queryTimeInMillis += stats.queryTimeInMillis; + queryFailure += stats.queryFailure; fetchCount += stats.fetchCount; fetchTimeInMillis += stats.fetchTimeInMillis; + fetchFailure += stats.fetchFailure; scrollCount += stats.scrollCount; scrollTimeInMillis += stats.scrollTimeInMillis; @@ -148,6 +189,10 @@ public long getQueryCurrent() { return queryCurrent; } + public long getQueryFailure() { + return queryFailure; + } + public long getFetchCount() { return fetchCount; } @@ -164,6 +209,10 @@ public long getFetchCurrent() { return fetchCurrent; } + public long getFetchFailure() { + return fetchFailure; + } + public long getScrollCount() { return scrollCount; } @@ -200,34 +249,17 @@ public static Stats readStats(StreamInput in) throws IOException { return new Stats(in); } - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVLong(queryCount); - out.writeVLong(queryTimeInMillis); - out.writeVLong(queryCurrent); - - out.writeVLong(fetchCount); - out.writeVLong(fetchTimeInMillis); - out.writeVLong(fetchCurrent); - - out.writeVLong(scrollCount); - out.writeVLong(scrollTimeInMillis); - out.writeVLong(scrollCurrent); - - out.writeVLong(suggestCount); - out.writeVLong(suggestTimeInMillis); - out.writeVLong(suggestCurrent); - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(Fields.QUERY_TOTAL, queryCount); builder.humanReadableField(Fields.QUERY_TIME_IN_MILLIS, Fields.QUERY_TIME, getQueryTime()); builder.field(Fields.QUERY_CURRENT, queryCurrent); + builder.field(Fields.QUERY_FAILURE, queryFailure); builder.field(Fields.FETCH_TOTAL, fetchCount); builder.humanReadableField(Fields.FETCH_TIME_IN_MILLIS, Fields.FETCH_TIME, getFetchTime()); builder.field(Fields.FETCH_CURRENT, fetchCurrent); + builder.field(Fields.FETCH_FAILURE, fetchFailure); builder.field(Fields.SCROLL_TOTAL, scrollCount); builder.humanReadableField(Fields.SCROLL_TIME_IN_MILLIS, Fields.SCROLL_TIME, getScrollTime()); @@ -248,9 +280,11 @@ public boolean equals(Object o) { return queryCount == that.queryCount && queryTimeInMillis == that.queryTimeInMillis && queryCurrent == that.queryCurrent + && queryFailure == that.queryFailure && fetchCount == that.fetchCount && fetchTimeInMillis == that.fetchTimeInMillis && fetchCurrent == that.fetchCurrent + && fetchFailure == that.fetchFailure && scrollCount == that.scrollCount && scrollTimeInMillis == that.scrollTimeInMillis && scrollCurrent == that.scrollCurrent @@ -265,9 +299,11 @@ public int hashCode() { queryCount, queryTimeInMillis, queryCurrent, + queryFailure, fetchCount, fetchTimeInMillis, fetchCurrent, + fetchCount, scrollCount, scrollTimeInMillis, scrollCurrent, @@ -377,10 +413,12 @@ static final class Fields { static final String QUERY_TIME = "query_time"; static final String QUERY_TIME_IN_MILLIS = "query_time_in_millis"; static final String QUERY_CURRENT = "query_current"; + static final String QUERY_FAILURE = "query_failure"; static final String FETCH_TOTAL = "fetch_total"; static final String FETCH_TIME = "fetch_time"; static final String FETCH_TIME_IN_MILLIS = "fetch_time_in_millis"; static final String FETCH_CURRENT = "fetch_current"; + static final String FETCH_FAILURE = "fetch_failure"; static final String SCROLL_TOTAL = "scroll_total"; static final String SCROLL_TIME = "scroll_time"; static final String SCROLL_TIME_IN_MILLIS = "scroll_time_in_millis"; diff --git a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java index f86727991a8b2..6e6f744f6b719 100644 --- a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java +++ b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java @@ -65,10 +65,14 @@ public void onPreQueryPhase(SearchContext searchContext) { @Override public void onFailedQueryPhase(SearchContext searchContext) { - computeStats( - searchContext, - searchContext.hasOnlySuggest() ? statsHolder -> statsHolder.suggestCurrent.dec() : statsHolder -> statsHolder.queryCurrent.dec() - ); + computeStats(searchContext, statsHolder -> { + if (searchContext.hasOnlySuggest()) { + statsHolder.suggestCurrent.dec(); + } else { + statsHolder.queryCurrent.dec(); + statsHolder.queryFailure.inc(); + } + }); } @Override @@ -89,7 +93,10 @@ public void onPreFetchPhase(SearchContext searchContext) { @Override public void onFailedFetchPhase(SearchContext searchContext) { - computeStats(searchContext, statsHolder -> statsHolder.fetchCurrent.dec()); + computeStats(searchContext, statsHolder -> { + statsHolder.fetchCurrent.dec(); + statsHolder.fetchFailure.inc(); + }); } @Override @@ -163,14 +170,19 @@ static final class StatsHolder { final CounterMetric scrollCurrent = new CounterMetric(); final CounterMetric suggestCurrent = new CounterMetric(); + final CounterMetric queryFailure = new CounterMetric(); + final CounterMetric fetchFailure = new CounterMetric(); + SearchStats.Stats stats() { return new SearchStats.Stats( queryMetric.count(), TimeUnit.NANOSECONDS.toMillis(queryMetric.sum()), queryCurrent.count(), + queryFailure.count(), fetchMetric.count(), TimeUnit.NANOSECONDS.toMillis(fetchMetric.sum()), fetchCurrent.count(), + fetchFailure.count(), scrollMetric.count(), TimeUnit.MICROSECONDS.toMillis(scrollMetric.sum()), scrollCurrent.count(), diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java index b4996db40f823..77d00f0e5a068 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -615,6 +615,8 @@ private static CommonStats createShardLevelCommonStats() { ++iota, ++iota, ++iota, + ++iota, + ++iota, ++iota ); Map groupStats = new HashMap<>(); diff --git a/server/src/test/java/org/elasticsearch/index/search/stats/SearchStatsTests.java b/server/src/test/java/org/elasticsearch/index/search/stats/SearchStatsTests.java index 06df262a090ae..a4430e1c1499d 100644 --- a/server/src/test/java/org/elasticsearch/index/search/stats/SearchStatsTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/stats/SearchStatsTests.java @@ -22,9 +22,9 @@ public void testShardLevelSearchGroupStats() throws Exception { // let's create two dummy search stats with groups Map groupStats1 = new HashMap<>(); Map groupStats2 = new HashMap<>(); - groupStats2.put("group1", new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)); - SearchStats searchStats1 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats1); - SearchStats searchStats2 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats2); + groupStats2.put("group1", new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)); + SearchStats searchStats1 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats1); + SearchStats searchStats2 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats2); // adding these two search stats and checking group stats are correct searchStats1.add(searchStats2); diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java index afeec7dd52b17..b6c059b7a0dcc 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java @@ -392,7 +392,7 @@ private static CommonStats mockCommonStats() { final IndexingStats.Stats indexingStats = new IndexingStats.Stats(++iota, ++iota, no, no, no, no, no, no, false, ++iota, no, no); commonStats.getIndexing().add(new IndexingStats(indexingStats)); - final SearchStats.Stats searchStats = new SearchStats.Stats(++iota, ++iota, no, no, no, no, no, no, no, no, no, no); + final SearchStats.Stats searchStats = new SearchStats.Stats(++iota, ++iota, no, no, no, no, no, no, no, no, no, no, no, no); commonStats.getSearch().add(new SearchStats(searchStats, no, null)); final SegmentsStats segmentsStats = new SegmentsStats(); diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java index 0d1a0374d4fc3..6822f54633bdc 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java @@ -186,7 +186,7 @@ private CommonStats mockCommonStats() { final IndexingStats.Stats indexingStats = new IndexingStats.Stats(3L, 4L, 0L, 0L, 0L, 0L, 0L, 0L, true, 5L, 0, 0); commonStats.getIndexing().add(new IndexingStats(indexingStats)); - final SearchStats.Stats searchStats = new SearchStats.Stats(6L, 7L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L); + final SearchStats.Stats searchStats = new SearchStats.Stats(6L, 7L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L); commonStats.getSearch().add(new SearchStats(searchStats, 0L, null)); final BulkStats bulkStats = new BulkStats(0L, 0L, 0L, 0L, 0L); diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java index 54f3ce634a25a..da23f27e1357e 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java @@ -352,7 +352,7 @@ private static NodeStats mockNodeStats() { indicesCommonStats.getQueryCache().add(new QueryCacheStats(++iota, ++iota, ++iota, ++iota, no)); indicesCommonStats.getRequestCache().add(new RequestCacheStats(++iota, ++iota, ++iota, ++iota)); - final SearchStats.Stats searchStats = new SearchStats.Stats(++iota, ++iota, no, no, no, no, no, no, no, no, no, no); + final SearchStats.Stats searchStats = new SearchStats.Stats(++iota, ++iota, no, no, no, no, no, no, no, no, no, no, no, no); indicesCommonStats.getSearch().add(new SearchStats(searchStats, no, null)); final SegmentsStats segmentsStats = new SegmentsStats(); From 372cbb1e97af2024e249be339de37a572ffeb4d9 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 3 Oct 2024 17:49:00 +0200 Subject: [PATCH 046/194] Change synthetic_source_keep setting's default (#113957) Incase of logsdb, the `index.mapping.synthetic_source_keep` setting should default to `arrays`. Otherwise it should default to `none`. --- .../common/settings/Setting.java | 21 ++++++++++ .../elasticsearch/index/mapper/Mapper.java | 13 +++++- x-pack/plugin/logsdb/build.gradle | 5 +++ .../xpack/logsdb/LogsdbTestSuiteIT.java | 42 +++++++++++++++++++ .../rest-api-spec/test/10_setting.yml | 39 +++++++++++++++++ .../rest-api-spec/test/20_logs_tests.yml | 6 +-- 6 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugin/logsdb/src/yamlRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbTestSuiteIT.java create mode 100644 x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/10_setting.yml diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index 6ad20b9fc6d16..a0b6e665042d0 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -1629,6 +1629,27 @@ public static > Setting enumSetting( return new Setting<>(key, defaultValue.toString(), e -> Enum.valueOf(clazz, e.toUpperCase(Locale.ROOT)), validator, properties); } + /** + * Creates a setting where the allowed values are defined as enum constants. All enum constants must be uppercase. + * + * @param the generics type parameter reflecting the actual type of the enum + * @param clazz the enum class + * @param defaultValue a default value function that returns the default values string representation. + * @param key the key for the setting + * @param validator validator for this setting + * @param properties properties for this setting like scope, filtering... + * @return the setting object + */ + public static > Setting enumSetting( + Class clazz, + Function defaultValue, + String key, + Validator validator, + Property... properties + ) { + return new Setting<>(key, defaultValue, e -> Enum.valueOf(clazz, e.toUpperCase(Locale.ROOT)), validator, properties); + } + /** * Creates a setting where the allowed values are defined as enum constants. All enum constants must be uppercase. * diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index d88bdf6e50615..0ecd3cc588d5b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -14,6 +14,8 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.util.StringLiteralDeduplicator; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.xcontent.ToXContentFragment; @@ -83,11 +85,18 @@ public void toXContent(XContentBuilder builder) throws IOException { // Setting to SourceKeepMode.ALL is equivalent to disabling synthetic source, so this is not allowed. public static final Setting SYNTHETIC_SOURCE_KEEP_INDEX_SETTING = Setting.enumSetting( SourceKeepMode.class, + settings -> { + var indexMode = IndexSettings.MODE.get(settings); + if (indexMode == IndexMode.LOGSDB) { + return SourceKeepMode.ARRAYS.toString(); + } else { + return SourceKeepMode.NONE.toString(); + } + }, "index.mapping.synthetic_source_keep", - SourceKeepMode.NONE, value -> { if (value == SourceKeepMode.ALL) { - throw new IllegalArgumentException("index.mapping.synthetic_source_keep can't be set to [" + value.toString() + "]"); + throw new IllegalArgumentException("index.mapping.synthetic_source_keep can't be set to [" + value + "]"); } }, Setting.Property.IndexScope, diff --git a/x-pack/plugin/logsdb/build.gradle b/x-pack/plugin/logsdb/build.gradle index 5b7e45a90149d..466cf69243c8e 100644 --- a/x-pack/plugin/logsdb/build.gradle +++ b/x-pack/plugin/logsdb/build.gradle @@ -11,6 +11,7 @@ evaluationDependsOn(xpackModule('core')) apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-java-rest-test' +apply plugin: 'elasticsearch.internal-yaml-rest-test' esplugin { name 'logsdb' @@ -30,3 +31,7 @@ dependencies { tasks.named("javaRestTest").configure { usesDefaultDistribution() } + +tasks.named('yamlRestTest') { + usesDefaultDistribution() +} diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbTestSuiteIT.java b/x-pack/plugin/logsdb/src/yamlRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbTestSuiteIT.java new file mode 100644 index 0000000000000..fcac791cb2057 --- /dev/null +++ b/x-pack/plugin/logsdb/src/yamlRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbTestSuiteIT.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; + +public class LogsdbTestSuiteIT extends ESClientYamlSuiteTestCase { + + @ClassRule + public static final ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .build(); + + public LogsdbTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + +} diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/10_setting.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/10_setting.yml new file mode 100644 index 0000000000000..d597859c1e4d8 --- /dev/null +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/10_setting.yml @@ -0,0 +1,39 @@ +--- +synthetic_source_keep defaults: + - requires: + test_runner_features: [ capabilities ] + capabilities: + - method: PUT + path: /{index} + capabilities: [ logsdb_index_mode ] + reason: "Support for 'logsdb' index mode capability required" + + - do: + indices.create: + index: test1 + body: + settings: + index: + mode: logsdb + + - do: + indices.create: + index: test2 + + - do: + indices.get_settings: + index: test1 + include_defaults: true + + - is_true: test1 + - match: { test1.settings.index.mode: "logsdb" } + - match: { test1.defaults.index.mapping.synthetic_source_keep: "arrays" } + + - do: + indices.get_settings: + index: test2 + include_defaults: true + + - is_true: test2 + - is_false: test2.settings.index.mode + - match: { test2.defaults.index.mapping.synthetic_source_keep: "none" } diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml index 95367c97ce8f2..657453bf4ae9f 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml @@ -48,9 +48,9 @@ setup: body: fields: ["*"] - length: { hits.hits: 1 } - - match: { hits.hits.0.fields.resource\.attributes\.host\.ip: ["0.0.0.0", "127.0.0.1"] } - - match: { hits.hits.0.fields.attributes\.foo: [1, 2, 3] } - - match: { hits.hits.0.fields.attributes\.bar: [a, b, c] } + - match: { hits.hits.0.fields.resource\.attributes\.host\.ip: ["127.0.0.1", "0.0.0.0"] } + - match: { hits.hits.0.fields.attributes\.foo: [3, 2, 1] } + - match: { hits.hits.0.fields.attributes\.bar: [b, c, a] } --- "Exception aliases": - do: From d3dbbb29b30d62609b1491900c4d372de8c2246d Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 3 Oct 2024 16:49:58 +0100 Subject: [PATCH 047/194] Revert "Assert that REST params are consumed iff supported (#113933)" This reverts commit 6d7ae82180fa30ee4b48bd58721cae2cda809b15. This reverts commit a5ef13961616a7ae74dc35320c6c160638910e22. --- .../rest/RestGetDataStreamsAction.java | 3 +-- muted-tests.yml | 2 -- .../org/elasticsearch/rest/BaseRestHandler.java | 17 ----------------- 3 files changed, 1 insertion(+), 21 deletions(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java index 7a27eddfaf8c7..da55376fb403b 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.set.Sets; @@ -36,12 +35,12 @@ public class RestGetDataStreamsAction extends BaseRestHandler { Set.of( "name", "include_defaults", + "timeout", "master_timeout", IndicesOptions.WildcardOptions.EXPAND_WILDCARDS, IndicesOptions.ConcreteTargetOptions.IGNORE_UNAVAILABLE, IndicesOptions.WildcardOptions.ALLOW_NO_INDICES, IndicesOptions.GatekeeperOptions.IGNORE_THROTTLED, - DataStream.isFailureStoreFeatureFlagEnabled() ? IndicesOptions.FailureStoreOptions.FAILURE_STORE : "name", "verbose" ) ) diff --git a/muted-tests.yml b/muted-tests.yml index 22ecb333a1908..4305ebe3d2e02 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -345,8 +345,6 @@ tests: - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT method: testPutE5Small_withPlatformAgnosticVariant issue: https://github.com/elastic/elasticsearch/issues/113983 -- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT - issue: https://github.com/elastic/elasticsearch/issues/114013 - class: org.elasticsearch.xpack.rank.rrf.RRFRankClientYamlTestSuiteIT method: test {yaml=rrf/700_rrf_retriever_search_api_compatibility/rrf retriever with top-level collapse} issue: https://github.com/elastic/elasticsearch/issues/114019 diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java index 2f7bb80a8d46a..99fa3e0166963 100644 --- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.CheckedConsumer; -import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.RestApiVersion; @@ -105,8 +104,6 @@ public final void handleRequest(RestRequest request, RestChannel channel, NodeCl // prepare the request for execution; has the side effect of touching the request parameters try (var action = prepareRequest(request, client)) { - assert assertConsumesSupportedParams(supported, request); - // validate unconsumed params, but we must exclude params used to format the response // use a sorted set so the unconsumed parameters appear in a reliable sorted order final SortedSet unconsumedParams = request.unconsumedParams() @@ -151,20 +148,6 @@ public void close() { } } - private boolean assertConsumesSupportedParams(@Nullable Set supported, RestRequest request) { - if (supported != null) { - final var supportedAndCommon = new TreeSet<>(supported); - supportedAndCommon.add("error_trace"); - supportedAndCommon.addAll(ALWAYS_SUPPORTED); - supportedAndCommon.removeAll(RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS); - final var consumed = new TreeSet<>(request.consumedParams()); - consumed.removeAll(RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS); - assert supportedAndCommon.equals(consumed) - : getName() + ": consumed params " + consumed + " while supporting " + supportedAndCommon; - } - return true; - } - protected static String unrecognized(RestRequest request, Set invalids, Set candidates, String detail) { StringBuilder message = new StringBuilder().append("request [") .append(request.path()) From fb39147d909eb379063e8ba2e2bac9fc64ae1358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?= Date: Thu, 3 Oct 2024 18:36:20 +0200 Subject: [PATCH 048/194] [DOCS] Removes link from semantic text tutorial. (#114038) --- .../search-your-data/semantic-search-semantic-text.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc index de9a35e0d29b8..f1bd238a64fbf 100644 --- a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc @@ -50,7 +50,7 @@ PUT _inference/sparse_embedding/my-elser-endpoint <1> be used and ELSER creates sparse vectors. The `inference_id` is `my-elser-endpoint`. <2> The `elser` service is used in this example. -<3> This setting enables and configures {ml-docs}/ml-nlp-elser.html#elser-adaptive-allocations[adaptive allocations]. +<3> This setting enables and configures adaptive allocations. Adaptive allocations make it possible for ELSER to automatically scale up or down resources based on the current load on the process. [NOTE] From 8fed8e63b0d05dbe975dbbcc391f3863563a2334 Mon Sep 17 00:00:00 2001 From: Athena Brown Date: Thu, 3 Oct 2024 11:38:48 -0600 Subject: [PATCH 049/194] Ensure all Security configuration is covered by enhanced file protections (#112408) This commit the work started in #108767 to other security config, adding additional protections to the other on-disk files used by the Security module. With this change, non-Security modules will be prevented from accessing Security's on-disk configuration files. --- .../org/elasticsearch/bootstrap/ESPolicy.java | 17 +++++++++--- .../org/elasticsearch/bootstrap/Security.java | 27 ++++++++++++------- .../xpack/security/authc/jwt/JwtUtil.java | 3 ++- .../authc/kerberos/KerberosRealm.java | 23 +++++++++++----- .../oidc/OpenIdConnectAuthenticator.java | 13 ++++++--- .../xpack/security/authc/saml/SamlRealm.java | 9 +++++-- .../plugin-metadata/plugin-security.policy | 4 +++ 7 files changed, 71 insertions(+), 25 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java b/server/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java index e07e0608c1383..845303abe6baf 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.net.SocketPermission; import java.net.URL; +import java.security.AllPermission; import java.security.CodeSource; import java.security.Permission; import java.security.PermissionCollection; @@ -39,6 +40,7 @@ final class ESPolicy extends Policy { static final String UNTRUSTED_RESOURCE = "untrusted.policy"; private static final String ALL_FILE_MASK = "read,readlink,write,delete,execute"; + private static final AllPermission ALL_PERMISSION = new AllPermission(); final Policy template; final Policy untrusted; @@ -124,7 +126,7 @@ public boolean implies(ProtectionDomain domain, Permission permission) { * It's helpful to use the infrastructure around FilePermission here to do the directory structure check with implies * so we use ALL_FILE_MASK mask to check if we can do something with this file, whatever the actual operation we're requesting */ - return canAccessSecuredFile(location, new FilePermission(permission.getName(), ALL_FILE_MASK)); + return canAccessSecuredFile(domain, new FilePermission(permission.getName(), ALL_FILE_MASK)); } if (location != null) { @@ -157,15 +159,24 @@ public boolean implies(ProtectionDomain domain, Permission permission) { } @SuppressForbidden(reason = "We get given an URL by the security infrastructure") - private boolean canAccessSecuredFile(URL location, FilePermission permission) { - if (location == null) { + private boolean canAccessSecuredFile(ProtectionDomain domain, FilePermission permission) { + if (domain == null || domain.getCodeSource() == null || domain.getCodeSource().getLocation() == null) { return false; } + // If the domain in question has AllPermission - only true of sources built into the JDK, as we prevent AllPermission from being + // configured in Elasticsearch - then it has access to this file. + + if (system.implies(domain, ALL_PERMISSION)) { + return true; + } + URL location = domain.getCodeSource().getLocation(); + // check the source Set accessibleSources = securedFiles.get(permission); if (accessibleSources != null) { // simple case - single-file referenced directly + return accessibleSources.contains(location); } else { // there's a directory reference in there somewhere diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Security.java b/server/src/main/java/org/elasticsearch/bootstrap/Security.java index f22413f9abd12..dc6de9a6b2c91 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Security.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Security.java @@ -47,6 +47,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.function.Consumer; @@ -236,17 +237,25 @@ private static Map> readSecuredConfigFiles( for (Map.Entry> ps : settingPatterns) { if (ps.getKey().matcher(setting).matches()) { // add the setting value to the secured files for these codebase URLs - Path file = environment.configFile().resolve(environment.settings().get(setting)); - if (file.startsWith(environment.configFile()) == false) { - throw new IllegalStateException(ps.getValue() + " tried to grant access to file outside config directory " + file); - } - if (logger.isDebugEnabled()) { - ps.getValue() - .forEach( - url -> logger.debug("Jar {} securing access to config file {} through setting {}", url, file, setting) + String settingValue = environment.settings().get(setting); + // Some settings can also be an HTTPS URL in addition to a file path; if that's the case just skip this one. + // If the setting shouldn't be an HTTPS URL, that'll be caught by that setting's validation later in the process. + // HTTP (no S) URLs are not supported. + if (settingValue.toLowerCase(Locale.ROOT).startsWith("https://") == false) { + Path file = environment.configFile().resolve(settingValue); + if (file.startsWith(environment.configFile()) == false) { + throw new IllegalStateException( + ps.getValue() + " tried to grant access to file outside config directory " + file ); + } + if (logger.isDebugEnabled()) { + ps.getValue() + .forEach( + url -> logger.debug("Jar {} securing access to config file {} through setting {}", url, file, setting) + ); + } + securedConfigFiles.computeIfAbsent(file.toString(), k -> new HashSet<>()).addAll(ps.getValue()); } - securedConfigFiles.computeIfAbsent(file.toString(), k -> new HashSet<>()).addAll(ps.getValue()); } } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java index b345178e205c3..8b3f8ec09675a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java @@ -228,7 +228,8 @@ public static byte[] readFileContents(final String jwkSetConfigKeyPkc, final Str throws SettingsException { try { final Path path = JwtUtil.resolvePath(environment, jwkSetPathPkc); - return Files.readAllBytes(path); + byte[] bytes = AccessController.doPrivileged((PrivilegedExceptionAction) () -> Files.readAllBytes(path)); + return bytes; } catch (Exception e) { throw new SettingsException( "Failed to read contents for setting [" + jwkSetConfigKeyPkc + "] value [" + jwkSetPathPkc + "].", diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java index 6601d27d5a431..d5ef90f7f1664 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java @@ -29,6 +29,8 @@ import java.nio.file.Files; import java.nio.file.Path; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.Collections; import java.util.List; import java.util.Map; @@ -101,19 +103,26 @@ public KerberosRealm(final RealmConfig config, final UserRoleMapper userRoleMapp this.threadPool = threadPool; this.keytabPath = config.env().configFile().resolve(config.getSetting(KerberosRealmSettings.HTTP_SERVICE_KEYTAB_PATH)); - if (Files.exists(keytabPath) == false) { + validateKeytab(this.keytabPath); + + this.enableKerberosDebug = config.getSetting(KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE); + this.removeRealmName = config.getSetting(KerberosRealmSettings.SETTING_REMOVE_REALM_NAME); + this.delegatedRealms = null; + } + + private static void validateKeytab(Path keytabPath) { + boolean fileExists = AccessController.doPrivileged((PrivilegedAction) () -> Files.exists(keytabPath)); + if (fileExists == false) { throw new IllegalArgumentException("configured service key tab file [" + keytabPath + "] does not exist"); } - if (Files.isDirectory(keytabPath)) { + boolean pathIsDir = AccessController.doPrivileged((PrivilegedAction) () -> Files.isDirectory(keytabPath)); + if (pathIsDir) { throw new IllegalArgumentException("configured service key tab file [" + keytabPath + "] is a directory"); } - if (Files.isReadable(keytabPath) == false) { + boolean isReadable = AccessController.doPrivileged((PrivilegedAction) () -> Files.isReadable(keytabPath)); + if (isReadable == false) { throw new IllegalArgumentException("configured service key tab file [" + keytabPath + "] must have read permission"); } - - this.enableKerberosDebug = config.getSetting(KerberosRealmSettings.SETTING_KRB_DEBUG_ENABLE); - this.removeRealmName = config.getSetting(KerberosRealmSettings.SETTING_REMOVE_REALM_NAME); - this.delegatedRealms = null; } @Override diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticator.java index c2e0caf7234cb..aa1946f445670 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticator.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthenticator.java @@ -93,6 +93,7 @@ import org.elasticsearch.xpack.core.security.authc.RealmSettings; import org.elasticsearch.xpack.core.security.authc.oidc.OpenIdConnectRealmSettings; import org.elasticsearch.xpack.core.ssl.SSLService; +import org.elasticsearch.xpack.security.PrivilegedFileWatcher; import org.elasticsearch.xpack.security.authc.jwt.JwtUtil; import java.io.IOException; @@ -366,8 +367,14 @@ private void validateAccessToken(AccessToken accessToken, JWT idToken) { private JWKSet readJwkSetFromFile(String jwkSetPath) throws IOException, ParseException { final Path path = realmConfig.env().configFile().resolve(jwkSetPath); // avoid using JWKSet.loadFile() as it does not close FileInputStream internally - String jwkSet = Files.readString(path, StandardCharsets.UTF_8); - return JWKSet.parse(jwkSet); + try { + String jwkSet = AccessController.doPrivileged( + (PrivilegedExceptionAction) () -> Files.readString(path, StandardCharsets.UTF_8) + ); + return JWKSet.parse(jwkSet); + } catch (PrivilegedActionException ex) { + throw (IOException) ex.getException(); + } } /** @@ -808,7 +815,7 @@ IDTokenValidator createIdTokenValidator(boolean addFileWatcherIfRequired) { private void setMetadataFileWatcher(String jwkSetPath) throws IOException { final Path path = realmConfig.env().configFile().resolve(jwkSetPath); - FileWatcher watcher = new FileWatcher(path); + FileWatcher watcher = new PrivilegedFileWatcher(path); watcher.addListener(new FileListener(LOGGER, () -> this.idTokenValidator.set(createIdTokenValidator(false)))); watcherService.add(watcher, ResourceWatcherService.Frequency.MEDIUM); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java index 704875efa18f6..9adfd15e23207 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java @@ -49,6 +49,7 @@ import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.ssl.CertParsingUtils; import org.elasticsearch.xpack.core.ssl.SSLService; +import org.elasticsearch.xpack.security.PrivilegedFileWatcher; import org.elasticsearch.xpack.security.authc.Realms; import org.elasticsearch.xpack.security.authc.TokenService; import org.elasticsearch.xpack.security.authc.support.DelegatedAuthorizationSupport; @@ -774,7 +775,11 @@ private static final class SamlFilesystemMetadataResolver extends FilesystemMeta @Override protected byte[] fetchMetadata() throws ResolverException { assert assertNotTransportThread("fetching SAML metadata from a file"); - return super.fetchMetadata(); + try { + return AccessController.doPrivileged((PrivilegedExceptionAction) () -> super.fetchMetadata()); + } catch (PrivilegedActionException e) { + throw (ResolverException) e.getException(); + } } } @@ -806,7 +811,7 @@ private static Tuple(resolver, () -> resolveEntityDescriptor(resolver, entityId, path.toString(), true)); diff --git a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy index b3d5e80e09dcd..d814dfbb1c117 100644 --- a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy +++ b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy @@ -6,6 +6,10 @@ grant { permission org.elasticsearch.SecuredConfigFileAccessPermission "x-pack/users"; // other security files specified by settings permission org.elasticsearch.SecuredConfigFileSettingAccessPermission "xpack.security.authc.realms.ldap.*.files.role_mapping"; + permission org.elasticsearch.SecuredConfigFileSettingAccessPermission "xpack.security.authc.realms.pki.*.files.role_mapping"; + permission org.elasticsearch.SecuredConfigFileSettingAccessPermission "xpack.security.authc.realms.jwt.*.pkc_jwkset_path"; + permission org.elasticsearch.SecuredConfigFileSettingAccessPermission "xpack.security.authc.realms.saml.*.idp.metadata.path"; + permission org.elasticsearch.SecuredConfigFileSettingAccessPermission "xpack.security.authc.realms.kerberos.*.keytab.path"; // needed for SAML permission java.util.PropertyPermission "org.apache.xml.security.ignoreLineBreaks", "read,write"; From b9adc701faa48321bf74e4a9da40d918eccc40c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?= Date: Thu, 3 Oct 2024 19:48:16 +0200 Subject: [PATCH 050/194] [DOCS] Expands param descriptions for semantic_text (#114024) Co-authored-by: Mike Pellegrini --- docs/reference/mapping/types/semantic-text.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/reference/mapping/types/semantic-text.asciidoc b/docs/reference/mapping/types/semantic-text.asciidoc index d0fdf0145aa58..07abbff986643 100644 --- a/docs/reference/mapping/types/semantic-text.asciidoc +++ b/docs/reference/mapping/types/semantic-text.asciidoc @@ -63,12 +63,14 @@ PUT my-index-000002 `inference_id`:: (Required, string) {infer-cap} endpoint that will be used to generate the embeddings for the field. +This parameter cannot be updated. Use the <> to create the endpoint. If `search_inference_id` is specified, the {infer} endpoint defined by `inference_id` will only be used at index time. `search_inference_id`:: (Optional, string) {infer-cap} endpoint that will be used to generate embeddings at query time. +You can update this parameter by using the <>. Use the <> to create the endpoint. If not specified, the {infer} endpoint defined by `inference_id` will be used at both index and query time. From eb1e89a503e29a7a7a127f91edcfe6cdc280aa0d Mon Sep 17 00:00:00 2001 From: Stef Nestor <26751266+stefnestor@users.noreply.github.com> Date: Thu, 3 Oct 2024 12:06:35 -0600 Subject: [PATCH 051/194] Add link to Circuit Breaker "Data too large" exception message (#113561) * Add link to CircuitBreaker exception message * Account #113866 * Update docs/changelog/113561.yaml --- docs/changelog/113561.yaml | 5 +++++ .../org/elasticsearch/common/ReferenceDocs.java | 1 + .../common/breaker/ChildMemoryCircuitBreaker.java | 4 +++- .../breaker/HierarchyCircuitBreakerService.java | 4 +++- .../elasticsearch/common/reference-docs-links.json | 2 +- .../elasticsearch/common/reference-docs-links.txt | 1 + .../HierarchyCircuitBreakerServiceTests.java | 13 +++++++++---- 7 files changed, 23 insertions(+), 7 deletions(-) create mode 100644 docs/changelog/113561.yaml diff --git a/docs/changelog/113561.yaml b/docs/changelog/113561.yaml new file mode 100644 index 0000000000000..d00eac7685bcc --- /dev/null +++ b/docs/changelog/113561.yaml @@ -0,0 +1,5 @@ +pr: 113561 +summary: Add link to Circuit Breaker "Data too large" exception message +area: Infra/Circuit Breakers +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java index 4b0a0c5e77ebb..b059113b4098c 100644 --- a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java +++ b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java @@ -80,6 +80,7 @@ public enum ReferenceDocs { FLOOD_STAGE_WATERMARK, X_OPAQUE_ID, FORMING_SINGLE_NODE_CLUSTERS, + CIRCUIT_BREAKER_ERRORS, // this comment keeps the ';' on the next line so every entry above has a trailing ',' which makes the diff for adding new links cleaner ; diff --git a/server/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java b/server/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java index 6d8510d27f27a..9669e78a119b9 100644 --- a/server/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java +++ b/server/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java @@ -10,6 +10,7 @@ package org.elasticsearch.common.breaker; import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.indices.breaker.BreakerSettings; import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; @@ -87,7 +88,8 @@ public void circuitBreak(String fieldName, long bytesNeeded) { + memoryBytesLimit + "/" + ByteSizeValue.ofBytes(memoryBytesLimit) - + "]"; + + "]; for more information, see " + + ReferenceDocs.CIRCUIT_BREAKER_ERRORS; logger.debug(() -> format("%s", message)); throw new CircuitBreakingException(message, bytesNeeded, memoryBytesLimit, durability); } diff --git a/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java b/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java index d72909806240c..b1b0f0201ebbe 100644 --- a/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java +++ b/server/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; @@ -474,7 +475,8 @@ public void accept(String key, CircuitBreaker breaker) { appendBytesSafe(message, (long) (breaker.getUsed() * breaker.getOverhead())); } }); - message.append("]"); + message.append("]; for more information, see "); + message.append(ReferenceDocs.CIRCUIT_BREAKER_ERRORS); return message.toString(); } diff --git a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json index 71be3d333ec3f..c8fa98b196c7b 100644 --- a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json +++ b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json @@ -2,4 +2,4 @@ "Content moved to reference-docs-links.txt", "This is a temporary placeholder to satisfy sub check_elasticsearch_links in the docs build", "Remove with @UpdateForV10 (if not before)" -] +] \ No newline at end of file diff --git a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt index 190bbd3c319b4..ab9a6b253be7a 100644 --- a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt +++ b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.txt @@ -42,3 +42,4 @@ MAX_SHARDS_PER_NODE size-your-shards FLOOD_STAGE_WATERMARK fix-watermark-errors.html X_OPAQUE_ID api-conventions.html#x-opaque-id FORMING_SINGLE_NODE_CLUSTERS modules-discovery-bootstrap-cluster.html#modules-discovery-bootstrap-cluster-joining +CIRCUIT_BREAKER_ERRORS circuit-breaker-errors.html \ No newline at end of file diff --git a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java index 156460d320ee2..610e87b50d365 100644 --- a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.indices.breaker; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; @@ -911,9 +912,11 @@ public double getOverhead() { ), oneOf( "[parent] Data too large, data for [test] would be [3/3b], which is larger than the limit of [6/6b], " - + "usages [child=7/7b, otherChild=8/8b]", + + "usages [child=7/7b, otherChild=8/8b]; for more information, see " + + ReferenceDocs.CIRCUIT_BREAKER_ERRORS, "[parent] Data too large, data for [test] would be [3/3b], which is larger than the limit of [6/6b], " - + "usages [otherChild=8/8b, child=7/7b]" + + "usages [otherChild=8/8b, child=7/7b]; for more information, see " + + ReferenceDocs.CIRCUIT_BREAKER_ERRORS ) ); @@ -928,7 +931,8 @@ public double getOverhead() { ), equalTo( "[parent] Data too large, data for [test] would be [3/3b], which is larger than the limit of [6/6b], " - + "real usage: [2/2b], new bytes reserved: [1/1b], usages []" + + "real usage: [2/2b], new bytes reserved: [1/1b], usages []; for more information, see " + + ReferenceDocs.CIRCUIT_BREAKER_ERRORS ) ); @@ -945,7 +949,8 @@ public double getOverhead() { ), equalTo( "[parent] Data too large, data for [test] would be [-3], which is larger than the limit of [-6], " - + "real usage: [-2], new bytes reserved: [-1/-1b], usages [child1=-7]" + + "real usage: [-2], new bytes reserved: [-1/-1b], usages [child1=-7]; for more information, see " + + ReferenceDocs.CIRCUIT_BREAKER_ERRORS ) ); } finally { From dd2024881d038d64532b8eea35cac88142b503cb Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Thu, 3 Oct 2024 21:19:04 +0300 Subject: [PATCH 052/194] Add object param for keeping synthetic source (#113690) * Add object param for keeping synthetic source * Update docs/changelog/113690.yaml * fix merging * add tests * merge * fix randomized tests * add documentation * dedup id in docs * update documentation * update documentation * fix bwc * fix bwc * fix unintended * Revert "fix bwc" This reverts commit 18dc913eee209b27c14f5cdaac4247c18b65c6b1. * Revert "fix bwc" This reverts commit f4ddb0e5e5bf26b841a3216aa5969ed60dabde59. * add missing test * fix transform * fix transform * fix transform * fix transform * fix transform --- docs/changelog/113690.yaml | 5 + .../mapping/fields/synthetic-source.asciidoc | 144 ++++++++++++++++-- rest-api-spec/build.gradle | 2 +- .../indices.create/20_synthetic_source.yml | 4 +- .../21_synthetic_source_stored.yml | 113 +++++++++++--- .../index/mapper/DocumentParser.java | 39 ++--- .../index/mapper/MapperFeatures.java | 6 + .../index/mapper/NestedObjectMapper.java | 29 ++-- .../index/mapper/ObjectMapper.java | 61 +++++--- .../index/mapper/PassThroughObjectMapper.java | 16 +- .../index/mapper/RootObjectMapper.java | 15 +- .../FieldAliasMapperValidationTests.java | 2 +- .../index/mapper/FieldTypeLookupTests.java | 2 + .../mapper/IgnoredSourceFieldMapperTests.java | 110 ++++++++++--- .../index/mapper/MappingLookupTests.java | 2 +- .../index/mapper/NestedObjectMapperTests.java | 4 +- .../index/mapper/ObjectMapperTests.java | 10 +- .../mapper/PassThroughObjectMapperTests.java | 2 + .../index/mapper/RootObjectMapperTests.java | 13 ++ .../index/mapper/MapperTestCase.java | 6 +- .../datasource/DataSourceRequest.java | 7 +- .../DefaultMappingParametersHandler.java | 6 + .../fields/NestedFieldDataGenerator.java | 2 +- .../fields/ObjectFieldDataGenerator.java | 2 +- .../TopLevelObjectFieldDataGenerator.java | 2 +- .../traces-otel@mappings.yaml | 2 +- ...cument_level_security_synthetic_source.yml | 2 +- 27 files changed, 477 insertions(+), 131 deletions(-) create mode 100644 docs/changelog/113690.yaml diff --git a/docs/changelog/113690.yaml b/docs/changelog/113690.yaml new file mode 100644 index 0000000000000..bd5f1245f471e --- /dev/null +++ b/docs/changelog/113690.yaml @@ -0,0 +1,5 @@ +pr: 113690 +summary: Add object param for keeping synthetic source +area: Mapping +type: enhancement +issues: [] diff --git a/docs/reference/mapping/fields/synthetic-source.asciidoc b/docs/reference/mapping/fields/synthetic-source.asciidoc index ccea38cf602da..902b6c26611e5 100644 --- a/docs/reference/mapping/fields/synthetic-source.asciidoc +++ b/docs/reference/mapping/fields/synthetic-source.asciidoc @@ -32,18 +32,25 @@ space. Additional latency can be avoided by not loading `_source` field in queri [[synthetic-source-fields]] ===== Supported fields -Synthetic `_source` is supported by all field types. Depending on implementation details, field types have different properties when used with synthetic `_source`. +Synthetic `_source` is supported by all field types. Depending on implementation details, field types have different +properties when used with synthetic `_source`. -<> construct synthetic `_source` using existing data, most commonly <> and <>. For these field types, no additional space is needed to store the contents of `_source` field. Due to the storage layout of <>, the generated `_source` field undergoes <> compared to original document. +<> construct synthetic `_source` using existing data, most +commonly <> and <>. For these field types, no additional space +is needed to store the contents of `_source` field. Due to the storage layout of <>, the +generated `_source` field undergoes <> compared to original document. -For all other field types, the original value of the field is stored as is, in the same way as the `_source` field in non-synthetic mode. In this case there are no modifications and field data in `_source` is the same as in the original document. Similarly, malformed values of fields that use <> or <> need to be stored as is. This approach is less storage efficient since data needed for `_source` reconstruction is stored in addition to other data required to index the field (like `doc_values`). +For all other field types, the original value of the field is stored as is, in the same way as the `_source` field in +non-synthetic mode. In this case there are no modifications and field data in `_source` is the same as in the original +document. Similarly, malformed values of fields that use <> or +<> need to be stored as is. This approach is less storage efficient since data needed for +`_source` reconstruction is stored in addition to other data required to index the field (like `doc_values`). [[synthetic-source-restrictions]] ===== Synthetic `_source` restrictions -Synthetic `_source` cannot be used together with field mappings that use <>. - -Some field types have additional restrictions. These restrictions are documented in the **synthetic `_source`** section of the field type's <>. +Some field types have additional restrictions. These restrictions are documented in the **synthetic `_source`** section +of the field type's <>. [[synthetic-source-modifications]] ===== Synthetic `_source` modifications @@ -144,6 +151,42 @@ Will become: ---- // TEST[s/^/{"_source":/ s/\n$/}/] +This impacts how source contents can be referenced in <>. For instance, referencing +a script in its original source form will return null: + +[source,js] +---- +"script": { "source": """ emit(params._source['foo.bar.baz']) """ } +---- +// NOTCONSOLE + +Instead, source references need to be in line with the mapping structure: + +[source,js] +---- +"script": { "source": """ emit(params._source['foo']['bar']['baz']) """ } +---- +// NOTCONSOLE + +or simply + +[source,js] +---- +"script": { "source": """ emit(params._source.foo.bar.baz) """ } +---- +// NOTCONSOLE + +The following <> are preferable as, in addition to being agnostic to the +mapping structure, they make use of docvalues if available and fall back to synthetic source only when needed. This +reduces source synthesizing, a slow and costly operation. + +[source,js] +---- +"script": { "source": """ emit(field('foo.bar.baz').get(null)) """ } +"script": { "source": """ emit($('foo.bar.baz', null)) """ } +---- +// NOTCONSOLE + [[synthetic-source-modifications-alphabetical]] ====== Alphabetical sorting Synthetic `_source` fields are sorted alphabetically. The @@ -155,18 +198,99 @@ that ordering. [[synthetic-source-modifications-ranges]] ====== Representation of ranges -Range field values (e.g. `long_range`) are always represented as inclusive on both sides with bounds adjusted accordingly. See <>. +Range field values (e.g. `long_range`) are always represented as inclusive on both sides with bounds adjusted +accordingly. See <>. [[synthetic-source-precision-loss-for-point-types]] ====== Reduced precision of `geo_point` values -Values of `geo_point` fields are represented in synthetic `_source` with reduced precision. See <>. +Values of `geo_point` fields are represented in synthetic `_source` with reduced precision. See +<>. + +[[synthetic-source-keep]] +====== Minimizing source modifications + +It is possible to avoid synthetic source modifications for a particular object or field, at extra storage cost. +This is controlled through param `synthetic_source_keep` with the following option: + + - `none`: synthetic source diverges from the original source as described above (default). + - `arrays`: arrays of the corresponding field or object preserve the original element ordering and duplicate elements. +The synthetic source fragment for such arrays is not guaranteed to match the original source exactly, e.g. array +`[1, 2, [5], [[4, [3]]], 5]` may appear as-is or in an equivalent format like `[1, 2, 5, 4, 3, 5]`. The exact format +may change in the future, in an effort to reduce the storage overhead of this option. +- `all`: the source for both singleton instances and arrays of the corresponding field or object gets recorded. When +applied to objects, the source of all sub-objects and sub-fields gets captured. Furthermore, the original source of +arrays gets captured and appears in synthetic source with no modifications. + +For instance: + +[source,console,id=create-index-with-synthetic-source-keep] +---- +PUT idx_keep +{ + "mappings": { + "_source": { + "mode": "synthetic" + }, + "properties": { + "path": { + "type": "object", + "synthetic_source_keep": "all" + }, + "ids": { + "type": "integer", + "synthetic_source_keep": "arrays" + } + } + } +} +---- +// TEST + +[source,console,id=synthetic-source-keep-example] +---- +PUT idx_keep/_doc/1 +{ + "path": { + "to": [ + { "foo": [3, 2, 1] }, + { "foo": [30, 20, 10] } + ], + "bar": "baz" + }, + "ids": [ 200, 100, 300, 100 ] +} +---- +// TEST[s/$/\nGET idx_keep\/_doc\/1?filter_path=_source\n/] + +returns the original source, with no array deduplication and sorting: + +[source,console-result] +---- +{ + "path": { + "to": [ + { "foo": [3, 2, 1] }, + { "foo": [30, 20, 10] } + ], + "bar": "baz" + }, + "ids": [ 200, 100, 300, 100 ] +} +---- +// TEST[s/^/{"_source":/ s/\n$/}/] +The option for capturing the source of arrays can be applied at index level, by setting +`index.mapping.synthetic_source_keep` to `arrays`. This applies to all objects and fields in the index, except for +the ones with explicit overrides of `synthetic_source_keep` set to `none`. In this case, the storage overhead grows +with the number and sizes of arrays present in source of each document, naturally. [[synthetic-source-fields-native-list]] ===== Field types that support synthetic source with no storage overhead -The following field types support synthetic source using data from <> or <>, and require no additional storage space to construct the `_source` field. +The following field types support synthetic source using data from <> or +>, and require no additional storage space to construct the `_source` field. -NOTE: If you enable the <> or <> settings, then additional storage is required to store ignored field values for these types. +NOTE: If you enable the <> or <> settings, then +additional storage is required to store ignored field values for these types. ** <> ** {plugins}/mapper-annotated-text-usage.html#annotated-text-synthetic-source[`annotated-text`] diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index ed1cf905f7e9d..05ed09e40f6bc 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -57,5 +57,5 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTestTransform").configure({task -> task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") - task.skipTestsByFilePattern("indices.create/synthetic_source*.yml", "@UpdateForV9 -> tests do not pass after bumping API version to 9 [ES-9597]") + task.skipTest("indices.create/21_synthetic_source_stored/object param - nested object with stored array", "skip test to submit #113690") }) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index a999bb7816065..62a248cae6b2d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -895,7 +895,7 @@ doubly nested object: --- subobjects auto: - requires: - cluster_features: ["mapper.subobjects_auto"] + cluster_features: ["mapper.subobjects_auto", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source and supporting subobjects auto setting - do: @@ -920,7 +920,7 @@ subobjects auto: id: type: keyword stored: - store_array_source: true + synthetic_source_keep: arrays properties: span: properties: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index 7d7be765631e5..07ab5259e6fd6 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -1,7 +1,71 @@ +--- +object param - store complex object: + - requires: + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + id: + type: integer + stored: + synthetic_source_keep: all + properties: + object_array: + properties: + trace: + type: keyword + nested: + type: nested + kw: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "id": 1, "stored": { "object_array": [ {"trace": "B"}, {"trace": "A"} ], "nested": [ {"foo": 20}, {"foo": 10} ], "kw": 100 } }' + - '{ "create": { } }' + - '{ "id": 2, "stored": { "object_array": { "trace": ["D", "C"] }, "nested": { "bar": [ 40, 30] }, "kw": 200, "baz": "2000" } }' + - '{ "create": { } }' + - '{ "id": 3, "stored": [ { "object_array": { "trace": "E" } }, { "nested": { "bar": [ 60, 50] } }, { "kw": 300 } ] }' + + - do: + search: + index: test + sort: id + + - match: { hits.hits.0._source.id: 1 } + - match: { hits.hits.0._source.stored.object_array.0.trace: B } + - match: { hits.hits.0._source.stored.object_array.1.trace: A } + - match: { hits.hits.0._source.stored.nested.0.foo: 20 } + - match: { hits.hits.0._source.stored.nested.1.foo: 10 } + - match: { hits.hits.0._source.stored.kw: 100 } + + - match: { hits.hits.1._source.id: 2 } + - match: { hits.hits.1._source.stored.object_array.trace: [D, C] } + - match: { hits.hits.1._source.stored.nested.bar: [40, 30] } + - match: { hits.hits.1._source.stored.kw: 200 } + - match: { hits.hits.1._source.stored.baz: "2000" } + + - match: { hits.hits.2._source.id: 3 } + - match: { hits.hits.2._source.stored.0.object_array.trace: E } + - match: { hits.hits.2._source.stored.1.nested.bar: [ 60, 50 ] } + - match: { hits.hits.2._source.stored.2.kw: 300 } + + --- object param - object array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -25,7 +89,7 @@ object param - object array: id: type: keyword stored: - store_array_source: true + synthetic_source_keep: arrays properties: span: properties: @@ -65,7 +129,7 @@ object param - object array: --- object param - object array within array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -77,10 +141,10 @@ object param - object array within array: mode: synthetic properties: stored: - store_array_source: true + synthetic_source_keep: arrays properties: path: - store_array_source: true + synthetic_source_keep: arrays properties: to: properties: @@ -108,7 +172,7 @@ object param - object array within array: --- object param - no object array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -120,7 +184,7 @@ object param - no object array: mode: synthetic properties: stored: - store_array_source: true + synthetic_source_keep: arrays properties: span: properties: @@ -150,7 +214,7 @@ object param - no object array: --- object param - field ordering in object array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -164,7 +228,7 @@ object param - field ordering in object array: a: type: keyword b: - store_array_source: true + synthetic_source_keep: arrays properties: aa: type: keyword @@ -173,7 +237,7 @@ object param - field ordering in object array: c: type: keyword d: - store_array_source: true + synthetic_source_keep: arrays properties: aa: type: keyword @@ -199,7 +263,7 @@ object param - field ordering in object array: --- object param - nested object array next to other fields: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -215,7 +279,7 @@ object param - nested object array next to other fields: b: properties: c: - store_array_source: true + synthetic_source_keep: arrays properties: aa: type: keyword @@ -255,7 +319,7 @@ object param - nested object array next to other fields: --- object param - nested object with stored array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -272,7 +336,7 @@ object param - nested object with stored array: type: nested nested_array_stored: type: nested - store_array_source: true + synthetic_source_keep: all - do: bulk: @@ -304,7 +368,7 @@ object param - nested object with stored array: --- index param - nested array within array: - requires: - cluster_features: ["mapper.synthetic_source_keep"] + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -322,7 +386,7 @@ index param - nested array within array: to: properties: some: - store_array_source: true + synthetic_source_keep: arrays properties: id: type: integer @@ -351,7 +415,7 @@ index param - nested array within array: # 112156 stored field under object with store_array_source: - requires: - cluster_features: ["mapper.source.synthetic_source_stored_fields_advance_fix"] + cluster_features: ["mapper.source.synthetic_source_stored_fields_advance_fix", "mapper.bwc_workaround_9_0"] reason: requires bug fix to be implemented - do: @@ -369,7 +433,7 @@ stored field under object with store_array_source: name: type: keyword obj: - store_array_source: true + synthetic_source_keep: arrays properties: foo: type: keyword @@ -740,7 +804,7 @@ field param - nested array within array: --- index param - root arrays: - requires: - cluster_features: ["mapper.synthetic_source_keep"] + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] reason: requires keeping array source - do: @@ -772,6 +836,9 @@ index param - root arrays: properties: id: type: keyword + obj_default: + type: object + synthetic_source_keep: none - do: bulk: @@ -782,6 +849,8 @@ index param - root arrays: - '{ "id": 1, "leaf": [30, 20, 10], "leaf_default": [30, 20, 10], "obj": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' - '{ "create": { } }' - '{ "id": 2, "leaf": [130, 120, 110], "leaf_default": [130, 120, 110], "obj": [ { "trace": { "id": "aa" }, "span": { "id": "2" } }, { "trace": { "id": "bb" }, "span": { "id": "2" } } ] }' + - '{ "create": { } }' + - '{ "id": 3, "obj_default": [ { "trace": { "id": "bb" }, "span": { "id": "2" } }, { "trace": { "id": "aa" }, "span": { "id": "2" } } ] }' - do: search: @@ -799,13 +868,17 @@ index param - root arrays: - match: { hits.hits.1._source.id: 2 } - match: { hits.hits.1._source.leaf: [ 130, 120, 110 ] } - - match: { hits.hits.0._source.leaf_default: [10, 20, 30] } + - match: { hits.hits.1._source.leaf_default: [110, 120, 130] } - length: { hits.hits.1._source.obj: 2 } - match: { hits.hits.1._source.obj.0.trace.id: aa } - match: { hits.hits.1._source.obj.0.span.id: "2" } - match: { hits.hits.1._source.obj.1.trace.id: bb } - match: { hits.hits.1._source.obj.1.span.id: "2" } + - match: { hits.hits.2._source.id: 3 } + - match: { hits.hits.2._source.obj_default.trace.id: [aa, bb] } + - match: { hits.hits.2._source.obj_default.span.id: "2" } + --- index param - dynamic root arrays: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index c82621baa717a..59a8c60e0d255 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -421,20 +421,21 @@ static void parseObjectOrNested(DocumentParserContext context) throws IOExceptio throwOnConcreteValue(context.parent(), currentFieldName, context); } + if (context.canAddIgnoredField() && getSourceKeepMode(context, context.parent().sourceKeepMode()) == Mapper.SourceKeepMode.ALL) { + context = context.addIgnoredFieldFromContext( + new IgnoredSourceFieldMapper.NameValue( + context.parent().fullPath(), + context.parent().fullPath().lastIndexOf(context.parent().leafName()), + null, + context.doc() + ) + ); + token = context.parser().currentToken(); + parser = context.parser(); + } + if (context.parent().isNested()) { // Handle a nested object that doesn't contain an array. Arrays are handled in #parseNonDynamicArray. - if (context.parent().storeArraySource() && context.canAddIgnoredField()) { - context = context.addIgnoredFieldFromContext( - new IgnoredSourceFieldMapper.NameValue( - context.parent().fullPath(), - context.parent().fullPath().lastIndexOf(context.parent().leafName()), - null, - context.doc() - ) - ); - token = context.parser().currentToken(); - parser = context.parser(); - } context = context.createNestedContext((NestedObjectMapper) context.parent()); } @@ -801,8 +802,8 @@ private static void parseNonDynamicArray( // Check if we need to record the array source. This only applies to synthetic source. if (context.canAddIgnoredField()) { boolean objectRequiresStoringSource = mapper instanceof ObjectMapper objectMapper - && (objectMapper.storeArraySource() - || (context.sourceKeepModeFromIndexSettings() == Mapper.SourceKeepMode.ARRAYS + && (getSourceKeepMode(context, objectMapper.sourceKeepMode()) == Mapper.SourceKeepMode.ALL + || (getSourceKeepMode(context, objectMapper.sourceKeepMode()) == Mapper.SourceKeepMode.ARRAYS && objectMapper instanceof NestedObjectMapper == false)); boolean fieldWithFallbackSyntheticSource = mapper instanceof FieldMapper fieldMapper && fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK; @@ -1115,15 +1116,7 @@ protected SyntheticSourceSupport syntheticSourceSupport() { private static class NoOpObjectMapper extends ObjectMapper { NoOpObjectMapper(String name, String fullPath) { - super( - name, - fullPath, - Explicit.IMPLICIT_TRUE, - Optional.empty(), - Explicit.IMPLICIT_FALSE, - Dynamic.RUNTIME, - Collections.emptyMap() - ); + super(name, fullPath, Explicit.IMPLICIT_TRUE, Optional.empty(), Optional.empty(), Dynamic.RUNTIME, Collections.emptyMap()); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index cf8f391813c09..a8fbb0e0ae95b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -21,9 +21,15 @@ * Spec for mapper-related features. */ public class MapperFeatures implements FeatureSpecification { + + // Used to avoid noise in mixed cluster and rest compatibility tests. Must not be backported to 8.x branch. + // This label gets added to tests with such failures before merging with main, then removed when backported to 8.x. + public static final NodeFeature BWC_WORKAROUND_9_0 = new NodeFeature("mapper.bwc_workaround_9_0"); + @Override public Set getFeatures() { return Set.of( + BWC_WORKAROUND_9_0, IgnoredSourceFieldMapper.TRACK_IGNORED_SOURCE, PassThroughObjectMapper.PASS_THROUGH_PRIORITY, RangeFieldMapper.NULL_VALUES_OFF_BY_ONE_FIX, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java index fc5f28dd51c9d..d0e0dcb6b97ba 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java @@ -98,6 +98,17 @@ public NestedObjectMapper build(MapperBuilderContext context) { } else { nestedTypePath = fullPath; } + if (sourceKeepMode.orElse(SourceKeepMode.NONE) == SourceKeepMode.ARRAYS) { + throw new MapperException( + "parameter [ " + + Mapper.SYNTHETIC_SOURCE_KEEP_PARAM + + " ] can't be set to [" + + SourceKeepMode.ARRAYS + + "] for nested object [" + + fullPath + + "]" + ); + } final Query nestedTypeFilter = NestedPathFieldMapper.filter(indexCreatedVersion, nestedTypePath); NestedMapperBuilderContext nestedContext = new NestedMapperBuilderContext( context.buildFullName(leafName()), @@ -115,7 +126,7 @@ public NestedObjectMapper build(MapperBuilderContext context) { buildMappers(nestedContext), enabled, dynamic, - storeArraySource, + sourceKeepMode, includeInParent, includeInRoot, parentTypeFilter, @@ -213,7 +224,7 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) { Map mappers, Explicit enabled, ObjectMapper.Dynamic dynamic, - Explicit storeArraySource, + Optional sourceKeepMode, Explicit includeInParent, Explicit includeInRoot, Query parentTypeFilter, @@ -222,7 +233,7 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) { Function bitsetProducer, IndexSettings indexSettings ) { - super(name, fullPath, enabled, Optional.empty(), storeArraySource, dynamic, mappers); + super(name, fullPath, enabled, Optional.empty(), sourceKeepMode, dynamic, mappers); this.parentTypeFilter = parentTypeFilter; this.nestedTypePath = nestedTypePath; this.nestedTypeFilter = nestedTypeFilter; @@ -283,7 +294,7 @@ NestedObjectMapper withoutMappers() { Map.of(), enabled, dynamic, - storeArraySource, + sourceKeepMode, includeInParent, includeInRoot, parentTypeFilter, @@ -310,8 +321,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (isEnabled() != Defaults.ENABLED) { builder.field("enabled", enabled.value()); } - if (storeArraySource != Defaults.STORE_ARRAY_SOURCE) { - builder.field(STORE_ARRAY_SOURCE_PARAM, storeArraySource.value()); + if (sourceKeepMode.isPresent()) { + builder.field(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, sourceKeepMode.get()); } serializeMappers(builder, params); return builder.endObject(); @@ -359,7 +370,7 @@ public ObjectMapper merge(Mapper mergeWith, MapperMergeContext parentMergeContex mergeResult.mappers(), mergeResult.enabled(), mergeResult.dynamic(), - mergeResult.trackArraySource(), + mergeResult.sourceKeepMode(), incInParent, incInRoot, parentTypeFilter, @@ -393,8 +404,8 @@ protected MapperMergeContext createChildContext(MapperMergeContext mapperMergeCo @Override public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (storeArraySource()) { - // IgnoredSourceFieldMapper integration takes care of writing the source for nested objects that enabled store_array_source. + if (sourceKeepMode.orElse(SourceKeepMode.NONE) == SourceKeepMode.ALL) { + // IgnoredSourceFieldMapper integration takes care of writing the source for the nested object. return SourceLoader.SyntheticFieldLoader.NOTHING; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 40019566adaa8..0b9727aa66c8a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -45,6 +45,7 @@ public class ObjectMapper extends Mapper { public static final String CONTENT_TYPE = "object"; static final String STORE_ARRAY_SOURCE_PARAM = "store_array_source"; static final NodeFeature SUBOBJECTS_AUTO = new NodeFeature("mapper.subobjects_auto"); + // No-op. All uses of this feature were reverted but node features can't be removed. static final NodeFeature SUBOBJECTS_AUTO_FIXES = new NodeFeature("mapper.subobjects_auto_fixes"); /** @@ -127,7 +128,7 @@ static Dynamic getRootDynamic(MappingLookup mappingLookup) { public static class Builder extends Mapper.Builder { protected Optional subobjects; protected Explicit enabled = Explicit.IMPLICIT_TRUE; - protected Explicit storeArraySource = Defaults.STORE_ARRAY_SOURCE; + protected Optional sourceKeepMode = Optional.empty(); protected Dynamic dynamic; protected final List mappersBuilders = new ArrayList<>(); @@ -141,8 +142,8 @@ public Builder enabled(boolean enabled) { return this; } - public Builder storeArraySource(boolean value) { - this.storeArraySource = Explicit.explicitBoolean(value); + public Builder sourceKeepMode(SourceKeepMode sourceKeepMode) { + this.sourceKeepMode = Optional.of(sourceKeepMode); return this; } @@ -245,7 +246,7 @@ public ObjectMapper build(MapperBuilderContext context) { context.buildFullName(leafName()), enabled, subobjects, - storeArraySource, + sourceKeepMode, dynamic, buildMappers(context.createChildContext(leafName(), dynamic)) ); @@ -307,7 +308,10 @@ protected static boolean parseObjectOrDocumentTypeProperties( builder.enabled(XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".enabled")); return true; } else if (fieldName.equals(STORE_ARRAY_SOURCE_PARAM)) { - builder.storeArraySource(XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".store_array_source")); + builder.sourceKeepMode(SourceKeepMode.ARRAYS); + return true; + } else if (fieldName.equals(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM)) { + builder.sourceKeepMode(SourceKeepMode.from(fieldNode.toString())); return true; } else if (fieldName.equals("properties")) { if (fieldNode instanceof Collection && ((Collection) fieldNode).isEmpty()) { @@ -434,7 +438,7 @@ private static void validateFieldName(String fieldName, IndexVersion indexCreate protected final Explicit enabled; protected final Optional subobjects; - protected final Explicit storeArraySource; + protected final Optional sourceKeepMode; protected final Dynamic dynamic; protected final Map mappers; @@ -444,7 +448,7 @@ private static void validateFieldName(String fieldName, IndexVersion indexCreate String fullPath, Explicit enabled, Optional subobjects, - Explicit storeArraySource, + Optional sourceKeepMode, Dynamic dynamic, Map mappers ) { @@ -454,7 +458,7 @@ private static void validateFieldName(String fieldName, IndexVersion indexCreate this.fullPath = internFieldName(fullPath); this.enabled = enabled; this.subobjects = subobjects; - this.storeArraySource = storeArraySource; + this.sourceKeepMode = sourceKeepMode; this.dynamic = dynamic; if (mappers == null) { this.mappers = Map.of(); @@ -482,7 +486,7 @@ public Builder newBuilder(IndexVersion indexVersionCreated) { * This is typically used in the context of a mapper merge when there's not enough budget to add the entire object. */ ObjectMapper withoutMappers() { - return new ObjectMapper(leafName(), fullPath, enabled, subobjects, storeArraySource, dynamic, Map.of()); + return new ObjectMapper(leafName(), fullPath, enabled, subobjects, sourceKeepMode, dynamic, Map.of()); } @Override @@ -520,8 +524,8 @@ public final Subobjects subobjects() { return subobjects.orElse(Subobjects.ENABLED); } - public final boolean storeArraySource() { - return storeArraySource.value(); + public final Optional sourceKeepMode() { + return sourceKeepMode; } @Override @@ -550,7 +554,7 @@ public ObjectMapper merge(Mapper mergeWith, MapperMergeContext parentMergeContex fullPath, mergeResult.enabled, mergeResult.subObjects, - mergeResult.trackArraySource, + mergeResult.sourceKeepMode, mergeResult.dynamic, mergeResult.mappers ); @@ -559,7 +563,7 @@ public ObjectMapper merge(Mapper mergeWith, MapperMergeContext parentMergeContex protected record MergeResult( Explicit enabled, Optional subObjects, - Explicit trackArraySource, + Optional sourceKeepMode, Dynamic dynamic, Map mappers ) { @@ -593,26 +597,31 @@ static MergeResult build(ObjectMapper existing, ObjectMapper mergeWithObject, Ma } else { subObjects = existing.subobjects; } - final Explicit trackArraySource; - if (mergeWithObject.storeArraySource.explicit()) { + final Optional sourceKeepMode; + if (mergeWithObject.sourceKeepMode.isPresent()) { if (reason == MergeReason.INDEX_TEMPLATE) { - trackArraySource = mergeWithObject.storeArraySource; - } else if (existing.storeArraySource != mergeWithObject.storeArraySource) { + sourceKeepMode = mergeWithObject.sourceKeepMode; + } else if (existing.sourceKeepMode.isEmpty() || existing.sourceKeepMode.get() != mergeWithObject.sourceKeepMode.get()) { throw new MapperException( - "the [store_array_source] parameter can't be updated for the object mapping [" + existing.fullPath() + "]" + "the [ " + + Mapper.SYNTHETIC_SOURCE_KEEP_PARAM + + " ] parameter can't be updated for the object mapping [" + + existing.fullPath() + + "]" ); } else { - trackArraySource = existing.storeArraySource; + sourceKeepMode = existing.sourceKeepMode; } } else { - trackArraySource = existing.storeArraySource; + sourceKeepMode = existing.sourceKeepMode; } + MapperMergeContext objectMergeContext = existing.createChildContext(parentMergeContext, existing.leafName()); Map mergedMappers = buildMergedMappers(existing, mergeWithObject, objectMergeContext, subObjects); return new MergeResult( enabled, subObjects, - trackArraySource, + sourceKeepMode, mergeWithObject.dynamic != null ? mergeWithObject.dynamic : existing.dynamic, mergedMappers ); @@ -733,6 +742,12 @@ private void ensureFlattenable(MapperBuilderContext context, ContentPath path) { + ")" ); } + if (sourceKeepMode.isPresent()) { + throwAutoFlatteningException( + path, + "the value of [" + Mapper.SYNTHETIC_SOURCE_KEEP_PARAM + "] is [ " + sourceKeepMode.get() + " ]" + ); + } if (isEnabled() == false) { throwAutoFlatteningException(path, "the value of [enabled] is [false]"); } @@ -774,8 +789,8 @@ void toXContent(XContentBuilder builder, Params params, ToXContent custom) throw if (subobjects.isPresent()) { builder.field("subobjects", subobjects.get().printedValue); } - if (storeArraySource != Defaults.STORE_ARRAY_SOURCE) { - builder.field(STORE_ARRAY_SOURCE_PARAM, storeArraySource.value()); + if (sourceKeepMode.isPresent()) { + builder.field("synthetic_source_keep", sourceKeepMode.get()); } if (custom != null) { custom.toXContent(builder, params); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java index 9ef36b99a57c5..80f845d626a2f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java @@ -82,6 +82,7 @@ public PassThroughObjectMapper build(MapperBuilderContext context) { leafName(), context.buildFullName(leafName()), enabled, + sourceKeepMode, dynamic, buildMappers(context.createChildContext(leafName(), timeSeriesDimensionSubFields.value(), dynamic)), timeSeriesDimensionSubFields, @@ -99,13 +100,14 @@ public PassThroughObjectMapper build(MapperBuilderContext context) { String name, String fullPath, Explicit enabled, + Optional sourceKeepMode, Dynamic dynamic, Map mappers, Explicit timeSeriesDimensionSubFields, int priority ) { // Subobjects are not currently supported. - super(name, fullPath, enabled, Optional.of(Subobjects.DISABLED), Explicit.IMPLICIT_FALSE, dynamic, mappers); + super(name, fullPath, enabled, Optional.of(Subobjects.DISABLED), sourceKeepMode, dynamic, mappers); this.timeSeriesDimensionSubFields = timeSeriesDimensionSubFields; this.priority = priority; if (priority < 0) { @@ -115,7 +117,16 @@ public PassThroughObjectMapper build(MapperBuilderContext context) { @Override PassThroughObjectMapper withoutMappers() { - return new PassThroughObjectMapper(leafName(), fullPath(), enabled, dynamic, Map.of(), timeSeriesDimensionSubFields, priority); + return new PassThroughObjectMapper( + leafName(), + fullPath(), + enabled, + sourceKeepMode, + dynamic, + Map.of(), + timeSeriesDimensionSubFields, + priority + ); } @Override @@ -158,6 +169,7 @@ public PassThroughObjectMapper merge(Mapper mergeWith, MapperMergeContext parent leafName(), fullPath(), mergeResult.enabled(), + mergeResult.sourceKeepMode(), mergeResult.dynamic(), mergeResult.mappers(), containsDimensions, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java index 878f9c92fa552..ce983e8a327c9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java @@ -113,7 +113,7 @@ public RootObjectMapper build(MapperBuilderContext context) { leafName(), enabled, subobjects, - storeArraySource, + sourceKeepMode, dynamic, buildMappers(context.createChildContext(null, dynamic)), new HashMap<>(runtimeFields), @@ -135,7 +135,7 @@ public RootObjectMapper build(MapperBuilderContext context) { String name, Explicit enabled, Optional subobjects, - Explicit trackArraySource, + Optional sourceKeepMode, Dynamic dynamic, Map mappers, Map runtimeFields, @@ -144,12 +144,17 @@ public RootObjectMapper build(MapperBuilderContext context) { Explicit dateDetection, Explicit numericDetection ) { - super(name, name, enabled, subobjects, trackArraySource, dynamic, mappers); + super(name, name, enabled, subobjects, sourceKeepMode, dynamic, mappers); this.runtimeFields = runtimeFields; this.dynamicTemplates = dynamicTemplates; this.dynamicDateTimeFormatters = dynamicDateTimeFormatters; this.dateDetection = dateDetection; this.numericDetection = numericDetection; + if (sourceKeepMode.orElse(SourceKeepMode.NONE) == SourceKeepMode.ALL) { + throw new MapperParsingException( + "root object can't be configured with [" + Mapper.SYNTHETIC_SOURCE_KEEP_PARAM + ":" + SourceKeepMode.ALL + "]" + ); + } } @Override @@ -166,7 +171,7 @@ RootObjectMapper withoutMappers() { leafName(), enabled, subobjects, - storeArraySource, + sourceKeepMode, dynamic, Map.of(), Map.of(), @@ -282,7 +287,7 @@ public RootObjectMapper merge(Mapper mergeWith, MapperMergeContext parentMergeCo leafName(), mergeResult.enabled(), mergeResult.subObjects(), - mergeResult.trackArraySource(), + mergeResult.sourceKeepMode(), mergeResult.dynamic(), mergeResult.mappers(), Map.copyOf(runtimeFields), diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index d48c5550631cd..e385177b87147 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -179,7 +179,7 @@ private static ObjectMapper createObjectMapper(String name) { name, Explicit.IMPLICIT_TRUE, Optional.empty(), - Explicit.IMPLICIT_FALSE, + Optional.empty(), ObjectMapper.Dynamic.FALSE, emptyMap() ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java index 18c4f393bc696..ae793bc3b329e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import static java.util.Collections.emptyList; @@ -433,6 +434,7 @@ private PassThroughObjectMapper createPassThroughMapper(String name, Map { b.startArray("path"); + b.startObject().field("int_value", 20).endObject(); b.startObject().field("int_value", 10).endObject(); + b.endArray(); + b.field("bool_value", true); + }); + assertEquals(""" + {"bool_value":true,"path":[{"int_value":20},{"int_value":10}]}""", syntheticSource); + } + + public void testIndexStoredArraySourceRootObjectArrayWithBypass() throws IOException { + DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { + b.startObject("path"); + { + b.field("type", "object"); + b.field("synthetic_source_keep", "none"); + b.startObject("properties"); + { + b.startObject("int_value").field("type", "integer").endObject(); + } + b.endObject(); + } + b.endObject(); + b.startObject("bool_value").field("type", "boolean").endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startArray("path"); b.startObject().field("int_value", 20).endObject(); + b.startObject().field("int_value", 10).endObject(); b.endArray(); b.field("bool_value", true); }); assertEquals(""" - {"bool_value":true,"path":[{"int_value":10},{"int_value":20}]}""", syntheticSource); + {"bool_value":true,"path":{"int_value":[10,20]}}""", syntheticSource); } public void testIndexStoredArraySourceNestedValueArray() throws IOException { @@ -622,6 +648,12 @@ public void testIndexStoredArraySourceNestedValueArrayDisabled() throws IOExcept { b.startObject("int_value").field("type", "integer").field(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, "none").endObject(); b.startObject("bool_value").field("type", "boolean").endObject(); + b.startObject("obj").field("type", "object").field(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, "none"); + b.startObject("properties"); + { + b.startObject("foo").field("type", "integer").endObject(); + } + b.endObject().endObject(); } b.endObject(); } @@ -632,11 +664,17 @@ public void testIndexStoredArraySourceNestedValueArrayDisabled() throws IOExcept { b.array("int_value", new int[] { 30, 20, 10 }); b.field("bool_value", true); + b.startArray("obj"); + { + b.startObject().field("foo", 2).endObject(); + b.startObject().field("foo", 1).endObject(); + } + b.endArray(); } b.endObject(); }); assertEquals(""" - {"path":{"bool_value":true,"int_value":[10,20,30]}}""", syntheticSource); + {"path":{"bool_value":true,"int_value":[10,20,30],"obj":{"foo":[1,2]}}}""", syntheticSource); } public void testFieldStoredArraySourceNestedValueArray() throws IOException { @@ -674,8 +712,8 @@ public void testFieldStoredSourceNestedValue() throws IOException { b.field("type", "object"); b.startObject("properties"); { - b.startObject("default").field("type", "float").field(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, "none").endObject(); - b.startObject("source_kept").field("type", "float").field(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, "all").endObject(); + b.startObject("default").field("type", "float").field("synthetic_source_keep", "none").endObject(); + b.startObject("source_kept").field("type", "float").field("synthetic_source_keep", "all").endObject(); b.startObject("bool_value").field("type", "boolean").endObject(); } b.endObject(); @@ -738,7 +776,7 @@ public void testRootArray() throws IOException { b.startObject("path"); { b.field("type", "object"); - b.field("store_array_source", true); + b.field("synthetic_source_keep", "arrays"); b.startObject("properties"); { b.startObject("int_value").field("type", "integer").endObject(); @@ -765,7 +803,7 @@ public void testNestedArray() throws IOException { b.field("type", "object"); b.startObject("properties"); { - b.startObject("to").field("type", "object").field("store_array_source", true); + b.startObject("to").field("type", "object").field("synthetic_source_keep", "arrays"); { b.startObject("properties"); { @@ -835,10 +873,10 @@ public void testArrayWithinArray() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); { - b.field("type", "object").field("store_array_source", true); + b.field("type", "object").field("synthetic_source_keep", "arrays"); b.startObject("properties"); { - b.startObject("to").field("type", "object").field("store_array_source", true); + b.startObject("to").field("type", "object").field("synthetic_source_keep", "arrays"); { b.startObject("properties"); { @@ -893,7 +931,7 @@ public void testObjectArrayAndValue() throws IOException { { b.startObject("stored"); { - b.field("type", "object").field("store_array_source", true); + b.field("type", "object").field("synthetic_source_keep", "arrays"); b.startObject("properties").startObject("leaf").field("type", "integer").endObject().endObject(); } b.endObject(); @@ -1061,7 +1099,7 @@ public void testStoredArrayWithinHigherLevelArray() throws IOException { b.field("type", "object"); b.startObject("properties"); { - b.startObject("to").field("type", "object").field("store_array_source", true); + b.startObject("to").field("type", "object").field("synthetic_source_keep", "arrays"); { b.startObject("properties"); { @@ -1107,6 +1145,42 @@ public void testStoredArrayWithinHigherLevelArray() throws IOException { {"path":{"to":[{"name":"A"},{"name":"B"},{"name":"C"},{"name":"D"}]}}""", booleanValue), syntheticSource); } + public void testObjectWithKeepAll() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("path"); + { + b.field("type", "object").field("synthetic_source_keep", "all"); + b.startObject("properties"); + { + b.startObject("a").field("type", "object").endObject(); + b.startObject("b").field("type", "integer").endObject(); + } + b.endObject(); + } + b.endObject(); + b.startObject("id").field("type", "integer").endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startObject("path"); + { + b.startArray("a"); + { + b.startObject().field("foo", 30).endObject(); + b.startObject().field("foo", 20).endObject(); + b.startObject().field("foo", 10).endObject(); + b.startObject().field("bar", 20).endObject(); + b.startObject().field("bar", 10).endObject(); + } + b.endArray(); + b.array("b", 4, 1, 3, 2); + } + b.endObject(); + b.field("id", 10); + }); + assertEquals(""" + {"id":10,"path":{"a":[{"foo":30},{"foo":20},{"foo":10},{"bar":20},{"bar":10}],"b":[4,1,3,2]}}""", syntheticSource); + } + public void testFallbackFieldWithinHigherLevelArray() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); @@ -1140,7 +1214,7 @@ public void testFallbackFieldWithinHigherLevelArray() throws IOException { public void testFieldOrdering() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("A").field("type", "integer").endObject(); - b.startObject("B").field("type", "object").field("store_array_source", true); + b.startObject("B").field("type", "object").field("synthetic_source_keep", "arrays"); { b.startObject("properties"); { @@ -1151,7 +1225,7 @@ public void testFieldOrdering() throws IOException { } b.endObject(); b.startObject("C").field("type", "integer").endObject(); - b.startObject("D").field("type", "object").field("store_array_source", true); + b.startObject("D").field("type", "object").field("synthetic_source_keep", "arrays"); { b.startObject("properties"); { @@ -1189,7 +1263,7 @@ public void testNestedObjectWithField() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { - b.field("store_array_source", true); + b.field("synthetic_source_keep", "all"); b.startObject("properties"); { b.startObject("foo").field("type", "keyword").endObject(); @@ -1211,7 +1285,7 @@ public void testNestedObjectWithArray() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { - b.field("store_array_source", true); + b.field("synthetic_source_keep", "all"); b.startObject("properties"); { b.startObject("foo").field("type", "keyword").endObject(); @@ -1244,7 +1318,7 @@ public void testNestedSubobjectWithField() throws IOException { b.startObject("int_value").field("type", "integer").endObject(); b.startObject("to").field("type", "nested"); { - b.field("store_array_source", true); + b.field("synthetic_source_keep", "all"); b.startObject("properties"); { b.startObject("foo").field("type", "keyword").endObject(); @@ -1285,7 +1359,7 @@ public void testNestedSubobjectWithArray() throws IOException { b.startObject("int_value").field("type", "integer").endObject(); b.startObject("to").field("type", "nested"); { - b.field("store_array_source", true); + b.field("synthetic_source_keep", "all"); b.startObject("properties"); { b.startObject("foo").field("type", "keyword").endObject(); @@ -1325,7 +1399,7 @@ public void testNestedSubobjectWithArray() throws IOException { public void testNestedObjectIncludeInRoot() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { - b.startObject("path").field("type", "nested").field("store_array_source", true).field("include_in_root", true); + b.startObject("path").field("type", "nested").field("synthetic_source_keep", "all").field("include_in_root", true); { b.startObject("properties"); { @@ -1599,7 +1673,7 @@ public void testStoredNestedSubObjectWithNameOverlappingParentName() throws IOEx b.startObject("path"); b.startObject("properties"); { - b.startObject("at").field("type", "nested").field("store_array_source", "true").endObject(); + b.startObject("at").field("type", "nested").field("synthetic_source_keep", "all").endObject(); } b.endObject(); b.endObject(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java index 1381df07789b5..fd44e68df19a8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java @@ -84,7 +84,7 @@ public void testSubfieldOverride() { "object", Explicit.EXPLICIT_TRUE, Optional.empty(), - Explicit.IMPLICIT_FALSE, + Optional.empty(), ObjectMapper.Dynamic.TRUE, Collections.singletonMap("object.subfield", fieldMapper) ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index 0a954115e77f6..be1469e25f24d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -1571,14 +1571,14 @@ public void testNestedMapperFilters() throws Exception { public void testStoreArraySourceinSyntheticSourceMode() throws IOException { DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { - b.startObject("o").field("type", "nested").field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true).endObject(); + b.startObject("o").field("type", "nested").field("synthetic_source_keep", "all").endObject(); })); assertNotNull(mapper.mapping().getRoot().getMapper("o")); } public void testStoreArraySourceNoopInNonSyntheticSourceMode() throws IOException { DocumentMapper mapper = createDocumentMapper(mapping(b -> { - b.startObject("o").field("type", "nested").field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true).endObject(); + b.startObject("o").field("type", "nested").field("synthetic_source_keep", "all").endObject(); })); assertNotNull(mapper.mapping().getRoot().getMapper("o")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index 3312c94e8a0e1..64eee39532c31 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -167,7 +167,7 @@ public void testMergeEnabledForIndexTemplates() throws IOException { assertNotNull(objectMapper); assertFalse(objectMapper.isEnabled()); assertEquals(ObjectMapper.Subobjects.ENABLED, objectMapper.subobjects()); - assertFalse(objectMapper.storeArraySource()); + assertTrue(objectMapper.sourceKeepMode().isEmpty()); // Setting 'enabled' to true is allowed, and updates the mapping. update = Strings.toString( @@ -189,7 +189,7 @@ public void testMergeEnabledForIndexTemplates() throws IOException { assertNotNull(objectMapper); assertTrue(objectMapper.isEnabled()); assertEquals(ObjectMapper.Subobjects.AUTO, objectMapper.subobjects()); - assertTrue(objectMapper.storeArraySource()); + assertEquals(Mapper.SourceKeepMode.ARRAYS, objectMapper.sourceKeepMode().orElse(Mapper.SourceKeepMode.NONE)); } public void testFieldReplacementForIndexTemplates() throws IOException { @@ -678,14 +678,14 @@ public void testSyntheticSourceDocValuesFieldWithout() throws IOException { public void testStoreArraySourceinSyntheticSourceMode() throws IOException { DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { - b.startObject("o").field("type", "object").field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true).endObject(); + b.startObject("o").field("type", "object").field("synthetic_source_keep", "arrays").endObject(); })); assertNotNull(mapper.mapping().getRoot().getMapper("o")); } public void testStoreArraySourceNoopInNonSyntheticSourceMode() throws IOException { DocumentMapper mapper = createDocumentMapper(mapping(b -> { - b.startObject("o").field("type", "object").field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true).endObject(); + b.startObject("o").field("type", "object").field("synthetic_source_keep", "arrays").endObject(); })); assertNotNull(mapper.mapping().getRoot().getMapper("o")); } @@ -727,7 +727,7 @@ private ObjectMapper createObjectMapperWithAllParametersSet(CheckedConsumer createMapperService(mapping)); + assertThat(e.getMessage(), containsString("root object can't be configured with [synthetic_source_keep:all]")); + } + public void testWithoutMappers() throws IOException { RootObjectMapper shallowRoot = createRootObjectMapperWithAllParametersSet(b -> {}, b -> {}); RootObjectMapper root = createRootObjectMapperWithAllParametersSet(b -> { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index a7d18ff782400..ca26779f3376d 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1537,7 +1537,7 @@ public void testSyntheticSourceKeepNone() throws IOException { SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed()).example(1); DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { b.startObject("field"); - b.field(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, "none"); + b.field("synthetic_source_keep", "none"); example.mapping().accept(b); b.endObject(); })); @@ -1548,7 +1548,7 @@ public void testSyntheticSourceKeepAll() throws IOException { SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed()).example(1); DocumentMapper mapperAll = createDocumentMapper(syntheticSourceMapping(b -> { b.startObject("field"); - b.field(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, "all"); + b.field("synthetic_source_keep", "all"); example.mapping().accept(b); b.endObject(); })); @@ -1565,7 +1565,7 @@ public void testSyntheticSourceKeepArrays() throws IOException { SyntheticSourceExample example = syntheticSourceSupportForKeepTests(shouldUseIgnoreMalformed()).example(1); DocumentMapper mapperAll = createDocumentMapper(syntheticSourceMapping(b -> { b.startObject("field"); - b.field(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, randomFrom("arrays", "all")); // Both options keep array source. + b.field("synthetic_source_keep", randomFrom("arrays", "all")); // Both options keep array source. example.mapping().accept(b); b.endObject(); })); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java index 81e120511a40f..067d1b96e965e 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java @@ -12,6 +12,7 @@ import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping; +import org.elasticsearch.test.ESTestCase; import java.util.Set; @@ -115,11 +116,15 @@ public DataSourceResponse.LeafMappingParametersGenerator accept(DataSourceHandle } } - record ObjectMappingParametersGenerator(boolean isNested) + record ObjectMappingParametersGenerator(boolean isRoot, boolean isNested) implements DataSourceRequest { public DataSourceResponse.ObjectMappingParametersGenerator accept(DataSourceHandler handler) { return handler.handle(this); } + + public String syntheticSourceKeepValue() { + return isRoot() ? ESTestCase.randomFrom("none", "arrays") : ESTestCase.randomFrom("none", "arrays", "all"); + } } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java index 89850cd56bbd0..69f839d461b40 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -83,6 +83,9 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequ if (ESTestCase.randomBoolean()) { parameters.put("dynamic", ESTestCase.randomFrom("true", "false", "strict")); } + if (ESTestCase.randomBoolean()) { + parameters.put(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, "all"); // [arrays] doesn't apply to nested objects + } return parameters; }); @@ -96,6 +99,9 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequ if (ESTestCase.randomBoolean()) { parameters.put("enabled", ESTestCase.randomFrom("true", "false")); } + if (ESTestCase.randomBoolean()) { + parameters.put(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, request.syntheticSourceKeepValue()); + } return parameters; }); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java index b5cd4f78aff95..ba168b221f572 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java @@ -28,7 +28,7 @@ public class NestedFieldDataGenerator implements FieldDataGenerator { this.mappingParameters = context.specification() .dataSource() - .get(new DataSourceRequest.ObjectMappingParametersGenerator(true)) + .get(new DataSourceRequest.ObjectMappingParametersGenerator(false, true)) .mappingGenerator() .get(); var dynamicMapping = context.determineDynamicMapping(mappingParameters); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java index 27c27e31702f7..084310ac967fc 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java @@ -28,7 +28,7 @@ public class ObjectFieldDataGenerator implements FieldDataGenerator { this.mappingParameters = context.specification() .dataSource() - .get(new DataSourceRequest.ObjectMappingParametersGenerator(false)) + .get(new DataSourceRequest.ObjectMappingParametersGenerator(false, false)) .mappingGenerator() .get(); var dynamicMapping = context.determineDynamicMapping(mappingParameters); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java index e85d18a1dac12..2c7aa65d8c6d1 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java @@ -37,7 +37,7 @@ public TopLevelObjectFieldDataGenerator(DataGeneratorSpecification specification this.mappingParameters = Map.of(); } else { this.mappingParameters = new HashMap<>( - specification.dataSource().get(new DataSourceRequest.ObjectMappingParametersGenerator(false)).mappingGenerator().get() + specification.dataSource().get(new DataSourceRequest.ObjectMappingParametersGenerator(true, false)).mappingGenerator().get() ); // Top-level object can't be disabled because @timestamp is a required field in data streams. this.mappingParameters.remove("enabled"); diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml index a4c62efeed7a4..0e77bc208eed4 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml @@ -44,7 +44,7 @@ template: dropped_events_count: type: long links: - store_array_source: true + synthetic_source_keep: arrays properties: trace_id: type: keyword diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/40_document_level_security_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/40_document_level_security_synthetic_source.yml index 769b9d848ba35..52abe0a3d83d7 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/40_document_level_security_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/40_document_level_security_synthetic_source.yml @@ -186,7 +186,7 @@ Filter on object with stored source: type: keyword obj: type: object - store_array_source: true + synthetic_source_keep: arrays properties: secret: type: keyword From 52fd836a7bbcad13c20ed1770766b756b10a424f Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 3 Oct 2024 19:35:23 +0100 Subject: [PATCH 053/194] Relax timeout in `waitForActiveLicense` (#113673) No sense in failing these tests just because the cluster took a bit too long to start up. Closes #113343 --- .../main/java/org/elasticsearch/test/rest/ESRestTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index e8c1aecb7abee..215973b5dece2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -2394,7 +2394,7 @@ protected static void waitForActiveLicense(final RestClient restClient) throws E assertThat("Expecting non-null license status", status, notNullValue()); assertThat("Expecting active license", status, equalTo("active")); } - }); + }, 10, TimeUnit.MINUTES); } // TODO: replace usages of this with warning_regex or allowed_warnings_regex From 4a7a5b7b3c1ad3426ffe96083212d68da88c4e1c Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 3 Oct 2024 15:46:40 -0400 Subject: [PATCH 054/194] [CI] Forward port Amazon Linux 2 dns fix from #107907 (#113902) --- .buildkite/hooks/pre-command | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 0c0ede8c3a076..a886220c84cda 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -109,3 +109,11 @@ EOF Agent information from gobld EOF fi + +# Amazon Linux 2 has DNS resolution issues with resource-based hostnames in EC2 +# We have many functional tests that try to lookup and resolve the hostname of the local machine in a particular way +# And they fail. This sets up a manual entry for the hostname in dnsmasq. +if [[ -f /etc/os-release ]] && grep -q '"Amazon Linux 2"' /etc/os-release; then + echo "$(hostname -i | cut -d' ' -f 2) $(hostname -f)." | sudo tee /etc/dnsmasq.hosts + sudo systemctl restart dnsmasq.service +fi From faa4c280aeec28f33323771304744ffe4e75b84d Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 3 Oct 2024 15:48:49 -0400 Subject: [PATCH 055/194] [CI] Reduce disk size used for most jobs/agents (#113488) --- .buildkite/pipelines/intake.template.yml | 8 ----- .buildkite/pipelines/intake.yml | 8 ----- .../lucene-snapshot/build-snapshot.yml | 1 - .../pipelines/lucene-snapshot/run-tests.yml | 8 ----- .../periodic-packaging.bwc.template.yml | 1 - .buildkite/pipelines/periodic-packaging.yml | 18 ----------- .../pipelines/periodic-platform-support.yml | 1 - .../pipelines/periodic.bwc.template.yml | 1 - .buildkite/pipelines/periodic.template.yml | 15 +-------- .buildkite/pipelines/periodic.yml | 31 ------------------- .../pull-request/build-benchmark.yml | 1 - .../pipelines/pull-request/bwc-snapshots.yml | 1 - .../pipelines/pull-request/cloud-deploy.yml | 1 - .../pipelines/pull-request/docs-check.yml | 1 - .../pull-request/eql-correctness.yml | 1 - .../pull-request/example-plugins.yml | 1 - .../pipelines/pull-request/full-bwc.yml | 1 - .../pull-request/packaging-upgrade-tests.yml | 1 - .../pipelines/pull-request/part-1-fips.yml | 1 - .buildkite/pipelines/pull-request/part-1.yml | 1 - .../pipelines/pull-request/part-2-fips.yml | 1 - .buildkite/pipelines/pull-request/part-2.yml | 1 - .../pipelines/pull-request/part-3-fips.yml | 1 - .buildkite/pipelines/pull-request/part-3.yml | 1 - .../pipelines/pull-request/part-4-fips.yml | 1 - .buildkite/pipelines/pull-request/part-4.yml | 1 - .../pipelines/pull-request/part-5-fips.yml | 1 - .buildkite/pipelines/pull-request/part-5.yml | 1 - .../pipelines/pull-request/precommit.yml | 1 - .../pull-request/rest-compatibility.yml | 1 - .../pull-request/validate-changelogs.yml | 1 - 31 files changed, 1 insertion(+), 112 deletions(-) diff --git a/.buildkite/pipelines/intake.template.yml b/.buildkite/pipelines/intake.template.yml index 1a513971b2c10..f530f237113a9 100644 --- a/.buildkite/pipelines/intake.template.yml +++ b/.buildkite/pipelines/intake.template.yml @@ -7,7 +7,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - wait - label: part1 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1 @@ -17,7 +16,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part2 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2 timeout_in_minutes: 300 @@ -26,7 +24,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part3 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3 timeout_in_minutes: 300 @@ -35,7 +32,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part4 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4 timeout_in_minutes: 300 @@ -44,7 +40,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part5 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart5 timeout_in_minutes: 300 @@ -53,7 +48,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - group: bwc-snapshots steps: - label: "{{matrix.BWC_VERSION}} / bwc-snapshots" @@ -67,7 +61,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: "{{matrix.BWC_VERSION}}" - label: rest-compat @@ -78,7 +71,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - wait - trigger: elasticsearch-dra-workflow label: Trigger DRA snapshot workflow diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index a6af8bd35c7a0..e44a1e67e9d59 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -8,7 +8,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - wait - label: part1 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1 @@ -18,7 +17,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part2 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2 timeout_in_minutes: 300 @@ -27,7 +25,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part3 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3 timeout_in_minutes: 300 @@ -36,7 +33,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part4 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4 timeout_in_minutes: 300 @@ -45,7 +41,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part5 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart5 timeout_in_minutes: 300 @@ -54,7 +49,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - group: bwc-snapshots steps: - label: "{{matrix.BWC_VERSION}} / bwc-snapshots" @@ -68,7 +62,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: "{{matrix.BWC_VERSION}}" - label: rest-compat @@ -79,7 +72,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - wait - trigger: elasticsearch-dra-workflow label: Trigger DRA snapshot workflow diff --git a/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml b/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml index 1f69b8faa7ab4..8cf2a8aacbece 100644 --- a/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml +++ b/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml @@ -15,7 +15,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - wait - trigger: "elasticsearch-lucene-snapshot-tests" build: diff --git a/.buildkite/pipelines/lucene-snapshot/run-tests.yml b/.buildkite/pipelines/lucene-snapshot/run-tests.yml index 49c3396488d82..c76c54a56494e 100644 --- a/.buildkite/pipelines/lucene-snapshot/run-tests.yml +++ b/.buildkite/pipelines/lucene-snapshot/run-tests.yml @@ -7,7 +7,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - wait: null - label: part1 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1 @@ -17,7 +16,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part2 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2 timeout_in_minutes: 300 @@ -26,7 +24,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part3 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3 timeout_in_minutes: 300 @@ -35,7 +32,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part4 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4 timeout_in_minutes: 300 @@ -44,7 +40,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: part5 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart5 timeout_in_minutes: 300 @@ -53,7 +48,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - group: bwc-snapshots steps: - label: "{{matrix.BWC_VERSION}} / bwc-snapshots" @@ -70,7 +64,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: "{{matrix.BWC_VERSION}}" - label: rest-compat @@ -81,4 +74,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/periodic-packaging.bwc.template.yml b/.buildkite/pipelines/periodic-packaging.bwc.template.yml index 8a6fa2553b204..b06bc80d3535d 100644 --- a/.buildkite/pipelines/periodic-packaging.bwc.template.yml +++ b/.buildkite/pipelines/periodic-packaging.bwc.template.yml @@ -11,6 +11,5 @@ image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: $BWC_VERSION diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 115873552e056..ac207fca5e3ed 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -44,7 +44,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.0.1 @@ -61,7 +60,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.1.3 @@ -78,7 +76,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.2.3 @@ -95,7 +92,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.3.3 @@ -112,7 +108,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.4.3 @@ -129,7 +124,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.5.3 @@ -146,7 +140,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.6.2 @@ -163,7 +156,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.7.1 @@ -180,7 +172,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.8.2 @@ -197,7 +188,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.9.2 @@ -214,7 +204,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.10.4 @@ -231,7 +220,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.11.4 @@ -248,7 +236,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.12.2 @@ -265,7 +252,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.13.4 @@ -282,7 +268,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.14.3 @@ -299,7 +284,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.15.3 @@ -316,7 +300,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 8.16.0 @@ -333,7 +316,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: 9.0.0 diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml index 29feb5b832ee2..86e0623ba5b87 100644 --- a/.buildkite/pipelines/periodic-platform-support.yml +++ b/.buildkite/pipelines/periodic-platform-support.yml @@ -28,7 +28,6 @@ steps: localSsds: 1 localSsdInterface: nvme machineType: custom-32-98304 - diskSizeGb: 250 env: {} - group: platform-support-windows steps: diff --git a/.buildkite/pipelines/periodic.bwc.template.yml b/.buildkite/pipelines/periodic.bwc.template.yml index b22270dbf221c..43a0a7438d656 100644 --- a/.buildkite/pipelines/periodic.bwc.template.yml +++ b/.buildkite/pipelines/periodic.bwc.template.yml @@ -7,7 +7,6 @@ machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: $BWC_VERSION retry: diff --git a/.buildkite/pipelines/periodic.template.yml b/.buildkite/pipelines/periodic.template.yml index 08648e2eedd3c..201c34058a409 100644 --- a/.buildkite/pipelines/periodic.template.yml +++ b/.buildkite/pipelines/periodic.template.yml @@ -25,7 +25,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: example-plugins command: |- cd $$WORKSPACE/plugins/examples @@ -37,7 +36,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - group: java-fips-matrix steps: - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-fips-matrix" @@ -59,7 +57,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" GRADLE_TASK: "{{matrix.GRADLE_TASK}}" @@ -76,7 +73,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" BWC_VERSION: "{{matrix.BWC_VERSION}}" @@ -102,7 +98,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" GRADLE_TASK: "{{matrix.GRADLE_TASK}}" @@ -120,7 +115,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" BWC_VERSION: "{{matrix.BWC_VERSION}}" @@ -156,7 +150,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: third-party / azure command: | export azure_storage_container=elasticsearch-ci-thirdparty @@ -171,7 +164,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: third-party / gcs command: | export google_storage_bucket=elasticsearch-ci-thirdparty @@ -186,7 +178,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: third-party / geoip command: | .ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest -Dtests.jvm.argline="-Dgeoip_use_service=true" @@ -196,7 +187,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: third-party / s3 command: | export amazon_s3_bucket=elasticsearch-ci.us-west-2 @@ -211,7 +201,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: Upload Snyk Dependency Graph command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH env: @@ -222,8 +211,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - if: build.branch == "main" || build.branch == "7.17" + if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17" - label: check-branch-consistency command: .ci/scripts/run-gradle.sh branchConsistency timeout_in_minutes: 15 @@ -231,7 +219,6 @@ steps: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-2 - diskSizeGb: 250 - label: check-branch-protection-rules command: .buildkite/scripts/branch-protection.sh timeout_in_minutes: 5 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 8e35d1561b6d7..cbca7f820c7b7 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -11,7 +11,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.0.1 retry: @@ -31,7 +30,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.1.3 retry: @@ -51,7 +49,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.2.3 retry: @@ -71,7 +68,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.3.3 retry: @@ -91,7 +87,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.4.3 retry: @@ -111,7 +106,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.5.3 retry: @@ -131,7 +125,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.6.2 retry: @@ -151,7 +144,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.7.1 retry: @@ -171,7 +163,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.8.2 retry: @@ -191,7 +182,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.9.2 retry: @@ -211,7 +201,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.10.4 retry: @@ -231,7 +220,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.11.4 retry: @@ -251,7 +239,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.12.2 retry: @@ -271,7 +258,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.13.4 retry: @@ -291,7 +277,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.14.3 retry: @@ -311,7 +296,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.15.3 retry: @@ -331,7 +315,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 8.16.0 retry: @@ -351,7 +334,6 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true - diskSizeGb: 250 env: BWC_VERSION: 9.0.0 retry: @@ -386,7 +368,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: example-plugins command: |- cd $$WORKSPACE/plugins/examples @@ -398,7 +379,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - group: java-fips-matrix steps: - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-fips-matrix" @@ -420,7 +400,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" GRADLE_TASK: "{{matrix.GRADLE_TASK}}" @@ -437,7 +416,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" BWC_VERSION: "{{matrix.BWC_VERSION}}" @@ -463,7 +441,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" GRADLE_TASK: "{{matrix.GRADLE_TASK}}" @@ -481,7 +458,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" BWC_VERSION: "{{matrix.BWC_VERSION}}" @@ -517,7 +493,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: third-party / azure command: | export azure_storage_container=elasticsearch-ci-thirdparty @@ -532,7 +507,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: third-party / gcs command: | export google_storage_bucket=elasticsearch-ci-thirdparty @@ -547,7 +521,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: third-party / geoip command: | .ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest -Dtests.jvm.argline="-Dgeoip_use_service=true" @@ -557,7 +530,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: third-party / s3 command: | export amazon_s3_bucket=elasticsearch-ci.us-west-2 @@ -572,7 +544,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 - label: Upload Snyk Dependency Graph command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH env: @@ -583,7 +554,6 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk - diskSizeGb: 250 if: build.branch == "main" || build.branch == "7.17" - label: check-branch-consistency command: .ci/scripts/run-gradle.sh branchConsistency @@ -592,7 +562,6 @@ steps: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-2 - diskSizeGb: 250 - label: check-branch-protection-rules command: .buildkite/scripts/branch-protection.sh timeout_in_minutes: 5 diff --git a/.buildkite/pipelines/pull-request/build-benchmark.yml b/.buildkite/pipelines/pull-request/build-benchmark.yml index 96330bee03638..8d3215b8393ce 100644 --- a/.buildkite/pipelines/pull-request/build-benchmark.yml +++ b/.buildkite/pipelines/pull-request/build-benchmark.yml @@ -22,4 +22,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/bwc-snapshots.yml b/.buildkite/pipelines/pull-request/bwc-snapshots.yml index 8f59e593b286f..5a9fc2d938ac0 100644 --- a/.buildkite/pipelines/pull-request/bwc-snapshots.yml +++ b/.buildkite/pipelines/pull-request/bwc-snapshots.yml @@ -18,4 +18,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/cloud-deploy.yml b/.buildkite/pipelines/pull-request/cloud-deploy.yml index 2932f874c5cf8..ce8e8206d51ff 100644 --- a/.buildkite/pipelines/pull-request/cloud-deploy.yml +++ b/.buildkite/pipelines/pull-request/cloud-deploy.yml @@ -11,4 +11,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/docs-check.yml b/.buildkite/pipelines/pull-request/docs-check.yml index 3bf1e43697a7c..2201eb2d1e4ea 100644 --- a/.buildkite/pipelines/pull-request/docs-check.yml +++ b/.buildkite/pipelines/pull-request/docs-check.yml @@ -12,4 +12,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/eql-correctness.yml b/.buildkite/pipelines/pull-request/eql-correctness.yml index d85827d10e886..8f7ca6942c0e9 100644 --- a/.buildkite/pipelines/pull-request/eql-correctness.yml +++ b/.buildkite/pipelines/pull-request/eql-correctness.yml @@ -7,4 +7,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/example-plugins.yml b/.buildkite/pipelines/pull-request/example-plugins.yml index fb4a17fb214cb..18d0de6594980 100644 --- a/.buildkite/pipelines/pull-request/example-plugins.yml +++ b/.buildkite/pipelines/pull-request/example-plugins.yml @@ -16,4 +16,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/full-bwc.yml b/.buildkite/pipelines/pull-request/full-bwc.yml index c404069bd0e60..d3fa8eccaf7d9 100644 --- a/.buildkite/pipelines/pull-request/full-bwc.yml +++ b/.buildkite/pipelines/pull-request/full-bwc.yml @@ -13,4 +13,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/packaging-upgrade-tests.yml b/.buildkite/pipelines/pull-request/packaging-upgrade-tests.yml index 970dafbb28647..c62cf23310422 100644 --- a/.buildkite/pipelines/pull-request/packaging-upgrade-tests.yml +++ b/.buildkite/pipelines/pull-request/packaging-upgrade-tests.yml @@ -18,6 +18,5 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk - diskSizeGb: 250 env: BWC_VERSION: $BWC_VERSION diff --git a/.buildkite/pipelines/pull-request/part-1-fips.yml b/.buildkite/pipelines/pull-request/part-1-fips.yml index 99544e7f5a80b..42f930c1bde9a 100644 --- a/.buildkite/pipelines/pull-request/part-1-fips.yml +++ b/.buildkite/pipelines/pull-request/part-1-fips.yml @@ -9,4 +9,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-1.yml b/.buildkite/pipelines/pull-request/part-1.yml index b4b9d5469ec41..3d467c6c41e43 100644 --- a/.buildkite/pipelines/pull-request/part-1.yml +++ b/.buildkite/pipelines/pull-request/part-1.yml @@ -7,4 +7,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-2-fips.yml b/.buildkite/pipelines/pull-request/part-2-fips.yml index 36a9801547d78..6a3647ceb50ae 100644 --- a/.buildkite/pipelines/pull-request/part-2-fips.yml +++ b/.buildkite/pipelines/pull-request/part-2-fips.yml @@ -9,4 +9,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-2.yml b/.buildkite/pipelines/pull-request/part-2.yml index 12bd78cf895fd..43de69bbcd945 100644 --- a/.buildkite/pipelines/pull-request/part-2.yml +++ b/.buildkite/pipelines/pull-request/part-2.yml @@ -7,4 +7,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-3-fips.yml b/.buildkite/pipelines/pull-request/part-3-fips.yml index 4a2df3026e782..cee3ea153acb9 100644 --- a/.buildkite/pipelines/pull-request/part-3-fips.yml +++ b/.buildkite/pipelines/pull-request/part-3-fips.yml @@ -9,4 +9,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-3.yml b/.buildkite/pipelines/pull-request/part-3.yml index 6991c05da85c6..12abae7634822 100644 --- a/.buildkite/pipelines/pull-request/part-3.yml +++ b/.buildkite/pipelines/pull-request/part-3.yml @@ -9,4 +9,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-4-fips.yml b/.buildkite/pipelines/pull-request/part-4-fips.yml index 734f8af816895..11a50456ca4c0 100644 --- a/.buildkite/pipelines/pull-request/part-4-fips.yml +++ b/.buildkite/pipelines/pull-request/part-4-fips.yml @@ -9,4 +9,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-4.yml b/.buildkite/pipelines/pull-request/part-4.yml index 59f2f2898a590..af11f08953d07 100644 --- a/.buildkite/pipelines/pull-request/part-4.yml +++ b/.buildkite/pipelines/pull-request/part-4.yml @@ -9,4 +9,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-5-fips.yml b/.buildkite/pipelines/pull-request/part-5-fips.yml index 801b812bb99c0..4e193ac751086 100644 --- a/.buildkite/pipelines/pull-request/part-5-fips.yml +++ b/.buildkite/pipelines/pull-request/part-5-fips.yml @@ -9,4 +9,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-5.yml b/.buildkite/pipelines/pull-request/part-5.yml index c7e50631d1cdd..306ce7533d0ed 100644 --- a/.buildkite/pipelines/pull-request/part-5.yml +++ b/.buildkite/pipelines/pull-request/part-5.yml @@ -9,4 +9,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/precommit.yml b/.buildkite/pipelines/pull-request/precommit.yml index 8d1458b1b60c8..f6548dfeed9b2 100644 --- a/.buildkite/pipelines/pull-request/precommit.yml +++ b/.buildkite/pipelines/pull-request/precommit.yml @@ -10,4 +10,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/rest-compatibility.yml b/.buildkite/pipelines/pull-request/rest-compatibility.yml index 16144a2a0780f..a69810e23d960 100644 --- a/.buildkite/pipelines/pull-request/rest-compatibility.yml +++ b/.buildkite/pipelines/pull-request/rest-compatibility.yml @@ -9,4 +9,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/validate-changelogs.yml b/.buildkite/pipelines/pull-request/validate-changelogs.yml index 296ef11637118..9451d321a9b39 100644 --- a/.buildkite/pipelines/pull-request/validate-changelogs.yml +++ b/.buildkite/pipelines/pull-request/validate-changelogs.yml @@ -7,4 +7,3 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 250 From 5531e5dcd2a3632685b4d27253c431fa11294cf8 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Thu, 3 Oct 2024 23:09:45 +0300 Subject: [PATCH 056/194] Use second pass for deeply nested array elements (#114060) Deeply nested array elements with ignored source need to use the second parsing pass, to avoid missing the source from siblings that go through stored source. --- .../index/mapper/DocumentParser.java | 4 +- .../index/mapper/DocumentParserContext.java | 30 ++++---- .../mapper/IgnoredSourceFieldMapperTests.java | 73 +++++++++++++++++++ 3 files changed, 90 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 59a8c60e0d255..91477590b6c84 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -827,8 +827,8 @@ private static void parseNonDynamicArray( // In synthetic source, if any array element requires storing its source as-is, it takes precedence over // elements from regular source loading that are then skipped from the synthesized array source. - // To prevent this, we track each array name, to check if it contains any sub-arrays in its elements. - context = context.cloneForArray(fullPath); + // To prevent this, we track that parsing sub-context is within array scope. + context = context.maybeCloneForArray(mapper); XContentParser parser = context.parser(); XContentParser.Token token; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index c2970d8716147..000c4118e9b36 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -110,7 +110,7 @@ public int get() { private final Set ignoredFields; private final List ignoredFieldValues; private final List ignoredFieldsMissingValues; - private String parentArrayField; + private boolean inArrayScope; private final Map> dynamicMappers; private final DynamicMapperSize dynamicMappersSize; @@ -142,7 +142,7 @@ private DocumentParserContext( Set ignoreFields, List ignoredFieldValues, List ignoredFieldsWithNoSource, - String parentArrayField, + boolean inArrayScope, Map> dynamicMappers, Map dynamicObjectMappers, Map> dynamicRuntimeFields, @@ -163,7 +163,7 @@ private DocumentParserContext( this.ignoredFields = ignoreFields; this.ignoredFieldValues = ignoredFieldValues; this.ignoredFieldsMissingValues = ignoredFieldsWithNoSource; - this.parentArrayField = parentArrayField; + this.inArrayScope = inArrayScope; this.dynamicMappers = dynamicMappers; this.dynamicObjectMappers = dynamicObjectMappers; this.dynamicRuntimeFields = dynamicRuntimeFields; @@ -187,7 +187,7 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, in.ignoredFields, in.ignoredFieldValues, in.ignoredFieldsMissingValues, - in.parentArrayField, + in.inArrayScope, in.dynamicMappers, in.dynamicObjectMappers, in.dynamicRuntimeFields, @@ -218,7 +218,7 @@ protected DocumentParserContext( new HashSet<>(), new ArrayList<>(), new ArrayList<>(), - null, + false, new HashMap<>(), new HashMap<>(), new HashMap<>(), @@ -323,10 +323,7 @@ public final void deduplicateIgnoredFieldValues(final Set fullNames) { public final DocumentParserContext addIgnoredFieldFromContext(IgnoredSourceFieldMapper.NameValue ignoredFieldWithNoSource) throws IOException { if (canAddIgnoredField()) { - if (parentArrayField != null - && parent != null - && parentArrayField.equals(parent.fullPath()) - && parent instanceof NestedObjectMapper == false) { + if (inArrayScope) { // The field is an array within an array, store all sub-array elements. ignoredFieldsMissingValues.add(ignoredFieldWithNoSource); return cloneWithRecordedSource(); @@ -349,14 +346,17 @@ public final Collection getIgnoredFieldsMiss } /** - * Clones the current context to mark it as an array. Records the full name of the array field, to check for sub-arrays. + * Clones the current context to mark it as an array, if it's not already marked, or restore it if it's within a nested object. * Applies to synthetic source only. */ - public final DocumentParserContext cloneForArray(String fullName) throws IOException { - if (canAddIgnoredField()) { - DocumentParserContext subcontext = switchParser(parser()); - subcontext.parentArrayField = fullName; - return subcontext; + public final DocumentParserContext maybeCloneForArray(Mapper mapper) throws IOException { + if (canAddIgnoredField() && mapper instanceof ObjectMapper) { + boolean isNested = mapper instanceof NestedObjectMapper; + if ((inArrayScope == false && isNested == false) || (inArrayScope && isNested)) { + DocumentParserContext subcontext = switchParser(parser()); + subcontext.inArrayScope = inArrayScope == false; + return subcontext; + } } return this; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index b0c6e4a5d6c95..d79815fa98e74 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -964,6 +964,79 @@ public void testObjectArrayAndValue() throws IOException { {"path":{"stored":[{"leaf":10},{"leaf":20}]}}""", syntheticSource); } + public void testDeeplyNestedObjectArrayAndValue() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("path").startObject("properties").startObject("to").startObject("properties"); + { + b.startObject("stored"); + { + b.field("type", "object").field("store_array_source", true); + b.startObject("properties").startObject("leaf").field("type", "integer").endObject().endObject(); + } + b.endObject(); + } + b.endObject().endObject().endObject().endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startArray("path"); + { + b.startObject(); + { + b.startObject("to").startArray("stored"); + { + b.startObject().field("leaf", 10).endObject(); + } + b.endArray().endObject(); + } + b.endObject(); + b.startObject(); + { + b.startObject("to").startObject("stored").field("leaf", 20).endObject().endObject(); + } + b.endObject(); + } + b.endArray(); + }); + assertEquals(""" + {"path":{"to":{"stored":[{"leaf":10},{"leaf":20}]}}}""", syntheticSource); + } + + public void testObjectArrayAndValueInNestedObject() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("path").startObject("properties").startObject("to").startObject("properties"); + { + b.startObject("stored"); + { + b.field("type", "nested").field("dynamic", false); + } + b.endObject(); + } + b.endObject().endObject().endObject().endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startArray("path"); + { + b.startObject(); + { + b.startObject("to").startArray("stored"); + { + b.startObject().field("leaf", 10).endObject(); + } + b.endArray().endObject(); + } + b.endObject(); + b.startObject(); + { + b.startObject("to").startObject("stored").field("leaf", 20).endObject().endObject(); + } + b.endObject(); + } + b.endArray(); + }); + assertEquals(""" + {"path":{"to":{"stored":[{"leaf":10},{"leaf":20}]}}}""", syntheticSource); + } + public void testObjectArrayAndValueDisabledObject() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "object").startObject("properties"); From 7facc94be6871804322a00a8e5b232cac958d788 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Thu, 3 Oct 2024 13:13:19 -0700 Subject: [PATCH 057/194] Do not expand dots when storing objects in ignored source (#113910) --- docs/changelog/113910.yaml | 5 + rest-api-spec/build.gradle | 4 + .../indices.create/20_synthetic_source.yml | 41 +++---- .../index/mapper/DocumentParser.java | 23 ++-- .../index/mapper/DocumentParserContext.java | 15 +++ .../mapper/IgnoredSourceFieldMapper.java | 1 + .../index/mapper/MapperFeatures.java | 2 +- .../mapper/IgnoredSourceFieldMapperTests.java | 101 ++++++++++++++++++ 8 files changed, 160 insertions(+), 32 deletions(-) create mode 100644 docs/changelog/113910.yaml diff --git a/docs/changelog/113910.yaml b/docs/changelog/113910.yaml new file mode 100644 index 0000000000000..aa9d3b61fe768 --- /dev/null +++ b/docs/changelog/113910.yaml @@ -0,0 +1,5 @@ +pr: 113910 +summary: Do not expand dots when storing objects in ignored source +area: Logs +type: bug +issues: [] diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 05ed09e40f6bc..a57757aa8fb71 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -58,4 +58,8 @@ tasks.named("yamlRestCompatTestTransform").configure({task -> task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") task.skipTest("indices.create/21_synthetic_source_stored/object param - nested object with stored array", "skip test to submit #113690") + task.skipTest("indices.create/20_synthetic_source/disabled object", "temporary until backported") + task.skipTest("indices.create/20_synthetic_source/disabled object contains array", "temporary until backported") + task.skipTest("indices.create/20_synthetic_source/object with dynamic override", "temporary until backported") + task.skipTest("indices.create/20_synthetic_source/disabled root object", "temporary until backported") }) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index 62a248cae6b2d..1c3c96a944501 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -197,7 +197,7 @@ empty object with unmapped fields: --- disabled root object: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.ignored_source.dont_expand_dots"] reason: requires tracking ignored source - do: @@ -222,17 +222,19 @@ disabled root object: index: test - match: { hits.total.value: 1 } - - match: { hits.hits.0._source.name: aaaa } - - match: { hits.hits.0._source.some_string: AaAa } - - match: { hits.hits.0._source.some_int: 1000 } - - match: { hits.hits.0._source.some_double: 123.456789 } - - match: { hits.hits.0._source.a.very.deeply.nested.field: AAAA } - + - match: + hits.hits.0._source: + name: aaaa + some_string: AaAa + some_int: 1000 + some_double: 123.456789 + some_bool: true + a.very.deeply.nested.field: AAAA --- disabled object: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.ignored_source.dont_expand_dots"] reason: requires tracking ignored source - do: @@ -261,14 +263,15 @@ disabled object: - match: { hits.total.value: 1 } - match: { hits.hits.0._source.name: aaaa } - - match: { hits.hits.0._source.path.some_int: 1000 } - - match: { hits.hits.0._source.path.to.a.very.deeply.nested.field: AAAA } - + - match: + hits.hits.0._source.path: + some_int: 1000 + to.a.very.deeply.nested.field: AAAA --- disabled object contains array: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.ignored_source.dont_expand_dots"] reason: requires tracking ignored source - do: @@ -297,10 +300,12 @@ disabled object contains array: - match: { hits.total.value: 1 } - match: { hits.hits.0._source.name: aaaa } - - match: { hits.hits.0._source.path.0.some_int: 1000 } - - match: { hits.hits.0._source.path.0.to.a.very.deeply.nested.field: AAAA } - - match: { hits.hits.0._source.path.1.some_double: 10.0 } - - match: { hits.hits.0._source.path.1.some_bool: true } + - match: + hits.hits.0._source.path: + - some_int: 1000 + to.a.very.deeply.nested.field: AAAA + - some_double: 10.0 + some_bool: true --- @@ -429,7 +434,7 @@ mixed disabled and enabled objects: --- object with dynamic override: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.ignored_source.dont_expand_dots"] reason: requires tracking ignored source - do: @@ -467,7 +472,7 @@ object with dynamic override: - match: { hits.hits.0._source.name: a } - match: { hits.hits.0._source.path_no.name: foo } - match: { hits.hits.0._source.path_no.some_int: 10 } - - match: { hits.hits.0._source.path_no.to.a.very.deeply.nested.field: A } + - match: { hits.hits.0._source.path_no.to: { a.very.deeply.nested.field: A } } - match: { hits.hits.0._source.path_runtime.name: bar } - match: { hits.hits.0._source.path_runtime.some_int: 20 } - match: { hits.hits.0._source.path_runtime.to.a.very.deeply.nested.field: B } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 91477590b6c84..19bd4f9980baf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -135,7 +135,7 @@ private void internalParseDocument(MetadataFieldMapper[] metadataFieldsMappers, new IgnoredSourceFieldMapper.NameValue( MapperService.SINGLE_MAPPING_NAME, 0, - XContentDataHelper.encodeToken(context.parser()), + context.encodeFlattenedToken(), context.doc() ) ); @@ -236,7 +236,7 @@ private static List parseDocForMissingValues var leaf = fields.get(fullName); // There may be multiple matches for array elements, don't use #remove. if (leaf != null) { parser.nextToken(); // Advance the parser to the value to be read. - result.add(leaf.cloneWithValue(XContentDataHelper.encodeToken(parser))); + result.add(leaf.cloneWithValue(context.encodeFlattenedToken())); parser.nextToken(); // Skip the token ending the value. fieldName = null; } @@ -402,7 +402,7 @@ static void parseObjectOrNested(DocumentParserContext context) throws IOExceptio new IgnoredSourceFieldMapper.NameValue( context.parent().fullPath(), context.parent().fullPath().lastIndexOf(context.parent().leafName()), - XContentDataHelper.encodeToken(parser), + context.encodeFlattenedToken(), context.doc() ) ); @@ -652,12 +652,11 @@ private static void parseObjectDynamic(DocumentParserContext context, String cur if (context.dynamic() == ObjectMapper.Dynamic.FALSE) { failIfMatchesRoutingPath(context, currentFieldName); if (context.canAddIgnoredField()) { - // read everything up to end object and store it context.addIgnoredField( IgnoredSourceFieldMapper.NameValue.fromContext( context, context.path().pathAsText(currentFieldName), - XContentDataHelper.encodeToken(context.parser()) + context.encodeFlattenedToken() ) ); } else { @@ -743,7 +742,7 @@ private static void parseArrayDynamic(DocumentParserContext context, String curr IgnoredSourceFieldMapper.NameValue.fromContext( context, context.path().pathAsText(currentFieldName), - XContentDataHelper.encodeToken(context.parser()) + context.encodeFlattenedToken() ) ); } else { @@ -761,7 +760,7 @@ private static void parseArrayDynamic(DocumentParserContext context, String curr IgnoredSourceFieldMapper.NameValue.fromContext( context, context.path().pathAsText(currentFieldName), - XContentDataHelper.encodeToken(context.parser()) + context.encodeFlattenedToken() ) ); } catch (IOException e) { @@ -818,9 +817,7 @@ private static void parseNonDynamicArray( } else if (mapper instanceof ObjectMapper objectMapper && (objectMapper.isEnabled() == false)) { // No need to call #addIgnoredFieldFromContext as both singleton and array instances of this object // get tracked through ignored source. - context.addIgnoredField( - IgnoredSourceFieldMapper.NameValue.fromContext(context, fullPath, XContentDataHelper.encodeToken(context.parser())) - ); + context.addIgnoredField(IgnoredSourceFieldMapper.NameValue.fromContext(context, fullPath, context.encodeFlattenedToken())); return; } } @@ -934,7 +931,7 @@ private static void parseDynamicValue(DocumentParserContext context, String curr IgnoredSourceFieldMapper.NameValue.fromContext( context, context.path().pathAsText(currentFieldName), - XContentDataHelper.encodeToken(context.parser()) + context.encodeFlattenedToken() ) ); } @@ -945,7 +942,7 @@ private static void parseDynamicValue(DocumentParserContext context, String curr IgnoredSourceFieldMapper.NameValue.fromContext( context, context.path().pathAsText(currentFieldName), - XContentDataHelper.encodeToken(context.parser()) + context.encodeFlattenedToken() ) ); } @@ -1044,7 +1041,7 @@ protected void parseCreateField(DocumentParserContext context) { if (context.dynamic() == ObjectMapper.Dynamic.RUNTIME && context.canAddIgnoredField()) { try { context.addIgnoredField( - IgnoredSourceFieldMapper.NameValue.fromContext(context, path, XContentDataHelper.encodeToken(context.parser())) + IgnoredSourceFieldMapper.NameValue.fromContext(context, path, context.encodeFlattenedToken()) ); } catch (IOException e) { throw new IllegalArgumentException( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 000c4118e9b36..eebe95e260dcf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -12,6 +12,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; @@ -338,6 +339,20 @@ public final DocumentParserContext addIgnoredFieldFromContext(IgnoredSourceField return this; } + /** + * Wraps {@link XContentDataHelper#encodeToken}, disabling dot expansion from {@link DotExpandingXContentParser}. + * This helps avoid producing duplicate names in the same scope, due to expanding dots to objects. + * For instance: { "a.b": "b", "a.c": "c" } => { "a": { "b": "b" }, "a": { "c": "c" } } + * This can happen when storing parts of document source that are not indexed (e.g. disabled objects). + */ + BytesRef encodeFlattenedToken() throws IOException { + boolean old = path().isWithinLeafObject(); + path().setWithinLeafObject(true); + BytesRef encoded = XContentDataHelper.encodeToken(parser()); + path().setWithinLeafObject(old); + return encoded; + } + /** * Return the collection of fields that are missing their source values. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index d57edb757ba10..296c2c5311d9a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -57,6 +57,7 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper { public static final TypeParser PARSER = new FixedTypeParser(context -> new IgnoredSourceFieldMapper(context.getIndexSettings())); static final NodeFeature TRACK_IGNORED_SOURCE = new NodeFeature("mapper.track_ignored_source"); + static final NodeFeature DONT_EXPAND_DOTS_IN_IGNORED_SOURCE = new NodeFeature("mapper.ignored_source.dont_expand_dots"); /* Setting to disable encoding and writing values for this field. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index a8fbb0e0ae95b..4f90bd6e6f2c9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -58,6 +58,6 @@ public Set getFeatures() { @Override public Set getTestFeatures() { - return Set.of(RangeFieldMapper.DATE_RANGE_INDEXING_FIX); + return Set.of(RangeFieldMapper.DATE_RANGE_INDEXING_FIX, IgnoredSourceFieldMapper.DONT_EXPAND_DOTS_IN_IGNORED_SOURCE); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index d79815fa98e74..8c65424fb8560 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -1793,6 +1793,107 @@ public void testCopyToLogicInsideObject() throws IOException { assertEquals("{\"path\":{\"at\":\"A\"}}", syntheticSource); } + public void testDynamicIgnoredObjectWithFlatFields() throws IOException { + DocumentMapper documentMapper = createMapperService(topMapping(b -> { + b.startObject("_source").field("mode", "synthetic").endObject(); + b.field("dynamic", false); + })).documentMapper(); + + CheckedConsumer document = b -> { + b.startObject("top"); + b.field("file.name", "A"); + b.field("file.line", 10); + b.endObject(); + }; + + var syntheticSource = syntheticSource(documentMapper, document); + assertEquals("{\"top\":{\"file.name\":\"A\",\"file.line\":10}}", syntheticSource); + + CheckedConsumer documentWithArray = b -> { + b.startArray("top"); + b.startObject(); + b.field("file.name", "A"); + b.field("file.line", 10); + b.endObject(); + b.startObject(); + b.field("file.name", "B"); + b.field("file.line", 20); + b.endObject(); + b.endArray(); + }; + + var syntheticSourceWithArray = syntheticSource(documentMapper, documentWithArray); + assertEquals(""" + {"top":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}""", syntheticSourceWithArray); + } + + public void testDisabledRootObjectWithFlatFields() throws IOException { + DocumentMapper documentMapper = createMapperService(topMapping(b -> { + b.startObject("_source").field("mode", "synthetic").endObject(); + b.field("enabled", false); + })).documentMapper(); + + CheckedConsumer document = b -> { + b.startObject("top"); + b.field("file.name", "A"); + b.field("file.line", 10); + b.endObject(); + }; + + var syntheticSource = syntheticSource(documentMapper, document); + assertEquals("{\"top\":{\"file.name\":\"A\",\"file.line\":10}}", syntheticSource); + + CheckedConsumer documentWithArray = b -> { + b.startArray("top"); + b.startObject(); + b.field("file.name", "A"); + b.field("file.line", 10); + b.endObject(); + b.startObject(); + b.field("file.name", "B"); + b.field("file.line", 20); + b.endObject(); + b.endArray(); + }; + + var syntheticSourceWithArray = syntheticSource(documentMapper, documentWithArray); + assertEquals(""" + {"top":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}""", syntheticSourceWithArray); + } + + public void testDisabledObjectWithFlatFields() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("top").field("type", "object").field("enabled", false).endObject(); + })).documentMapper(); + + CheckedConsumer document = b -> { + b.startObject("top"); + b.field("file.name", "A"); + b.field("file.line", 10); + b.endObject(); + }; + + var syntheticSource = syntheticSource(documentMapper, document); + assertEquals("{\"top\":{\"file.name\":\"A\",\"file.line\":10}}", syntheticSource); + + CheckedConsumer documentWithArray = b -> { + b.startArray("top"); + b.startObject(); + b.field("file.name", "A"); + b.field("file.line", 10); + b.endObject(); + b.startObject(); + b.field("file.name", "B"); + b.field("file.line", 20); + b.endObject(); + b.endArray(); + }; + + var syntheticSourceWithArray = syntheticSource(documentMapper, documentWithArray); + assertEquals(""" + {"top":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}""", syntheticSourceWithArray); + } + protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) throws IOException { // We exclude ignored source field since in some cases it contains an exact copy of a part of document source. From 867198a9b5c1f9818016d6be84005b43ac93df13 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 4 Oct 2024 06:40:09 +1000 Subject: [PATCH 058/194] Mute org.elasticsearch.xpack.rank.rrf.RRFRetrieverBuilderIT testRRFWithCollapse #114074 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 4305ebe3d2e02..558f27b96fb33 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -351,6 +351,9 @@ tests: - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT method: testPutE5WithTrainedModelAndInference issue: https://github.com/elastic/elasticsearch/issues/114023 +- class: org.elasticsearch.xpack.rank.rrf.RRFRetrieverBuilderIT + method: testRRFWithCollapse + issue: https://github.com/elastic/elasticsearch/issues/114074 # Examples: # From 3bfb30b8690edc5c2aa6567338c9cfecfdc2f8a1 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Thu, 3 Oct 2024 23:56:40 +0300 Subject: [PATCH 059/194] Update cluster features for tests with synthetic_source_keep (#114064) Applied to 8.x too in #114058 --- .../indices.create/21_synthetic_source_stored.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index 07ab5259e6fd6..dfe6c9820a16a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -65,7 +65,7 @@ object param - store complex object: --- object param - object array: - requires: - cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -129,7 +129,7 @@ object param - object array: --- object param - object array within array: - requires: - cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -172,7 +172,7 @@ object param - object array within array: --- object param - no object array: - requires: - cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -214,7 +214,7 @@ object param - no object array: --- object param - field ordering in object array: - requires: - cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -263,7 +263,7 @@ object param - field ordering in object array: --- object param - nested object array next to other fields: - requires: - cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -319,7 +319,7 @@ object param - nested object array next to other fields: --- object param - nested object with stored array: - requires: - cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: From d962285b2646f1d470be97009a6f876bcb0a772e Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 4 Oct 2024 07:13:43 +1000 Subject: [PATCH 060/194] Mute org.elasticsearch.xpack.rank.rrf.RRFRetrieverBuilderIT testMultipleRRFRetrievers #114079 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 558f27b96fb33..e6e0c901a0edd 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -354,6 +354,9 @@ tests: - class: org.elasticsearch.xpack.rank.rrf.RRFRetrieverBuilderIT method: testRRFWithCollapse issue: https://github.com/elastic/elasticsearch/issues/114074 +- class: org.elasticsearch.xpack.rank.rrf.RRFRetrieverBuilderIT + method: testMultipleRRFRetrievers + issue: https://github.com/elastic/elasticsearch/issues/114079 # Examples: # From 99e7dcc38c7fc1f180a1a9023c6b1aacd570db0e Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Thu, 3 Oct 2024 17:06:32 -0500 Subject: [PATCH 061/194] Listing all available databases in the _ingest/geoip/database API (#113498) --- docs/changelog/113498.yaml | 5 + modules/ingest-geoip/build.gradle | 5 + .../ingest/geoip/DatabaseNodeService.java | 34 +++ .../geoip/DatabaseReaderLazyLoader.java | 13 ++ .../geoip/EnterpriseGeoIpDownloader.java | 4 +- .../ingest/geoip/GeoIpTaskState.java | 6 +- .../ingest/geoip/IngestGeoIpPlugin.java | 18 +- .../geoip/direct/DatabaseConfiguration.java | 164 ++++++++++++++- .../PutDatabaseConfigurationAction.java | 7 +- ...portDeleteDatabaseConfigurationAction.java | 2 + ...ansportGetDatabaseConfigurationAction.java | 193 +++++++++++++++--- .../geoip/IngestGeoIpMetadataTests.java | 7 + .../DatabaseConfigurationMetadataTests.java | 7 + .../direct/DatabaseConfigurationTests.java | 38 +++- ...rtGetDatabaseConfigurationActionTests.java | 131 ++++++++++++ .../test/ingest_geoip/40_geoip_databases.yml | 35 +++- .../org/elasticsearch/TransportVersions.java | 1 + 17 files changed, 619 insertions(+), 51 deletions(-) create mode 100644 docs/changelog/113498.yaml create mode 100644 modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationActionTests.java diff --git a/docs/changelog/113498.yaml b/docs/changelog/113498.yaml new file mode 100644 index 0000000000000..93b21a1d171eb --- /dev/null +++ b/docs/changelog/113498.yaml @@ -0,0 +1,5 @@ +pr: 113498 +summary: Listing all available databases in the _ingest/geoip/database API +area: Ingest Node +type: enhancement +issues: [] diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index b50fc86282d1f..4312221b33937 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -88,3 +88,8 @@ tasks.named("dependencyLicenses").configure { artifacts { restTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test")) } + +tasks.named("yamlRestCompatTestTransform").configure({ task -> + task.skipTest("ingest_geoip/40_geoip_databases/Test adding, getting, and removing geoip databases", + "get databases behavior began returning more results in 8.16") +}) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java index ce15e02e6efcc..940231b12c894 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java @@ -20,11 +20,13 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.env.Environment; import org.elasticsearch.gateway.GatewayService; @@ -37,6 +39,7 @@ import org.elasticsearch.watcher.ResourceWatcherService; import java.io.Closeable; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.FileAlreadyExistsException; @@ -51,8 +54,10 @@ import java.security.MessageDigest; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -541,6 +546,35 @@ public Set getConfigDatabases() { return configDatabases.getConfigDatabases().keySet(); } + public Map getConfigDatabasesDetail() { + Map allDatabases = new HashMap<>(); + for (Map.Entry entry : configDatabases.getConfigDatabases().entrySet()) { + DatabaseReaderLazyLoader databaseReaderLazyLoader = entry.getValue(); + try { + allDatabases.put( + entry.getKey(), + new ConfigDatabaseDetail( + entry.getKey(), + databaseReaderLazyLoader.getMd5(), + databaseReaderLazyLoader.getBuildDateMillis(), + databaseReaderLazyLoader.getDatabaseType() + ) + ); + } catch (FileNotFoundException e) { + /* + * Since there is nothing to prevent a database from being deleted while this method is running, it is possible we get an + * exception here because the file no longer exists. We just log it and move on -- it's preferable to synchronization. + */ + logger.trace(Strings.format("Unable to get metadata for config database %s", entry.getKey()), e); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return allDatabases; + } + + public record ConfigDatabaseDetail(String name, @Nullable String md5, @Nullable Long buildDateInMillis, @Nullable String type) {} + public Set getFilesInTemp() { try (Stream files = Files.list(geoipTmpDirectory)) { return files.map(Path::getFileName).map(Path::toString).collect(Collectors.toSet()); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java index dff083ea0cde8..e160c8ad1543f 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java @@ -63,6 +63,7 @@ class DatabaseReaderLazyLoader implements IpDatabase { // cache the database type so that we do not re-read it on every pipeline execution final SetOnce databaseType; + final SetOnce buildDate; private volatile boolean deleteDatabaseFileOnShutdown; private final AtomicInteger currentUsages = new AtomicInteger(0); @@ -74,6 +75,7 @@ class DatabaseReaderLazyLoader implements IpDatabase { this.loader = createDatabaseLoader(databasePath); this.databaseReader = new SetOnce<>(); this.databaseType = new SetOnce<>(); + this.buildDate = new SetOnce<>(); } /** @@ -277,4 +279,15 @@ private Optional lookup(Reader reader, String ip, Class getNamedWriteables() { new NamedWriteableRegistry.Entry(PersistentTaskParams.class, GEOIP_DOWNLOADER, GeoIpTaskParams::new), new NamedWriteableRegistry.Entry(PersistentTaskState.class, ENTERPRISE_GEOIP_DOWNLOADER, EnterpriseGeoIpTaskState::new), new NamedWriteableRegistry.Entry(PersistentTaskParams.class, ENTERPRISE_GEOIP_DOWNLOADER, EnterpriseGeoIpTaskParams::new), - new NamedWriteableRegistry.Entry(Task.Status.class, GEOIP_DOWNLOADER, GeoIpDownloaderStats::new) + new NamedWriteableRegistry.Entry(Task.Status.class, GEOIP_DOWNLOADER, GeoIpDownloaderStats::new), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Maxmind.NAME, + DatabaseConfiguration.Maxmind::new + ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Local.NAME, + DatabaseConfiguration.Local::new + ), + new NamedWriteableRegistry.Entry( + DatabaseConfiguration.Provider.class, + DatabaseConfiguration.Web.NAME, + DatabaseConfiguration.Web::new + ) ); } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java index b8b48e0f738a5..3399b71879e26 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java @@ -9,13 +9,16 @@ package org.elasticsearch.ingest.geoip.direct; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -34,19 +37,19 @@ * That is, it has an id e.g. "my_db_config_1" and it says "download the file named XXXX from SomeCompany, and here's the * magic token to use to do that." */ -public record DatabaseConfiguration(String id, String name, Maxmind maxmind) implements Writeable, ToXContentObject { +public record DatabaseConfiguration(String id, String name, Provider provider) implements Writeable, ToXContentObject { // id is a user selected signifier like 'my_domain_db' // name is the name of a file that can be downloaded (like 'GeoIP2-Domain') - // a configuration will have a 'type' like "maxmind", and that might have some more details, + // a configuration will have a 'provider' like "maxmind", and that might have some more details, // for now, though the important thing is that the json has to have it even though we don't model it meaningfully in this class public DatabaseConfiguration { // these are invariants, not actual validation Objects.requireNonNull(id); Objects.requireNonNull(name); - Objects.requireNonNull(maxmind); + Objects.requireNonNull(provider); } /** @@ -76,25 +79,49 @@ public record DatabaseConfiguration(String id, String name, Maxmind maxmind) imp ); private static final ParseField NAME = new ParseField("name"); - private static final ParseField MAXMIND = new ParseField("maxmind"); + private static final ParseField MAXMIND = new ParseField(Maxmind.NAME); + private static final ParseField WEB = new ParseField(Web.NAME); + private static final ParseField LOCAL = new ParseField(Local.NAME); private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "database", false, (a, id) -> { String name = (String) a[0]; - Maxmind maxmind = (Maxmind) a[1]; - return new DatabaseConfiguration(id, name, maxmind); + Provider provider; + if (a[1] != null) { + provider = (Maxmind) a[1]; + } else if (a[2] != null) { + provider = (Web) a[2]; + } else { + provider = (Local) a[3]; + } + return new DatabaseConfiguration(id, name, provider); } ); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME); - PARSER.declareObject(ConstructingObjectParser.constructorArg(), (parser, id) -> Maxmind.PARSER.apply(parser, null), MAXMIND); + PARSER.declareObject( + ConstructingObjectParser.optionalConstructorArg(), + (parser, id) -> Maxmind.PARSER.apply(parser, null), + MAXMIND + ); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, id) -> Web.PARSER.apply(parser, null), WEB); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (parser, id) -> Local.PARSER.apply(parser, null), LOCAL); } public DatabaseConfiguration(StreamInput in) throws IOException { - this(in.readString(), in.readString(), new Maxmind(in)); + this(in.readString(), in.readString(), readProvider(in)); + } + + private static Provider readProvider(StreamInput in) throws IOException { + if (in.getTransportVersion().onOrAfter(TransportVersions.INGEST_GEO_DATABASE_PROVIDERS)) { + return in.readNamedWriteable(Provider.class); + } else { + // prior to the above version, everything was always a maxmind, so this half of the if is logical + return new Maxmind(in.readString()); + } } public static DatabaseConfiguration parse(XContentParser parser, String id) { @@ -105,14 +132,27 @@ public static DatabaseConfiguration parse(XContentParser parser, String id) { public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeString(name); - maxmind.writeTo(out); + if (out.getTransportVersion().onOrAfter(TransportVersions.INGEST_GEO_DATABASE_PROVIDERS)) { + out.writeNamedWriteable(provider); + } else { + if (provider instanceof Maxmind maxmind) { + out.writeString(maxmind.accountId); + } else { + /* + * The existence of a non-Maxmind providers is gated on the feature get_database_configuration_action.multi_node, and + * get_database_configuration_action.multi_node is only available on or after + * TransportVersions.INGEST_GEO_DATABASE_PROVIDERS. + */ + assert false : "non-maxmind DatabaseConfiguration.Provider [" + provider.getWriteableName() + "]"; + } + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field("name", name); - builder.field("maxmind", maxmind); + builder.field(provider.getWriteableName(), provider); builder.endObject(); return builder; } @@ -168,7 +208,24 @@ public ActionRequestValidationException validate() { return err.validationErrors().isEmpty() ? null : err; } - public record Maxmind(String accountId) implements Writeable, ToXContentObject { + public boolean isReadOnly() { + return provider.isReadOnly(); + } + + /** + * A marker interface that all providers need to implement. + */ + public interface Provider extends NamedWriteable, ToXContentObject { + boolean isReadOnly(); + } + + public record Maxmind(String accountId) implements Provider { + public static final String NAME = "maxmind"; + + @Override + public String getWriteableName() { + return NAME; + } public Maxmind { // this is an invariant, not actual validation @@ -206,5 +263,90 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); return builder; } + + @Override + public boolean isReadOnly() { + return false; + } + } + + public record Local(String type) implements Provider { + public static final String NAME = "local"; + + private static final ParseField TYPE = new ParseField("type"); + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("database", false, (a, id) -> { + String type = (String) a[0]; + return new Local(type); + }); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE); + } + + public Local(StreamInput in) throws IOException { + this(in.readString()); + } + + public static Local parse(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(type); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("type", type); + builder.endObject(); + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public boolean isReadOnly() { + return true; + } + } + + public record Web() implements Provider { + public static final String NAME = "web"; + + private static final ObjectParser PARSER = new ObjectParser<>("database", Web::new); + + public Web(StreamInput in) throws IOException { + this(); + } + + public static Web parse(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public void writeTo(StreamOutput out) throws IOException {} + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.endObject(); + return builder; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public boolean isReadOnly() { + return true; + } } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/PutDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/PutDatabaseConfigurationAction.java index 41be25987a31b..b5343f17e47b6 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/PutDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/PutDatabaseConfigurationAction.java @@ -49,7 +49,12 @@ public DatabaseConfiguration getDatabase() { } public static Request parseRequest(TimeValue masterNodeTimeout, TimeValue ackTimeout, String id, XContentParser parser) { - return new Request(masterNodeTimeout, ackTimeout, DatabaseConfiguration.parse(parser, id)); + DatabaseConfiguration database = DatabaseConfiguration.parse(parser, id); + if (database.isReadOnly()) { + throw new IllegalArgumentException("Database " + id + " is read only"); + } else { + return new Request(masterNodeTimeout, ackTimeout, database); + } } @Override diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportDeleteDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportDeleteDatabaseConfigurationAction.java index 088cea04cef87..b73b2fd4beb08 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportDeleteDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportDeleteDatabaseConfigurationAction.java @@ -91,6 +91,8 @@ protected void masterOperation(Task task, Request request, ClusterState state, A final IngestGeoIpMetadata geoIpMeta = state.metadata().custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY); if (geoIpMeta.getDatabases().containsKey(id) == false) { throw new ResourceNotFoundException("Database configuration not found: {}", id); + } else if (geoIpMeta.getDatabases().get(id).database().isReadOnly()) { + throw new IllegalArgumentException("Database " + id + " is read only"); } deleteDatabaseConfigurationTaskQueue.submitTask( Strings.format("delete-geoip-database-configuration-[%s]", id), diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java index 0660a9ff0491d..c83c40e56b749 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java @@ -9,7 +9,6 @@ package org.elasticsearch.ingest.geoip.direct; -import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; @@ -19,19 +18,28 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.features.FeatureService; +import org.elasticsearch.ingest.geoip.DatabaseNodeService; +import org.elasticsearch.ingest.geoip.GeoIpTaskState; import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata; import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; +import java.util.Collection; +import java.util.Comparator; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import static org.elasticsearch.ingest.IngestGeoIpFeatures.GET_DATABASE_CONFIGURATION_ACTION_MULTI_NODE; @@ -43,6 +51,7 @@ public class TransportGetDatabaseConfigurationAction extends TransportNodesActio List> { private final FeatureService featureService; + private final DatabaseNodeService databaseNodeService; @Inject public TransportGetDatabaseConfigurationAction( @@ -50,7 +59,8 @@ public TransportGetDatabaseConfigurationAction( ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - FeatureService featureService + FeatureService featureService, + DatabaseNodeService databaseNodeService ) { super( GetDatabaseConfigurationAction.NAME, @@ -61,6 +71,7 @@ public TransportGetDatabaseConfigurationAction( threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.featureService = featureService; + this.databaseNodeService = databaseNodeService; } @Override @@ -74,9 +85,19 @@ protected void doExecute( * TransportGetDatabaseConfigurationAction used to be a TransportMasterNodeAction, and not all nodes in the cluster have been * updated. So we don't want to send node requests to the other nodes because they will blow up. Instead, we just return * the information that we used to return from the master node (it doesn't make any difference that this might not be the master - * node, because we're only reading the cluster state). + * node, because we're only reading the cluster state). Because older nodes only know about the Maxmind provider type, we filter + * out all others here to avoid causing problems on those nodes. */ - newResponseAsync(task, request, createActionContext(task, request), List.of(), List.of(), listener); + newResponseAsync( + task, + request, + createActionContext(task, request).stream() + .filter(database -> database.database().provider() instanceof DatabaseConfiguration.Maxmind) + .toList(), + List.of(), + List.of(), + listener + ); } else { super.doExecute(task, request, listener); } @@ -97,28 +118,79 @@ protected List createActionContext(Task task, Get ); } - final IngestGeoIpMetadata geoIpMeta = clusterService.state().metadata().custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY); List results = new ArrayList<>(); - + PersistentTasksCustomMetadata tasksMetadata = PersistentTasksCustomMetadata.getPersistentTasksCustomMetadata( + clusterService.state() + ); for (String id : ids) { - if (Regex.isSimpleMatchPattern(id)) { - for (Map.Entry entry : geoIpMeta.getDatabases().entrySet()) { - if (Regex.simpleMatch(id, entry.getKey())) { - results.add(entry.getValue()); + results.addAll(getWebDatabases(tasksMetadata, id)); + results.addAll(getMaxmindDatabases(clusterService, id)); + } + return results; + } + + /* + * This returns read-only database information about the databases managed by the standard downloader + */ + private static Collection getWebDatabases(PersistentTasksCustomMetadata tasksMetadata, String id) { + List webDatabases = new ArrayList<>(); + if (tasksMetadata != null) { + PersistentTasksCustomMetadata.PersistentTask maybeGeoIpTask = tasksMetadata.getTask("geoip-downloader"); + if (maybeGeoIpTask != null) { + GeoIpTaskState geoIpTaskState = (GeoIpTaskState) maybeGeoIpTask.getState(); + if (geoIpTaskState != null) { + Map databases = geoIpTaskState.getDatabases(); + for (String databaseFileName : databases.keySet()) { + String databaseName = getDatabaseNameForFileName(databaseFileName); + String databaseId = getDatabaseIdForFileName(DatabaseConfiguration.Web.NAME, databaseFileName); + if ((Regex.isSimpleMatchPattern(id) && Regex.simpleMatch(id, databaseId)) || id.equals(databaseId)) { + webDatabases.add( + new DatabaseConfigurationMetadata( + new DatabaseConfiguration(databaseId, databaseName, new DatabaseConfiguration.Web()), + -1, + databases.get(databaseFileName).lastUpdate() + ) + ); + } } } - } else { - DatabaseConfigurationMetadata meta = geoIpMeta.getDatabases().get(id); - if (meta == null) { - throw new ResourceNotFoundException("database configuration not found: {}", id); - } else { - results.add(meta); + } + } + return webDatabases; + } + + private static String getDatabaseIdForFileName(String providerType, String databaseFileName) { + return "_" + providerType + "_" + Base64.getEncoder().encodeToString(databaseFileName.getBytes(StandardCharsets.UTF_8)); + } + + private static String getDatabaseNameForFileName(String databaseFileName) { + return databaseFileName.endsWith(".mmdb") + ? databaseFileName.substring(0, databaseFileName.length() + 1 - ".mmmdb".length()) + : databaseFileName; + } + + /* + * This returns information about databases that are downloaded from maxmind. + */ + private static Collection getMaxmindDatabases(ClusterService clusterService, String id) { + List maxmindDatabases = new ArrayList<>(); + final IngestGeoIpMetadata geoIpMeta = clusterService.state().metadata().custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY); + if (Regex.isSimpleMatchPattern(id)) { + for (Map.Entry entry : geoIpMeta.getDatabases().entrySet()) { + if (Regex.simpleMatch(id, entry.getKey())) { + maxmindDatabases.add(entry.getValue()); } } + } else { + DatabaseConfigurationMetadata meta = geoIpMeta.getDatabases().get(id); + if (meta != null) { + maxmindDatabases.add(meta); + } } - return results; + return maxmindDatabases; } + @Override protected void newResponseAsync( Task task, GetDatabaseConfigurationAction.Request request, @@ -127,13 +199,47 @@ protected void newResponseAsync( List failures, ActionListener listener ) { - ActionListener.run( - listener, - l -> ActionListener.respondAndRelease( + ActionListener.run(listener, l -> { + List combinedResults = new ArrayList<>(results); + combinedResults.addAll( + deduplicateNodeResponses(responses, results.stream().map(result -> result.database().name()).collect(Collectors.toSet())) + ); + ActionListener.respondAndRelease( l, - new GetDatabaseConfigurationAction.Response(results, clusterService.getClusterName(), responses, failures) + new GetDatabaseConfigurationAction.Response(combinedResults, clusterService.getClusterName(), responses, failures) + ); + }); + } + + /* + * This deduplicates the nodeResponses by name, favoring the most recent. This is because each node is reporting the local databases + * that it has, and we don't want to report duplicates to the user. It also filters out any that already exist in the set of + * preExistingNames. This is because the non-local databases take precedence, so any local database with the same name as a non-local + * one will not be used. + * Non-private for unit testing + */ + static Collection deduplicateNodeResponses( + List nodeResponses, + Set preExistingNames + ) { + /* + * Each node reports the list of databases that are in its config/ingest-geoip directory. For the sake of this API we assume all + * local databases with the same name are the same database, and deduplicate by name and just return the newest. + */ + return nodeResponses.stream() + .flatMap(response -> response.getDatabases().stream()) + .collect( + Collectors.groupingBy( + database -> database.database().name(), + Collectors.maxBy(Comparator.comparing(DatabaseConfigurationMetadata::modifiedDate)) + ) ) - ); + .values() + .stream() + .filter(Optional::isPresent) + .map(Optional::get) + .filter(database -> preExistingNames.contains(database.database().name()) == false) + .toList(); } @Override @@ -157,7 +263,48 @@ protected GetDatabaseConfigurationAction.NodeResponse newNodeResponse(StreamInpu @Override protected GetDatabaseConfigurationAction.NodeResponse nodeOperation(GetDatabaseConfigurationAction.NodeRequest request, Task task) { - return new GetDatabaseConfigurationAction.NodeResponse(transportService.getLocalNode(), List.of()); + final Set ids; + if (request.getDatabaseIds().length == 0) { + // if we did not ask for a specific name, then return all databases + ids = Set.of("*"); + } else { + ids = new LinkedHashSet<>(Arrays.asList(request.getDatabaseIds())); + } + if (ids.size() > 1 && ids.stream().anyMatch(Regex::isSimpleMatchPattern)) { + throw new IllegalArgumentException( + "wildcard only supports a single value, please use comma-separated values or a single wildcard value" + ); + } + + List results = new ArrayList<>(); + for (String id : ids) { + results.addAll(getLocalDatabases(databaseNodeService, id)); + } + return new GetDatabaseConfigurationAction.NodeResponse(transportService.getLocalNode(), results); } + /* + * This returns information about the databases that users have put in the config/ingest-geoip directory on the node. + */ + private static List getLocalDatabases(DatabaseNodeService databaseNodeService, String id) { + List localDatabases = new ArrayList<>(); + Map configDatabases = databaseNodeService.getConfigDatabasesDetail(); + for (DatabaseNodeService.ConfigDatabaseDetail configDatabase : configDatabases.values()) { + String databaseId = getDatabaseIdForFileName(DatabaseConfiguration.Local.NAME, configDatabase.name()); + if ((Regex.isSimpleMatchPattern(id) && Regex.simpleMatch(id, databaseId)) || id.equals(databaseId)) { + localDatabases.add( + new DatabaseConfigurationMetadata( + new DatabaseConfiguration( + databaseId, + getDatabaseNameForFileName(configDatabase.name()), + new DatabaseConfiguration.Local(configDatabase.type()) + ), + -1, + configDatabase.buildDateInMillis() == null ? -1 : configDatabase.buildDateInMillis() + ) + ); + } + } + return localDatabases; + } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadataTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadataTests.java index 231a2a856815c..6a98cd532604b 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadataTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadataTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.geoip; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration; import org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationMetadata; @@ -21,6 +22,12 @@ import java.util.Map; public class IngestGeoIpMetadataTests extends AbstractChunkedSerializingTestCase { + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(new IngestGeoIpPlugin().getNamedWriteables()); + } + @Override protected IngestGeoIpMetadata doParseInstance(XContentParser parser) throws IOException { return IngestGeoIpMetadata.fromXContent(parser); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadataTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadataTests.java index 847f9c5bf7d4a..476a30d86ee05 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadataTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationMetadataTests.java @@ -9,7 +9,9 @@ package org.elasticsearch.ingest.geoip.direct; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.ingest.geoip.IngestGeoIpPlugin; import org.elasticsearch.test.AbstractXContentSerializingTestCase; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentParser; @@ -21,6 +23,11 @@ public class DatabaseConfigurationMetadataTests extends AbstractXContentSerializingTestCase { + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(new IngestGeoIpPlugin().getNamedWriteables()); + } + private String id; @Override diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java index bb11f71b26d03..33356ad4235dc 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfigurationTests.java @@ -9,8 +9,12 @@ package org.elasticsearch.ingest.geoip.direct; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.ingest.geoip.IngestGeoIpPlugin; +import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Local; import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Maxmind; +import org.elasticsearch.ingest.geoip.direct.DatabaseConfiguration.Web; import org.elasticsearch.test.AbstractXContentSerializingTestCase; import org.elasticsearch.xcontent.XContentParser; @@ -21,6 +25,11 @@ public class DatabaseConfigurationTests extends AbstractXContentSerializingTestCase { + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(new IngestGeoIpPlugin().getNamedWriteables()); + } + private String id; @Override @@ -35,26 +44,39 @@ protected DatabaseConfiguration createTestInstance() { } public static DatabaseConfiguration randomDatabaseConfiguration(String id) { - return new DatabaseConfiguration(id, randomFrom(MAXMIND_NAMES), new Maxmind(randomAlphaOfLength(5))); + DatabaseConfiguration.Provider provider = switch (between(0, 2)) { + case 0 -> new Maxmind(randomAlphaOfLength(5)); + case 1 -> new Web(); + case 2 -> new Local(randomAlphaOfLength(10)); + default -> throw new AssertionError("failure, got illegal switch case"); + }; + return new DatabaseConfiguration(id, randomFrom(MAXMIND_NAMES), provider); } @Override protected DatabaseConfiguration mutateInstance(DatabaseConfiguration instance) { switch (between(0, 2)) { case 0: - return new DatabaseConfiguration(instance.id() + randomAlphaOfLength(2), instance.name(), instance.maxmind()); + return new DatabaseConfiguration(instance.id() + randomAlphaOfLength(2), instance.name(), instance.provider()); case 1: return new DatabaseConfiguration( instance.id(), randomValueOtherThan(instance.name(), () -> randomFrom(MAXMIND_NAMES)), - instance.maxmind() + instance.provider() ); case 2: - return new DatabaseConfiguration( - instance.id(), - instance.name(), - new Maxmind(instance.maxmind().accountId() + randomAlphaOfLength(2)) - ); + DatabaseConfiguration.Provider provider = instance.provider(); + DatabaseConfiguration.Provider modifiedProvider; + if (provider instanceof Maxmind maxmind) { + modifiedProvider = new Maxmind(((Maxmind) instance.provider()).accountId() + randomAlphaOfLength(2)); + } else if (provider instanceof Web) { + modifiedProvider = new Maxmind(randomAlphaOfLength(20)); // can't modify a Web + } else if (provider instanceof Local local) { + modifiedProvider = new Local(local.type() + randomAlphaOfLength(2)); + } else { + throw new AssertionError("Unexpected provider type: " + provider.getClass()); + } + return new DatabaseConfiguration(instance.id(), instance.name(), modifiedProvider); default: throw new AssertionError("failure, got illegal switch case"); } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationActionTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationActionTests.java new file mode 100644 index 0000000000000..988b50311186d --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationActionTests.java @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip.direct; + +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.test.ESTestCase; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.elasticsearch.ingest.geoip.direct.GetDatabaseConfigurationAction.NodeResponse; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; + +public class TransportGetDatabaseConfigurationActionTests extends ESTestCase { + public void testDeduplicateNodeResponses() { + { + List nodeResponses = new ArrayList<>(); + Set preExistingNames = Set.of(); + Collection deduplicated = TransportGetDatabaseConfigurationAction.deduplicateNodeResponses( + nodeResponses, + preExistingNames + ); + assertTrue(deduplicated.isEmpty()); + } + { + List nodeResponses = List.of( + generateTestNodeResponse(List.of()), + generateTestNodeResponse(List.of()), + generateTestNodeResponse(List.of()) + ); + Set preExistingNames = Set.of(); + Collection deduplicated = TransportGetDatabaseConfigurationAction.deduplicateNodeResponses( + nodeResponses, + preExistingNames + ); + assertTrue(deduplicated.isEmpty()); + } + { + // 3 nodes with 3 overlapping responses. We expect the deduplicated collection to include 1, 2, 3, and 4. + List nodeResponses = List.of( + generateTestNodeResponse(List.of("1", "2", "3")), + generateTestNodeResponse(List.of("1", "2", "3")), + generateTestNodeResponse(List.of("1", "4")) + ); + Set preExistingNames = Set.of(); + Collection deduplicated = TransportGetDatabaseConfigurationAction.deduplicateNodeResponses( + nodeResponses, + preExistingNames + ); + assertThat(deduplicated.size(), equalTo(4)); + assertThat( + deduplicated.stream().map(database -> database.database().name()).collect(Collectors.toSet()), + equalTo(Set.of("1", "2", "3", "4")) + ); + } + { + /* + * 3 nodes with 3 overlapping responses, but this time we're also passing in a set of pre-existing names that overlap with + * two of them. So we expect the deduplicated collection to include 1 and 4. + */ + List nodeResponses = List.of( + generateTestNodeResponse(List.of("1", "2", "3")), + generateTestNodeResponse(List.of("1", "2", "3")), + generateTestNodeResponse(List.of("1", "4")) + ); + Set preExistingNames = Set.of("2", "3", "5"); + Collection deduplicated = TransportGetDatabaseConfigurationAction.deduplicateNodeResponses( + nodeResponses, + preExistingNames + ); + assertThat(deduplicated.size(), equalTo(2)); + assertThat( + deduplicated.stream().map(database -> database.database().name()).collect(Collectors.toSet()), + equalTo(Set.of("1", "4")) + ); + } + { + /* + * Here 3 nodes report the same database, but with different modified dates and versions. We expect the one with the highest + * modified date to win out. + */ + List nodeResponses = List.of( + generateTestNodeResponseFromDatabases(List.of(generateTestDatabase("1", 1))), + generateTestNodeResponseFromDatabases(List.of(generateTestDatabase("1", 1000))), + generateTestNodeResponseFromDatabases(List.of(generateTestDatabase("1", 3))) + ); + Set preExistingNames = Set.of("2", "3", "5"); + Collection deduplicated = TransportGetDatabaseConfigurationAction.deduplicateNodeResponses( + nodeResponses, + preExistingNames + ); + assertThat(deduplicated.size(), equalTo(1)); + DatabaseConfigurationMetadata result = deduplicated.iterator().next(); + assertThat(result, equalTo(nodeResponses.get(1).getDatabases().get(0))); + } + } + + private NodeResponse generateTestNodeResponse(List databaseNames) { + List databases = databaseNames.stream().map(this::generateTestDatabase).toList(); + return generateTestNodeResponseFromDatabases(databases); + } + + private NodeResponse generateTestNodeResponseFromDatabases(List databases) { + DiscoveryNode discoveryNode = mock(DiscoveryNode.class); + return new NodeResponse(discoveryNode, databases); + } + + private DatabaseConfigurationMetadata generateTestDatabase(String databaseName) { + return generateTestDatabase(databaseName, randomLongBetween(0, Long.MAX_VALUE)); + } + + private DatabaseConfigurationMetadata generateTestDatabase(String databaseName, long modifiedDate) { + DatabaseConfiguration databaseConfiguration = new DatabaseConfiguration( + randomAlphaOfLength(50), + databaseName, + new DatabaseConfiguration.Local(randomAlphaOfLength(20)) + ); + return new DatabaseConfigurationMetadata(databaseConfiguration, randomLongBetween(0, Long.MAX_VALUE), modifiedDate); + } +} diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml index 6809443fdfbc3..04fd2ac6a8189 100644 --- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml @@ -1,7 +1,7 @@ setup: - requires: - cluster_features: ["geoip.downloader.database.configuration"] - reason: "geoip downloader database configuration APIs added in 8.15" + cluster_features: ["geoip.downloader.database.configuration", "get_database_configuration_action.multi_node"] + reason: "geoip downloader database configuration APIs added in 8.15, and updated in 8.16 to return more results" --- "Test adding, getting, and removing geoip databases": @@ -41,6 +41,17 @@ setup: } - match: { acknowledged: true } + - do: + catch: /illegal_argument_exception/ + ingest.put_geoip_database: + id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" + body: > + { + "name": "GeoIP2-City", + "web": { + } + } + - do: ingest.get_geoip_database: id: "my_database_1" @@ -52,19 +63,37 @@ setup: - do: ingest.get_geoip_database: {} - - length: { databases: 2 } + - length: { databases: 6 } - do: ingest.get_geoip_database: id: "my_database_1,my_database_2" - length: { databases: 2 } + - do: + ingest.get_geoip_database: + id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" + - length: { databases: 1 } + - match: { databases.0.id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" } + - gte: { databases.0.modified_date_millis: -1 } + - match: { databases.0.database.name: "MyCustomGeoLite2-City" } + - do: ingest.delete_geoip_database: id: "my_database_1" + - do: + catch: /resource_not_found_exception/ + ingest.delete_geoip_database: + id: "_web_TXlDdXN0b21HZW9MaXRlMi1DaXR5Lm1tZGI=" + - do: ingest.get_geoip_database: {} + - length: { databases: 5 } + + - do: + ingest.get_geoip_database: + id: "my_database_2" - length: { databases: 1 } - match: { databases.0.id: "my_database_2" } - gte: { databases.0.modified_date_millis: 0 } diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index b82afe2a22fa6..c55436f85a6e3 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -233,6 +233,7 @@ static TransportVersion def(int id) { public static final TransportVersion REGEX_AND_RANGE_INTERVAL_QUERIES = def(8_757_00_0); public static final TransportVersion RRF_QUERY_REWRITE = def(8_758_00_0); public static final TransportVersion SEARCH_FAILURE_STATS = def(8_759_00_0); + public static final TransportVersion INGEST_GEO_DATABASE_PROVIDERS = def(8_760_00_0); /* * STOP! READ THIS FIRST! No, really, From 987c5a6641a69df3590bcbcb470783c3cee529cd Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Thu, 3 Oct 2024 15:29:47 -0700 Subject: [PATCH 062/194] Fix features for synthetic source test (#114070) --- .../rest-api-spec/test/indices.create/20_synthetic_source.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index 1c3c96a944501..a871d2ac0ae15 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -529,7 +529,7 @@ subobject with dynamic override: --- object array in object with dynamic override: - requires: - cluster_features: ["mapper.track_ignored_source"] + cluster_features: ["mapper.synthetic_source_keep"] reason: requires tracking ignored source - do: From 353ab06655073780214693eb6ed64dbdb182edbe Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Thu, 3 Oct 2024 17:00:55 -0700 Subject: [PATCH 063/194] Remove temporary compatibility test mutes (#114083) --- rest-api-spec/build.gradle | 5 ----- 1 file changed, 5 deletions(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index a57757aa8fb71..a742e83255bbb 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -57,9 +57,4 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTestTransform").configure({task -> task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility") task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility") - task.skipTest("indices.create/21_synthetic_source_stored/object param - nested object with stored array", "skip test to submit #113690") - task.skipTest("indices.create/20_synthetic_source/disabled object", "temporary until backported") - task.skipTest("indices.create/20_synthetic_source/disabled object contains array", "temporary until backported") - task.skipTest("indices.create/20_synthetic_source/object with dynamic override", "temporary until backported") - task.skipTest("indices.create/20_synthetic_source/disabled root object", "temporary until backported") }) From fc1bee290a4f9a6e225421d1e9706f9494dfb47b Mon Sep 17 00:00:00 2001 From: Mikhail Berezovskiy Date: Thu, 3 Oct 2024 21:49:51 -0700 Subject: [PATCH 064/194] Add max_multipart_parts setting to S3 repository (#113989) --- docs/changelog/113989.yaml | 5 +++ .../snapshot-restore/repository-s3.asciidoc | 16 +++++++-- .../repositories/s3/S3Repository.java | 23 ++++++++++++- .../repositories/s3/S3RepositoryTests.java | 33 +++++++++++++++++++ 4 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 docs/changelog/113989.yaml diff --git a/docs/changelog/113989.yaml b/docs/changelog/113989.yaml new file mode 100644 index 0000000000000..7bf50b52d9e07 --- /dev/null +++ b/docs/changelog/113989.yaml @@ -0,0 +1,5 @@ +pr: 113989 +summary: Add `max_multipart_parts` setting to S3 repository +area: Snapshot/Restore +type: enhancement +issues: [] diff --git a/docs/reference/snapshot-restore/repository-s3.asciidoc b/docs/reference/snapshot-restore/repository-s3.asciidoc index a75a1a3ce1042..1f55296139cd3 100644 --- a/docs/reference/snapshot-restore/repository-s3.asciidoc +++ b/docs/reference/snapshot-restore/repository-s3.asciidoc @@ -261,9 +261,11 @@ multiple deployments may share the same bucket. `chunk_size`:: - (<>) Big files can be broken down into chunks during snapshotting if needed. - Specify the chunk size as a value and unit, for example: - `1TB`, `1GB`, `10MB`. Defaults to the maximum size of a blob in the S3 which is `5TB`. + (<>) The maximum size of object that {es} will write to the repository + when creating a snapshot. Files which are larger than `chunk_size` will be chunked into several + smaller objects. {es} may also split a file across multiple objects to satisfy other constraints + such as the `max_multipart_parts` limit. Defaults to `5TB` which is the + https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html[maximum size of an object in AWS S3]. `compress`:: @@ -292,6 +294,14 @@ include::repository-shared-settings.asciidoc[] size allowed by S3. Defaults to `100mb` or `5%` of JVM heap, whichever is smaller. +`max_multipart_parts` :: + + (<>) The maximum number of parts that {es} will write during a multipart upload + of a single object. Files which are larger than `buffer_size × max_multipart_parts` will be + chunked into several smaller objects. {es} may also split a file across multiple objects to + satisfy other constraints such as the `chunk_size` limit. Defaults to `10000` which is the + https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html[maximum number of parts in a multipart upload in AWS S3]. + `canned_acl`:: The S3 repository supports all diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index f919284d8e897..af385eeac6a5b 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -140,6 +140,11 @@ class S3Repository extends MeteredBlobStoreRepository { MAX_FILE_SIZE_USING_MULTIPART ); + /** + * Maximum parts number for multipart upload. (see https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html) + */ + static final Setting MAX_MULTIPART_PARTS = Setting.intSetting("max_multipart_parts", 10_000, 1, 10_000); + /** * Sets the S3 storage class type for the backup files. Values may be standard, reduced_redundancy, * standard_ia, onezone_ia and intelligent_tiering. Defaults to standard. @@ -253,7 +258,9 @@ class S3Repository extends MeteredBlobStoreRepository { } this.bufferSize = BUFFER_SIZE_SETTING.get(metadata.settings()); - this.chunkSize = CHUNK_SIZE_SETTING.get(metadata.settings()); + var maxChunkSize = CHUNK_SIZE_SETTING.get(metadata.settings()); + var maxPartsNum = MAX_MULTIPART_PARTS.get(metadata.settings()); + this.chunkSize = objectSizeLimit(maxChunkSize, bufferSize, maxPartsNum); // We make sure that chunkSize is bigger or equal than/to bufferSize if (this.chunkSize.getBytes() < bufferSize.getBytes()) { @@ -302,6 +309,20 @@ private static Map buildLocation(RepositoryMetadata metadata) { return Map.of("base_path", BASE_PATH_SETTING.get(metadata.settings()), "bucket", BUCKET_SETTING.get(metadata.settings())); } + /** + * Calculates S3 object size limit based on 2 constraints: maximum object(chunk) size + * and maximum number of parts for multipart upload. + * https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html + * + * @param chunkSize s3 object size + * @param bufferSize s3 multipart upload part size + * @param maxPartsNum s3 multipart upload max parts number + */ + private static ByteSizeValue objectSizeLimit(ByteSizeValue chunkSize, ByteSizeValue bufferSize, int maxPartsNum) { + var bytes = Math.min(chunkSize.getBytes(), bufferSize.getBytes() * maxPartsNum); + return ByteSizeValue.ofBytes(bytes); + } + /** * Holds a reference to delayed repository operation {@link Scheduler.Cancellable} so it can be cancelled should the repository be * closed concurrently. diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java index 1eab59ebb0eb7..3817af4def888 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java @@ -175,4 +175,37 @@ public void testAnalysisFailureDetail() { } } + // ensures that chunkSize is limited to chunk_size setting, when buffer_size * parts_num is bigger + public void testChunkSizeLimit() { + var meta = new RepositoryMetadata( + "dummy-repo", + "mock", + Settings.builder() + .put(S3Repository.BUCKET_SETTING.getKey(), "bucket") + .put(S3Repository.CHUNK_SIZE_SETTING.getKey(), "1GB") + .put(S3Repository.BUFFER_SIZE_SETTING.getKey(), "100MB") + .put(S3Repository.MAX_MULTIPART_PARTS.getKey(), 10_000) // ~1TB + .build() + ); + try (var repo = createS3Repo(meta)) { + assertEquals(ByteSizeValue.ofGb(1), repo.chunkSize()); + } + } + + // ensures that chunkSize is limited to buffer_size * parts_num, when chunk_size setting is bigger + public void testPartsNumLimit() { + var meta = new RepositoryMetadata( + "dummy-repo", + "mock", + Settings.builder() + .put(S3Repository.BUCKET_SETTING.getKey(), "bucket") + .put(S3Repository.CHUNK_SIZE_SETTING.getKey(), "5TB") + .put(S3Repository.BUFFER_SIZE_SETTING.getKey(), "100MB") + .put(S3Repository.MAX_MULTIPART_PARTS.getKey(), 10_000) + .build() + ); + try (var repo = createS3Repo(meta)) { + assertEquals(ByteSizeValue.ofMb(1_000_000), repo.chunkSize()); + } + } } From f27411ee1239a53c9bef32e078d9c4ec7b304ddb Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 4 Oct 2024 05:55:59 +0100 Subject: [PATCH 065/194] Assert that REST params are consumed iff supported (#114040) REST APIs which declare their supported parameters must consume exactly those parameters: consuming an unsupported parameter means that requests including that parameter will be rejected, whereas failing to consume a supported parameter means that this parameter has no effect and should be removed. This commit adds an assertion to verify that we are consuming the correct parameters. Closes #113854 --- .../rest/RestGetDataStreamsAction.java | 5 +++-- .../org/elasticsearch/rest/BaseRestHandler.java | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java index da55376fb403b..3456f4b679474 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.set.Sets; @@ -35,14 +36,14 @@ public class RestGetDataStreamsAction extends BaseRestHandler { Set.of( "name", "include_defaults", - "timeout", "master_timeout", IndicesOptions.WildcardOptions.EXPAND_WILDCARDS, IndicesOptions.ConcreteTargetOptions.IGNORE_UNAVAILABLE, IndicesOptions.WildcardOptions.ALLOW_NO_INDICES, IndicesOptions.GatekeeperOptions.IGNORE_THROTTLED, "verbose" - ) + ), + DataStream.isFailureStoreFeatureFlagEnabled() ? Set.of(IndicesOptions.FailureStoreOptions.FAILURE_STORE) : Set.of() ) ); diff --git a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java index 99fa3e0166963..2f7bb80a8d46a 100644 --- a/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/BaseRestHandler.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.RestApiVersion; @@ -104,6 +105,8 @@ public final void handleRequest(RestRequest request, RestChannel channel, NodeCl // prepare the request for execution; has the side effect of touching the request parameters try (var action = prepareRequest(request, client)) { + assert assertConsumesSupportedParams(supported, request); + // validate unconsumed params, but we must exclude params used to format the response // use a sorted set so the unconsumed parameters appear in a reliable sorted order final SortedSet unconsumedParams = request.unconsumedParams() @@ -148,6 +151,20 @@ public void close() { } } + private boolean assertConsumesSupportedParams(@Nullable Set supported, RestRequest request) { + if (supported != null) { + final var supportedAndCommon = new TreeSet<>(supported); + supportedAndCommon.add("error_trace"); + supportedAndCommon.addAll(ALWAYS_SUPPORTED); + supportedAndCommon.removeAll(RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS); + final var consumed = new TreeSet<>(request.consumedParams()); + consumed.removeAll(RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS); + assert supportedAndCommon.equals(consumed) + : getName() + ": consumed params " + consumed + " while supporting " + supportedAndCommon; + } + return true; + } + protected static String unrecognized(RestRequest request, Set invalids, Set candidates, String detail) { StringBuilder message = new StringBuilder().append("request [") .append(request.path()) From 38a0711df6d6a1581d82af12c67dddd7aba11f60 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 4 Oct 2024 15:39:04 +1000 Subject: [PATCH 066/194] Mute org.elasticsearch.xpack.inference.TextEmbeddingCrudIT testPutE5Small_withPlatformSpecificVariant #113950 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index e6e0c901a0edd..9fc98499d7950 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -357,6 +357,9 @@ tests: - class: org.elasticsearch.xpack.rank.rrf.RRFRetrieverBuilderIT method: testMultipleRRFRetrievers issue: https://github.com/elastic/elasticsearch/issues/114079 +- class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT + method: testPutE5Small_withPlatformSpecificVariant + issue: https://github.com/elastic/elasticsearch/issues/113950 # Examples: # From ddfdd40b167b7cb6ddaf682997c44859d62e6379 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 4 Oct 2024 08:34:24 +0200 Subject: [PATCH 067/194] Reduce footprint of codec instances further (#114072) Lots of effectively singleton objects here and fields that can be made static, saves a little more on the per-index overhead and might reveal further simplifications. --- .../codec/tsdb/internal/DecodeBenchmark.java | 3 ++- .../index/codec/Elasticsearch814Codec.java | 2 +- .../index/codec/Elasticsearch816Codec.java | 15 +++++++-------- .../index/codec/tsdb/DocValuesForUtil.java | 17 +++++------------ .../codec/tsdb/ES87TSDBDocValuesEncoder.java | 4 ++-- .../codec/tsdb/ES87TSDBDocValuesProducer.java | 2 +- .../codec/vectors/ES813FlatVectorFormat.java | 2 +- .../ES814ScalarQuantizedVectorsFormat.java | 6 ++++-- .../codec/vectors/ES815BitFlatVectorFormat.java | 2 +- .../vectors/ES815BitFlatVectorsFormat.java | 2 +- .../vectors/ES815HnswBitVectorsFormat.java | 2 +- .../codec/zstd/Zstd814StoredFieldsFormat.java | 8 +++++++- .../index/codec/tsdb/DocValuesForUtilTests.java | 3 +-- 13 files changed, 34 insertions(+), 34 deletions(-) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecodeBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecodeBenchmark.java index 284324b3d9206..b8f0a11e21c8f 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecodeBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/internal/DecodeBenchmark.java @@ -12,6 +12,7 @@ import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ByteArrayDataOutput; import org.apache.lucene.store.DataOutput; +import org.elasticsearch.index.codec.tsdb.DocValuesForUtil; import org.openjdk.jmh.infra.Blackhole; import java.io.IOException; @@ -43,7 +44,7 @@ public void setupInvocation(int bitsPerValue) { @Override public void benchmark(int bitsPerValue, Blackhole bh) throws IOException { - forUtil.decode(bitsPerValue, this.dataInput, this.output); + DocValuesForUtil.decode(bitsPerValue, this.dataInput, this.output); bh.consume(this.output); } } diff --git a/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java index f3d758f4fc8b7..ae372ea8194bc 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java +++ b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch814Codec.java @@ -67,7 +67,7 @@ public Elasticsearch814Codec() { */ public Elasticsearch814Codec(Zstd814StoredFieldsFormat.Mode mode) { super("Elasticsearch814", lucene99Codec); - this.storedFieldsFormat = new Zstd814StoredFieldsFormat(mode); + this.storedFieldsFormat = mode.getFormat(); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch816Codec.java b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch816Codec.java index 00711c7ecc306..27ff19a9d8e40 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch816Codec.java +++ b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch816Codec.java @@ -28,9 +28,13 @@ */ public class Elasticsearch816Codec extends CodecService.DeduplicateFieldInfosCodec { + private static final Lucene912Codec LUCENE_912_CODEC = new Lucene912Codec(); + private static final PostingsFormat defaultPostingsFormat = new Lucene912PostingsFormat(); + private static final DocValuesFormat defaultDVFormat = new Lucene90DocValuesFormat(); + private static final KnnVectorsFormat defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat(); + private final StoredFieldsFormat storedFieldsFormat; - private final PostingsFormat defaultPostingsFormat; private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { @Override public PostingsFormat getPostingsFormatForField(String field) { @@ -38,7 +42,6 @@ public PostingsFormat getPostingsFormatForField(String field) { } }; - private final DocValuesFormat defaultDVFormat; private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { @Override public DocValuesFormat getDocValuesFormatForField(String field) { @@ -46,7 +49,6 @@ public DocValuesFormat getDocValuesFormatForField(String field) { } }; - private final KnnVectorsFormat defaultKnnVectorsFormat; private final KnnVectorsFormat knnVectorsFormat = new PerFieldKnnVectorsFormat() { @Override public KnnVectorsFormat getKnnVectorsFormatForField(String field) { @@ -64,11 +66,8 @@ public Elasticsearch816Codec() { * worse space-efficiency or vice-versa. */ public Elasticsearch816Codec(Zstd814StoredFieldsFormat.Mode mode) { - super("Elasticsearch816", new Lucene912Codec()); - this.storedFieldsFormat = new Zstd814StoredFieldsFormat(mode); - this.defaultPostingsFormat = new Lucene912PostingsFormat(); - this.defaultDVFormat = new Lucene90DocValuesFormat(); - this.defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat(); + super("Elasticsearch816", LUCENE_912_CODEC); + this.storedFieldsFormat = mode.getFormat(); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtil.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtil.java index 671931ac7154a..648913098ff0d 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtil.java +++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtil.java @@ -21,17 +21,10 @@ public class DocValuesForUtil { private static final int BITS_IN_FIVE_BYTES = 5 * Byte.SIZE; private static final int BITS_IN_SIX_BYTES = 6 * Byte.SIZE; private static final int BITS_IN_SEVEN_BYTES = 7 * Byte.SIZE; - private final int blockSize; - private final byte[] encoded; + private static final int blockSize = ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE; + private final byte[] encoded = new byte[1024]; - public DocValuesForUtil() { - this(ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE); - } - - private DocValuesForUtil(int blockSize) { - this.blockSize = blockSize; - this.encoded = new byte[1024]; - } + public DocValuesForUtil() {} public static int roundBits(int bitsPerValue) { if (bitsPerValue > 24 && bitsPerValue <= 32) { @@ -74,7 +67,7 @@ private void encodeFiveSixOrSevenBytesPerValue(long[] in, int bitsPerValue, fina out.writeBytes(this.encoded, bytesPerValue * in.length); } - public void decode(int bitsPerValue, final DataInput in, long[] out) throws IOException { + public static void decode(int bitsPerValue, final DataInput in, long[] out) throws IOException { if (bitsPerValue <= 24) { ForUtil.decode(bitsPerValue, in, out); } else if (bitsPerValue <= 32) { @@ -88,7 +81,7 @@ public void decode(int bitsPerValue, final DataInput in, long[] out) throws IOEx } } - private void decodeFiveSixOrSevenBytesPerValue(int bitsPerValue, final DataInput in, long[] out) throws IOException { + private static void decodeFiveSixOrSevenBytesPerValue(int bitsPerValue, final DataInput in, long[] out) throws IOException { // NOTE: we expect multibyte values to be written "least significant byte" first int bytesPerValue = bitsPerValue / Byte.SIZE; long mask = (1L << bitsPerValue) - 1; diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoder.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoder.java index f152a0b0601a2..4e95ce34dc410 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoder.java +++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesEncoder.java @@ -275,7 +275,7 @@ void decodeOrdinals(DataInput in, long[] out, int bitsPerOrd) throws IOException Arrays.fill(out, runLen, out.length, v2); } else if (encoding == 2) { // bit-packed - forUtil.decode(bitsPerOrd, in, out); + DocValuesForUtil.decode(bitsPerOrd, in, out); } else if (encoding == 3) { // cycle encoding int cycleLength = (int) v1; @@ -299,7 +299,7 @@ void decode(DataInput in, long[] out) throws IOException { final int bitsPerValue = token >>> 3; if (bitsPerValue != 0) { - forUtil.decode(bitsPerValue, in, out); + DocValuesForUtil.decode(bitsPerValue, in, out); } else { Arrays.fill(out, 0L); } diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java index a887516e5e7cc..e3c2daddba80e 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java +++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java @@ -355,7 +355,7 @@ public TermsEnum termsEnum() throws IOException { } } - private abstract class BaseSortedSetDocValues extends SortedSetDocValues { + private abstract static class BaseSortedSetDocValues extends SortedSetDocValues { final SortedSetEntry entry; final IndexInput data; diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java index 7a8d09c02ba3b..b1e91ad75e9a2 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java @@ -36,7 +36,7 @@ public class ES813FlatVectorFormat extends KnnVectorsFormat { static final String NAME = "ES813FlatVectorFormat"; - private final FlatVectorsFormat format = new Lucene99FlatVectorsFormat(DefaultFlatVectorScorer.INSTANCE); + private static final FlatVectorsFormat format = new Lucene99FlatVectorsFormat(DefaultFlatVectorScorer.INSTANCE); /** * Sole constructor diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java index 4313aa40cf13e..4bf396e8d5ad1 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java @@ -49,6 +49,10 @@ public class ES814ScalarQuantizedVectorsFormat extends FlatVectorsFormat { private static final FlatVectorsFormat rawVectorFormat = new Lucene99FlatVectorsFormat(DefaultFlatVectorScorer.INSTANCE); + static final FlatVectorsScorer flatVectorScorer = new ESFlatVectorsScorer( + new ScalarQuantizedVectorScorer(DefaultFlatVectorScorer.INSTANCE) + ); + /** The minimum confidence interval */ private static final float MINIMUM_CONFIDENCE_INTERVAL = 0.9f; @@ -60,7 +64,6 @@ public class ES814ScalarQuantizedVectorsFormat extends FlatVectorsFormat { * calculated as `1-1/(vector_dimensions + 1)` */ public final Float confidenceInterval; - final FlatVectorsScorer flatVectorScorer; private final byte bits; private final boolean compress; @@ -83,7 +86,6 @@ public ES814ScalarQuantizedVectorsFormat(Float confidenceInterval, int bits, boo throw new IllegalArgumentException("bits must be one of: 4, 7, 8; bits=" + bits); } this.confidenceInterval = confidenceInterval; - this.flatVectorScorer = new ESFlatVectorsScorer(new ScalarQuantizedVectorScorer(DefaultFlatVectorScorer.INSTANCE)); this.bits = (byte) bits; this.compress = compress; } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormat.java index 2df0757a8b8ee..af771b6a27f19 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorFormat.java @@ -22,7 +22,7 @@ public class ES815BitFlatVectorFormat extends KnnVectorsFormat { static final String NAME = "ES815BitFlatVectorFormat"; - private final FlatVectorsFormat format = new ES815BitFlatVectorsFormat(); + private static final FlatVectorsFormat format = new ES815BitFlatVectorsFormat(); /** * Sole constructor diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java index f1ae4e3fdeded..5969c9d5db6d7 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java @@ -27,7 +27,7 @@ class ES815BitFlatVectorsFormat extends FlatVectorsFormat { - private final FlatVectorsFormat delegate = new Lucene99FlatVectorsFormat(FlatBitVectorScorer.INSTANCE); + private static final FlatVectorsFormat delegate = new Lucene99FlatVectorsFormat(FlatBitVectorScorer.INSTANCE); protected ES815BitFlatVectorsFormat() { super("ES815BitFlatVectorsFormat"); diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormat.java index 55271719a4574..5e4656ea94c5b 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815HnswBitVectorsFormat.java @@ -30,7 +30,7 @@ public class ES815HnswBitVectorsFormat extends KnnVectorsFormat { private final int maxConn; private final int beamWidth; - private final FlatVectorsFormat flatVectorsFormat = new ES815BitFlatVectorsFormat(); + private static final FlatVectorsFormat flatVectorsFormat = new ES815BitFlatVectorsFormat(); public ES815HnswBitVectorsFormat() { this(16, 100); diff --git a/server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd814StoredFieldsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd814StoredFieldsFormat.java index 84871b5c811dd..6aa77b7222696 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd814StoredFieldsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd814StoredFieldsFormat.java @@ -52,17 +52,23 @@ public enum Mode { BEST_COMPRESSION(3, BEST_COMPRESSION_BLOCK_SIZE, 2048); final int level, blockSizeInBytes, blockDocCount; + final Zstd814StoredFieldsFormat format; Mode(int level, int blockSizeInBytes, int blockDocCount) { this.level = level; this.blockSizeInBytes = blockSizeInBytes; this.blockDocCount = blockDocCount; + this.format = new Zstd814StoredFieldsFormat(this); + } + + public Zstd814StoredFieldsFormat getFormat() { + return format; } } private final Mode mode; - public Zstd814StoredFieldsFormat(Mode mode) { + private Zstd814StoredFieldsFormat(Mode mode) { super("ZstdStoredFields814", new ZstdCompressionMode(mode.level), mode.blockSizeInBytes, mode.blockDocCount, 10); this.mode = mode; } diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtilTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtilTests.java index 72295743608c3..7da5463ea46ff 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtilTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/DocValuesForUtilTests.java @@ -70,11 +70,10 @@ public void testEncodeDecode() throws IOException { { // decode IndexInput in = d.openInput("test.bin", IOContext.READONCE); - final DocValuesForUtil forUtil = new DocValuesForUtil(); final long[] restored = new long[ES87TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE]; for (int i = 0; i < iterations; ++i) { final int bitsPerValue = in.readByte(); - forUtil.decode(bitsPerValue, in, restored); + DocValuesForUtil.decode(bitsPerValue, in, restored); assertArrayEquals( Arrays.toString(restored), ArrayUtil.copyOfSubArray( From 9852458f254e8d616a3fe0f8c697e13aaea8b2d4 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Fri, 4 Oct 2024 09:00:14 +0100 Subject: [PATCH 068/194] Unmute DotPrefixClientYamlTestSuiteIT again (#113944) The test was accidentally re-muted in merge conflict --- muted-tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 9fc98499d7950..5b2c59c1dc8c0 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -278,9 +278,6 @@ tests: - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testCreateJobsWithIndexNameOption issue: https://github.com/elastic/elasticsearch/issues/113528 -- class: org.elasticsearch.validation.DotPrefixClientYamlTestSuiteIT - method: test {p0=dot_prefix/10_basic/Deprecated index template with a dot prefix index pattern} - issue: https://github.com/elastic/elasticsearch/issues/113529 - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testCantCreateJobWithSameID issue: https://github.com/elastic/elasticsearch/issues/113581 From 41894eada8939ce3c7f48a390a351b854a30d0de Mon Sep 17 00:00:00 2001 From: Panagiotis Bailis Date: Fri, 4 Oct 2024 11:14:31 +0300 Subject: [PATCH 069/194] Fix for RRF and RankDoc tests due to different shard and doc ordering (#114084) --- muted-tests.yml | 3 --- .../search/retriever/RankDocRetrieverBuilderIT.java | 3 ++- .../elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java | 5 +++-- .../xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java | 3 ++- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 5b2c59c1dc8c0..0ced48b2a9bb4 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -351,9 +351,6 @@ tests: - class: org.elasticsearch.xpack.rank.rrf.RRFRetrieverBuilderIT method: testRRFWithCollapse issue: https://github.com/elastic/elasticsearch/issues/114074 -- class: org.elasticsearch.xpack.rank.rrf.RRFRetrieverBuilderIT - method: testMultipleRRFRetrievers - issue: https://github.com/elastic/elasticsearch/issues/114079 - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT method: testPutE5Small_withPlatformSpecificVariant issue: https://github.com/elastic/elasticsearch/issues/113950 diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java index 891096dfa67a9..f42e52b4fe561 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java @@ -53,6 +53,7 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.hamcrest.Matchers.equalTo; @@ -112,7 +113,7 @@ public void setup() throws Exception { } } """; - createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).build()); + createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).build()); admin().indices().preparePutMapping(INDEX).setSource(mapping, XContentType.JSON).get(); indexDoc( INDEX, diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java index 8b924af48c631..2e7bc44811bf6 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java @@ -33,6 +33,7 @@ import java.util.Collection; import java.util.List; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -97,7 +98,7 @@ protected void setupIndex() { } } """; - createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 5).build()); + createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).build()); admin().indices().preparePutMapping(INDEX).setSource(mapping, XContentType.JSON).get(); indexDoc(INDEX, "doc_1", DOC_FIELD, "doc_1", TOPIC_FIELD, "technology", TEXT_FIELD, "term"); indexDoc( @@ -301,7 +302,7 @@ public void testRRFWithCollapse() { }); } - public void testRankDocsRetrieverWithCollapseAndAggs() { + public void testRRFRetrieverWithCollapseAndAggs() { final int rankWindowSize = 100; final int rankConstant = 10; SearchSourceBuilder source = new SearchSourceBuilder(); diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java index 3a4ace9b6754a..512874e5009f3 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java @@ -21,6 +21,7 @@ import java.util.Arrays; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.hamcrest.Matchers.equalTo; @@ -67,7 +68,7 @@ protected void setupIndex() { } } """; - createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 5).build()); + createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).build()); admin().indices().preparePutMapping(INDEX).setSource(mapping, XContentType.JSON).get(); indexDoc(INDEX, "doc_1", DOC_FIELD, "doc_1", TOPIC_FIELD, "technology", TEXT_FIELD, "term", LAST_30D_FIELD, 100); indexDoc( From 24d46700f246a5a02bfb1c5ab5c9a85f147eb395 Mon Sep 17 00:00:00 2001 From: Panagiotis Bailis Date: Fri, 4 Oct 2024 11:14:57 +0300 Subject: [PATCH 070/194] Updating expected values in RankDocRetrieverBuilderIT testRankDocsRetrieverWithNestedQuery (#114047) --- muted-tests.yml | 3 --- .../search/retriever/RankDocRetrieverBuilderIT.java | 7 +++---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 0ced48b2a9bb4..77c6d6b8fa4c1 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -121,9 +121,6 @@ tests: - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testMultiIndexDelete issue: https://github.com/elastic/elasticsearch/issues/112381 -- class: org.elasticsearch.search.retriever.RankDocRetrieverBuilderIT - method: testRankDocsRetrieverWithNestedQuery - issue: https://github.com/elastic/elasticsearch/issues/112421 - class: org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialCentroidTests method: "testAggregateIntermediate {TestCase= #2}" issue: https://github.com/elastic/elasticsearch/issues/112461 diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java index f42e52b4fe561..b78448bfd873f 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java @@ -418,8 +418,7 @@ public void testRankDocsRetrieverWithNestedQuery() { SearchSourceBuilder source = new SearchSourceBuilder(); StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(LAST_30D_FIELD).gt(10L), ScoreMode.Avg) - .innerHit(new InnerHitBuilder("a").addSort(new FieldSortBuilder(DOC_FIELD).order(SortOrder.DESC)).setSize(10)); + standard0.queryBuilder = QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(LAST_30D_FIELD).gt(10L), ScoreMode.Avg); StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); // this one retrieves docs 2 and 6 due to prefilter standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); @@ -456,9 +455,9 @@ public void testRankDocsRetrieverWithNestedQuery() { assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6")); assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_2")); assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_7")); + assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_3")); assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_4")); - assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_3")); + assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_7")); }); } From 7dfd3f5f8c00b8f5115853e50de523ce2c6da894 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 4 Oct 2024 18:21:34 +1000 Subject: [PATCH 071/194] Mute org.elasticsearch.xpack.inference.services.openai.OpenAiServiceTests testInfer_StreamRequest_ErrorResponse #114105 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 77c6d6b8fa4c1..fc24018173291 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -351,6 +351,9 @@ tests: - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT method: testPutE5Small_withPlatformSpecificVariant issue: https://github.com/elastic/elasticsearch/issues/113950 +- class: org.elasticsearch.xpack.inference.services.openai.OpenAiServiceTests + method: testInfer_StreamRequest_ErrorResponse + issue: https://github.com/elastic/elasticsearch/issues/114105 # Examples: # From 11c2eb299b7857164999c93616ccd36aa0c58696 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Fri, 4 Oct 2024 09:51:26 +0100 Subject: [PATCH 072/194] Convert a few more implementations to ChunkedXContentBuilder (#113125) Remove the complex methods from ChunkedXContentHelper --- .../ingest/geoip/IngestGeoIpMetadata.java | 7 +- .../mustache/SearchTemplateResponse.java | 2 +- .../action/bulk/BulkResponse.java | 14 +- .../action/search/SearchResponse.java | 41 ++---- .../broadcast/ChunkedBroadcastResponse.java | 9 +- .../metadata/ComponentTemplateMetadata.java | 6 +- .../ComposableIndexTemplateMetadata.java | 6 +- .../cluster/metadata/DataStreamMetadata.java | 2 +- .../metadata/NodesShutdownMetadata.java | 6 +- .../xcontent/ChunkedToXContentBuilder.java | 121 ++++++++++++++---- .../xcontent/ChunkedToXContentHelper.java | 52 -------- .../rest/action/RestActions.java | 8 +- .../upgrades/FeatureMigrationResults.java | 6 +- .../segments/IndicesSegmentResponseTests.java | 2 +- .../stats/FieldUsageStatsResponseTests.java | 2 +- .../stats/IndicesStatsResponseTests.java | 4 +- .../cluster/metadata/MetadataTests.java | 47 ++++--- .../ClusterSerializationTests.java | 18 +-- .../test/TestCustomMetadata.java | 6 +- .../autoscaling/AutoscalingMetadata.java | 6 +- .../xpack/core/ccr/AutoFollowMetadata.java | 6 +- .../xpack/core/enrich/EnrichMetadata.java | 6 +- .../core/ilm/IndexLifecycleMetadata.java | 12 +- .../core/ml/inference/ModelAliasMetadata.java | 6 +- .../core/slm/SnapshotLifecycleMetadata.java | 17 +-- .../xpack/esql/action/EsqlExecutionInfo.java | 26 ++-- .../xpack/esql/action/EsqlQueryResponse.java | 8 +- ...rverSentEventsRestActionListenerTests.java | 10 +- .../action/GetStackTracesResponse.java | 43 +++---- 29 files changed, 240 insertions(+), 259 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java index c7d3db5f5b572..b6e73f3f33f7c 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java @@ -15,10 +15,9 @@ import org.elasticsearch.cluster.DiffableUtils; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.ingest.geoip.direct.DatabaseConfigurationMetadata; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; @@ -91,8 +90,8 @@ public static IngestGeoIpMetadata fromXContent(XContentParser parser) throws IOE } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return Iterators.concat(ChunkedToXContentHelper.xContentValuesMap(DATABASES_FIELD.getPreferredName(), databases)); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params).xContentObjectFields(DATABASES_FIELD.getPreferredName(), databases); } @Override diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java index ecc84ddca2e13..9bb80d5688b5f 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateResponse.java @@ -107,7 +107,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws void innerToXContent(XContentBuilder builder, Params params) throws IOException { if (hasResponse()) { - ChunkedToXContent.wrapAsToXContent(p -> response.innerToXContentChunked(p)).toXContent(builder, params); + ChunkedToXContent.wrapAsToXContent(response::innerToXContentChunked).toXContent(builder, params); } else { // we can assume the template is always json as we convert it before compiling it try (InputStream stream = source.streamInput()) { diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java index b02d7acf66d14..88a9fb56b8edb 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.core.TimeValue; import org.elasticsearch.xcontent.ToXContent; @@ -157,14 +158,13 @@ public void writeTo(StreamOutput out) throws IOException { @Override public Iterator toXContentChunked(ToXContent.Params params) { - return Iterators.concat(Iterators.single((builder, p) -> { - builder.startObject(); - builder.field(ERRORS, hasFailures()); - builder.field(TOOK, tookInMillis); + return ChunkedToXContent.builder(params).object(ob -> { + ob.field(ERRORS, hasFailures()); + ob.field(TOOK, tookInMillis); if (ingestTookInMillis != BulkResponse.NO_INGEST_TOOK) { - builder.field(INGEST_TOOK, ingestTookInMillis); + ob.field(INGEST_TOOK, ingestTookInMillis); } - return builder.startArray(ITEMS); - }), Iterators.forArray(responses), Iterators.single((builder, p) -> builder.endArray().endObject())); + ob.array(ITEMS, Iterators.forArray(responses)); + }); } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java index eb272a9302e85..83ee6c216ad49 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -14,13 +14,12 @@ import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RefCounted; @@ -383,39 +382,17 @@ public Clusters getClusters() { @Override public Iterator toXContentChunked(ToXContent.Params params) { assert hasReferences(); - return Iterators.concat( - ChunkedToXContentHelper.startObject(), - this.innerToXContentChunked(params), - ChunkedToXContentHelper.endObject() - ); + return ChunkedToXContent.builder(params).xContentObject(innerToXContentChunked(params)); } public Iterator innerToXContentChunked(ToXContent.Params params) { - return Iterators.concat( - ChunkedToXContentHelper.singleChunk(SearchResponse.this::headerToXContent), - Iterators.single(clusters), - Iterators.concat( - Iterators.flatMap(Iterators.single(hits), r -> r.toXContentChunked(params)), - Iterators.single((ToXContent) (b, p) -> { - if (aggregations != null) { - aggregations.toXContent(b, p); - } - return b; - }), - Iterators.single((b, p) -> { - if (suggest != null) { - suggest.toXContent(b, p); - } - return b; - }), - Iterators.single((b, p) -> { - if (profileResults != null) { - profileResults.toXContent(b, p); - } - return b; - }) - ) - ); + return ChunkedToXContent.builder(params) + .append(SearchResponse.this::headerToXContent) + .append(clusters) + .append(hits) + .appendIfPresent(aggregations) + .appendIfPresent(suggest) + .appendIfPresent(profileResults); } public XContentBuilder headerToXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/action/support/broadcast/ChunkedBroadcastResponse.java b/server/src/main/java/org/elasticsearch/action/support/broadcast/ChunkedBroadcastResponse.java index e07ad7bac4037..4b65af23c622a 100644 --- a/server/src/main/java/org/elasticsearch/action/support/broadcast/ChunkedBroadcastResponse.java +++ b/server/src/main/java/org/elasticsearch/action/support/broadcast/ChunkedBroadcastResponse.java @@ -9,8 +9,8 @@ package org.elasticsearch.action.support.broadcast; import org.elasticsearch.action.support.DefaultShardOperationFailedException; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.xcontent.ToXContent; @@ -35,11 +35,8 @@ public ChunkedBroadcastResponse( @Override public final Iterator toXContentChunked(ToXContent.Params params) { - return Iterators.concat(Iterators.single((b, p) -> { - b.startObject(); - RestActions.buildBroadcastShardsHeader(b, p, this); - return b; - }), customXContentChunks(params), Iterators.single((builder, p) -> builder.endObject())); + return ChunkedToXContent.builder(params) + .object(ob -> ob.append((b, p) -> RestActions.buildBroadcastShardsHeader(b, p, this)).append(this::customXContentChunks)); } protected abstract Iterator customXContentChunks(ToXContent.Params params); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java index fda66e230c52a..1151a99a24403 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java @@ -17,7 +17,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; @@ -102,8 +102,8 @@ public static ComponentTemplateMetadata fromXContent(XContentParser parser) thro } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return ChunkedToXContentHelper.xContentValuesMap(COMPONENT_TEMPLATE.getPreferredName(), componentTemplates); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params).xContentObjectFields(COMPONENT_TEMPLATE.getPreferredName(), componentTemplates); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java index c786d9c6ea71c..e798b0f6add4f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java @@ -17,7 +17,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; @@ -103,8 +103,8 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return ChunkedToXContentHelper.xContentValuesMap(INDEX_TEMPLATE.getPreferredName(), indexTemplates); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params).xContentObjectFields(INDEX_TEMPLATE.getPreferredName(), indexTemplates); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java index 44f3101395b88..3f5e7a2e0c4aa 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java @@ -233,7 +233,7 @@ public static DataStreamMetadata fromXContent(XContentParser parser) throws IOEx @Override public Iterator toXContentChunked(ToXContent.Params params) { return ChunkedToXContent.builder(params) - .object(DATA_STREAM.getPreferredName(), b -> b.appendXContentFields(dataStreams)) + .xContentObjectFields(DATA_STREAM.getPreferredName(), dataStreams) .xContentObject(DATA_STREAM_ALIASES.getPreferredName(), dataStreamAliases.values().iterator()); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java index eb7e36a8e9c7d..272f321e386fc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java @@ -17,7 +17,7 @@ import org.elasticsearch.cluster.SimpleDiffable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.core.Nullable; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; @@ -190,8 +190,8 @@ public int hashCode() { } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return ChunkedToXContentHelper.xContentValuesMap(NODES_FIELD.getPreferredName(), nodes); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params).xContentObjectFields(NODES_FIELD.getPreferredName(), nodes); } /** diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilder.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilder.java index ce0c2bd30f160..0868a7fa303ae 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilder.java @@ -110,6 +110,46 @@ public ChunkedToXContentBuilder xContentObject(String name, Iterator map) { + startObject(); + map.forEach(this::field); + endObject(); + return this; + } + + /** + * Creates an object named {@code name}, with the contents of each field created from each entry in {@code map} + */ + public ChunkedToXContentBuilder xContentObjectFields(String name, Map map) { + startObject(name); + map.forEach(this::field); + endObject(); + return this; + } + + /** + * Creates an object with the contents of each field each another object created from each entry in {@code map} + */ + public ChunkedToXContentBuilder xContentObjectFieldObjects(Map map) { + startObject(); + map.forEach(this::xContentObject); + endObject(); + return this; + } + + /** + * Creates an object named {@code name}, with the contents of each field each another object created from each entry in {@code map} + */ + public ChunkedToXContentBuilder xContentObjectFieldObjects(String name, Map map) { + startObject(name); + map.forEach(this::xContentObject); + endObject(); + return this; + } + /** * Creates an object, with the contents set by {@code contents} */ @@ -172,6 +212,26 @@ public ChunkedToXContentBuilder object(String name, Iterator items, Funct return this; } + /** + * Creates an object with the contents of each field set by {@code map} + */ + public ChunkedToXContentBuilder object(Map map) { + startObject(); + map.forEach(this::field); + endObject(); + return this; + } + + /** + * Creates an object named {@code name}, with the contents of each field set by {@code map} + */ + public ChunkedToXContentBuilder object(String name, Map map) { + startObject(name); + map.forEach(this::field); + endObject(); + return this; + } + private void startArray() { addChunk((b, p) -> b.startArray()); } @@ -223,7 +283,7 @@ public ChunkedToXContentBuilder array(String name, Iterator items, BiCons /** * Creates an array named {@code name}, with the contents set by appending together the contents of {@code items} */ - public ChunkedToXContentBuilder array(String name, Iterator items) { + public ChunkedToXContentBuilder array(String name, Iterator items) { startArray(name); items.forEachRemaining(this::append); endArray(); @@ -246,16 +306,51 @@ public ChunkedToXContentBuilder field(String name) { return this; } + public ChunkedToXContentBuilder field(String name, boolean value) { + addChunk((b, p) -> b.field(name, value)); + return this; + } + + public ChunkedToXContentBuilder field(String name, Boolean value) { + addChunk((b, p) -> b.field(name, value)); + return this; + } + + public ChunkedToXContentBuilder field(String name, int value) { + addChunk((b, p) -> b.field(name, value)); + return this; + } + + public ChunkedToXContentBuilder field(String name, Integer value) { + addChunk((b, p) -> b.field(name, value)); + return this; + } + public ChunkedToXContentBuilder field(String name, long value) { addChunk((b, p) -> b.field(name, value)); return this; } + public ChunkedToXContentBuilder field(String name, Long value) { + addChunk((b, p) -> b.field(name, value)); + return this; + } + public ChunkedToXContentBuilder field(String name, String value) { addChunk((b, p) -> b.field(name, value)); return this; } + public ChunkedToXContentBuilder field(String name, Enum value) { + addChunk((b, p) -> b.field(name, value)); + return this; + } + + public ChunkedToXContentBuilder field(String name, ToXContent value) { + addChunk((b, p) -> b.field(name, value, p)); + return this; + } + public ChunkedToXContentBuilder field(String name, Object value) { addChunk((b, p) -> b.field(name, value)); return this; @@ -289,30 +384,6 @@ public ChunkedToXContentBuilder forEach(Iterator items, Function - * Note that any {@link ToXContent} objects in {@code map} will be passed an empty {@link ToXContent.Params}, - * rather than the {@code params} given to this builder in the constructor. - */ - public ChunkedToXContentBuilder appendEntries(Map map) { - return forEach(map.entrySet().iterator(), (b, e) -> b.field(e.getKey(), e.getValue())); - } - - /** - * Each value in {@code map} is added to the builder as a separate object, named by its key. - */ - public ChunkedToXContentBuilder appendXContentObjects(Map map) { - return forEach(map.entrySet().iterator(), (b, e) -> b.xContentObject(e.getKey(), e.getValue())); - } - - /** - * Each value in {@code map} is added to the builder as a separate field, named by its key. - */ - public ChunkedToXContentBuilder appendXContentFields(Map map) { - return forEach(map.entrySet().iterator(), (b, e) -> b.field(e.getKey()).append(e.getValue())); - } - public ChunkedToXContentBuilder append(ToXContent xContent) { if (xContent != ToXContent.EMPTY) { addChunk(xContent); diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java index 8755139ad84b7..940d4495ae909 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java @@ -13,8 +13,6 @@ import org.elasticsearch.xcontent.ToXContent; import java.util.Iterator; -import java.util.Map; -import java.util.function.Function; public enum ChunkedToXContentHelper { ; @@ -43,36 +41,6 @@ public static Iterator endArray() { return Iterators.single(((builder, params) -> builder.endArray())); } - public static Iterator map(String name, Map map) { - return map(name, map, entry -> (ToXContent) (builder, params) -> builder.field(entry.getKey(), entry.getValue())); - } - - public static Iterator xContentFragmentValuesMap(String name, Map map) { - return map( - name, - map, - entry -> (ToXContent) (builder, params) -> entry.getValue().toXContent(builder.startObject(entry.getKey()), params).endObject() - ); - } - - public static Iterator xContentValuesMap(String name, Map map) { - return map( - name, - map, - entry -> (ToXContent) (builder, params) -> entry.getValue().toXContent(builder.field(entry.getKey()), params) - ); - } - - /** - * Like xContentFragmentValuesMap, but allows the underlying XContent object to define its own "name" with startObject(string) - * and endObject, rather than assuming that the key in the map should be the name in the XContent output. - * @param name name to use in the XContent for the outer object wrapping the map being rendered to XContent - * @param map map being rendered to XContent - */ - public static Iterator xContentFragmentValuesMapCreateOwnName(String name, Map map) { - return map(name, map, entry -> (ToXContent) (builder, params) -> entry.getValue().toXContent(builder, params)); - } - public static Iterator field(String name, boolean value) { return Iterators.single(((builder, params) -> builder.field(name, value))); } @@ -101,30 +69,10 @@ public static Iterator array(String name, Iterator array(String name, Iterator contents, ToXContent.Params params) { - return Iterators.concat( - ChunkedToXContentHelper.startArray(name), - Iterators.flatMap(contents, c -> c.toXContentChunked(params)), - ChunkedToXContentHelper.endArray() - ); - } - public static Iterator wrapWithObject(String name, Iterator iterator) { return Iterators.concat(startObject(name), iterator, endObject()); } - public static Iterator map(String name, Map map, Function, ToXContent> toXContent) { - return wrapWithObject(name, Iterators.map(map.entrySet().iterator(), toXContent)); - } - /** * Creates an Iterator of a single ToXContent object that serializes the given object as a single chunk. Just wraps {@link * Iterators#single}, but still useful because it avoids any type ambiguity. diff --git a/server/src/main/java/org/elasticsearch/rest/action/RestActions.java b/server/src/main/java/org/elasticsearch/rest/action/RestActions.java index dc1ab92a66a84..69cf5bbb1b89d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/RestActions.java +++ b/server/src/main/java/org/elasticsearch/rest/action/RestActions.java @@ -64,9 +64,9 @@ public static long parseVersion(RestRequest request, long defaultVersion) { return (version == Versions.MATCH_ANY) ? defaultVersion : version; } - public static void buildBroadcastShardsHeader(XContentBuilder builder, Params params, BaseBroadcastResponse response) + public static XContentBuilder buildBroadcastShardsHeader(XContentBuilder builder, Params params, BaseBroadcastResponse response) throws IOException { - buildBroadcastShardsHeader( + return buildBroadcastShardsHeader( builder, params, response.getTotalShards(), @@ -77,7 +77,7 @@ public static void buildBroadcastShardsHeader(XContentBuilder builder, Params pa ); } - public static void buildBroadcastShardsHeader( + public static XContentBuilder buildBroadcastShardsHeader( XContentBuilder builder, Params params, int total, @@ -100,7 +100,7 @@ public static void buildBroadcastShardsHeader( } builder.endArray(); } - builder.endObject(); + return builder.endObject(); } /** diff --git a/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java b/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java index d8badd6847c93..1aeab06146834 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java +++ b/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java @@ -18,7 +18,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; @@ -57,8 +57,8 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return ChunkedToXContentHelper.xContentValuesMap(RESULTS_FIELD.getPreferredName(), featureStatuses); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params).xContentObjectFields(RESULTS_FIELD.getPreferredName(), featureStatuses); } /** diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java index e33f97e7cb580..144db693b8ab3 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponseTests.java @@ -68,7 +68,7 @@ public void testChunking() { 0, Collections.emptyList() ), - response -> 11 * response.getIndices().size() + 4 + response -> 11 * response.getIndices().size() + 5 ); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java index 7b707eb4c31ba..a34e36e81dd83 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponseTests.java @@ -47,7 +47,7 @@ public void testToXContentChunkPerIndex() { AbstractChunkedSerializingTestCase.assertChunkCount( new FieldUsageStatsResponse(indices, indices, 0, List.of(), perIndex), - ignored -> 3 * indices + 2 + ignored -> 3 * indices + 3 ); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java index 2e915083476b8..bce4d20a06fbe 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java @@ -138,12 +138,12 @@ public void testChunkedEncodingPerIndex() { AbstractChunkedSerializingTestCase.assertChunkCount( indicesStatsResponse, new ToXContent.MapParams(Map.of("level", "cluster")), - ignored1 -> 3 + ignored1 -> 4 ); AbstractChunkedSerializingTestCase.assertChunkCount( indicesStatsResponse, new ToXContent.MapParams(Map.of("level", "indices")), - ignored -> 4 + 2 * shards + ignored -> 5 + 2 * shards ); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index 892b6aa3bf176..00e21603ec8b4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -20,7 +20,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; @@ -28,8 +27,8 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Predicates; @@ -71,7 +70,6 @@ import java.util.Set; import java.util.SortedMap; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -2308,30 +2306,32 @@ public static int expectedChunkCount(ToXContent.Params params, Metadata metadata chunkCount += 2; if (custom instanceof ComponentTemplateMetadata componentTemplateMetadata) { - chunkCount += 2 + componentTemplateMetadata.componentTemplates().size(); + chunkCount += checkChunkSize(custom, params, 2 + componentTemplateMetadata.componentTemplates().size()); } else if (custom instanceof ComposableIndexTemplateMetadata composableIndexTemplateMetadata) { - chunkCount += 2 + composableIndexTemplateMetadata.indexTemplates().size(); + chunkCount += checkChunkSize(custom, params, 2 + composableIndexTemplateMetadata.indexTemplates().size()); } else if (custom instanceof DataStreamMetadata dataStreamMetadata) { - chunkCount += 4 + (dataStreamMetadata.dataStreams().size() * 2L) + dataStreamMetadata.getDataStreamAliases().size(); + chunkCount += checkChunkSize( + custom, + params, + 4 + dataStreamMetadata.dataStreams().size() + dataStreamMetadata.getDataStreamAliases().size() + ); } else if (custom instanceof DesiredNodesMetadata) { - chunkCount += 1; + chunkCount += checkChunkSize(custom, params, 1); } else if (custom instanceof FeatureMigrationResults featureMigrationResults) { - chunkCount += 2 + featureMigrationResults.getFeatureStatuses().size(); + chunkCount += checkChunkSize(custom, params, 2 + featureMigrationResults.getFeatureStatuses().size()); } else if (custom instanceof IndexGraveyard indexGraveyard) { - chunkCount += 2 + indexGraveyard.getTombstones().size(); + chunkCount += checkChunkSize(custom, params, 2 + indexGraveyard.getTombstones().size()); } else if (custom instanceof IngestMetadata ingestMetadata) { - chunkCount += 2 + ingestMetadata.getPipelines().size(); + chunkCount += checkChunkSize(custom, params, 2 + ingestMetadata.getPipelines().size()); } else if (custom instanceof NodesShutdownMetadata nodesShutdownMetadata) { - chunkCount += 2 + nodesShutdownMetadata.getAll().size(); + chunkCount += checkChunkSize(custom, params, 2 + nodesShutdownMetadata.getAll().size()); } else if (custom instanceof PersistentTasksCustomMetadata persistentTasksCustomMetadata) { - chunkCount += 3 + persistentTasksCustomMetadata.tasks().size(); + chunkCount += checkChunkSize(custom, params, 3 + persistentTasksCustomMetadata.tasks().size()); } else if (custom instanceof RepositoriesMetadata repositoriesMetadata) { - chunkCount += repositoriesMetadata.repositories().size(); + chunkCount += checkChunkSize(custom, params, repositoriesMetadata.repositories().size()); } else { // could be anything, we have to just try it - chunkCount += Iterables.size( - (Iterable) (() -> Iterators.map(custom.toXContentChunked(params), Function.identity())) - ); + chunkCount += count(custom.toXContentChunked(params)); } } @@ -2343,6 +2343,21 @@ public static int expectedChunkCount(ToXContent.Params params, Metadata metadata return Math.toIntExact(chunkCount); } + private static long count(Iterator it) { + long count = 0; + while (it.hasNext()) { + count++; + it.next(); + } + return count; + } + + private static long checkChunkSize(ChunkedToXContent custom, ToXContent.Params params, long expectedSize) { + long actualSize = count(custom.toXContentChunked(params)); + assertThat(actualSize, equalTo(expectedSize)); + return actualSize; + } + /** * With this test we ensure that we consider whether a new field added to Metadata should be checked * in Metadata.isGlobalStateEquals. We force the instance fields to be either in the checked list diff --git a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java index 37d75d923a7a7..ca5b9295adfd7 100644 --- a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java @@ -30,12 +30,12 @@ import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.shard.IndexLongFieldRange; import org.elasticsearch.index.shard.ShardLongFieldRange; @@ -398,12 +398,8 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return Iterators.concat( - Iterators.single((builder, params) -> builder.startObject()), - Iterators.single((builder, params) -> builder.field("custom_string_object", strObject)), - Iterators.single((builder, params) -> builder.endObject()) - ); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params).object(b -> b.field("custom_string_object", strObject)); } @Override @@ -441,12 +437,8 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return Iterators.concat( - Iterators.single((builder, params) -> builder.startObject()), - Iterators.single((builder, params) -> builder.field("custom_integer_object", intObject)), - Iterators.single((builder, params) -> builder.endObject()) - ); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params).object(b -> b.field("custom_integer_object", intObject)); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetadata.java b/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetadata.java index 9954739e4a9c3..3a24b7ae0f14f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetadata.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestCustomMetadata.java @@ -13,9 +13,9 @@ import org.elasticsearch.cluster.AbstractNamedDiffable; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentParser; @@ -94,8 +94,8 @@ public static T fromXContent(Function sup } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return Iterators.single((builder, params) -> builder.field("data", getData())); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params).field("data", getData()); } @Override diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java index 5c885ad718d8c..38c654f94fff3 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java @@ -16,7 +16,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; @@ -118,8 +118,8 @@ public TransportVersion getMinimalSupportedVersion() { } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return ChunkedToXContentHelper.xContentValuesMap(POLICIES_FIELD.getPreferredName(), policies); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params).xContentObjectFields(POLICIES_FIELD.getPreferredName(), policies); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java index b7795fd3b579e..6b54f7a7dddce 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java @@ -150,9 +150,9 @@ public void writeTo(StreamOutput out) throws IOException { @Override public Iterator toXContentChunked(ToXContent.Params params) { return ChunkedToXContent.builder(params) - .object(PATTERNS_FIELD.getPreferredName(), b -> b.appendXContentObjects(patterns)) - .object(FOLLOWED_LEADER_INDICES_FIELD.getPreferredName(), b -> b.appendEntries(followedLeaderIndexUUIDs)) - .object(HEADERS.getPreferredName(), b -> b.appendEntries(headers)); + .xContentObjectFieldObjects(PATTERNS_FIELD.getPreferredName(), patterns) + .object(FOLLOWED_LEADER_INDICES_FIELD.getPreferredName(), followedLeaderIndexUUIDs) + .object(HEADERS.getPreferredName(), headers); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichMetadata.java index 0cd5d617752f4..b949e44ef036a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichMetadata.java @@ -13,7 +13,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; @@ -98,8 +98,8 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return ChunkedToXContentHelper.xContentFragmentValuesMap(POLICIES.getPreferredName(), policies); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params).xContentObjectFieldObjects(POLICIES.getPreferredName(), policies); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java index d4f2ecb36e95d..3674103eda215 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java @@ -15,10 +15,9 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.Metadata.Custom; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; @@ -105,11 +104,10 @@ public Diff diff(Custom previousState) { } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return Iterators.concat( - ChunkedToXContentHelper.xContentValuesMap(POLICIES_FIELD.getPreferredName(), policyMetadatas), - Iterators.single((builder, params) -> builder.field(OPERATION_MODE_FIELD.getPreferredName(), operationMode)) - ); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params) + .xContentObjectFields(POLICIES_FIELD.getPreferredName(), policyMetadatas) + .field(OPERATION_MODE_FIELD.getPreferredName(), operationMode); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/ModelAliasMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/ModelAliasMetadata.java index 1d6c5e564a442..fd07688b1578d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/ModelAliasMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/ModelAliasMetadata.java @@ -17,7 +17,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; @@ -92,8 +92,8 @@ public Map modelAliases() { } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return ChunkedToXContentHelper.xContentValuesMap(MODEL_ALIASES.getPreferredName(), modelAliases); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params).xContentObjectFields(MODEL_ALIASES.getPreferredName(), modelAliases); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java index 9610f70689897..0d6aba9406a3f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java @@ -15,10 +15,9 @@ import org.elasticsearch.cluster.SimpleDiffable; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; @@ -143,15 +142,11 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public Iterator toXContentChunked(ToXContent.Params ignored) { - return Iterators.concat( - ChunkedToXContentHelper.xContentValuesMap(POLICIES_FIELD.getPreferredName(), this.snapshotConfigurations), - Iterators.single((builder, params) -> { - builder.field(OPERATION_MODE_FIELD.getPreferredName(), operationMode); - builder.field(STATS_FIELD.getPreferredName(), this.slmStats); - return builder; - }) - ); + public Iterator toXContentChunked(ToXContent.Params params) { + return ChunkedToXContent.builder(params) + .xContentObjectFields(POLICIES_FIELD.getPreferredName(), snapshotConfigurations) + .field(OPERATION_MODE_FIELD.getPreferredName(), operationMode) + .field(STATS_FIELD.getPreferredName(), slmStats); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java index b01aff2a09bd4..f7966ff5ae9ec 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java @@ -7,12 +7,11 @@ package org.elasticsearch.xpack.esql.action; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.core.Predicates; import org.elasticsearch.core.TimeValue; @@ -167,19 +166,18 @@ public Cluster swapCluster(String clusterAlias, BiFunction toXContentChunked(ToXContent.Params params) { if (isCrossClusterSearch() == false || clusterInfo.isEmpty()) { - return Iterators.concat(); + return Collections.emptyIterator(); } - return Iterators.concat( - ChunkedToXContentHelper.startObject(), - ChunkedToXContentHelper.field(TOTAL_FIELD.getPreferredName(), clusterInfo.size()), - ChunkedToXContentHelper.field(SUCCESSFUL_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.SUCCESSFUL)), - ChunkedToXContentHelper.field(RUNNING_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.RUNNING)), - ChunkedToXContentHelper.field(SKIPPED_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.SKIPPED)), - ChunkedToXContentHelper.field(PARTIAL_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.PARTIAL)), - ChunkedToXContentHelper.field(FAILED_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.FAILED)), - ChunkedToXContentHelper.xContentFragmentValuesMapCreateOwnName("details", clusterInfo), - ChunkedToXContentHelper.endObject() - ); + return ChunkedToXContent.builder(params).object(b -> { + b.field(TOTAL_FIELD.getPreferredName(), clusterInfo.size()); + b.field(SUCCESSFUL_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.SUCCESSFUL)); + b.field(RUNNING_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.RUNNING)); + b.field(SKIPPED_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.SKIPPED)); + b.field(PARTIAL_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.PARTIAL)); + b.field(FAILED_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.FAILED)); + // each clusterinfo defines its own field object name + b.xContentObject("details", clusterInfo.values().iterator()); + }); } /** diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java index 146a88128da35..8e4da3f138a6f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ChunkedToXContent; +import org.elasticsearch.common.xcontent.ChunkedToXContentBuilder; import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.compute.data.BlockFactory; @@ -375,11 +376,8 @@ public int hashCode() { @Override public Iterator toXContentChunked(ToXContent.Params params) { - return Iterators.concat( - ChunkedToXContentHelper.startObject(), - ChunkedToXContentHelper.array("drivers", drivers.iterator(), params), - ChunkedToXContentHelper.endObject() - ); + return ChunkedToXContent.builder(params) + .object(ob -> ob.array("drivers", drivers.iterator(), ChunkedToXContentBuilder::append)); } List drivers() { diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListenerTests.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListenerTests.java index d56b9fe21cd50..f3cefa04c2911 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListenerTests.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListenerTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.ClusterSettings; @@ -34,7 +33,6 @@ import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.ChunkedToXContent; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.inference.InferenceResults; import org.elasticsearch.inference.InferenceServiceResults; @@ -234,11 +232,7 @@ private static class RandomString implements ChunkedToXContent { @Override public Iterator toXContentChunked(ToXContent.Params params) { var randomString = randomUnicodeOfLengthBetween(2, 20); - return Iterators.concat( - ChunkedToXContentHelper.startObject(), - ChunkedToXContentHelper.field("delta", randomString), - ChunkedToXContentHelper.endObject() - ); + return ChunkedToXContent.builder(params).object(b -> b.field("delta", randomString)); } } @@ -271,7 +265,7 @@ public void writeTo(StreamOutput out) { @Override public Iterator toXContentChunked(ToXContent.Params params) { - return ChunkedToXContentHelper.field("result", randomUnicodeOfLengthBetween(2, 20)); + return ChunkedToXContent.builder(params).field("result", randomUnicodeOfLengthBetween(2, 20)); } } diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java index e9757d3806b69..19747ec6e49f3 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java @@ -9,19 +9,19 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.TriConsumer; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; +import org.elasticsearch.common.xcontent.ChunkedToXContentBuilder; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.xcontent.ToXContent; -import java.util.Collections; import java.util.Iterator; import java.util.Map; import java.util.Objects; -import java.util.function.BiFunction; +import java.util.function.Consumer; public class GetStackTracesResponse extends ActionResponse implements ChunkedToXContentObject { @Nullable @@ -91,34 +91,33 @@ public long getTotalSamples() { @Override public Iterator toXContentChunked(ToXContent.Params params) { - return Iterators.concat( - ChunkedToXContentHelper.startObject(), - optional("stack_traces", stackTraces, ChunkedToXContentHelper::xContentValuesMap), - optional("stack_frames", stackFrames, ChunkedToXContentHelper::xContentValuesMap), - optional("executables", executables, ChunkedToXContentHelper::map), + return ChunkedToXContent.builder(params).object(ob -> { + ob.execute(optional("stack_traces", stackTraces, ChunkedToXContentBuilder::xContentObjectFields)); + ob.execute(optional("stack_frames", stackFrames, ChunkedToXContentBuilder::xContentObjectFields)); + ob.execute(optional("executables", executables, ChunkedToXContentBuilder::object)); // render only count for backwards-compatibility - optional( - "stack_trace_events", - stackTraceEvents, - (n, v) -> ChunkedToXContentHelper.map(n, v, entry -> (b, p) -> b.field(entry.getKey(), entry.getValue().count)) - ), - Iterators.single((b, p) -> b.field("total_frames", totalFrames)), - Iterators.single((b, p) -> b.field("sampling_rate", samplingRate)), + ob.execute( + optional( + "stack_trace_events", + stackTraceEvents, + (steb, n, v) -> steb.object(n, v.entrySet().iterator(), e -> (b, p) -> b.field(e.getKey(), e.getValue().count)) + ) + ); + ob.field("total_frames", totalFrames); + ob.field("sampling_rate", samplingRate); // the following fields are intentionally not written to the XContent representation (only needed on the transport layer): - // // * start // * end // * totalSamples - ChunkedToXContentHelper.endObject() - ); + }); } - private static Iterator optional( + private static Consumer optional( String name, Map values, - BiFunction, Iterator> supplier + TriConsumer> function ) { - return (values != null) ? supplier.apply(name, values) : Collections.emptyIterator(); + return values != null ? b -> function.apply(b, name, values) : b -> {}; } @Override From 54c83d7fa7ca346b4044f2b70a05b589bc01f0bd Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Fri, 4 Oct 2024 11:01:49 +0200 Subject: [PATCH 073/194] Add wolfi ess docker image (#113810) --- .../pipelines/periodic-packaging.template.yml | 2 +- .buildkite/pipelines/periodic-packaging.yml | 2 +- .../gradle/internal/DockerBase.java | 6 +- .../InternalDistributionDownloadPlugin.java | 3 + ...WolfiEssElasticsearchDistributionType.java | 27 +++++ ...nternalElasticsearchDistributionTypes.java | 4 +- .../internal/test/DistroTestPlugin.java | 2 + distribution/docker/README.md | 13 ++- distribution/docker/build.gradle | 101 +++++++++++++----- .../docker/src/docker/Dockerfile.cloud-ess | 13 --- distribution/docker/src/docker/Dockerfile.ess | 51 +++++++++ .../build.gradle | 2 + .../wolfi-ess-docker-export/build.gradle | 2 + .../packaging/test/DockerTests.java | 28 +++-- .../test/KeystoreManagementTests.java | 2 +- .../packaging/test/PackagingTestCase.java | 7 +- .../packaging/util/Distribution.java | 7 +- .../packaging/util/docker/Docker.java | 6 +- .../packaging/util/docker/DockerRun.java | 1 + settings.gradle | 2 + 20 files changed, 220 insertions(+), 61 deletions(-) create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiEssElasticsearchDistributionType.java delete mode 100644 distribution/docker/src/docker/Dockerfile.cloud-ess create mode 100644 distribution/docker/src/docker/Dockerfile.ess create mode 100644 distribution/docker/wolfi-ess-docker-aarch64-export/build.gradle create mode 100644 distribution/docker/wolfi-ess-docker-export/build.gradle diff --git a/.buildkite/pipelines/periodic-packaging.template.yml b/.buildkite/pipelines/periodic-packaging.template.yml index 14a2fd7ba1bc4..e0da1f46486ea 100644 --- a/.buildkite/pipelines/periodic-packaging.template.yml +++ b/.buildkite/pipelines/periodic-packaging.template.yml @@ -3,7 +3,7 @@ steps: steps: - label: "{{matrix.image}} / packaging-tests-unix" command: ./.ci/scripts/packaging-test.sh destructivePackagingTest - timeout_in_minutes: 300 + timeout_in_minutes: 420 matrix: setup: image: diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index ac207fca5e3ed..5a05d75cf95ac 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -4,7 +4,7 @@ steps: steps: - label: "{{matrix.image}} / packaging-tests-unix" command: ./.ci/scripts/packaging-test.sh destructivePackagingTest - timeout_in_minutes: 300 + timeout_in_minutes: 420 matrix: setup: image: diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index ed1689cfb0eb9..793ff6049e10e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -33,7 +33,11 @@ public enum DockerBase { "docker.elastic.co/wolfi/chainguard-base:latest@sha256:c16d3ad6cebf387e8dd2ad769f54320c4819fbbaa21e729fad087c7ae223b4d0", "-wolfi", "apk" - ); + ), + + // Based on WOLFI above, with more extras. We don't set a base image because + // we programmatically extend from the Wolfi image. + WOLFI_ESS(null, "-wolfi-ess", "apk"); private final String image; private final String suffix; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java index 19309fe2da8a3..6b93ea10283ae 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java @@ -181,6 +181,9 @@ private static String distributionProjectName(ElasticsearchDistribution distribu if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_WOLFI) { return projectName + "wolfi-docker" + archString + "-export"; } + if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_WOLFI_ESS) { + return projectName + "wolfi-ess-docker" + archString + "-export"; + } return projectName + distribution.getType().getName(); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiEssElasticsearchDistributionType.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiEssElasticsearchDistributionType.java new file mode 100644 index 0000000000000..550c43d43a536 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiEssElasticsearchDistributionType.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.distribution; + +import org.elasticsearch.gradle.ElasticsearchDistributionType; + +public class DockerWolfiEssElasticsearchDistributionType implements ElasticsearchDistributionType { + + DockerWolfiEssElasticsearchDistributionType() {} + + @Override + public String getName() { + return "dockerWolfiEss"; + } + + @Override + public boolean isDocker() { + return true; + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java index ba0e76b3f5b99..077a47041861f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java @@ -22,6 +22,7 @@ public class InternalElasticsearchDistributionTypes { public static ElasticsearchDistributionType DOCKER_CLOUD = new DockerCloudElasticsearchDistributionType(); public static ElasticsearchDistributionType DOCKER_CLOUD_ESS = new DockerCloudEssElasticsearchDistributionType(); public static ElasticsearchDistributionType DOCKER_WOLFI = new DockerWolfiElasticsearchDistributionType(); + public static ElasticsearchDistributionType DOCKER_WOLFI_ESS = new DockerWolfiEssElasticsearchDistributionType(); public static List ALL_INTERNAL = List.of( DEB, @@ -31,6 +32,7 @@ public class InternalElasticsearchDistributionTypes { DOCKER_IRONBANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, - DOCKER_WOLFI + DOCKER_WOLFI, + DOCKER_WOLFI_ESS ); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java index 77ab9557eac33..cc852e615726a 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java @@ -54,6 +54,7 @@ import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_IRONBANK; import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_UBI; import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI; +import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI_ESS; import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.RPM; /** @@ -152,6 +153,7 @@ private static Map> lifecycleTask lifecyleTasks.put(DOCKER_CLOUD, project.getTasks().register(taskPrefix + ".docker-cloud")); lifecyleTasks.put(DOCKER_CLOUD_ESS, project.getTasks().register(taskPrefix + ".docker-cloud-ess")); lifecyleTasks.put(DOCKER_WOLFI, project.getTasks().register(taskPrefix + ".docker-wolfi")); + lifecyleTasks.put(DOCKER_WOLFI_ESS, project.getTasks().register(taskPrefix + ".docker-wolfi-ess")); lifecyleTasks.put(ARCHIVE, project.getTasks().register(taskPrefix + ".archives")); lifecyleTasks.put(DEB, project.getTasks().register(taskPrefix + ".packages")); lifecyleTasks.put(RPM, lifecyleTasks.get(DEB)); diff --git a/distribution/docker/README.md b/distribution/docker/README.md index eb0e7b296097d..28e6ff314d91a 100644 --- a/distribution/docker/README.md +++ b/distribution/docker/README.md @@ -7,6 +7,17 @@ the [DockerBase] enum. * UBI - the same as the default image, but based upon [RedHat's UBI images][ubi], specifically their minimal flavour. * Wolfi - the same as the default image, but based upon [Wolfi](https://github.com/wolfi-dev) + * Wolfi ESS - this directly extends the Wolfi image, and adds all ES plugins + that the ES build generates in an archive directory. It also sets an + environment variable that points at this directory. This allows plugins to + be installed from the archive instead of the internet, speeding up + deployment times. Furthermore this image has + * `filebeat` and `metricbeat` included + * `wget` included + * The `ENTRYPOINT` is just `/sbin/tini`, and the `CMD` is + `/app/elasticsearch.sh`. In normal use this file would be bind-mounted + in, but the image ships a stub version of this file so that the image + can still be tested. * Iron Bank - this is the US Department of Defence's repository of digitally signed, binary container images including both Free and Open-Source software (FOSS) and Commercial off-the-shelf (COTS). In practice, this is @@ -17,7 +28,7 @@ the [DockerBase] enum. * `filebeat` and `metricbeat` are included * `wget` is included * The `ENTRYPOINT` is just `/bin/tini`, and the `CMD` is - `/app/elasticsearc.sh`. In normal use this file would be bind-mounted + `/app/elasticsearch.sh`. In normal use this file would be bind-mounted in, but the image ships a stub version of this file so that the image can still be tested. * Cloud ESS - this directly extends the Cloud image, and adds all ES plugins diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 30974ed2396a8..99c482d91085a 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -1,4 +1,3 @@ -import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.LoggedExec import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.internal.DockerBase @@ -8,8 +7,10 @@ import org.elasticsearch.gradle.internal.docker.DockerSupportPlugin import org.elasticsearch.gradle.internal.docker.DockerSupportService import org.elasticsearch.gradle.internal.docker.ShellRetry import org.elasticsearch.gradle.internal.docker.TransformLog4jConfigFilter +import org.elasticsearch.gradle.internal.docker.* import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.util.GradleUtils +import org.elasticsearch.gradle.Architecture import java.nio.file.Path import java.time.temporal.ChronoUnit @@ -99,9 +100,9 @@ String tiniArch = Architecture.current() == Architecture.AARCH64 ? 'arm64' : 'am dependencies { aarch64DockerSource project(":distribution:archives:linux-aarch64-tar") - aarch64DockerSourceTar project(path: ":distribution:archives:linux-aarch64-tar", configuration:"default") + aarch64DockerSourceTar project(path: ":distribution:archives:linux-aarch64-tar", configuration: "default") dockerSource project(":distribution:archives:linux-tar") - dockerSourceTar project(path: ":distribution:archives:linux-tar", configuration:"default") + dockerSourceTar project(path: ":distribution:archives:linux-tar", configuration: "default") log4jConfig project(path: ":distribution", configuration: 'log4jConfig') tini "krallin:tini:0.19.0:${tiniArch}" allPlugins project(path: ':plugins', configuration: 'allPlugins') @@ -112,7 +113,7 @@ dependencies { } ext.expansions = { Architecture architecture, DockerBase base -> - def (major,minor) = VersionProperties.elasticsearch.split("\\.") + def (major, minor) = VersionProperties.elasticsearch.split("\\.") // We tag our Docker images with various pieces of information, including a timestamp // for when the image was built. However, this makes it impossible completely cache @@ -216,7 +217,8 @@ elasticsearch_distributions { } interface Injected { - @Inject FileSystemOperations getFs() + @Inject + FileSystemOperations getFs() } tasks.named("preProcessFixture").configure { @@ -300,7 +302,10 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) { // For some reason, the artifact name can differ depending on what repository we used. rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz" } - Provider serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME) + Provider serviceProvider = GradleUtils.getBuildService( + project.gradle.sharedServices, + DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME + ) onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) } } @@ -337,9 +342,9 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) { into "${project.buildDir}/docker-context/${archiveName}" // Since we replaced the remote URL in the Dockerfile, copy in the required file - if(base == DockerBase.IRON_BANK) { + if (base == DockerBase.IRON_BANK) { from(architecture == Architecture.AARCH64 ? configurations.aarch64DockerSourceTar : configurations.dockerSourceTar) - from (configurations.tini) { + from(configurations.tini) { rename { _ -> 'tini' } } } else { @@ -349,7 +354,10 @@ void addTransformDockerContextTask(Architecture architecture, DockerBase base) { expansions(architecture, base).findAll { it.key != 'build_date' }.each { k, v -> inputs.property(k, { v.toString() }) } - Provider serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME) + Provider serviceProvider = GradleUtils.getBuildService( + project.gradle.sharedServices, + DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME + ) onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) } } @@ -373,7 +381,7 @@ private static List generateTags(DockerBase base, Architecture architect String image = "elasticsearch${base.suffix}" String namespace = 'elasticsearch' - if (base == DockerBase.CLOUD || base == DockerBase.CLOUD_ESS) { + if (base == DockerBase.CLOUD || base == DockerBase.CLOUD_ESS || base == DockerBase.WOLFI_ESS) { namespace += '-ci' } @@ -423,7 +431,10 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) { baseImages = [base.image] } - Provider serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME) + Provider serviceProvider = GradleUtils.getBuildService( + project.gradle.sharedServices, + DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME + ) onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) } } @@ -435,13 +446,12 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) { } } -void addBuildEssDockerImageTask(Architecture architecture) { - DockerBase base = DockerBase.CLOUD_ESS +void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase) { String arch = architecture == Architecture.AARCH64 ? '-aarch64' : '' - String contextDir = "${project.buildDir}/docker-context/elasticsearch${base.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}" + String contextDir = "${project.buildDir}/docker-context/elasticsearch${dockerBase.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}" final TaskProvider buildContextTask = - tasks.register(taskName('build', architecture, base, 'DockerContext'), Sync) { + tasks.register(taskName('build', architecture, dockerBase, 'DockerContext'), Sync) { into contextDir final Path projectDir = project.projectDir.toPath() @@ -450,28 +460,54 @@ void addBuildEssDockerImageTask(Architecture architecture) { from configurations.allPlugins } - from(projectDir.resolve("src/docker/Dockerfile.cloud-ess")) { - expand([ - base_image: "elasticsearch${DockerBase.CLOUD.suffix}:${architecture.classifier}" - ]) + if (dockerBase == DockerBase.WOLFI_ESS) { + // If we're performing a release build, but `build.id` hasn't been set, we can + // infer that we're not at the Docker building stage of the build, and therefore + // we should skip the beats part of the build. + String buildId = providers.systemProperty('build.id').getOrNull() + boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra + + if (includeBeats) { + from configurations.getByName("filebeat_${architecture.classifier}") + from configurations.getByName("metricbeat_${architecture.classifier}") + } + // For some reason, the artifact name can differ depending on what repository we used. + rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz" + } + + String baseSuffix = dockerBase == DockerBase.CLOUD_ESS ? DockerBase.CLOUD.suffix : DockerBase.WOLFI.suffix + from(projectDir.resolve("src/docker/Dockerfile.ess")) { + expand( + [ + base_image: "elasticsearch${baseSuffix}:${architecture.classifier}", + docker_base: "${dockerBase.name().toLowerCase()}", + version: "${VersionProperties.elasticsearch}", + retry: ShellRetry + ] + ) filter SquashNewlinesFilter - rename ~/Dockerfile\.cloud-ess$/, 'Dockerfile' + rename ~/Dockerfile\.ess$/, 'Dockerfile' } } final TaskProvider buildDockerImageTask = - tasks.register(taskName("build", architecture, base, "DockerImage"), DockerBuildTask) { + tasks.register(taskName("build", architecture, dockerBase, "DockerImage"), DockerBuildTask) { - TaskProvider buildCloudTask = tasks.named(taskName("build", architecture, DockerBase.CLOUD, "DockerImage")) - inputs.files(buildCloudTask) + DockerBase base = dockerBase == DockerBase.CLOUD_ESS ? DockerBase.CLOUD : DockerBase.WOLFI + + TaskProvider buildBaseTask = tasks.named(taskName("build", architecture, base, "DockerImage")) + inputs.files(buildBaseTask) dockerContext.fileProvider(buildContextTask.map { it.getDestinationDir() }) noCache = BuildParams.isCi() baseImages = [] - tags = generateTags(base, architecture) + tags = generateTags(dockerBase, architecture) platforms.add(architecture.dockerPlatform) - Provider serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME) + Provider serviceProvider = GradleUtils.getBuildService( + project.gradle.sharedServices, + DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME + ) onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) } } @@ -483,7 +519,7 @@ void addBuildEssDockerImageTask(Architecture architecture) { for (final Architecture architecture : Architecture.values()) { for (final DockerBase base : DockerBase.values()) { - if (base == DockerBase.CLOUD_ESS) { + if (base == DockerBase.CLOUD_ESS || base == DockerBase.WOLFI_ESS) { continue } addBuildDockerContextTask(architecture, base) @@ -491,7 +527,8 @@ for (final Architecture architecture : Architecture.values()) { addBuildDockerImageTask(architecture, base) } - addBuildEssDockerImageTask(architecture) + addBuildEssDockerImageTask(architecture, DockerBase.CLOUD_ESS) + addBuildEssDockerImageTask(architecture, DockerBase.WOLFI_ESS) } def exportDockerImages = tasks.register("exportDockerImages") @@ -515,6 +552,8 @@ subprojects { Project subProject -> base = DockerBase.CLOUD_ESS } else if (subProject.name.contains('cloud-')) { base = DockerBase.CLOUD + } else if (subProject.name.contains('wolfi-ess')) { + base = DockerBase.WOLFI_ESS } else if (subProject.name.contains('wolfi-')) { base = DockerBase.WOLFI } @@ -525,7 +564,8 @@ subprojects { Project subProject -> (base == DockerBase.CLOUD ? 'cloud.tar' : (base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' : (base == DockerBase.WOLFI ? 'wolfi.tar' : - 'docker.tar')))) + (base == DockerBase.WOLFI_ESS ? 'wolfi-ess.tar' : + 'docker.tar'))))) final String artifactName = "elasticsearch${arch}${base.suffix}_test" final String exportTaskName = taskName("export", architecture, base, 'DockerImage') @@ -541,7 +581,10 @@ subprojects { Project subProject -> tarFile, "elasticsearch${base.suffix}:${architecture.classifier}" dependsOn(parent.path + ":" + buildTaskName) - Provider serviceProvider = GradleUtils.getBuildService(project.gradle.sharedServices, DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME) + Provider serviceProvider = GradleUtils.getBuildService( + project.gradle.sharedServices, + DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME + ) onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) } } diff --git a/distribution/docker/src/docker/Dockerfile.cloud-ess b/distribution/docker/src/docker/Dockerfile.cloud-ess deleted file mode 100644 index f82752d67a284..0000000000000 --- a/distribution/docker/src/docker/Dockerfile.cloud-ess +++ /dev/null @@ -1,13 +0,0 @@ -FROM ${base_image} AS builder - -USER root - -COPY plugins/*.zip /opt/plugins/archive/ - -RUN chown root.root /opt/plugins/archive/* -RUN chmod 0444 /opt/plugins/archive/* - -FROM ${base_image} - -COPY --from=builder /opt/plugins /opt/plugins -ENV ES_PLUGIN_ARCHIVE_DIR /opt/plugins/archive diff --git a/distribution/docker/src/docker/Dockerfile.ess b/distribution/docker/src/docker/Dockerfile.ess new file mode 100644 index 0000000000000..4a230bb562786 --- /dev/null +++ b/distribution/docker/src/docker/Dockerfile.ess @@ -0,0 +1,51 @@ +FROM ${base_image} AS builder + +USER root + +<% if (docker_base == "wolfi_ess") { %> + # Add plugins infrastructure + RUN mkdir -p /opt/plugins/archive + RUN chmod -R 0555 /opt/plugins + + COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/ + RUN set -eux ; \\ + for beat in filebeat metricbeat ; do \\ + if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\ + echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\ + exit 1 ; \\ + fi ; \\ + if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\ + echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\ + exit 1 ; \\ + fi ; \\ + mkdir -p /opt/\$beat ; \\ + tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\ + done +<% } %> + +COPY plugins/*.zip /opt/plugins/archive/ + +RUN chown root.root /opt/plugins/archive/* +RUN chmod 0444 /opt/plugins/archive/* + +FROM ${base_image} +<% if (docker_base == "wolfi_ess") { %> +USER root + +RUN <%= retry.loop("apk", "export DEBIAN_FRONTEND=noninteractive && apk update && apk update && apk add --no-cache wget") %> + +# tweak entry point for ESS specific wolfi image +ENTRYPOINT ["/sbin/tini", "--"] +CMD ["/app/elasticsearch.sh"] +# Generate a stub command that will be overwritten at runtime +RUN mkdir /app && \\ + echo -e '#!/bin/bash\\nexec /usr/local/bin/docker-entrypoint.sh eswrapper' > /app/elasticsearch.sh && \\ + chmod 0555 /app/elasticsearch.sh + +COPY --from=builder --chown=0:0 /opt /opt +USER 1000:0 +<% } else { %> +COPY --from=builder /opt/plugins /opt/plugins +<% } %> + +ENV ES_PLUGIN_ARCHIVE_DIR /opt/plugins/archive diff --git a/distribution/docker/wolfi-ess-docker-aarch64-export/build.gradle b/distribution/docker/wolfi-ess-docker-aarch64-export/build.gradle new file mode 100644 index 0000000000000..537b5a093683e --- /dev/null +++ b/distribution/docker/wolfi-ess-docker-aarch64-export/build.gradle @@ -0,0 +1,2 @@ +// This file is intentionally blank. All configuration of the +// export is done in the parent project. diff --git a/distribution/docker/wolfi-ess-docker-export/build.gradle b/distribution/docker/wolfi-ess-docker-export/build.gradle new file mode 100644 index 0000000000000..537b5a093683e --- /dev/null +++ b/distribution/docker/wolfi-ess-docker-export/build.gradle @@ -0,0 +1,2 @@ +// This file is intentionally blank. All configuration of the +// export is done in the parent project. diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index f588b78c78cc8..2a3e0c16fdc2f 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -170,7 +170,9 @@ public void test012SecurityCanBeDisabled() throws Exception { public void test020PluginsListWithNoPlugins() { assumeTrue( "Only applies to non-Cloud images", - distribution.packaging != Packaging.DOCKER_CLOUD && distribution().packaging != Packaging.DOCKER_CLOUD_ESS + distribution.packaging != Packaging.DOCKER_CLOUD + && distribution().packaging != Packaging.DOCKER_CLOUD_ESS + && distribution().packaging != Packaging.DOCKER_WOLFI_ESS ); final Installation.Executables bin = installation.executables(); @@ -201,7 +203,10 @@ public void test021InstallPlugin() { * Checks that ESS images can install plugins from the local archive. */ public void test022InstallPluginsFromLocalArchive() { - assumeTrue("Only ESS images have a local archive", distribution().packaging == Packaging.DOCKER_CLOUD_ESS); + assumeTrue( + "Only ESS images have a local archive", + distribution().packaging == Packaging.DOCKER_CLOUD_ESS || distribution().packaging == Packaging.DOCKER_WOLFI_ESS + ); final String plugin = "analysis-icu"; final Installation.Executables bin = installation.executables(); @@ -254,7 +259,10 @@ public void test023InstallPluginUsingConfigFile() { * Checks that ESS images can manage plugins from the local archive by deploying a plugins config file. */ public void test024InstallPluginFromArchiveUsingConfigFile() { - assumeTrue("Only ESS image has a plugin archive", distribution().packaging == Packaging.DOCKER_CLOUD_ESS); + assumeTrue( + "Only ESS image has a plugin archive", + distribution().packaging == Packaging.DOCKER_CLOUD_ESS || distribution().packaging == Packaging.DOCKER_WOLFI_ESS + ); final String filename = "elasticsearch-plugins.yml"; append(tempDir.resolve(filename), """ @@ -386,7 +394,7 @@ public void test040JavaUsesTheOsProvidedKeystore() { if (distribution.packaging == Packaging.DOCKER_UBI || distribution.packaging == Packaging.DOCKER_IRON_BANK) { // In these images, the `cacerts` file ought to be a symlink here assertThat(path, equalTo("/etc/pki/ca-trust/extracted/java/cacerts")); - } else if (distribution.packaging == Packaging.DOCKER_WOLFI) { + } else if (distribution.packaging == Packaging.DOCKER_WOLFI || distribution.packaging == Packaging.DOCKER_WOLFI_ESS) { // In these images, the `cacerts` file ought to be a symlink here assertThat(path, equalTo("/etc/ssl/certs/java/cacerts")); } else { @@ -1113,8 +1121,10 @@ public void test170DefaultShellIsBash() { */ public void test171AdditionalCliOptionsAreForwarded() throws Exception { assumeTrue( - "Does not apply to Cloud images, because they don't use the default entrypoint", - distribution.packaging != Packaging.DOCKER_CLOUD && distribution().packaging != Packaging.DOCKER_CLOUD_ESS + "Does not apply to Cloud and wolfi ess images, because they don't use the default entrypoint", + distribution.packaging != Packaging.DOCKER_CLOUD + && distribution().packaging != Packaging.DOCKER_CLOUD_ESS + && distribution().packaging != Packaging.DOCKER_WOLFI_ESS ); runContainer(distribution(), builder().runArgs("bin/elasticsearch", "-Ecluster.name=kimchy").envVar("ELASTIC_PASSWORD", PASSWORD)); @@ -1201,7 +1211,11 @@ public void test310IronBankImageHasNoAdditionalLabels() throws Exception { * Check that the Cloud image contains the required Beats */ public void test400CloudImageBundlesBeats() { - assumeTrue(distribution.packaging == Packaging.DOCKER_CLOUD || distribution.packaging == Packaging.DOCKER_CLOUD_ESS); + assumeTrue( + distribution.packaging == Packaging.DOCKER_CLOUD + || distribution.packaging == Packaging.DOCKER_CLOUD_ESS + || distribution.packaging == Packaging.DOCKER_WOLFI_ESS + ); final List contents = listContents("/opt"); assertThat("Expected beats in /opt", contents, hasItems("filebeat", "metricbeat")); diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java index a988a446f561f..2aff1f258ed65 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java @@ -436,7 +436,7 @@ private void verifyKeystorePermissions() { switch (distribution.packaging) { case TAR, ZIP -> assertThat(keystore, file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660)); case DEB, RPM -> assertThat(keystore, file(File, "root", "elasticsearch", p660)); - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> assertThat( + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI, DOCKER_WOLFI_ESS -> assertThat( keystore, DockerFileMatcher.file(p660) ); diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java index 644990105f60f..487a00bdecac9 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java @@ -245,7 +245,7 @@ protected static void install() throws Exception { installation = Packages.installPackage(sh, distribution); Packages.verifyPackageInstallation(installation, distribution, sh); } - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> { + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI, DOCKER_WOLFI_ESS -> { installation = Docker.runContainer(distribution); Docker.verifyContainerInstallation(installation); } @@ -338,6 +338,7 @@ public Shell.Result runElasticsearchStartCommand(String password, boolean daemon case DOCKER_CLOUD: case DOCKER_CLOUD_ESS: case DOCKER_WOLFI: + case DOCKER_WOLFI_ESS: // nothing, "installing" docker image is running it return Shell.NO_OP; default: @@ -361,6 +362,7 @@ public void stopElasticsearch() throws Exception { case DOCKER_CLOUD: case DOCKER_CLOUD_ESS: case DOCKER_WOLFI: + case DOCKER_WOLFI_ESS: // nothing, "installing" docker image is running it break; default: @@ -373,7 +375,8 @@ public void awaitElasticsearchStartup(Shell.Result result) throws Exception { switch (distribution.packaging) { case TAR, ZIP -> Archives.assertElasticsearchStarted(installation); case DEB, RPM -> Packages.assertElasticsearchStarted(sh, installation); - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> Docker.waitForElasticsearchToStart(); + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI, DOCKER_WOLFI_ESS -> Docker + .waitForElasticsearchToStart(); default -> throw new IllegalStateException("Unknown Elasticsearch packaging type."); } } diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java index 05cef4a0818ba..d63d956dc5199 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java @@ -39,6 +39,8 @@ public Distribution(Path path) { this.packaging = Packaging.DOCKER_CLOUD_ESS; } else if (filename.endsWith(".wolfi.tar")) { this.packaging = Packaging.DOCKER_WOLFI; + } else if (filename.endsWith(".wolfi-ess.tar")) { + this.packaging = Packaging.DOCKER_WOLFI_ESS; } else { int lastDot = filename.lastIndexOf('.'); this.packaging = Packaging.valueOf(filename.substring(lastDot + 1).toUpperCase(Locale.ROOT)); @@ -63,7 +65,7 @@ public boolean isPackage() { */ public boolean isDocker() { return switch (packaging) { - case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> true; + case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI, DOCKER_WOLFI_ESS -> true; default -> false; }; } @@ -79,7 +81,8 @@ public enum Packaging { DOCKER_IRON_BANK(".ironbank.tar", Platforms.isDocker()), DOCKER_CLOUD(".cloud.tar", Platforms.isDocker()), DOCKER_CLOUD_ESS(".cloud-ess.tar", Platforms.isDocker()), - DOCKER_WOLFI(".wolfi.tar", Platforms.isDocker()); + DOCKER_WOLFI(".wolfi.tar", Platforms.isDocker()), + DOCKER_WOLFI_ESS(".wolfi-ess.tar", Platforms.isDocker()); /** The extension of this distribution's file */ public final String extension; diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java index c38eaa58f0552..6f7827663d46c 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java @@ -532,7 +532,9 @@ public static void verifyContainerInstallation(Installation es) { ) ); - if (es.distribution.packaging == Packaging.DOCKER_CLOUD || es.distribution.packaging == Packaging.DOCKER_CLOUD_ESS) { + if (es.distribution.packaging == Packaging.DOCKER_CLOUD + || es.distribution.packaging == Packaging.DOCKER_CLOUD_ESS + || es.distribution.packaging == Packaging.DOCKER_WOLFI_ESS) { verifyCloudContainerInstallation(es); } } @@ -541,7 +543,7 @@ private static void verifyCloudContainerInstallation(Installation es) { final String pluginArchive = "/opt/plugins/archive"; final List plugins = listContents(pluginArchive); - if (es.distribution.packaging == Packaging.DOCKER_CLOUD_ESS) { + if (es.distribution.packaging == Packaging.DOCKER_CLOUD_ESS || es.distribution.packaging == Packaging.DOCKER_WOLFI_ESS) { assertThat("ESS image should come with plugins in " + pluginArchive, plugins, not(empty())); final List repositoryPlugins = plugins.stream() diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java index e562e7591564e..a1529de825804 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java @@ -168,6 +168,7 @@ public static String getImageName(Distribution distribution) { case DOCKER_CLOUD -> "-cloud"; case DOCKER_CLOUD_ESS -> "-cloud-ess"; case DOCKER_WOLFI -> "-wolfi"; + case DOCKER_WOLFI_ESS -> "-wolfi-ess"; default -> throw new IllegalStateException("Unexpected distribution packaging type: " + distribution.packaging); }; diff --git a/settings.gradle b/settings.gradle index 2926a9a303375..6767ce4a3e3c8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -75,6 +75,8 @@ List projects = [ 'distribution:docker:ubi-docker-export', 'distribution:docker:wolfi-docker-aarch64-export', 'distribution:docker:wolfi-docker-export', + 'distribution:docker:wolfi-ess-docker-aarch64-export', + 'distribution:docker:wolfi-ess-docker-export', 'distribution:packages:aarch64-deb', 'distribution:packages:deb', 'distribution:packages:aarch64-rpm', From 4438e4955093bb8a5cf4b7712b16c6d245618b74 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Fri, 4 Oct 2024 11:25:04 +0200 Subject: [PATCH 074/194] Add attribute type conflict handling test (#114100) --- .../rest-api-spec/test/20_traces_tests.yml | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml index abdb8d49d774c..6e51f3f91ddb5 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml @@ -92,3 +92,28 @@ setup: fields: ["data_stream.type"] - length: { hits.hits: 1 } - match: { hits.hits.0.fields.data_stream\.type: ["traces"] } +--- +Conflicting attribute types: + - do: + bulk: + index: traces-generic.otel-default + refresh: true + body: + - create: {} + - "@timestamp": 2024-10-04T00:00:00 + attributes: + http.status_code: 200 + - create: {} + - "@timestamp": 2024-10-04T01:00:00 + attributes: + http.status_code: "foo" + - is_false: errors + - do: + search: + index: traces-generic.otel-default + body: + fields: ["*"] + "sort" : [ "@timestamp" ] + - length: { hits.hits: 2 } + - match: { hits.hits.0.fields.attributes\.http\.status_code: [200] } + - match: { hits.hits.1._ignored: ["attributes.http.status_code"] } From 4d5906422b6696f1e083fae716ac2b1486de39ea Mon Sep 17 00:00:00 2001 From: David Kyle Date: Fri, 4 Oct 2024 10:27:10 +0100 Subject: [PATCH 075/194] [ML] Default inference endpoint for ELSER (#113873) Adds a default configuration for the ELSER model. The config uses adaptive allocations to automatically scale. Min number of allocations is set to 1 for this PR, a follow up with change that to 0 and enable scale from 0. This end point is always visible in the GET API. ``` GET _inference { "endpoints": [ { "inference_id": ".elser-2", "task_type": "sparse_embedding", "service": "elser", "service_settings": { "num_threads": 1, "model_id": ".elser_model_2", "adaptive_allocations": { "enabled": true, "min_number_of_allocations": 1, "max_number_of_allocations": 8 } }, "task_settings": {} } ] } ``` The default configuration can be used against without any prior setup. If the model is not downloaded it is automatically downloaded. If it is not deployed it is deployed ``` POST _inference/.elser-2 { "input": "Automagically deploy and infer" } ... { "sparse_embedding": [ { "is_truncated": false, "embedding": { "##fer": 2.2107008, "deployment": 2.1624098, "deploy": 2.144009, "auto": 1.9384763, ``` ### Follow up tasks - [ ] Add default config for the E5 text embedding model - [ ] Select platform specific version - [ ] Scale from 0 - [ ] Chunking settings - What happens when the end point is deleted, can it be deleted? - Can the default config be modified - chunking settings for example? Probably not --- docs/changelog/113873.yaml | 5 + docs/reference/rest-api/usage.asciidoc | 7 +- .../inference/InferenceService.java | 9 + .../inference/UnparsedModel.java | 24 ++ .../test/cluster/FeatureFlag.java | 3 +- x-pack/plugin/build.gradle | 1 + .../StartTrainedModelDeploymentAction.java | 4 +- .../AdaptiveAllocationsSettings.java | 4 +- .../AdaptiveAllocationSettingsTests.java | 2 +- .../xpack/inference/CustomElandModelIT.java | 2 +- .../xpack/inference/DefaultElserIT.java | 70 ++++++ .../inference/InferenceBaseRestTest.java | 21 +- .../xpack/inference/InferenceCrudIT.java | 8 +- .../MockDenseInferenceServiceIT.java | 11 +- .../MockSparseInferenceServiceIT.java | 15 +- .../xpack/inference/TextEmbeddingCrudIT.java | 4 +- .../integration/ModelRegistryIT.java | 215 +++++++++++++++++- .../inference/DefaultElserFeatureFlag.java | 21 ++ .../xpack/inference/InferencePlugin.java | 3 + ...ransportDeleteInferenceEndpointAction.java | 3 +- .../TransportGetInferenceModelAction.java | 3 +- .../action/TransportInferenceAction.java | 41 ++-- .../ShardBulkInferenceActionFilter.java | 5 +- .../inference/registry/ModelRegistry.java | 161 ++++++++++--- .../BaseElasticsearchInternalService.java | 28 ++- .../ElasticsearchInternalService.java | 106 +++++++-- .../ShardBulkInferenceActionFilterTests.java | 6 +- .../registry/ModelRegistryTests.java | 90 +++++++- .../test/inference/inference_crud.yml | 9 +- 29 files changed, 745 insertions(+), 136 deletions(-) create mode 100644 docs/changelog/113873.yaml create mode 100644 server/src/main/java/org/elasticsearch/inference/UnparsedModel.java create mode 100644 x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultElserIT.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/DefaultElserFeatureFlag.java diff --git a/docs/changelog/113873.yaml b/docs/changelog/113873.yaml new file mode 100644 index 0000000000000..ac52aaf94d518 --- /dev/null +++ b/docs/changelog/113873.yaml @@ -0,0 +1,5 @@ +pr: 113873 +summary: Default inference endpoint for ELSER +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc index 4a8895807f2fa..4dcf0d328e4f1 100644 --- a/docs/reference/rest-api/usage.asciidoc +++ b/docs/reference/rest-api/usage.asciidoc @@ -206,7 +206,12 @@ GET /_xpack/usage "inference": { "available" : true, "enabled" : true, - "models" : [] + "models" : [{ + "service": "elasticsearch", + "task_type": "SPARSE_EMBEDDING", + "count": 1 + } + ] }, "logstash" : { "available" : true, diff --git a/server/src/main/java/org/elasticsearch/inference/InferenceService.java b/server/src/main/java/org/elasticsearch/inference/InferenceService.java index aba644b392cec..cbbfef2cc65fa 100644 --- a/server/src/main/java/org/elasticsearch/inference/InferenceService.java +++ b/server/src/main/java/org/elasticsearch/inference/InferenceService.java @@ -191,4 +191,13 @@ default Set supportedStreamingTasks() { default boolean canStream(TaskType taskType) { return supportedStreamingTasks().contains(taskType); } + + /** + * A service can define default configurations that can be + * used out of the box without creating an endpoint first. + * @return Default configurations provided by this service + */ + default List defaultConfigs() { + return List.of(); + } } diff --git a/server/src/main/java/org/elasticsearch/inference/UnparsedModel.java b/server/src/main/java/org/elasticsearch/inference/UnparsedModel.java new file mode 100644 index 0000000000000..30a7c6aa2bf9c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/inference/UnparsedModel.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.inference; + +import java.util.Map; + +/** + * Semi parsed model where inference entity id, task type and service + * are known but the settings are not parsed. + */ +public record UnparsedModel( + String inferenceEntityId, + TaskType taskType, + String service, + Map settings, + Map secrets +) {} diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java index 7df791bf11559..5adf01a2a0e7d 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java @@ -19,7 +19,8 @@ public enum FeatureFlag { TIME_SERIES_MODE("es.index_mode_feature_flag_registered=true", Version.fromString("8.0.0"), null), FAILURE_STORE_ENABLED("es.failure_store_feature_flag_enabled=true", Version.fromString("8.12.0"), null), CHUNKING_SETTINGS_ENABLED("es.inference_chunking_settings_feature_flag_enabled=true", Version.fromString("8.16.0"), null), - INFERENCE_SCALE_TO_ZERO("es.inference_scale_to_zero_feature_flag_enabled=true", Version.fromString("8.16.0"), null); + INFERENCE_SCALE_TO_ZERO("es.inference_scale_to_zero_feature_flag_enabled=true", Version.fromString("8.16.0"), null), + INFERENCE_DEFAULT_ELSER("es.inference_default_elser_feature_flag_enabled=true", Version.fromString("8.16.0"), null); public final String systemProperty; public final Version from; diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index eb0796672a174..158cccb1b6ea2 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -83,5 +83,6 @@ tasks.named("precommit").configure { tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("security/10_forbidden/Test bulk response with invalid credentials", "warning does not exist for compatibility") task.skipTest("wildcard/30_ignore_above_synthetic_source/wildcard field type ignore_above", "Temporary until backported") + task.skipTest("inference/inference_crud/Test get all", "Assertions on number of inference models break due to default configs") }) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java index 4a570bfde99a4..34ebdcb7f9f9f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java @@ -237,7 +237,9 @@ public int computeNumberOfAllocations() { if (numberOfAllocations != null) { return numberOfAllocations; } else { - if (adaptiveAllocationsSettings == null || adaptiveAllocationsSettings.getMinNumberOfAllocations() == null) { + if (adaptiveAllocationsSettings == null + || adaptiveAllocationsSettings.getMinNumberOfAllocations() == null + || adaptiveAllocationsSettings.getMinNumberOfAllocations() == 0) { return DEFAULT_NUM_ALLOCATIONS; } else { return adaptiveAllocationsSettings.getMinNumberOfAllocations(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AdaptiveAllocationsSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AdaptiveAllocationsSettings.java index 19af6a3a4ef4c..d4eace8e96621 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AdaptiveAllocationsSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/assignment/AdaptiveAllocationsSettings.java @@ -147,8 +147,8 @@ public AdaptiveAllocationsSettings merge(AdaptiveAllocationsSettings updates) { public ActionRequestValidationException validate() { ActionRequestValidationException validationException = new ActionRequestValidationException(); boolean hasMinNumberOfAllocations = (minNumberOfAllocations != null && minNumberOfAllocations != -1); - if (hasMinNumberOfAllocations && minNumberOfAllocations < 1) { - validationException.addValidationError("[" + MIN_NUMBER_OF_ALLOCATIONS + "] must be a positive integer or null"); + if (hasMinNumberOfAllocations && minNumberOfAllocations < 0) { + validationException.addValidationError("[" + MIN_NUMBER_OF_ALLOCATIONS + "] must be a non-negative integer or null"); } boolean hasMaxNumberOfAllocations = (maxNumberOfAllocations != null && maxNumberOfAllocations != -1); if (hasMaxNumberOfAllocations && maxNumberOfAllocations < 1) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AdaptiveAllocationSettingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AdaptiveAllocationSettingsTests.java index c86648f10f08b..d59fbb2a24ee0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AdaptiveAllocationSettingsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/assignment/AdaptiveAllocationSettingsTests.java @@ -17,7 +17,7 @@ public class AdaptiveAllocationSettingsTests extends AbstractWireSerializingTest public static AdaptiveAllocationsSettings testInstance() { return new AdaptiveAllocationsSettings( randomBoolean() ? null : randomBoolean(), - randomBoolean() ? null : randomIntBetween(1, 2), + randomBoolean() ? null : randomIntBetween(0, 2), randomBoolean() ? null : randomIntBetween(2, 4) ); } diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CustomElandModelIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CustomElandModelIT.java index 65b7a138e7e1e..c05d08fa33692 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CustomElandModelIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/CustomElandModelIT.java @@ -85,7 +85,7 @@ public void testSparse() throws IOException { var inferenceId = "sparse-inf"; putModel(inferenceId, inferenceConfig, TaskType.SPARSE_EMBEDDING); - var results = inferOnMockService(inferenceId, List.of("washing", "machine")); + var results = infer(inferenceId, List.of("washing", "machine")); deleteModel(inferenceId); assertNotNull(results.get("sparse_embedding")); } diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultElserIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultElserIT.java new file mode 100644 index 0000000000000..5d84aad4b7344 --- /dev/null +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/DefaultElserIT.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference; + +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.oneOf; + +public class DefaultElserIT extends InferenceBaseRestTest { + + private TestThreadPool threadPool; + + @Before + public void createThreadPool() { + threadPool = new TestThreadPool(DefaultElserIT.class.getSimpleName()); + } + + @After + public void tearDown() throws Exception { + threadPool.close(); + super.tearDown(); + } + + @SuppressWarnings("unchecked") + public void testInferCreatesDefaultElser() throws IOException { + assumeTrue("Default config requires a feature flag", DefaultElserFeatureFlag.isEnabled()); + var model = getModel(ElasticsearchInternalService.DEFAULT_ELSER_ID); + assertDefaultElserConfig(model); + + var inputs = List.of("Hello World", "Goodnight moon"); + var queryParams = Map.of("timeout", "120s"); + var results = infer(ElasticsearchInternalService.DEFAULT_ELSER_ID, TaskType.SPARSE_EMBEDDING, inputs, queryParams); + var embeddings = (List>) results.get("sparse_embedding"); + assertThat(results.toString(), embeddings, hasSize(2)); + } + + @SuppressWarnings("unchecked") + private static void assertDefaultElserConfig(Map modelConfig) { + assertEquals(modelConfig.toString(), ElasticsearchInternalService.DEFAULT_ELSER_ID, modelConfig.get("inference_id")); + assertEquals(modelConfig.toString(), ElasticsearchInternalService.NAME, modelConfig.get("service")); + assertEquals(modelConfig.toString(), TaskType.SPARSE_EMBEDDING.toString(), modelConfig.get("task_type")); + + var serviceSettings = (Map) modelConfig.get("service_settings"); + assertThat(modelConfig.toString(), serviceSettings.get("model_id"), is(oneOf(".elser_model_2", ".elser_model_2_linux-x86_64"))); + assertEquals(modelConfig.toString(), 1, serviceSettings.get("num_threads")); + + var adaptiveAllocations = (Map) serviceSettings.get("adaptive_allocations"); + assertThat( + modelConfig.toString(), + adaptiveAllocations, + Matchers.is(Map.of("enabled", true, "min_number_of_allocations", 1, "max_number_of_allocations", 8)) + ); + } +} diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java index 3fa6159661b7e..f82b6f155c0a0 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBaseRestTest.java @@ -270,7 +270,7 @@ protected Map deployE5TrainedModels() throws IOException { @SuppressWarnings("unchecked") protected Map getModel(String modelId) throws IOException { - var endpoint = Strings.format("_inference/%s", modelId); + var endpoint = Strings.format("_inference/%s?error_trace", modelId); return ((List>) getInternal(endpoint).get("endpoints")).get(0); } @@ -293,9 +293,9 @@ private Map getInternal(String endpoint) throws IOException { return entityAsMap(response); } - protected Map inferOnMockService(String modelId, List input) throws IOException { + protected Map infer(String modelId, List input) throws IOException { var endpoint = Strings.format("_inference/%s", modelId); - return inferOnMockServiceInternal(endpoint, input); + return inferInternal(endpoint, input, Map.of()); } protected Deque streamInferOnMockService(String modelId, TaskType taskType, List input) throws Exception { @@ -324,14 +324,23 @@ public void onFailure(Exception exception) { return responseConsumer.events(); } - protected Map inferOnMockService(String modelId, TaskType taskType, List input) throws IOException { + protected Map infer(String modelId, TaskType taskType, List input) throws IOException { var endpoint = Strings.format("_inference/%s/%s", taskType, modelId); - return inferOnMockServiceInternal(endpoint, input); + return inferInternal(endpoint, input, Map.of()); + } + + protected Map infer(String modelId, TaskType taskType, List input, Map queryParameters) + throws IOException { + var endpoint = Strings.format("_inference/%s/%s?error_trace", taskType, modelId); + return inferInternal(endpoint, input, queryParameters); } - private Map inferOnMockServiceInternal(String endpoint, List input) throws IOException { + private Map inferInternal(String endpoint, List input, Map queryParameters) throws IOException { var request = new Request("POST", endpoint); request.setJsonEntity(jsonBody(input)); + if (queryParameters.isEmpty() == false) { + request.addParameters(queryParameters); + } var response = client().performRequest(request); assertOkOrCreated(response); return entityAsMap(response); diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java index 92affbc043669..5a84fd8985504 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java @@ -38,10 +38,12 @@ public void testGet() throws IOException { } var getAllModels = getAllModels(); - assertThat(getAllModels, hasSize(9)); + int numModels = DefaultElserFeatureFlag.isEnabled() ? 10 : 9; + assertThat(getAllModels, hasSize(numModels)); var getSparseModels = getModels("_all", TaskType.SPARSE_EMBEDDING); - assertThat(getSparseModels, hasSize(5)); + int numSparseModels = DefaultElserFeatureFlag.isEnabled() ? 6 : 5; + assertThat(getSparseModels, hasSize(numSparseModels)); for (var sparseModel : getSparseModels) { assertEquals("sparse_embedding", sparseModel.get("task_type")); } @@ -99,7 +101,7 @@ public void testApisWithoutTaskType() throws IOException { assertEquals(modelId, singleModel.get("inference_id")); assertEquals(TaskType.SPARSE_EMBEDDING.toString(), singleModel.get("task_type")); - var inference = inferOnMockService(modelId, List.of(randomAlphaOfLength(10))); + var inference = infer(modelId, List.of(randomAlphaOfLength(10))); assertNonEmptyInferenceResults(inference, 1, TaskType.SPARSE_EMBEDDING); deleteModel(modelId); } diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/MockDenseInferenceServiceIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/MockDenseInferenceServiceIT.java index 5f6bad5687407..1077bfec8bbbd 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/MockDenseInferenceServiceIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/MockDenseInferenceServiceIT.java @@ -28,15 +28,12 @@ public void testMockService() throws IOException { } List input = List.of(randomAlphaOfLength(10)); - var inference = inferOnMockService(inferenceEntityId, input); + var inference = infer(inferenceEntityId, input); assertNonEmptyInferenceResults(inference, 1, TaskType.TEXT_EMBEDDING); // Same input should return the same result - assertEquals(inference, inferOnMockService(inferenceEntityId, input)); + assertEquals(inference, infer(inferenceEntityId, input)); // Different input values should not - assertNotEquals( - inference, - inferOnMockService(inferenceEntityId, randomValueOtherThan(input, () -> List.of(randomAlphaOfLength(10)))) - ); + assertNotEquals(inference, infer(inferenceEntityId, randomValueOtherThan(input, () -> List.of(randomAlphaOfLength(10))))); } public void testMockServiceWithMultipleInputs() throws IOException { @@ -44,7 +41,7 @@ public void testMockServiceWithMultipleInputs() throws IOException { putModel(inferenceEntityId, mockDenseServiceModelConfig(), TaskType.TEXT_EMBEDDING); // The response is randomly generated, the input can be anything - var inference = inferOnMockService( + var inference = infer( inferenceEntityId, TaskType.TEXT_EMBEDDING, List.of(randomAlphaOfLength(5), randomAlphaOfLength(10), randomAlphaOfLength(15)) diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/MockSparseInferenceServiceIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/MockSparseInferenceServiceIT.java index 24ba2708f5de4..9a17d8edc0768 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/MockSparseInferenceServiceIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/MockSparseInferenceServiceIT.java @@ -30,15 +30,12 @@ public void testMockService() throws IOException { } List input = List.of(randomAlphaOfLength(10)); - var inference = inferOnMockService(inferenceEntityId, input); + var inference = infer(inferenceEntityId, input); assertNonEmptyInferenceResults(inference, 1, TaskType.SPARSE_EMBEDDING); // Same input should return the same result - assertEquals(inference, inferOnMockService(inferenceEntityId, input)); + assertEquals(inference, infer(inferenceEntityId, input)); // Different input values should not - assertNotEquals( - inference, - inferOnMockService(inferenceEntityId, randomValueOtherThan(input, () -> List.of(randomAlphaOfLength(10)))) - ); + assertNotEquals(inference, infer(inferenceEntityId, randomValueOtherThan(input, () -> List.of(randomAlphaOfLength(10))))); } public void testMockServiceWithMultipleInputs() throws IOException { @@ -46,7 +43,7 @@ public void testMockServiceWithMultipleInputs() throws IOException { putModel(inferenceEntityId, mockSparseServiceModelConfig(), TaskType.SPARSE_EMBEDDING); // The response is randomly generated, the input can be anything - var inference = inferOnMockService( + var inference = infer( inferenceEntityId, TaskType.SPARSE_EMBEDDING, List.of(randomAlphaOfLength(5), randomAlphaOfLength(10), randomAlphaOfLength(15)) @@ -84,7 +81,7 @@ public void testMockService_DoesNotReturnHiddenField_InModelResponses() throws I } // The response is randomly generated, the input can be anything - var inference = inferOnMockService(inferenceEntityId, List.of(randomAlphaOfLength(10))); + var inference = infer(inferenceEntityId, List.of(randomAlphaOfLength(10))); assertNonEmptyInferenceResults(inference, 1, TaskType.SPARSE_EMBEDDING); } @@ -102,7 +99,7 @@ public void testMockService_DoesReturnHiddenField_InModelResponses() throws IOEx } // The response is randomly generated, the input can be anything - var inference = inferOnMockService(inferenceEntityId, List.of(randomAlphaOfLength(10))); + var inference = infer(inferenceEntityId, List.of(randomAlphaOfLength(10))); assertNonEmptyInferenceResults(inference, 1, TaskType.SPARSE_EMBEDDING); } } diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/TextEmbeddingCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/TextEmbeddingCrudIT.java index 01e8c30e3bf27..8d9c859f129cb 100644 --- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/TextEmbeddingCrudIT.java +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/TextEmbeddingCrudIT.java @@ -38,7 +38,7 @@ public void testPutE5Small_withPlatformAgnosticVariant() throws IOException { var models = getTrainedModel("_all"); assertThat(models.toString(), containsString("deployment_id=" + inferenceEntityId)); - Map results = inferOnMockService( + Map results = infer( inferenceEntityId, TaskType.TEXT_EMBEDDING, List.of("hello world", "this is the second document") @@ -57,7 +57,7 @@ public void testPutE5Small_withPlatformSpecificVariant() throws IOException { var models = getTrainedModel("_all"); assertThat(models.toString(), containsString("deployment_id=" + inferenceEntityId)); - Map results = inferOnMockService( + Map results = infer( inferenceEntityId, TaskType.TEXT_EMBEDDING, List.of("hello world", "this is the second document") diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java index ea8b32f36f54c..8e68ca9dfa565 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java @@ -20,6 +20,7 @@ import org.elasticsearch.inference.ServiceSettings; import org.elasticsearch.inference.TaskSettings; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.reindex.ReindexPlugin; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -38,6 +39,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; @@ -110,7 +112,7 @@ public void testGetModel() throws Exception { assertThat(putModelHolder.get(), is(true)); // now get the model - AtomicReference modelHolder = new AtomicReference<>(); + AtomicReference modelHolder = new AtomicReference<>(); blockingCall(listener -> modelRegistry.getModelWithSecrets(inferenceEntityId, listener), modelHolder, exceptionHolder); assertThat(exceptionHolder.get(), is(nullValue())); assertThat(modelHolder.get(), not(nullValue())); @@ -168,7 +170,7 @@ public void testDeleteModel() throws Exception { // get should fail deleteResponseHolder.set(false); - AtomicReference modelHolder = new AtomicReference<>(); + AtomicReference modelHolder = new AtomicReference<>(); blockingCall(listener -> modelRegistry.getModelWithSecrets("model1", listener), modelHolder, exceptionHolder); assertThat(exceptionHolder.get(), not(nullValue())); @@ -194,7 +196,7 @@ public void testGetModelsByTaskType() throws InterruptedException { } AtomicReference exceptionHolder = new AtomicReference<>(); - AtomicReference> modelHolder = new AtomicReference<>(); + AtomicReference> modelHolder = new AtomicReference<>(); blockingCall(listener -> modelRegistry.getModelsByTaskType(TaskType.SPARSE_EMBEDDING, listener), modelHolder, exceptionHolder); assertThat(modelHolder.get(), hasSize(3)); var sparseIds = sparseAndTextEmbeddingModels.stream() @@ -235,8 +237,9 @@ public void testGetAllModels() throws InterruptedException { assertNull(exceptionHolder.get()); } - AtomicReference> modelHolder = new AtomicReference<>(); + AtomicReference> modelHolder = new AtomicReference<>(); blockingCall(listener -> modelRegistry.getAllModels(listener), modelHolder, exceptionHolder); + assertNull(exceptionHolder.get()); assertThat(modelHolder.get(), hasSize(modelCount)); var getAllModels = modelHolder.get(); @@ -264,15 +267,213 @@ public void testGetModelWithSecrets() throws InterruptedException { assertThat(putModelHolder.get(), is(true)); assertNull(exceptionHolder.get()); - AtomicReference modelHolder = new AtomicReference<>(); + AtomicReference modelHolder = new AtomicReference<>(); blockingCall(listener -> modelRegistry.getModelWithSecrets(inferenceEntityId, listener), modelHolder, exceptionHolder); assertThat(modelHolder.get().secrets().keySet(), hasSize(1)); var secretSettings = (Map) modelHolder.get().secrets().get("secret_settings"); assertThat(secretSettings.get("secret"), equalTo(secret)); + assertReturnModelIsModifiable(modelHolder.get()); // get model without secrets blockingCall(listener -> modelRegistry.getModel(inferenceEntityId, listener), modelHolder, exceptionHolder); assertThat(modelHolder.get().secrets().keySet(), empty()); + assertReturnModelIsModifiable(modelHolder.get()); + } + + public void testGetAllModels_WithDefaults() throws Exception { + var service = "foo"; + var secret = "abc"; + int configuredModelCount = 10; + int defaultModelCount = 2; + int totalModelCount = 12; + + var defaultConfigs = new HashMap(); + for (int i = 0; i < defaultModelCount; i++) { + var id = "default-" + i; + defaultConfigs.put(id, createUnparsedConfig(id, randomFrom(TaskType.values()), service, secret)); + } + defaultConfigs.values().forEach(modelRegistry::addDefaultConfiguration); + + AtomicReference putModelHolder = new AtomicReference<>(); + AtomicReference exceptionHolder = new AtomicReference<>(); + + var createdModels = new HashMap(); + for (int i = 0; i < configuredModelCount; i++) { + var id = randomAlphaOfLength(5) + i; + var model = createModel(id, randomFrom(TaskType.values()), service); + createdModels.put(id, model); + blockingCall(listener -> modelRegistry.storeModel(model, listener), putModelHolder, exceptionHolder); + assertThat(putModelHolder.get(), is(true)); + assertNull(exceptionHolder.get()); + } + + AtomicReference> modelHolder = new AtomicReference<>(); + blockingCall(listener -> modelRegistry.getAllModels(listener), modelHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertThat(modelHolder.get(), hasSize(totalModelCount)); + var getAllModels = modelHolder.get(); + assertReturnModelIsModifiable(modelHolder.get().get(0)); + + // sort in the same order as the returned models + var ids = new ArrayList<>(defaultConfigs.keySet().stream().toList()); + ids.addAll(createdModels.keySet().stream().toList()); + ids.sort(String::compareTo); + for (int i = 0; i < totalModelCount; i++) { + var id = ids.get(i); + assertEquals(id, getAllModels.get(i).inferenceEntityId()); + if (id.startsWith("default")) { + assertEquals(defaultConfigs.get(id).taskType(), getAllModels.get(i).taskType()); + assertEquals(defaultConfigs.get(id).service(), getAllModels.get(i).service()); + } else { + assertEquals(createdModels.get(id).getTaskType(), getAllModels.get(i).taskType()); + assertEquals(createdModels.get(id).getConfigurations().getService(), getAllModels.get(i).service()); + } + } + } + + public void testGetAllModels_OnlyDefaults() throws Exception { + var service = "foo"; + var secret = "abc"; + int defaultModelCount = 2; + + var defaultConfigs = new HashMap(); + for (int i = 0; i < defaultModelCount; i++) { + var id = "default-" + i; + defaultConfigs.put(id, createUnparsedConfig(id, randomFrom(TaskType.values()), service, secret)); + } + defaultConfigs.values().forEach(modelRegistry::addDefaultConfiguration); + + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference> modelHolder = new AtomicReference<>(); + blockingCall(listener -> modelRegistry.getAllModels(listener), modelHolder, exceptionHolder); + assertNull(exceptionHolder.get()); + assertThat(modelHolder.get(), hasSize(2)); + var getAllModels = modelHolder.get(); + assertReturnModelIsModifiable(modelHolder.get().get(0)); + + // sort in the same order as the returned models + var ids = new ArrayList<>(defaultConfigs.keySet().stream().toList()); + ids.sort(String::compareTo); + for (int i = 0; i < defaultModelCount; i++) { + var id = ids.get(i); + assertEquals(id, getAllModels.get(i).inferenceEntityId()); + assertEquals(defaultConfigs.get(id).taskType(), getAllModels.get(i).taskType()); + assertEquals(defaultConfigs.get(id).service(), getAllModels.get(i).service()); + } + } + + public void testGet_WithDefaults() throws InterruptedException { + var service = "foo"; + var secret = "abc"; + + var defaultSparse = createUnparsedConfig("default-sparse", TaskType.SPARSE_EMBEDDING, service, secret); + var defaultText = createUnparsedConfig("default-text", TaskType.TEXT_EMBEDDING, service, secret); + + modelRegistry.addDefaultConfiguration(defaultSparse); + modelRegistry.addDefaultConfiguration(defaultText); + + AtomicReference putModelHolder = new AtomicReference<>(); + AtomicReference exceptionHolder = new AtomicReference<>(); + + var configured1 = createModel(randomAlphaOfLength(5) + 1, randomFrom(TaskType.values()), service); + var configured2 = createModel(randomAlphaOfLength(5) + 1, randomFrom(TaskType.values()), service); + blockingCall(listener -> modelRegistry.storeModel(configured1, listener), putModelHolder, exceptionHolder); + assertThat(putModelHolder.get(), is(true)); + blockingCall(listener -> modelRegistry.storeModel(configured2, listener), putModelHolder, exceptionHolder); + assertThat(putModelHolder.get(), is(true)); + assertNull(exceptionHolder.get()); + + AtomicReference modelHolder = new AtomicReference<>(); + blockingCall(listener -> modelRegistry.getModel("default-sparse", listener), modelHolder, exceptionHolder); + assertEquals("default-sparse", modelHolder.get().inferenceEntityId()); + assertEquals(TaskType.SPARSE_EMBEDDING, modelHolder.get().taskType()); + assertReturnModelIsModifiable(modelHolder.get()); + + blockingCall(listener -> modelRegistry.getModel("default-text", listener), modelHolder, exceptionHolder); + assertEquals("default-text", modelHolder.get().inferenceEntityId()); + assertEquals(TaskType.TEXT_EMBEDDING, modelHolder.get().taskType()); + + blockingCall(listener -> modelRegistry.getModel(configured1.getInferenceEntityId(), listener), modelHolder, exceptionHolder); + assertEquals(configured1.getInferenceEntityId(), modelHolder.get().inferenceEntityId()); + assertEquals(configured1.getTaskType(), modelHolder.get().taskType()); + } + + public void testGetByTaskType_WithDefaults() throws Exception { + var service = "foo"; + var secret = "abc"; + + var defaultSparse = createUnparsedConfig("default-sparse", TaskType.SPARSE_EMBEDDING, service, secret); + var defaultText = createUnparsedConfig("default-text", TaskType.TEXT_EMBEDDING, service, secret); + var defaultChat = createUnparsedConfig("default-chat", TaskType.COMPLETION, service, secret); + + modelRegistry.addDefaultConfiguration(defaultSparse); + modelRegistry.addDefaultConfiguration(defaultText); + modelRegistry.addDefaultConfiguration(defaultChat); + + AtomicReference putModelHolder = new AtomicReference<>(); + AtomicReference exceptionHolder = new AtomicReference<>(); + + var configuredSparse = createModel("configured-sparse", TaskType.SPARSE_EMBEDDING, service); + var configuredText = createModel("configured-text", TaskType.TEXT_EMBEDDING, service); + var configuredRerank = createModel("configured-rerank", TaskType.RERANK, service); + blockingCall(listener -> modelRegistry.storeModel(configuredSparse, listener), putModelHolder, exceptionHolder); + assertThat(putModelHolder.get(), is(true)); + blockingCall(listener -> modelRegistry.storeModel(configuredText, listener), putModelHolder, exceptionHolder); + assertThat(putModelHolder.get(), is(true)); + blockingCall(listener -> modelRegistry.storeModel(configuredRerank, listener), putModelHolder, exceptionHolder); + assertThat(putModelHolder.get(), is(true)); + assertNull(exceptionHolder.get()); + + AtomicReference> modelHolder = new AtomicReference<>(); + blockingCall(listener -> modelRegistry.getModelsByTaskType(TaskType.SPARSE_EMBEDDING, listener), modelHolder, exceptionHolder); + if (exceptionHolder.get() != null) { + throw exceptionHolder.get(); + } + assertNull(exceptionHolder.get()); + assertThat(modelHolder.get(), hasSize(2)); + assertEquals("configured-sparse", modelHolder.get().get(0).inferenceEntityId()); + assertEquals("default-sparse", modelHolder.get().get(1).inferenceEntityId()); + + blockingCall(listener -> modelRegistry.getModelsByTaskType(TaskType.TEXT_EMBEDDING, listener), modelHolder, exceptionHolder); + assertThat(modelHolder.get(), hasSize(2)); + assertEquals("configured-text", modelHolder.get().get(0).inferenceEntityId()); + assertEquals("default-text", modelHolder.get().get(1).inferenceEntityId()); + assertReturnModelIsModifiable(modelHolder.get().get(0)); + + blockingCall(listener -> modelRegistry.getModelsByTaskType(TaskType.RERANK, listener), modelHolder, exceptionHolder); + assertThat(modelHolder.get(), hasSize(1)); + assertEquals("configured-rerank", modelHolder.get().get(0).inferenceEntityId()); + + blockingCall(listener -> modelRegistry.getModelsByTaskType(TaskType.COMPLETION, listener), modelHolder, exceptionHolder); + assertThat(modelHolder.get(), hasSize(1)); + assertEquals("default-chat", modelHolder.get().get(0).inferenceEntityId()); + assertReturnModelIsModifiable(modelHolder.get().get(0)); + } + + @SuppressWarnings("unchecked") + private void assertReturnModelIsModifiable(UnparsedModel unparsedModel) { + var settings = unparsedModel.settings(); + if (settings != null) { + var serviceSettings = (Map) settings.get("service_settings"); + if (serviceSettings != null && serviceSettings.size() > 0) { + var itr = serviceSettings.entrySet().iterator(); + itr.next(); + itr.remove(); + } + + var taskSettings = (Map) settings.get("task_settings"); + if (taskSettings != null && taskSettings.size() > 0) { + var itr = taskSettings.entrySet().iterator(); + itr.next(); + itr.remove(); + } + + if (unparsedModel.secrets() != null && unparsedModel.secrets().size() > 0) { + var itr = unparsedModel.secrets().entrySet().iterator(); + itr.next(); + itr.remove(); + } + } } private Model buildElserModelConfig(String inferenceEntityId, TaskType taskType) { @@ -327,6 +528,10 @@ public static Model createModelWithSecrets(String inferenceEntityId, TaskType ta ); } + public static UnparsedModel createUnparsedConfig(String inferenceEntityId, TaskType taskType, String service, String secret) { + return new UnparsedModel(inferenceEntityId, taskType, service, Map.of("a", "b"), Map.of("secret", secret)); + } + private static class TestModelOfAnyKind extends ModelConfigurations { record TestModelServiceSettings() implements ServiceSettings { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/DefaultElserFeatureFlag.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/DefaultElserFeatureFlag.java new file mode 100644 index 0000000000000..2a764dabd62ae --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/DefaultElserFeatureFlag.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference; + +import org.elasticsearch.common.util.FeatureFlag; + +public class DefaultElserFeatureFlag { + + private DefaultElserFeatureFlag() {} + + private static final FeatureFlag FEATURE_FLAG = new FeatureFlag("inference_default_elser"); + + public static boolean isEnabled() { + return FEATURE_FLAG.isEnabled(); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index 0ab395f4bfa39..dbb9130ab91e1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -210,6 +210,9 @@ public Collection createComponents(PluginServices services) { // reference correctly var registry = new InferenceServiceRegistry(inferenceServices, factoryContext); registry.init(services.client()); + for (var service : registry.getServices().values()) { + service.defaultConfigs().forEach(modelRegistry::addDefaultConfiguration); + } inferenceServiceRegistry.set(registry); var actionFilter = new ShardBulkInferenceActionFilter(registry, modelRegistry); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java index 3c893f8870627..829a6b6c67ff9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.inference.InferenceServiceRegistry; +import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; @@ -91,7 +92,7 @@ private void doExecuteForked( ClusterState state, ActionListener masterListener ) { - SubscribableListener.newForked(modelConfigListener -> { + SubscribableListener.newForked(modelConfigListener -> { // Get the model from the registry modelRegistry.getModel(request.getInferenceEndpointId(), modelConfigListener); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java index a1f33afa05b5c..5ee1e40869dbc 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportGetInferenceModelAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.inference.InferenceServiceRegistry; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; @@ -112,7 +113,7 @@ private void getModelsByTaskType(TaskType taskType, ActionListener unparsedModels) { + private GetInferenceModelAction.Response parseModels(List unparsedModels) { var parsedModels = new ArrayList(); for (var unparsedModel : unparsedModels) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java index d2a73b7df77c1..e046e2aad463b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.inference.InferenceService; @@ -19,6 +18,7 @@ import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; @@ -43,7 +43,6 @@ public class TransportInferenceAction extends HandledTransportAction listener) { - ActionListener getModelListener = listener.delegateFailureAndWrap((delegate, unparsedModel) -> { + ActionListener getModelListener = listener.delegateFailureAndWrap((delegate, unparsedModel) -> { var service = serviceRegistry.getService(unparsedModel.service()); if (service.isEmpty()) { - delegate.onFailure( - new ElasticsearchStatusException( - "Unknown service [{}] for model [{}]. ", - RestStatus.INTERNAL_SERVER_ERROR, - unparsedModel.service(), - unparsedModel.inferenceEntityId() - ) - ); + listener.onFailure(unknownServiceException(unparsedModel.service(), request.getInferenceEntityId())); return; } if (request.getTaskType().isAnyOrSame(unparsedModel.taskType()) == false) { // not the wildcard task type and not the model task type - delegate.onFailure( - new ElasticsearchStatusException( - "Incompatible task_type, the requested type [{}] does not match the model type [{}]", - RestStatus.BAD_REQUEST, - request.getTaskType(), - unparsedModel.taskType() - ) - ); + listener.onFailure(incompatibleTaskTypeException(request.getTaskType(), unparsedModel.taskType())); return; } @@ -98,7 +83,6 @@ protected void doExecute(Task task, InferenceAction.Request request, ActionListe unparsedModel.settings(), unparsedModel.secrets() ); - inferenceStats.incrementRequestCount(model); inferOnService(model, request, service.get(), delegate); }); @@ -112,6 +96,7 @@ private void inferOnService( ActionListener listener ) { if (request.isStreaming() == false || service.canStream(request.getTaskType())) { + inferenceStats.incrementRequestCount(model); service.infer( model, request.getQuery(), @@ -160,5 +145,19 @@ private ActionListener createListener( }); } return listener.delegateFailureAndWrap((l, inferenceResults) -> l.onResponse(new InferenceAction.Response(inferenceResults))); - }; + } + + private static ElasticsearchStatusException unknownServiceException(String service, String inferenceId) { + return new ElasticsearchStatusException("Unknown service [{}] for model [{}]. ", RestStatus.BAD_REQUEST, service, inferenceId); + } + + private static ElasticsearchStatusException incompatibleTaskTypeException(TaskType requested, TaskType expected) { + return new ElasticsearchStatusException( + "Incompatible task_type, the requested type [{}] does not match the model type [{}]", + RestStatus.BAD_REQUEST, + requested, + expected + ); + } + } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java index ade0748ef10bf..a4eb94c2674d1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java @@ -35,6 +35,7 @@ import org.elasticsearch.inference.InferenceServiceRegistry; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; +import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; @@ -211,9 +212,9 @@ private void executeShardBulkInferenceAsync( final Releasable onFinish ) { if (inferenceProvider == null) { - ActionListener modelLoadingListener = new ActionListener<>() { + ActionListener modelLoadingListener = new ActionListener<>() { @Override - public void onResponse(ModelRegistry.UnparsedModel unparsedModel) { + public void onResponse(UnparsedModel unparsedModel) { var service = inferenceServiceRegistry.getService(unparsedModel.service()); if (service.isEmpty() == false) { var provider = new InferenceProvider( diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java index a6e4fcae7169f..d756c0ef26f14 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/registry/ModelRegistry.java @@ -32,6 +32,7 @@ import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -48,6 +49,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -58,32 +61,19 @@ public class ModelRegistry { public record ModelConfigMap(Map config, Map secrets) {} - /** - * Semi parsed model where inference entity id, task type and service - * are known but the settings are not parsed. - */ - public record UnparsedModel( - String inferenceEntityId, - TaskType taskType, - String service, - Map settings, - Map secrets - ) { - - public static UnparsedModel unparsedModelFromMap(ModelConfigMap modelConfigMap) { - if (modelConfigMap.config() == null) { - throw new ElasticsearchStatusException("Missing config map", RestStatus.BAD_REQUEST); - } - String inferenceEntityId = ServiceUtils.removeStringOrThrowIfNull( - modelConfigMap.config(), - ModelConfigurations.INDEX_ONLY_ID_FIELD_NAME - ); - String service = ServiceUtils.removeStringOrThrowIfNull(modelConfigMap.config(), ModelConfigurations.SERVICE); - String taskTypeStr = ServiceUtils.removeStringOrThrowIfNull(modelConfigMap.config(), TaskType.NAME); - TaskType taskType = TaskType.fromString(taskTypeStr); - - return new UnparsedModel(inferenceEntityId, taskType, service, modelConfigMap.config(), modelConfigMap.secrets()); + public static UnparsedModel unparsedModelFromMap(ModelConfigMap modelConfigMap) { + if (modelConfigMap.config() == null) { + throw new ElasticsearchStatusException("Missing config map", RestStatus.BAD_REQUEST); } + String inferenceEntityId = ServiceUtils.removeStringOrThrowIfNull( + modelConfigMap.config(), + ModelConfigurations.INDEX_ONLY_ID_FIELD_NAME + ); + String service = ServiceUtils.removeStringOrThrowIfNull(modelConfigMap.config(), ModelConfigurations.SERVICE); + String taskTypeStr = ServiceUtils.removeStringOrThrowIfNull(modelConfigMap.config(), TaskType.NAME); + TaskType taskType = TaskType.fromString(taskTypeStr); + + return new UnparsedModel(inferenceEntityId, taskType, service, modelConfigMap.config(), modelConfigMap.secrets()); } private static final String TASK_TYPE_FIELD = "task_type"; @@ -91,9 +81,27 @@ public static UnparsedModel unparsedModelFromMap(ModelConfigMap modelConfigMap) private static final Logger logger = LogManager.getLogger(ModelRegistry.class); private final OriginSettingClient client; + private Map defaultConfigs; public ModelRegistry(Client client) { this.client = new OriginSettingClient(client, ClientHelper.INFERENCE_ORIGIN); + this.defaultConfigs = new HashMap<>(); + } + + public void addDefaultConfiguration(UnparsedModel serviceDefaultConfig) { + if (defaultConfigs.containsKey(serviceDefaultConfig.inferenceEntityId())) { + throw new IllegalStateException( + "Cannot add default endpoint to the inference endpoint registry with duplicate inference id [" + + serviceDefaultConfig.inferenceEntityId() + + "] declared by service [" + + serviceDefaultConfig.service() + + "]. The inference Id is already use by [" + + defaultConfigs.get(serviceDefaultConfig.inferenceEntityId()).service() + + "] service." + ); + } + + defaultConfigs.put(serviceDefaultConfig.inferenceEntityId(), serviceDefaultConfig); } /** @@ -102,6 +110,11 @@ public ModelRegistry(Client client) { * @param listener Model listener */ public void getModelWithSecrets(String inferenceEntityId, ActionListener listener) { + if (defaultConfigs.containsKey(inferenceEntityId)) { + listener.onResponse(deepCopyDefaultConfig(defaultConfigs.get(inferenceEntityId))); + return; + } + ActionListener searchListener = listener.delegateFailureAndWrap((delegate, searchResponse) -> { // There should be a hit for the configurations and secrets if (searchResponse.getHits().getHits().length == 0) { @@ -109,7 +122,7 @@ public void getModelWithSecrets(String inferenceEntityId, ActionListener listener) { + if (defaultConfigs.containsKey(inferenceEntityId)) { + listener.onResponse(deepCopyDefaultConfig(defaultConfigs.get(inferenceEntityId))); + return; + } + ActionListener searchListener = listener.delegateFailureAndWrap((delegate, searchResponse) -> { // There should be a hit for the configurations and secrets if (searchResponse.getHits().getHits().length == 0) { @@ -135,7 +153,7 @@ public void getModel(String inferenceEntityId, ActionListener lis return; } - var modelConfigs = parseHitsAsModels(searchResponse.getHits()).stream().map(UnparsedModel::unparsedModelFromMap).toList(); + var modelConfigs = parseHitsAsModels(searchResponse.getHits()).stream().map(ModelRegistry::unparsedModelFromMap).toList(); assert modelConfigs.size() == 1; delegate.onResponse(modelConfigs.get(0)); }); @@ -162,14 +180,29 @@ private ResourceNotFoundException inferenceNotFoundException(String inferenceEnt */ public void getModelsByTaskType(TaskType taskType, ActionListener> listener) { ActionListener searchListener = listener.delegateFailureAndWrap((delegate, searchResponse) -> { + var defaultConfigsForTaskType = defaultConfigs.values() + .stream() + .filter(m -> m.taskType() == taskType) + .map(ModelRegistry::deepCopyDefaultConfig) + .toList(); + // Not an error if no models of this task_type - if (searchResponse.getHits().getHits().length == 0) { + if (searchResponse.getHits().getHits().length == 0 && defaultConfigsForTaskType.isEmpty()) { delegate.onResponse(List.of()); return; } - var modelConfigs = parseHitsAsModels(searchResponse.getHits()).stream().map(UnparsedModel::unparsedModelFromMap).toList(); - delegate.onResponse(modelConfigs); + var modelConfigs = parseHitsAsModels(searchResponse.getHits()).stream().map(ModelRegistry::unparsedModelFromMap).toList(); + + if (defaultConfigsForTaskType.isEmpty() == false) { + var allConfigs = new ArrayList(); + allConfigs.addAll(modelConfigs); + allConfigs.addAll(defaultConfigsForTaskType); + allConfigs.sort(Comparator.comparing(UnparsedModel::inferenceEntityId)); + delegate.onResponse(allConfigs); + } else { + delegate.onResponse(modelConfigs); + } }); QueryBuilder queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(TASK_TYPE_FIELD, taskType.toString())); @@ -191,14 +224,19 @@ public void getModelsByTaskType(TaskType taskType, ActionListener> listener) { ActionListener searchListener = listener.delegateFailureAndWrap((delegate, searchResponse) -> { - // Not an error if no models of this task_type - if (searchResponse.getHits().getHits().length == 0) { + var defaults = defaultConfigs.values().stream().map(ModelRegistry::deepCopyDefaultConfig).toList(); + + if (searchResponse.getHits().getHits().length == 0 && defaults.isEmpty()) { delegate.onResponse(List.of()); return; } - var modelConfigs = parseHitsAsModels(searchResponse.getHits()).stream().map(UnparsedModel::unparsedModelFromMap).toList(); - delegate.onResponse(modelConfigs); + var foundConfigs = parseHitsAsModels(searchResponse.getHits()).stream().map(ModelRegistry::unparsedModelFromMap).toList(); + var allConfigs = new ArrayList(); + allConfigs.addAll(foundConfigs); + allConfigs.addAll(defaults); + allConfigs.sort(Comparator.comparing(UnparsedModel::inferenceEntityId)); + delegate.onResponse(allConfigs); }); // In theory the index should only contain model config documents @@ -216,7 +254,7 @@ public void getAllModels(ActionListener> listener) { client.search(modelSearch, searchListener); } - private List parseHitsAsModels(SearchHits hits) { + private ArrayList parseHitsAsModels(SearchHits hits) { var modelConfigs = new ArrayList(); for (var hit : hits) { modelConfigs.add(new ModelConfigMap(hit.getSourceAsMap(), Map.of())); @@ -393,4 +431,57 @@ private static IndexRequest createIndexRequest(String docId, String indexName, T private QueryBuilder documentIdQuery(String inferenceEntityId) { return QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds(Model.documentId(inferenceEntityId))); } + + static UnparsedModel deepCopyDefaultConfig(UnparsedModel other) { + // Because the default config uses immutable maps + return new UnparsedModel( + other.inferenceEntityId(), + other.taskType(), + other.service(), + copySettingsMap(other.settings()), + copySecretsMap(other.secrets()) + ); + } + + @SuppressWarnings("unchecked") + static Map copySettingsMap(Map other) { + var result = new HashMap(); + + var serviceSettings = (Map) other.get(ModelConfigurations.SERVICE_SETTINGS); + if (serviceSettings != null) { + var copiedServiceSettings = copyMap1LevelDeep(serviceSettings); + result.put(ModelConfigurations.SERVICE_SETTINGS, copiedServiceSettings); + } + + var taskSettings = (Map) other.get(ModelConfigurations.TASK_SETTINGS); + if (taskSettings != null) { + var copiedTaskSettings = copyMap1LevelDeep(taskSettings); + result.put(ModelConfigurations.TASK_SETTINGS, copiedTaskSettings); + } + + var chunkSettings = (Map) other.get(ModelConfigurations.CHUNKING_SETTINGS); + if (chunkSettings != null) { + var copiedChunkSettings = copyMap1LevelDeep(chunkSettings); + result.put(ModelConfigurations.CHUNKING_SETTINGS, copiedChunkSettings); + } + + return result; + } + + static Map copySecretsMap(Map other) { + return copyMap1LevelDeep(other); + } + + @SuppressWarnings("unchecked") + static Map copyMap1LevelDeep(Map other) { + var result = new HashMap(); + for (var entry : other.entrySet()) { + if (entry.getValue() instanceof Map) { + result.put(entry.getKey(), new HashMap<>((Map) entry.getValue())); + } else { + result.put(entry.getKey(), entry.getValue()); + } + } + return result; + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java index 23e806e01300a..0dd41db2f016c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.SubscribableListener; @@ -31,6 +32,7 @@ import org.elasticsearch.xpack.core.ml.inference.TrainedModelPrefixStrings; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfigUpdate; import org.elasticsearch.xpack.core.ml.utils.MlPlatformArchitecturesUtil; +import org.elasticsearch.xpack.inference.DefaultElserFeatureFlag; import org.elasticsearch.xpack.inference.InferencePlugin; import java.io.IOException; @@ -80,7 +82,6 @@ public BaseElasticsearchInternalService( @Override public void start(Model model, ActionListener finalListener) { if (model instanceof ElasticsearchInternalModel esModel) { - if (supportedTaskTypes().contains(model.getTaskType()) == false) { finalListener.onFailure( new IllegalStateException(TaskType.unsupportedTaskTypeErrorMsg(model.getConfigurations().getTaskType(), name())) @@ -149,7 +150,7 @@ public void putModel(Model model, ActionListener listener) { } } - private void putBuiltInModel(String modelId, ActionListener listener) { + protected void putBuiltInModel(String modelId, ActionListener listener) { var input = new TrainedModelInput(List.of("text_field")); // by convention text_field is used var config = TrainedModelConfig.builder().setInput(input).setModelId(modelId).validate(true).build(); PutTrainedModelAction.Request putRequest = new PutTrainedModelAction.Request(config, false, true); @@ -258,4 +259,27 @@ public static InferModelAction.Request buildInferenceRequest( request.setChunked(chunk); return request; } + + protected abstract boolean isDefaultId(String inferenceId); + + protected void maybeStartDeployment( + ElasticsearchInternalModel model, + Exception e, + InferModelAction.Request request, + ActionListener listener + ) { + if (DefaultElserFeatureFlag.isEnabled() == false) { + listener.onFailure(e); + return; + } + + if (isDefaultId(model.getInferenceEntityId()) && ExceptionsHelper.unwrapCause(e) instanceof ResourceNotFoundException) { + this.start( + model, + listener.delegateFailureAndWrap((l, started) -> { client.execute(InferModelAction.INSTANCE, request, listener); }) + ); + } else { + listener.onFailure(e); + } + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java index e274c641e30be..dd14e16412996 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java @@ -26,6 +26,7 @@ import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; @@ -73,6 +74,8 @@ public class ElasticsearchInternalService extends BaseElasticsearchInternalServi MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86 ); + public static final String DEFAULT_ELSER_ID = ".elser-2"; + private static final Logger logger = LogManager.getLogger(ElasticsearchInternalService.class); private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(ElasticsearchInternalService.class); @@ -100,6 +103,17 @@ public void parseRequestConfig( Map config, ActionListener modelListener ) { + if (inferenceEntityId.equals(DEFAULT_ELSER_ID)) { + modelListener.onFailure( + new ElasticsearchStatusException( + "[{}] is a reserved inference Id. Cannot create a new inference endpoint with a reserved Id", + RestStatus.BAD_REQUEST, + inferenceEntityId + ) + ); + return; + } + try { Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); Map taskSettingsMap = removeFromMap(config, ModelConfigurations.TASK_SETTINGS); @@ -459,20 +473,24 @@ public void infer( TimeValue timeout, ActionListener listener ) { - var taskType = model.getConfigurations().getTaskType(); - if (TaskType.TEXT_EMBEDDING.equals(taskType)) { - inferTextEmbedding(model, input, inputType, timeout, listener); - } else if (TaskType.RERANK.equals(taskType)) { - inferRerank(model, query, input, inputType, timeout, taskSettings, listener); - } else if (TaskType.SPARSE_EMBEDDING.equals(taskType)) { - inferSparseEmbedding(model, input, inputType, timeout, listener); + if (model instanceof ElasticsearchInternalModel esModel) { + var taskType = model.getConfigurations().getTaskType(); + if (TaskType.TEXT_EMBEDDING.equals(taskType)) { + inferTextEmbedding(esModel, input, inputType, timeout, listener); + } else if (TaskType.RERANK.equals(taskType)) { + inferRerank(esModel, query, input, inputType, timeout, taskSettings, listener); + } else if (TaskType.SPARSE_EMBEDDING.equals(taskType)) { + inferSparseEmbedding(esModel, input, inputType, timeout, listener); + } else { + throw new ElasticsearchStatusException(TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), RestStatus.BAD_REQUEST); + } } else { - throw new ElasticsearchStatusException(TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), RestStatus.BAD_REQUEST); + listener.onFailure(notElasticsearchModelException(model)); } } public void inferTextEmbedding( - Model model, + ElasticsearchInternalModel model, List inputs, InputType inputType, TimeValue timeout, @@ -487,17 +505,19 @@ public void inferTextEmbedding( false ); - client.execute( - InferModelAction.INSTANCE, - request, - listener.delegateFailureAndWrap( - (l, inferenceResult) -> l.onResponse(InferenceTextEmbeddingFloatResults.of(inferenceResult.getInferenceResults())) - ) + ActionListener mlResultsListener = listener.delegateFailureAndWrap( + (l, inferenceResult) -> l.onResponse(InferenceTextEmbeddingFloatResults.of(inferenceResult.getInferenceResults())) + ); + + var maybeDeployListener = mlResultsListener.delegateResponse( + (l, exception) -> maybeStartDeployment(model, exception, request, mlResultsListener) ); + + client.execute(InferModelAction.INSTANCE, request, maybeDeployListener); } public void inferSparseEmbedding( - Model model, + ElasticsearchInternalModel model, List inputs, InputType inputType, TimeValue timeout, @@ -512,17 +532,19 @@ public void inferSparseEmbedding( false ); - client.execute( - InferModelAction.INSTANCE, - request, - listener.delegateFailureAndWrap( - (l, inferenceResult) -> l.onResponse(SparseEmbeddingResults.of(inferenceResult.getInferenceResults())) - ) + ActionListener mlResultsListener = listener.delegateFailureAndWrap( + (l, inferenceResult) -> l.onResponse(SparseEmbeddingResults.of(inferenceResult.getInferenceResults())) + ); + + var maybeDeployListener = mlResultsListener.delegateResponse( + (l, exception) -> maybeStartDeployment(model, exception, request, mlResultsListener) ); + + client.execute(InferModelAction.INSTANCE, request, maybeDeployListener); } public void inferRerank( - Model model, + ElasticsearchInternalModel model, String query, List inputs, InputType inputType, @@ -671,4 +693,42 @@ private RankedDocsResults textSimilarityResultsToRankedDocs( Collections.sort(rankings); return new RankedDocsResults(rankings); } + + @Override + public List defaultConfigs() { + // TODO Chunking settings + Map elserSettings = Map.of( + ModelConfigurations.SERVICE_SETTINGS, + Map.of( + ElasticsearchInternalServiceSettings.MODEL_ID, + ElserModels.ELSER_V2_MODEL, // TODO pick model depending on platform + ElasticsearchInternalServiceSettings.NUM_THREADS, + 1, + ElasticsearchInternalServiceSettings.ADAPTIVE_ALLOCATIONS, + Map.of( + "enabled", + Boolean.TRUE, + "min_number_of_allocations", + 1, + "max_number_of_allocations", + 8 // no max? + ) + ) + ); + + return List.of( + new UnparsedModel( + DEFAULT_ELSER_ID, + TaskType.SPARSE_EMBEDDING, + NAME, + elserSettings, + Map.of() // no secrets + ) + ); + } + + @Override + protected boolean isDefaultId(String inferenceId) { + return DEFAULT_ELSER_ID.equals(inferenceId); + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java index d78ea7933e836..770e6e3cb9cf4 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.inference.InferenceServiceRegistry; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; @@ -266,12 +267,11 @@ private static ShardBulkInferenceActionFilter createFilter(ThreadPool threadPool ModelRegistry modelRegistry = mock(ModelRegistry.class); Answer unparsedModelAnswer = invocationOnMock -> { String id = (String) invocationOnMock.getArguments()[0]; - ActionListener listener = (ActionListener) invocationOnMock - .getArguments()[1]; + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[1]; var model = modelMap.get(id); if (model != null) { listener.onResponse( - new ModelRegistry.UnparsedModel( + new UnparsedModel( model.getInferenceEntityId(), model.getTaskType(), model.getServiceSettings().model(), diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java index fbd8ccd621559..75c370fd4d3fb 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/registry/ModelRegistryTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.SearchResponseUtils; @@ -38,9 +39,12 @@ import java.util.concurrent.TimeUnit; import static org.elasticsearch.core.Strings.format; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.sameInstance; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -68,7 +72,7 @@ public void testGetUnparsedModelMap_ThrowsResourceNotFound_WhenNoHitsReturned() var registry = new ModelRegistry(client); - var listener = new PlainActionFuture(); + var listener = new PlainActionFuture(); registry.getModelWithSecrets("1", listener); ResourceNotFoundException exception = expectThrows(ResourceNotFoundException.class, () -> listener.actionGet(TIMEOUT)); @@ -82,7 +86,7 @@ public void testGetUnparsedModelMap_ThrowsIllegalArgumentException_WhenInvalidIn var registry = new ModelRegistry(client); - var listener = new PlainActionFuture(); + var listener = new PlainActionFuture(); registry.getModelWithSecrets("1", listener); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> listener.actionGet(TIMEOUT)); @@ -99,7 +103,7 @@ public void testGetUnparsedModelMap_ThrowsIllegalStateException_WhenUnableToFind var registry = new ModelRegistry(client); - var listener = new PlainActionFuture(); + var listener = new PlainActionFuture(); registry.getModelWithSecrets("1", listener); IllegalStateException exception = expectThrows(IllegalStateException.class, () -> listener.actionGet(TIMEOUT)); @@ -116,7 +120,7 @@ public void testGetUnparsedModelMap_ThrowsIllegalStateException_WhenUnableToFind var registry = new ModelRegistry(client); - var listener = new PlainActionFuture(); + var listener = new PlainActionFuture(); registry.getModelWithSecrets("1", listener); IllegalStateException exception = expectThrows(IllegalStateException.class, () -> listener.actionGet(TIMEOUT)); @@ -150,7 +154,7 @@ public void testGetModelWithSecrets() { var registry = new ModelRegistry(client); - var listener = new PlainActionFuture(); + var listener = new PlainActionFuture(); registry.getModelWithSecrets("1", listener); var modelConfig = listener.actionGet(TIMEOUT); @@ -179,7 +183,7 @@ public void testGetModelNoSecrets() { var registry = new ModelRegistry(client); - var listener = new PlainActionFuture(); + var listener = new PlainActionFuture(); registry.getModel("1", listener); registry.getModel("1", listener); @@ -288,6 +292,80 @@ public void testStoreModel_ThrowsException_WhenFailureIsNotAVersionConflict() { ); } + @SuppressWarnings("unchecked") + public void testDeepCopyDefaultConfig() { + { + var toCopy = new UnparsedModel("tocopy", randomFrom(TaskType.values()), "service-a", Map.of(), Map.of()); + var copied = ModelRegistry.deepCopyDefaultConfig(toCopy); + assertThat(copied, not(sameInstance(toCopy))); + assertThat(copied.taskType(), is(toCopy.taskType())); + assertThat(copied.service(), is(toCopy.service())); + assertThat(copied.secrets(), not(sameInstance(toCopy.secrets()))); + assertThat(copied.secrets(), is(toCopy.secrets())); + // Test copied is a modifiable map + copied.secrets().put("foo", "bar"); + + assertThat(copied.settings(), not(sameInstance(toCopy.settings()))); + assertThat(copied.settings(), is(toCopy.settings())); + // Test copied is a modifiable map + copied.settings().put("foo", "bar"); + } + + { + Map secretsMap = Map.of("secret", "value"); + Map chunking = Map.of("strategy", "word"); + Map task = Map.of("user", "name"); + Map service = Map.of("num_threads", 1, "adaptive_allocations", Map.of("enabled", true)); + Map settings = Map.of("chunking_settings", chunking, "service_settings", service, "task_settings", task); + + var toCopy = new UnparsedModel("tocopy", randomFrom(TaskType.values()), "service-a", settings, secretsMap); + var copied = ModelRegistry.deepCopyDefaultConfig(toCopy); + assertThat(copied, not(sameInstance(toCopy))); + + assertThat(copied.secrets(), not(sameInstance(toCopy.secrets()))); + assertThat(copied.secrets(), is(toCopy.secrets())); + // Test copied is a modifiable map + copied.secrets().remove("secret"); + + assertThat(copied.settings(), not(sameInstance(toCopy.settings()))); + assertThat(copied.settings(), is(toCopy.settings())); + // Test copied is a modifiable map + var chunkOut = (Map) copied.settings().get("chunking_settings"); + assertThat(chunkOut, is(chunking)); + chunkOut.remove("strategy"); + + var taskOut = (Map) copied.settings().get("task_settings"); + assertThat(taskOut, is(task)); + taskOut.remove("user"); + + var serviceOut = (Map) copied.settings().get("service_settings"); + assertThat(serviceOut, is(service)); + var adaptiveOut = (Map) serviceOut.remove("adaptive_allocations"); + assertThat(adaptiveOut, is(Map.of("enabled", true))); + adaptiveOut.remove("enabled"); + } + } + + public void testDuplicateDefaultIds() { + var client = mockBulkClient(); + var registry = new ModelRegistry(client); + + var id = "my-inference"; + + registry.addDefaultConfiguration(new UnparsedModel(id, randomFrom(TaskType.values()), "service-a", Map.of(), Map.of())); + var ise = expectThrows( + IllegalStateException.class, + () -> registry.addDefaultConfiguration(new UnparsedModel(id, randomFrom(TaskType.values()), "service-b", Map.of(), Map.of())) + ); + assertThat( + ise.getMessage(), + containsString( + "Cannot add default endpoint to the inference endpoint registry with duplicate inference id [my-inference] declared by " + + "service [service-b]. The inference Id is already use by [service-a] service." + ) + ); + } + private Client mockBulkClient() { var client = mockClient(); when(client.prepareBulk()).thenReturn(new BulkRequestBuilder(client)); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml index 6aec721b35418..11be68cc764e2 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml @@ -44,15 +44,18 @@ - do: inference.get: inference_id: "*" - - length: { endpoints: 0} + - length: { endpoints: 1} + - match: { endpoints.0.inference_id: ".elser-2" } - do: inference.get: inference_id: _all - - length: { endpoints: 0} + - length: { endpoints: 1} + - match: { endpoints.0.inference_id: ".elser-2" } - do: inference.get: inference_id: "" - - length: { endpoints: 0} + - length: { endpoints: 1} + - match: { endpoints.0.inference_id: ".elser-2" } From 2f4b96f2c017013329326436ea93b4330efa2e98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Fred=C3=A9n?= <109296772+jfreden@users.noreply.github.com> Date: Fri, 4 Oct 2024 12:01:42 +0200 Subject: [PATCH 076/194] Remove testMetadataMigratedAfterUpgrade mute (does not exist) (#114108) This test changed name and was fixed in https://github.com/elastic/elasticsearch/pull/110228 --- muted-tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index fc24018173291..e83f15d445c93 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -26,9 +26,6 @@ tests: - class: org.elasticsearch.index.store.FsDirectoryFactoryTests method: testPreload issue: https://github.com/elastic/elasticsearch/issues/110211 -- class: org.elasticsearch.upgrades.SecurityIndexRolesMetadataMigrationIT - method: testMetadataMigratedAfterUpgrade - issue: https://github.com/elastic/elasticsearch/issues/110232 - class: org.elasticsearch.backwards.SearchWithMinCompatibleSearchNodeIT method: testMinVersionAsNewVersion issue: https://github.com/elastic/elasticsearch/issues/95384 From 2adb52ae5616820da7abcab23e1a829030058a34 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 4 Oct 2024 11:27:28 +0100 Subject: [PATCH 077/194] Flush response stream in `Ec2DiscoveryTests` (#114101) This is apparently necessary when running with JDK23. Also cleans up some warnings in the same test suite. Closes #114088 Closes #114089 Closes #114090 --- .../discovery/ec2/Ec2DiscoveryTests.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java index 8a6f6b84fec0d..135ddcee8da44 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java @@ -59,7 +59,7 @@ public class Ec2DiscoveryTests extends AbstractEC2MockAPITestCase { private static final String PREFIX_PUBLIC_IP = "8.8.8."; private static final String PREFIX_PRIVATE_IP = "10.0.0."; - private Map poorMansDNS = new ConcurrentHashMap<>(); + private final Map poorMansDNS = new ConcurrentHashMap<>(); protected MockTransportService createTransportService() { final Transport transport = new Netty4Transport( @@ -133,7 +133,7 @@ protected List buildDynamicHosts(Settings nodeSettings, int no .stream() .filter(t -> t.getKey().equals(entry.getKey())) .map(Tag::getValue) - .collect(Collectors.toList()) + .toList() .containsAll(entry.getValue()) ) ) @@ -144,6 +144,7 @@ protected List buildDynamicHosts(Settings nodeSettings, int no exchange.getResponseHeaders().set("Content-Type", "text/xml; charset=UTF-8"); exchange.sendResponseHeaders(HttpStatus.SC_OK, responseBody.length); exchange.getResponseBody().write(responseBody); + exchange.getResponseBody().flush(); return; } } @@ -160,14 +161,14 @@ protected List buildDynamicHosts(Settings nodeSettings, int no } } - public void testDefaultSettings() throws InterruptedException { + public void testDefaultSettings() { int nodes = randomInt(10); Settings nodeSettings = Settings.builder().build(); List discoveryNodes = buildDynamicHosts(nodeSettings, nodes); assertThat(discoveryNodes, hasSize(nodes)); } - public void testPrivateIp() throws InterruptedException { + public void testPrivateIp() { int nodes = randomInt(10); for (int i = 0; i < nodes; i++) { poorMansDNS.put(PREFIX_PRIVATE_IP + (i + 1), buildNewFakeTransportAddress()); @@ -183,7 +184,7 @@ public void testPrivateIp() throws InterruptedException { } } - public void testPublicIp() throws InterruptedException { + public void testPublicIp() { int nodes = randomInt(10); for (int i = 0; i < nodes; i++) { poorMansDNS.put(PREFIX_PUBLIC_IP + (i + 1), buildNewFakeTransportAddress()); @@ -199,7 +200,7 @@ public void testPublicIp() throws InterruptedException { } } - public void testPrivateDns() throws InterruptedException { + public void testPrivateDns() { int nodes = randomInt(10); for (int i = 0; i < nodes; i++) { String instanceId = "node" + (i + 1); @@ -217,7 +218,7 @@ public void testPrivateDns() throws InterruptedException { } } - public void testPublicDns() throws InterruptedException { + public void testPublicDns() { int nodes = randomInt(10); for (int i = 0; i < nodes; i++) { String instanceId = "node" + (i + 1); @@ -235,14 +236,14 @@ public void testPublicDns() throws InterruptedException { } } - public void testInvalidHostType() throws InterruptedException { + public void testInvalidHostType() { Settings nodeSettings = Settings.builder().put(AwsEc2Service.HOST_TYPE_SETTING.getKey(), "does_not_exist").build(); IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { buildDynamicHosts(nodeSettings, 1); }); assertThat(exception.getMessage(), containsString("does_not_exist is unknown for discovery.ec2.host_type")); } - public void testFilterByTags() throws InterruptedException { + public void testFilterByTags() { int nodes = randomIntBetween(5, 10); Settings nodeSettings = Settings.builder().put(AwsEc2Service.TAG_SETTING.getKey() + "stage", "prod").build(); @@ -265,7 +266,7 @@ public void testFilterByTags() throws InterruptedException { assertThat(dynamicHosts, hasSize(prodInstances)); } - public void testFilterByMultipleTags() throws InterruptedException { + public void testFilterByMultipleTags() { int nodes = randomIntBetween(5, 10); Settings nodeSettings = Settings.builder().putList(AwsEc2Service.TAG_SETTING.getKey() + "stage", "prod", "preprod").build(); From ddf019a1a50885425318631ba585545f92ce0366 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Slobodan=20Adamovi=C4=87?= Date: Fri, 4 Oct 2024 12:47:39 +0200 Subject: [PATCH 078/194] Upgrade protobufer to 3.25.5 (#113869) Updating `com.google.protobuf:protobuf-java-util` and `com.google.protobuf:protobuf-java` dependencies to the latest 3.x version. --- build-tools-internal/version.properties | 2 +- docs/changelog/113869.yaml | 5 +++++ gradle/verification-metadata.xml | 12 ++++++------ 3 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 docs/changelog/113869.yaml diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index ac75a3a968ed1..169c187ef115a 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -30,7 +30,7 @@ httpcore = 4.4.13 httpasyncclient = 4.1.5 commonslogging = 1.2 commonscodec = 1.15 -protobuf = 3.21.9 +protobuf = 3.25.5 # test dependencies randomizedrunner = 2.8.0 diff --git a/docs/changelog/113869.yaml b/docs/changelog/113869.yaml new file mode 100644 index 0000000000000..f1cd1ec423966 --- /dev/null +++ b/docs/changelog/113869.yaml @@ -0,0 +1,5 @@ +pr: 113869 +summary: Upgrade protobufer to 3.25.5 +area: Snapshot/Restore +type: upgrade +issues: [] diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index f1c4b15ea5702..53a65e217ed18 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -749,9 +749,9 @@ - - - + + + @@ -759,9 +759,9 @@ - - - + + + From 9f7d7b12ce713fea572c3f588ae72019abfe9172 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 4 Oct 2024 21:05:08 +1000 Subject: [PATCH 079/194] Forward port release notes for v8.15.2 (#113634) --- docs/reference/release-notes/8.15.2.asciidoc | 42 ++++++++++++++++ .../release-notes/highlights.asciidoc | 50 ++++++++++++++++--- 2 files changed, 86 insertions(+), 6 deletions(-) create mode 100644 docs/reference/release-notes/8.15.2.asciidoc diff --git a/docs/reference/release-notes/8.15.2.asciidoc b/docs/reference/release-notes/8.15.2.asciidoc new file mode 100644 index 0000000000000..7dfd8690109b2 --- /dev/null +++ b/docs/reference/release-notes/8.15.2.asciidoc @@ -0,0 +1,42 @@ +[[release-notes-8.15.2]] +== {es} version 8.15.2 + +Also see <>. + +[[bug-8.15.2]] +[float] +=== Bug fixes + +Authorization:: +* Fix remote cluster credential secure settings reload {es-pull}111535[#111535] + +ES|QL:: +* ESQL: Don't mutate the `BoolQueryBuilder` in plan {es-pull}111519[#111519] +* ES|QL: Fix `ResolvedEnrichPolicy` serialization (bwc) in v 8.15 {es-pull}112985[#112985] (issue: {es-issue}112968[#112968]) +* Fix union-types where one index is missing the field {es-pull}111932[#111932] (issue: {es-issue}111912[#111912]) +* Support widening of numeric types in union-types {es-pull}112610[#112610] (issue: {es-issue}111277[#111277]) + +Infra/Core:: +* JSON parse failures should be 4xx codes {es-pull}112703[#112703] +* Json parsing exceptions should not cause 500 errors {es-pull}111548[#111548] (issue: {es-issue}111542[#111542]) +* Make sure file accesses in `DnRoleMapper` are done in stack frames with permissions {es-pull}112400[#112400] + +Ingest Node:: +* Fix missing header in `put_geoip_database` JSON spec {es-pull}112581[#112581] + +Logs:: +* Fix encoding of dynamic arrays in ignored source {es-pull}112713[#112713] + +Mapping:: +* Full coverage of ECS by ecs@mappings when `date_detection` is disabled {es-pull}112444[#112444] (issue: {es-issue}112398[#112398]) + +Search:: +* Fix parsing error in `_terms_enum` API {es-pull}112872[#112872] (issue: {es-issue}94378[#94378]) + +Security:: +* Allowlist `tracestate` header on remote server port {es-pull}112649[#112649] + +Vector Search:: +* Fix NPE in `dense_vector` stats {es-pull}112720[#112720] + + diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index bf5260928797c..1e0018f590ac0 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -72,16 +72,54 @@ version 8.16 `allow_rebalance` setting defaults to `always` unless the legacy al [discrete] [[add_global_retention_in_data_stream_lifecycle]] === Add global retention in data stream lifecycle -Data stream lifecycle now supports configuring retention on a cluster level, namely global retention. Global retention -allows us to configure two different retentions: +Data stream lifecycle now supports configuring retention on a cluster level, +namely global retention. Global retention \nallows us to configure two different +retentions: -- `data_streams.lifecycle.retention.default` is applied to all data streams managed by the data stream lifecycle that do not have retention -defined on the data stream level. -- `data_streams.lifecycle.retention.max` is applied to all data streams managed by the data stream lifecycle and it allows any data stream -data to be deleted after the `max_retention` has passed. +- `data_streams.lifecycle.retention.default` is applied to all data streams managed +by the data stream lifecycle that do not have retention defined on the data stream level. +- `data_streams.lifecycle.retention.max` is applied to all data streams managed by the +data stream lifecycle and it allows any data stream \ndata to be deleted after the `max_retention` has passed. {es-pull}111972[#111972] +[discrete] +[[enable_zstandard_compression_for_indices_with_index_codec_set_to_best_compression]] +=== Enable ZStandard compression for indices with index.codec set to best_compression +Before DEFLATE compression was used to compress stored fields in indices with index.codec index setting set to +best_compression, with this change ZStandard is used as compression algorithm to stored fields for indices with +index.codec index setting set to best_compression. The usage ZStandard results in less storage usage with a +similar indexing throughput depending on what options are used. Experiments with indexing logs have shown that +ZStandard offers ~12% lower storage usage and a ~14% higher indexing throughput compared to DEFLATE. + +{es-pull}112665[#112665] + // end::notable-highlights[] +[discrete] +[[esql_multi_value_fields_supported_in_geospatial_predicates]] +=== ESQL: Multi-value fields supported in Geospatial predicates +Supporting multi-value fields in `WHERE` predicates is a challenge due to not knowing whether `ALL` or `ANY` +of the values in the field should pass the predicate. +For example, should the field `age:[10,30]` pass the predicate `WHERE age>20` or not? +This ambiguity does not exist with the spatial predicates +`ST_INTERSECTS` and `ST_DISJOINT`, because the choice between `ANY` or `ALL` +is implied by the predicate itself. +Consider a predicate checking a field named `location` against a test geometry named `shape`: + +* `ST_INTERSECTS(field, shape)` - true if `ANY` value can intersect the shape +* `ST_DISJOINT(field, shape)` - true only if `ALL` values are disjoint from the shape + +This works even if the shape argument is itself a complex or compound geometry. + +Similar logic exists for `ST_CONTAINS` and `ST_WITHIN` predicates, but these are not as easily solved +with `ANY` or `ALL`, because a collection of geometries contains another collection if each of the contained +geometries is within at least one of the containing geometries. Evaluating this requires that the multi-value +field is first combined into a single geometry before performing the predicate check. + +* `ST_CONTAINS(field, shape)` - true if the combined geometry contains the shape +* `ST_WITHIN(field, shape)` - true if the combined geometry is within the shape + +{es-pull}112063[#112063] + From 3cd809ac7107f82e20235340293c9bad5bcffd66 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 4 Oct 2024 12:31:42 +0100 Subject: [PATCH 080/194] Flush response stream in `EC2RetriesTests` (#114115) This is apparently necessary when running with JDK23. Relates #114101 Closes #114078 --- .../java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java index aa669a45bc0c7..78ea619d81f84 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java @@ -101,6 +101,7 @@ public void testEC2DiscoveryRetriesOnRateLimiting() throws IOException { exchange.getResponseHeaders().set("Content-Type", "text/xml; charset=UTF-8"); exchange.sendResponseHeaders(HttpStatus.SC_OK, responseBody.length); exchange.getResponseBody().write(responseBody); + exchange.getResponseBody().flush(); return; } } From ed4764a7b0417089032b4c8071f28983aa2c11c9 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Fri, 4 Oct 2024 13:59:14 +0200 Subject: [PATCH 081/194] [CI] Add workaround for no space left on device failures (#114123) --- .buildkite/pipelines/dra-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipelines/dra-workflow.yml b/.buildkite/pipelines/dra-workflow.yml index bcc6c9c57d756..5ef0a2ffb3df9 100644 --- a/.buildkite/pipelines/dra-workflow.yml +++ b/.buildkite/pipelines/dra-workflow.yml @@ -7,7 +7,7 @@ steps: image: family/elasticsearch-ubuntu-2204 machineType: custom-32-98304 buildDirectory: /dev/shm/bk - diskSizeGb: 350 + diskSizeGb: 500 - wait # The hadoop build depends on the ES artifact # So let's trigger the hadoop build any time we build a new staging artifact From c20f5f9a5d1081d3d1abd832f57ac80d58a34305 Mon Sep 17 00:00:00 2001 From: Gergely Kalapos Date: Fri, 4 Oct 2024 14:22:28 +0200 Subject: [PATCH 082/194] [otel-data] Add @timestamp as sort field for logs and traces (#114049) * Add @timestamp as sort field for logs and traces * Change order of sort fields * Adapt tests --- .../logs-otel@mappings.yaml | 2 +- .../traces-otel@mappings.yaml | 2 +- .../rest-api-spec/test/20_logs_tests.yml | 20 ++++++++++ .../rest-api-spec/test/20_traces_tests.yml | 37 ++++++++++--------- 4 files changed, 42 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml index f350997de9e01..107901adb834f 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/logs-otel@mappings.yaml @@ -8,7 +8,7 @@ template: index: mode: logsdb sort: - field: [ "resource.attributes.host.name" ] + field: [ "resource.attributes.host.name", "@timestamp" ] mappings: properties: attributes: diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml index 0e77bc208eed4..2b0d1ec536fa6 100644 --- a/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/traces-otel@mappings.yaml @@ -8,7 +8,7 @@ template: index: mode: logsdb sort: - field: [ "resource.attributes.host.name" ] + field: [ "resource.attributes.host.name", "@timestamp" ] mappings: _source: mode: synthetic diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml index 657453bf4ae9f..fc162d0647d08 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml @@ -70,3 +70,23 @@ setup: - match: { hits.hits.0.fields.error\.exception\.type: ["MyException"] } - match: { hits.hits.0.fields.error\.exception\.message: ["foo"] } - match: { hits.hits.0.fields.error\.stack_trace: ["Exception in thread \"main\" java.lang.RuntimeException: Test exception\n at com.example.GenerateTrace.methodB(GenerateTrace.java:13)\n at com.example.GenerateTrace.methodA(GenerateTrace.java:9)\n at com.example.GenerateTrace.main(GenerateTrace.java:5)"] } +--- +"resource.attributes.host.name @timestamp should be used as sort fields": + - do: + bulk: + index: logs-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:49:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "body_text":"error1"}' + - is_false: errors + - do: + indices.get_data_stream: + name: logs-generic.otel-default + - set: { data_streams.0.indices.0.index_name: datastream-backing-index } + - do: + indices.get_settings: + index: $datastream-backing-index + - is_true: $datastream-backing-index + - match: { .$datastream-backing-index.settings.index.sort.field.0: "resource.attributes.host.name" } + - match: { .$datastream-backing-index.settings.index.sort.field.1: "@timestamp" } diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml index 6e51f3f91ddb5..d5b87c9b45116 100644 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_traces_tests.yml @@ -76,23 +76,6 @@ setup: - match: { hits.hits.0._source.links.1.trace_id: "4aaa9f33312b3dbb8b2c2c62bb7abe1a1" } - match: { hits.hits.0._source.links.1.span_id: "b3b7d1f1f1b4e1e1" } --- -"Default data_stream.type must be traces": - - do: - bulk: - index: traces-generic.otel-default - refresh: true - body: - - create: {} - - '{"@timestamp":"2024-02-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","type":"traces","namespace":"default"},"resource":{"attributes":{"service.name":"OtelSample","telemetry.sdk.language":"dotnet","telemetry.sdk.name":"opentelemetry"}},"name":"foo","trace_id":"7bba9f33312b3dbb8b2c2c62bb7abe2d","span_id":"086e83747d0e381e","kind":"SERVER","status":{"code":"2xx"}}' - - is_false: errors - - do: - search: - index: traces-generic.otel-default - body: - fields: ["data_stream.type"] - - length: { hits.hits: 1 } - - match: { hits.hits.0.fields.data_stream\.type: ["traces"] } ---- Conflicting attribute types: - do: bulk: @@ -117,3 +100,23 @@ Conflicting attribute types: - length: { hits.hits: 2 } - match: { hits.hits.0.fields.attributes\.http\.status_code: [200] } - match: { hits.hits.1._ignored: ["attributes.http.status_code"] } +--- +"resource.attributes.host.name @timestamp should be used as sort fields": + - do: + bulk: + index: traces-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:49:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "span_id":"1"}' + - is_false: errors + - do: + indices.get_data_stream: + name: traces-generic.otel-default + - set: { data_streams.0.indices.0.index_name: datastream-backing-index } + - do: + indices.get_settings: + index: $datastream-backing-index + - is_true: $datastream-backing-index + - match: { .$datastream-backing-index.settings.index.sort.field.0: "resource.attributes.host.name" } + - match: { .$datastream-backing-index.settings.index.sort.field.1: "@timestamp" } From 89978229f7862a8eff930d197c16beaac728a6c7 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 4 Oct 2024 07:40:55 -0500 Subject: [PATCH 083/194] Simplifying the way the simulate ingest API uses component template substitutions for pipeline lookup and mapping validation (#113908) --- .../test/ingest/80_ingest_simulate.yml | 201 ++++++++++++++++++ .../TransportSimulateIndexTemplateAction.java | 8 +- .../post/TransportSimulateTemplateAction.java | 3 +- .../bulk/TransportAbstractBulkAction.java | 43 +++- .../bulk/TransportSimulateBulkAction.java | 37 +++- .../metadata/MetadataCreateIndexService.java | 10 +- .../MetadataIndexTemplateService.java | 19 +- .../elasticsearch/ingest/IngestService.java | 47 +--- ...sportSimulateIndexTemplateActionTests.java | 3 +- .../MetadataIndexTemplateServiceTests.java | 22 +- .../ingest/IngestServiceTests.java | 84 +------- 11 files changed, 293 insertions(+), 184 deletions(-) diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index 9c6a1ca2e96d2..b4672b1d8924d 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -606,3 +606,204 @@ setup: - match: { docs.0.doc._source.foo: "FOO" } - match: { docs.0.doc.executed_pipelines: [] } - not_exists: docs.0.doc.error + +--- +"Test ingest simulate with component template substitutions for data streams": + # In this test, we make sure that when the index template is a data stream template, simulte ingest works the same whether the data stream + # has been created or not -- either way, we expect it to use the template rather than the data stream / index mappings and settings. + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.component.template.substitutions"] + reason: "ingest simulate component template substitutions added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: mappings_template + body: + template: + mappings: + dynamic: strict + properties: + foo: + type: keyword + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo-pipeline" + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - mappings_template + - settings_template + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + composed_of: + - mappings_template + - settings_template + data_stream: {} + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": false + } + } + ], + "pipeline_substitutions": { + "foo-pipeline-2": { + "processors": [ + { + "set": { + "field": "foo", + "value": "FOO" + } + } + ] + } + }, + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": "foo-pipeline-2" + } + } + } + }, + "mappings_template": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } + - not_exists: docs.0.doc.error + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + cluster.health: + wait_for_status: yellow + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": false + } + } + ], + "pipeline_substitutions": { + "foo-pipeline-2": { + "processors": [ + { + "set": { + "field": "foo", + "value": "FOO" + } + } + ] + } + }, + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": "foo-pipeline-2" + } + } + } + }, + "mappings_template": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } + - not_exists: docs.0.doc.error diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 3561a4d0e2cb4..fdced5fc18ac9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -16,7 +16,6 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.AliasMetadata; -import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; @@ -157,8 +156,7 @@ protected void masterOperation( xContentRegistry, indicesService, systemIndices, - indexSettingProviders, - Map.of() + indexSettingProviders ); final Map> overlapping = new HashMap<>(); @@ -235,8 +233,7 @@ public static Template resolveTemplate( final NamedXContentRegistry xContentRegistry, final IndicesService indicesService, final SystemIndices systemIndices, - Set indexSettingProviders, - Map componentTemplateSubstitutions + Set indexSettingProviders ) throws Exception { var metadata = simulatedState.getMetadata(); Settings templateSettings = resolveSettings(simulatedState.metadata(), matchingTemplate); @@ -266,7 +263,6 @@ public static Template resolveTemplate( null, // empty request mapping as the user can't specify any explicit mappings via the simulate api simulatedState, matchingTemplate, - componentTemplateSubstitutions, xContentRegistry, simulatedIndexName ); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java index af7a253b5a042..30bbad0b57df0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -170,8 +170,7 @@ protected void masterOperation( xContentRegistry, indicesService, systemIndices, - indexSettingProviders, - Map.of() + indexSettingProviders ); if (request.includeDefaults()) { listener.onResponse( diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java index 5eae1c660d7d0..8c6565e52daa7 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java @@ -41,6 +41,7 @@ import org.elasticsearch.transport.TransportService; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.concurrent.Executor; @@ -180,12 +181,48 @@ protected void doRun() throws IOException { private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor executor, ActionListener listener) throws IOException { boolean hasIndexRequestsWithPipelines = false; - final Metadata metadata = clusterService.state().getMetadata(); - Map templateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); + final Metadata metadata; + Map componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); + if (bulkRequest.isSimulated() && componentTemplateSubstitutions.isEmpty() == false) { + /* + * If this is a simulated request, and there are template substitutions, then we want to create and use a new metadata that has + * those templates. That is, we want to add the new templates (which will replace any that already existed with the same name), + * and remove the indices and data streams that are referred to from the bulkRequest so that we get settings from the templates + * rather than from the indices/data streams. + */ + Metadata.Builder simulatedMetadataBuilder = Metadata.builder(clusterService.state().getMetadata()); + if (componentTemplateSubstitutions.isEmpty() == false) { + Map updatedComponentTemplates = new HashMap<>(); + updatedComponentTemplates.putAll(clusterService.state().metadata().componentTemplates()); + updatedComponentTemplates.putAll(componentTemplateSubstitutions); + simulatedMetadataBuilder.componentTemplates(updatedComponentTemplates); + } + /* + * We now remove the index from the simulated metadata to force the templates to be used. Note that simulated requests are + * always index requests -- no other type of request is supported. + */ + for (DocWriteRequest actionRequest : bulkRequest.requests) { + assert actionRequest != null : "Requests cannot be null in simulate mode"; + assert actionRequest instanceof IndexRequest + : "Only IndexRequests are supported in simulate mode, but got " + actionRequest.getClass(); + if (actionRequest != null) { + IndexRequest indexRequest = (IndexRequest) actionRequest; + String indexName = indexRequest.index(); + if (indexName != null) { + simulatedMetadataBuilder.remove(indexName); + simulatedMetadataBuilder.removeDataStream(indexName); + } + } + } + metadata = simulatedMetadataBuilder.build(); + } else { + metadata = clusterService.state().getMetadata(); + } + for (DocWriteRequest actionRequest : bulkRequest.requests) { IndexRequest indexRequest = getIndexWriteRequest(actionRequest); if (indexRequest != null) { - IngestService.resolvePipelinesAndUpdateIndexRequest(actionRequest, indexRequest, metadata, templateSubstitutions); + IngestService.resolvePipelinesAndUpdateIndexRequest(actionRequest, indexRequest, metadata); hasIndexRequestsWithPipelines |= IngestService.hasPipeline(indexRequest); } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index c860c49809cb5..713116c4cf98e 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -51,6 +51,7 @@ import org.elasticsearch.xcontent.NamedXContentRegistry; import java.io.IOException; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -197,12 +198,33 @@ private Exception validateMappings(Map componentTempl * path for when the index does not exist). And it does not deal with system indices since we do not intend for users to * simulate writing to system indices. */ - // First, we remove the index from the cluster state if necessary (since we're going to use the templates) - ClusterState simulatedState = indexAbstraction == null - ? state - : new ClusterState.Builder(state).metadata(Metadata.builder(state.metadata()).remove(request.index()).build()).build(); + ClusterState.Builder simulatedClusterStateBuilder = new ClusterState.Builder(state); + Metadata.Builder simulatedMetadata = Metadata.builder(state.metadata()); + if (indexAbstraction != null) { + /* + * We remove the index or data stream from the cluster state so that we are forced to fall back to the templates to get + * mappings. + */ + String indexRequest = request.index(); + assert indexRequest != null : "Index requests cannot be null in a simulate bulk call"; + if (indexRequest != null) { + simulatedMetadata.remove(indexRequest); + simulatedMetadata.removeDataStream(indexRequest); + } + } + if (componentTemplateSubstitutions.isEmpty() == false) { + /* + * We put the template substitutions into the cluster state. If they have the same name as an existing one, the + * existing one is replaced. + */ + Map updatedComponentTemplates = new HashMap<>(); + updatedComponentTemplates.putAll(state.metadata().componentTemplates()); + updatedComponentTemplates.putAll(componentTemplateSubstitutions); + simulatedMetadata.componentTemplates(updatedComponentTemplates); + } + ClusterState simulatedState = simulatedClusterStateBuilder.metadata(simulatedMetadata).build(); - String matchingTemplate = findV2Template(state.metadata(), request.index(), false); + String matchingTemplate = findV2Template(simulatedState.metadata(), request.index(), false); if (matchingTemplate != null) { final Template template = TransportSimulateIndexTemplateAction.resolveTemplate( matchingTemplate, @@ -212,8 +234,7 @@ private Exception validateMappings(Map componentTempl xContentRegistry, indicesService, systemIndices, - indexSettingProviders, - componentTemplateSubstitutions + indexSettingProviders ); CompressedXContent mappings = template.mappings(); if (mappings != null) { @@ -247,7 +268,7 @@ private Exception validateMappings(Map componentTempl }); } } else { - List matchingTemplates = findV1Templates(state.metadata(), request.index(), false); + List matchingTemplates = findV1Templates(simulatedState.metadata(), request.index(), false); final Map mappingsMap = MetadataCreateIndexService.parseV1Mappings( "{}", matchingTemplates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()), diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 4cdf1508a7987..f43f1c6b05a15 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -665,7 +665,6 @@ private ClusterState applyCreateIndexRequestWithV2Template( request.mappings(), currentState, templateName, - Map.of(), xContentRegistry, request.index() ); @@ -824,7 +823,6 @@ private static List collectSystemV2Mappings( List templateMappings = MetadataIndexTemplateService.collectMappings( composableIndexTemplate, componentTemplates, - Map.of(), indexName ); return collectV2Mappings(null, templateMappings, xContentRegistry); @@ -834,16 +832,10 @@ public static List collectV2Mappings( @Nullable final String requestMappings, final ClusterState currentState, final String templateName, - Map componentTemplateSubstitutions, final NamedXContentRegistry xContentRegistry, final String indexName ) throws Exception { - List templateMappings = MetadataIndexTemplateService.collectMappings( - currentState, - templateName, - componentTemplateSubstitutions, - indexName - ); + List templateMappings = MetadataIndexTemplateService.collectMappings(currentState, templateName, indexName); return collectV2Mappings(requestMappings, templateMappings, xContentRegistry); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 1f9f6f636c1cf..abeb3279b7b50 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -698,7 +698,7 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT final var now = Instant.now(); final var metadata = currentState.getMetadata(); - final var combinedMappings = collectMappings(indexTemplate, metadata.componentTemplates(), Map.of(), "tmp_idx"); + final var combinedMappings = collectMappings(indexTemplate, metadata.componentTemplates(), "tmp_idx"); final var combinedSettings = resolveSettings(indexTemplate, metadata.componentTemplates()); // First apply settings sourced from index setting providers: for (var provider : indexSettingProviders) { @@ -1341,12 +1341,7 @@ private static boolean isGlobalAndHasIndexHiddenSetting(Metadata metadata, Compo /** * Collect the given v2 template into an ordered list of mappings. */ - public static List collectMappings( - final ClusterState state, - final String templateName, - Map componentTemplateSubstitutions, - final String indexName - ) { + public static List collectMappings(final ClusterState state, final String templateName, final String indexName) { final ComposableIndexTemplate template = state.metadata().templatesV2().get(templateName); assert template != null : "attempted to resolve mappings for a template [" + templateName + "] that did not exist in the cluster state"; @@ -1355,7 +1350,7 @@ public static List collectMappings( } final Map componentTemplates = state.metadata().componentTemplates(); - return collectMappings(template, componentTemplates, componentTemplateSubstitutions, indexName); + return collectMappings(template, componentTemplates, indexName); } /** @@ -1364,7 +1359,6 @@ public static List collectMappings( public static List collectMappings( final ComposableIndexTemplate template, final Map componentTemplates, - final Map componentTemplateSubstitutions, final String indexName ) { Objects.requireNonNull(template, "Composable index template must be provided"); @@ -1375,12 +1369,9 @@ public static List collectMappings( ComposableIndexTemplate.DataStreamTemplate.DATA_STREAM_MAPPING_SNIPPET ); } - final Map combinedComponentTemplates = new HashMap<>(); - combinedComponentTemplates.putAll(componentTemplates); - combinedComponentTemplates.putAll(componentTemplateSubstitutions); List mappings = template.composedOf() .stream() - .map(combinedComponentTemplates::get) + .map(componentTemplates::get) .filter(Objects::nonNull) .map(ComponentTemplate::template) .map(Template::mappings) @@ -1716,7 +1707,7 @@ private static void validateCompositeTemplate( String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName; // Parse mappings to ensure they are valid after being composed - List mappings = collectMappings(stateWithIndex, templateName, Map.of(), indexName); + List mappings = collectMappings(stateWithIndex, templateName, indexName); try { MapperService mapperService = tempIndexService.mapperService(); mapperService.merge(MapperService.SINGLE_MAPPING_NAME, mappings, MapperService.MergeReason.INDEX_TEMPLATE); diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 0275e988ce39d..0f63d2a8dcc1b 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -33,7 +33,6 @@ import org.elasticsearch.cluster.ClusterStateApplier; import org.elasticsearch.cluster.ClusterStateTaskExecutor; import org.elasticsearch.cluster.ClusterStateTaskListener; -import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -271,30 +270,14 @@ public static void resolvePipelinesAndUpdateIndexRequest( final IndexRequest indexRequest, final Metadata metadata ) { - resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, Map.of()); - } - - public static void resolvePipelinesAndUpdateIndexRequest( - final DocWriteRequest originalRequest, - final IndexRequest indexRequest, - final Metadata metadata, - Map componentTemplateSubstitutions - ) { - resolvePipelinesAndUpdateIndexRequest( - originalRequest, - indexRequest, - metadata, - System.currentTimeMillis(), - componentTemplateSubstitutions - ); + resolvePipelinesAndUpdateIndexRequest(originalRequest, indexRequest, metadata, System.currentTimeMillis()); } static void resolvePipelinesAndUpdateIndexRequest( final DocWriteRequest originalRequest, final IndexRequest indexRequest, final Metadata metadata, - final long epochMillis, - final Map componentTemplateSubstitutions + final long epochMillis ) { if (indexRequest.isPipelineResolved()) { return; @@ -302,21 +285,11 @@ static void resolvePipelinesAndUpdateIndexRequest( /* * Here we look for the pipelines associated with the index if the index exists. If the index does not exist we fall back to using - * templates to find the pipelines. But if a user has passed in component template substitutions, they want the settings from those - * used in place of the settings used to create any previous indices. So in that case we use the templates to find the pipelines -- - * we don't fall back to the existing index if we don't find any because it is possible the user has intentionally removed the - * pipeline. + * templates to find the pipelines. */ - final Pipelines pipelines; - if (componentTemplateSubstitutions.isEmpty()) { - pipelines = resolvePipelinesFromMetadata(originalRequest, indexRequest, metadata, epochMillis) // - .or(() -> resolvePipelinesFromIndexTemplates(indexRequest, metadata, Map.of())) - .orElse(Pipelines.NO_PIPELINES_DEFINED); - } else { - pipelines = resolvePipelinesFromIndexTemplates(indexRequest, metadata, componentTemplateSubstitutions).orElse( - Pipelines.NO_PIPELINES_DEFINED - ); - } + final Pipelines pipelines = resolvePipelinesFromMetadata(originalRequest, indexRequest, metadata, epochMillis).or( + () -> resolvePipelinesFromIndexTemplates(indexRequest, metadata) + ).orElse(Pipelines.NO_PIPELINES_DEFINED); // The pipeline coming as part of the request always has priority over the resolved one from metadata or templates String requestPipeline = indexRequest.getPipeline(); @@ -1466,11 +1439,7 @@ private static Optional resolvePipelinesFromMetadata( return Optional.of(new Pipelines(IndexSettings.DEFAULT_PIPELINE.get(settings), IndexSettings.FINAL_PIPELINE.get(settings))); } - private static Optional resolvePipelinesFromIndexTemplates( - IndexRequest indexRequest, - Metadata metadata, - Map componentTemplateSubstitutions - ) { + private static Optional resolvePipelinesFromIndexTemplates(IndexRequest indexRequest, Metadata metadata) { if (indexRequest.index() == null) { return Optional.empty(); } @@ -1480,7 +1449,7 @@ private static Optional resolvePipelinesFromIndexTemplates( // precedence), or if a V2 template does not match, any V1 templates String v2Template = MetadataIndexTemplateService.findV2Template(metadata, indexRequest.index(), false); if (v2Template != null) { - final Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template, componentTemplateSubstitutions); + final Settings settings = MetadataIndexTemplateService.resolveSettings(metadata, v2Template); return Optional.of(new Pipelines(IndexSettings.DEFAULT_PIPELINE.get(settings), IndexSettings.FINAL_PIPELINE.get(settings))); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java index 9b1d8c15619ad..8f0ff82beab4b 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java @@ -87,8 +87,7 @@ public Settings getAdditionalIndexSettings( xContentRegistry(), indicesService, systemIndices, - indexSettingsProviders, - Map.of() + indexSettingsProviders ); assertThat(resolvedTemplate.settings().getAsInt("test-setting", -1), is(1)); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 5fadd8f263f7c..8d4b04746e7a4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -1074,7 +1074,7 @@ public void testResolveConflictingMappings() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "my-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", Map.of(), "my-index"); + List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(3)); @@ -1136,7 +1136,7 @@ public void testResolveMappings() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "my-template", it); - List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", Map.of(), "my-index"); + List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(3)); @@ -1190,7 +1190,6 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex List mappings = MetadataIndexTemplateService.collectMappings( state, "logs-data-stream-template", - Map.of(), DataStream.getDefaultBackingIndexName("logs", 1L) ); @@ -1242,12 +1241,7 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex .build(); state = service.addIndexTemplateV2(state, true, "timeseries-template", it); - List mappings = MetadataIndexTemplateService.collectMappings( - state, - "timeseries-template", - Map.of(), - "timeseries" - ); + List mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template", "timeseries"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(2)); @@ -1269,7 +1263,6 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex mappings = MetadataIndexTemplateService.collectMappings( state, "timeseries-template", - Map.of(), DataStream.getDefaultBackingIndexName("timeseries", 1L) ); @@ -1318,7 +1311,6 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception List mappings = MetadataIndexTemplateService.collectMappings( state, "logs-template", - Map.of(), DataStream.getDefaultBackingIndexName("logs", 1L) ); @@ -1375,7 +1367,6 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception List mappings = MetadataIndexTemplateService.collectMappings( state, "timeseries-template", - Map.of(), DataStream.getDefaultBackingIndexName("timeseries-template", 1L) ); @@ -2442,12 +2433,7 @@ public void testComposableTemplateWithSubobjectsFalse() throws Exception { .build(); state = service.addIndexTemplateV2(state, true, "composable-template", it); - List mappings = MetadataIndexTemplateService.collectMappings( - state, - "composable-template", - Map.of(), - "test-index" - ); + List mappings = MetadataIndexTemplateService.collectMappings(state, "composable-template", "test-index"); assertNotNull(mappings); assertThat(mappings.size(), equalTo(2)); diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 49e75a71aa7f7..3adaf398624de 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -32,20 +32,16 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasMetadata; -import org.elasticsearch.cluster.metadata.ComponentTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.ClusterStateTaskExecutorUtils; import org.elasticsearch.common.TriConsumer; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.util.Maps; @@ -77,7 +73,6 @@ import org.mockito.ArgumentMatcher; import org.mockito.invocation.InvocationOnMock; -import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -2493,7 +2488,7 @@ public void testResolveFinalPipelineWithDateMathExpression() { // index name matches with IDM: IndexRequest indexRequest = new IndexRequest(""); - IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, epochMillis, Map.of()); + IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, epochMillis); assertTrue(hasPipeline(indexRequest)); assertTrue(indexRequest.isPipelineResolved()); assertThat(indexRequest.getPipeline(), equalTo("_none")); @@ -2858,83 +2853,6 @@ public void testResolvePipelinesWithNonePipeline() { } } - public void testResolvePipelinesAndUpdateIndexRequestWithComponentTemplateSubstitutions() throws IOException { - final String componentTemplateName = "test-component-template"; - final String indexName = "my-index-1"; - final String indexPipeline = "index-pipeline"; - final String realTemplatePipeline = "template-pipeline"; - final String substitutePipeline = "substitute-pipeline"; - - Metadata metadata; - { - // Build up cluster state metadata - IndexMetadata.Builder builder = IndexMetadata.builder(indexName) - .settings(settings(IndexVersion.current())) - .numberOfShards(1) - .numberOfReplicas(0); - ComponentTemplate realComponentTemplate = new ComponentTemplate( - new Template( - Settings.builder().put("index.default_pipeline", realTemplatePipeline).build(), - CompressedXContent.fromJSON("{}"), - null - ), - null, - null - ); - ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.builder() - .indexPatterns(List.of("my-index-*")) - .componentTemplates(List.of(componentTemplateName)) - .build(); - metadata = Metadata.builder() - .put(builder) - .indexTemplates(Map.of("my-index-template", composableIndexTemplate)) - .componentTemplates(Map.of("test-component-template", realComponentTemplate)) - .build(); - } - - Map componentTemplateSubstitutions; - { - ComponentTemplate simulatedComponentTemplate = new ComponentTemplate( - new Template( - Settings.builder().put("index.default_pipeline", substitutePipeline).build(), - CompressedXContent.fromJSON("{}"), - null - ), - null, - null - ); - componentTemplateSubstitutions = Map.of(componentTemplateName, simulatedComponentTemplate); - } - - { - /* - * Here there is a pipeline in the request. This takes precedence over anything in the index or templates or component template - * substitutions. - */ - IndexRequest indexRequest = new IndexRequest(indexName).setPipeline(indexPipeline); - IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, 0, componentTemplateSubstitutions); - assertThat(indexRequest.getPipeline(), equalTo(indexPipeline)); - } - { - /* - * Here there is no pipeline in the request, but there is one in the substitute component template. So it takes precedence. - */ - IndexRequest indexRequest = new IndexRequest(indexName); - IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, 0, componentTemplateSubstitutions); - assertThat(indexRequest.getPipeline(), equalTo(substitutePipeline)); - } - { - /* - * This one is tricky. Since the index exists and there are no component template substitutions, we're going to use the actual - * index in this case rather than its template. The index does not have a default pipeline set, so it's "_none" instead of - * realTemplatePipeline. - */ - IndexRequest indexRequest = new IndexRequest(indexName); - IngestService.resolvePipelinesAndUpdateIndexRequest(indexRequest, indexRequest, metadata, 0, Map.of()); - assertThat(indexRequest.getPipeline(), equalTo("_none")); - } - } - private static Tuple randomMapEntry() { return tuple(randomAlphaOfLength(5), randomObject()); } From c3f4f4e1869e3adf1830993a3ca283bfe4bc012b Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 4 Oct 2024 09:01:53 -0400 Subject: [PATCH 084/194] ESQL: Rework building evaluators (#113979) This reworks `EvaluatorMapper` to use a named interface instead of a `Function`. It's shorter to type and it allows us to add extra information to the builder. --- .../evaluator/mapper/EvaluatorMapper.java | 8 ++++--- .../expression/function/grouping/Bucket.java | 3 +-- .../function/grouping/Categorize.java | 3 +-- .../function/scalar/conditional/Case.java | 4 ++-- .../function/scalar/conditional/Greatest.java | 3 +-- .../function/scalar/conditional/Least.java | 3 +-- .../convert/AbstractConvertFunction.java | 3 +-- .../function/scalar/convert/FromBase64.java | 5 +--- .../function/scalar/convert/ToBase64.java | 5 +--- .../function/scalar/date/DateDiff.java | 3 +-- .../function/scalar/date/DateExtract.java | 3 +-- .../function/scalar/date/DateFormat.java | 3 +-- .../function/scalar/date/DateParse.java | 3 +-- .../function/scalar/date/DateTrunc.java | 3 +-- .../expression/function/scalar/date/Now.java | 3 +-- .../function/scalar/ip/CIDRMatch.java | 5 ++-- .../function/scalar/ip/IpPrefix.java | 3 +-- .../expression/function/scalar/math/Abs.java | 3 +-- .../math/AbstractTrigonometricFunction.java | 3 +-- .../function/scalar/math/Atan2.java | 3 +-- .../expression/function/scalar/math/Cbrt.java | 3 +-- .../expression/function/scalar/math/Ceil.java | 3 +-- .../expression/function/scalar/math/Exp.java | 5 +--- .../function/scalar/math/Floor.java | 3 +-- .../expression/function/scalar/math/Log.java | 3 +-- .../function/scalar/math/Log10.java | 3 +-- .../expression/function/scalar/math/Pow.java | 3 +-- .../function/scalar/math/Round.java | 5 ++-- .../function/scalar/math/Signum.java | 5 +--- .../expression/function/scalar/math/Sqrt.java | 3 +-- .../AbstractMultivalueFunction.java | 2 +- .../function/scalar/multivalue/MvAppend.java | 5 +--- .../function/scalar/multivalue/MvConcat.java | 3 +-- .../multivalue/MvPSeriesWeightedSum.java | 4 +--- .../scalar/multivalue/MvPercentile.java | 3 +-- .../function/scalar/multivalue/MvSlice.java | 5 +--- .../function/scalar/multivalue/MvSort.java | 5 +--- .../function/scalar/multivalue/MvZip.java | 5 +--- .../function/scalar/nulls/Coalesce.java | 5 ++-- .../spatial/SpatialEvaluatorFactory.java | 24 +++++-------------- .../spatial/SpatialRelatesFunction.java | 5 +--- .../function/scalar/spatial/StDistance.java | 9 +++---- .../function/scalar/spatial/StX.java | 5 +--- .../function/scalar/spatial/StY.java | 5 +--- .../function/scalar/string/Concat.java | 5 ++-- .../function/scalar/string/EndsWith.java | 3 +-- .../function/scalar/string/LTrim.java | 3 +-- .../function/scalar/string/Left.java | 3 +-- .../function/scalar/string/Length.java | 3 +-- .../function/scalar/string/Locate.java | 3 +-- .../function/scalar/string/RLike.java | 5 +--- .../function/scalar/string/RTrim.java | 3 +-- .../function/scalar/string/Repeat.java | 3 +-- .../function/scalar/string/Replace.java | 3 +-- .../function/scalar/string/Right.java | 3 +-- .../function/scalar/string/Space.java | 3 +-- .../function/scalar/string/Split.java | 3 +-- .../function/scalar/string/StartsWith.java | 3 +-- .../function/scalar/string/Substring.java | 3 +-- .../function/scalar/string/ToLower.java | 3 +-- .../function/scalar/string/ToUpper.java | 3 +-- .../function/scalar/string/Trim.java | 3 +-- .../function/scalar/string/WildcardLike.java | 5 +--- .../DateTimeArithmeticOperation.java | 3 +-- .../arithmetic/EsqlArithmeticOperation.java | 3 +-- .../predicate/operator/arithmetic/Neg.java | 3 +-- .../comparison/EsqlBinaryComparison.java | 5 +--- .../predicate/operator/comparison/In.java | 7 ++---- .../esql/expression/function/DeepCopy.java | 5 +--- .../function/EsqlFunctionRegistryTests.java | 5 +--- .../function/scalar/nulls/CoalesceTests.java | 6 ++--- 71 files changed, 90 insertions(+), 203 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java index 5888e30747557..d8692faef5290 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java @@ -14,8 +14,6 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.planner.Layout; -import java.util.function.Function; - import static org.elasticsearch.compute.data.BlockUtils.fromArrayRow; import static org.elasticsearch.compute.data.BlockUtils.toJavaObject; @@ -23,6 +21,10 @@ * Expressions that have a mapping to an {@link ExpressionEvaluator}. */ public interface EvaluatorMapper { + interface ToEvaluator { + ExpressionEvaluator.Factory apply(Expression expression); + } + /** *

* Note for implementors: @@ -50,7 +52,7 @@ public interface EvaluatorMapper { * garbage. Or return an evaluator that throws when run. *

*/ - ExpressionEvaluator.Factory toEvaluator(Function toEvaluator); + ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator); /** * Fold using {@link #toEvaluator} so you don't need a "by hand" diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java index 5fabfe0e03d89..3357b2abf0e0f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java @@ -39,7 +39,6 @@ import java.time.ZoneOffset; import java.util.ArrayList; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; @@ -241,7 +240,7 @@ public boolean foldable() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (field.dataType() == DataType.DATETIME) { Rounding.Prepared preparedRounding; if (buckets.dataType().isWholeNumber()) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java index 0865e070aecd4..75a9883a77102 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java @@ -36,7 +36,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -107,7 +106,7 @@ static int process( } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new CategorizeEvaluator.Factory( source(), toEvaluator.apply(field), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java index 6acb8ea974ed0..62e5651d07dca 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java @@ -50,7 +50,7 @@ public final class Case extends EsqlScalarFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Case", Case::new); record Condition(Expression condition, Expression value) { - ConditionEvaluatorSupplier toEvaluator(Function toEvaluator) { + ConditionEvaluatorSupplier toEvaluator(ToEvaluator toEvaluator) { return new ConditionEvaluatorSupplier(condition.source(), toEvaluator.apply(condition), toEvaluator.apply(value)); } } @@ -311,7 +311,7 @@ private Expression finishPartialFold(List newChildren) { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { List conditionsFactories = conditions.stream().map(c -> c.toEvaluator(toEvaluator)).toList(); ExpressionEvaluator.Factory elseValueFactory = toEvaluator.apply(elseValue); ElementType resultType = PlannerUtils.toElementType(dataType()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java index 7c0427a95d478..c374eac32acc0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import java.util.stream.Stream; import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; @@ -138,7 +137,7 @@ public boolean foldable() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { // force datatype initialization var dataType = dataType(); ExpressionEvaluator.Factory[] factories = children().stream() diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java index 272e65106e7de..90d0d33995cb0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import java.util.stream.Stream; import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; @@ -136,7 +135,7 @@ public boolean foldable() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { // force datatype initialization var dataType = dataType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java index f3ce6b1465d6b..43c9a0864118d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java @@ -34,7 +34,6 @@ import java.util.Locale; import java.util.Map; import java.util.Set; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isTypeOrUnionType; @@ -124,7 +123,7 @@ interface BuildFactory { protected abstract Map factories(); @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return evaluator(toEvaluator.apply(field())); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64.java index 873d496bfc8fd..7f9d0d3f2e647 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.Base64; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; @@ -96,9 +95,7 @@ static BytesRef process(BytesRef field, @Fixed(includeInToString = false, build } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return switch (PlannerUtils.toElementType(field.dataType())) { case BYTES_REF -> new FromBase64Evaluator.Factory(source(), toEvaluator.apply(field), context -> new BytesRefBuilder()); case NULL -> EvalOperator.CONSTANT_NULL_FACTORY; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64.java index ab8287413c614..c23cef31f32f5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.Base64; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; @@ -90,9 +89,7 @@ static BytesRef process(BytesRef field, @Fixed(includeInToString = false, build } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return switch (PlannerUtils.toElementType(field.dataType())) { case BYTES_REF -> new ToBase64Evaluator.Factory(source(), toEvaluator.apply(field), context -> new BytesRefBuilder()); case NULL -> EvalOperator.CONSTANT_NULL_FACTORY; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java index f9039417e48a6..f6a23a5d5962e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java @@ -37,7 +37,6 @@ import java.util.Map; import java.util.Set; import java.util.function.BiFunction; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -227,7 +226,7 @@ static int process(BytesRef unit, long startTimestamp, long endTimestamp) throws } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { ExpressionEvaluator.Factory startTimestampEvaluator = toEvaluator.apply(startTimestamp); ExpressionEvaluator.Factory endTimestampEvaluator = toEvaluator.apply(endTimestamp); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java index bbb19ca0eecab..501dfd431f106 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java @@ -31,7 +31,6 @@ import java.time.ZoneId; import java.time.temporal.ChronoField; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isDate; import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isStringAndExact; @@ -108,7 +107,7 @@ public String getWriteableName() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var fieldEvaluator = toEvaluator.apply(children().get(1)); if (children().get(0).foldable()) { ChronoField chrono = chronoField(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java index bfca72a563c05..60bc014ccbeec 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.util.List; import java.util.Locale; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -139,7 +138,7 @@ static BytesRef process(long val, BytesRef formatter, @Fixed Locale locale) { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var fieldEvaluator = toEvaluator.apply(field); if (format == null) { return new DateFormatConstantEvaluator.Factory(source(), fieldEvaluator, DEFAULT_DATE_TIME_FORMATTER); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java index 27b613e0e6d5a..1aaa227c3846e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.time.ZoneId; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.common.time.DateFormatter.forPattern; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; @@ -136,7 +135,7 @@ static long process(BytesRef val, BytesRef formatter, @Fixed ZoneId zoneId) thro } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { ZoneId zone = UTC; // TODO session timezone? ExpressionEvaluator.Factory fieldEvaluator = toEvaluator.apply(field); if (format == null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java index d5ec3d1d96fae..35a705f418906 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java @@ -32,7 +32,6 @@ import java.time.ZoneOffset; import java.util.List; import java.util.concurrent.TimeUnit; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -199,7 +198,7 @@ private static Rounding.Prepared createRounding(final Duration duration, final Z } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var fieldEvaluator = toEvaluator.apply(timestampField); if (interval.foldable() == false) { throw new IllegalArgumentException("Function [" + sourceText() + "] has invalid interval [" + interval.sourceText() + "]."); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java index 0654aec3a0522..d259fc6ae57ce 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java @@ -25,7 +25,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; public class Now extends EsqlConfigurationFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Now", Now::new); @@ -90,7 +89,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return dvrCtx -> new NowEvaluator(source(), now, dvrCtx); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java index c141beeefb1ea..51430603a4077 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; @@ -113,12 +112,12 @@ public boolean foldable() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var ipEvaluatorSupplier = toEvaluator.apply(ipField); return new CIDRMatchEvaluator.Factory( source(), ipEvaluatorSupplier, - matches.stream().map(x -> toEvaluator.apply(x)).toArray(EvalOperator.ExpressionEvaluator.Factory[]::new) + matches.stream().map(toEvaluator::apply).toArray(EvalOperator.ExpressionEvaluator.Factory[]::new) ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java index 60b464b26750a..26e75e752f681 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefix.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -120,7 +119,7 @@ public boolean foldable() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var ipEvaluatorSupplier = toEvaluator.apply(ipField); var prefixLengthV4EvaluatorSupplier = toEvaluator.apply(prefixLengthV4Field); var prefixLengthV6EvaluatorSupplier = toEvaluator.apply(prefixLengthV6Field); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Abs.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Abs.java index 363b70ef5ed12..ba47fd15e9c9d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Abs.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Abs.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; public class Abs extends UnaryScalarFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Abs", Abs::new); @@ -69,7 +68,7 @@ static int process(int fieldVal) { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var field = toEvaluator.apply(field()); if (dataType() == DataType.DOUBLE) { return new AbsDoubleEvaluator.Factory(source(), field); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbstractTrigonometricFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbstractTrigonometricFunction.java index 8353fe24b3dd0..f44e7b029643c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbstractTrigonometricFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AbstractTrigonometricFunction.java @@ -16,7 +16,6 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; import java.io.IOException; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric; @@ -39,7 +38,7 @@ protected AbstractTrigonometricFunction(StreamInput in) throws IOException { protected abstract EvalOperator.ExpressionEvaluator.Factory doubleEvaluator(EvalOperator.ExpressionEvaluator.Factory field); @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return doubleEvaluator(Cast.cast(source(), field().dataType(), DataType.DOUBLE, toEvaluator.apply(field()))); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2.java index f940cb6d68554..7dbc0001f4b3d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Atan2.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric; @@ -118,7 +117,7 @@ public boolean foldable() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var yEval = Cast.cast(source(), y.dataType(), DataType.DOUBLE, toEvaluator.apply(y)); var xEval = Cast.cast(source(), x.dataType(), DataType.DOUBLE, toEvaluator.apply(x)); return new Atan2Evaluator.Factory(source(), yEval, xEval); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Cbrt.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Cbrt.java index 364e91aad8b1b..dcc704318ca5f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Cbrt.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Cbrt.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric; @@ -56,7 +55,7 @@ public String getWriteableName() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var field = toEvaluator.apply(field()); var fieldType = field().dataType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Ceil.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Ceil.java index 909de387c62ff..f7295421de8aa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Ceil.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Ceil.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric; @@ -64,7 +63,7 @@ public String getWriteableName() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (dataType().isWholeNumber()) { return toEvaluator.apply(field()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Exp.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Exp.java index a0d9937fc87b8..7abef8bba711d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Exp.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Exp.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; /** * Returns the value of e raised to the power of tbe number specified as parameter @@ -58,9 +57,7 @@ public String getWriteableName() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var field = toEvaluator.apply(field()); var fieldType = field().dataType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Floor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Floor.java index 638770f2f079a..7e727c1c2cada 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Floor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Floor.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric; @@ -66,7 +65,7 @@ public String getWriteableName() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (dataType().isWholeNumber()) { return toEvaluator.apply(field()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log.java index da11d1e77885b..4528897c194ca 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -145,7 +144,7 @@ public DataType dataType() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var valueEval = Cast.cast(source(), value.dataType(), DataType.DOUBLE, toEvaluator.apply(value)); if (base != null) { var baseEval = Cast.cast(source(), base.dataType(), DataType.DOUBLE, toEvaluator.apply(base)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10.java index ae725f6ed6498..1e987651b686c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric; @@ -62,7 +61,7 @@ public String getWriteableName() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var field = toEvaluator.apply(field()); var fieldType = field().dataType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pow.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pow.java index 46d80635823ca..3f5249e3e8cb3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pow.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pow.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -128,7 +127,7 @@ public DataType dataType() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var baseEval = Cast.cast(source(), base.dataType(), DataType.DOUBLE, toEvaluator.apply(base)); var expEval = Cast.cast(source(), exponent.dataType(), DataType.DOUBLE, toEvaluator.apply(exponent)); return new PowEvaluator.Factory(source(), baseEval, expEval); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Round.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Round.java index 8fcb04d021e7a..b1baa6c55ce47 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Round.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Round.java @@ -31,7 +31,6 @@ import java.util.Arrays; import java.util.List; import java.util.function.BiFunction; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -167,7 +166,7 @@ public DataType dataType() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { DataType fieldType = dataType(); if (fieldType == DataType.DOUBLE) { return toEvaluator(toEvaluator, RoundDoubleNoDecimalsEvaluator.Factory::new, RoundDoubleEvaluator.Factory::new); @@ -185,7 +184,7 @@ public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator, + ToEvaluator toEvaluator, BiFunction noDecimals, TriFunction withDecimals ) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Signum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Signum.java index e78c2ce90e6c1..9c7a5fdcaa236 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Signum.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Signum.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; public class Signum extends UnaryScalarFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Signum", Signum::new); @@ -56,9 +55,7 @@ public String getWriteableName() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var field = toEvaluator.apply(field()); var fieldType = field().dataType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Sqrt.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Sqrt.java index d1af693d8aa7f..080c0448e082c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Sqrt.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Sqrt.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNumeric; @@ -56,7 +55,7 @@ public String getWriteableName() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var field = toEvaluator.apply(field()); var fieldType = field().dataType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java index 998a1815cbada..6a3b58728b192 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java @@ -84,7 +84,7 @@ protected final TypeResolution resolveType() { protected abstract TypeResolution resolveFieldType(); @Override - public final ExpressionEvaluator.Factory toEvaluator(java.util.function.Function toEvaluator) { + public final ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return evaluator(toEvaluator.apply(field())); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java index deb170d9e569c..72d96a86d31eb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java @@ -35,7 +35,6 @@ import java.util.Arrays; import java.util.List; import java.util.Objects; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -149,9 +148,7 @@ public boolean foldable() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return switch (PlannerUtils.toElementType(dataType())) { case BOOLEAN -> new MvAppendBooleanEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2)); case BYTES_REF -> new MvAppendBytesRefEvaluator.Factory(source(), toEvaluator.apply(field1), toEvaluator.apply(field2)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java index fa9475055515f..1996744a76567 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java @@ -28,7 +28,6 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import java.io.IOException; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -87,7 +86,7 @@ public DataType dataType() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new EvaluatorFactory(toEvaluator.apply(left()), toEvaluator.apply(right())); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java index 212f626090789..cf49607893aae 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java @@ -14,7 +14,6 @@ import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.operator.EvalOperator; -import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.search.aggregations.metrics.CompensatedSum; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -33,7 +32,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -110,7 +108,7 @@ public boolean foldable() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return switch (PlannerUtils.toElementType(field.dataType())) { case DOUBLE -> new MvPSeriesWeightedSumDoubleEvaluator.Factory( source(), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java index 1eb0c70a7b08e..f3a63c835bd34 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPercentile.java @@ -34,7 +34,6 @@ import java.math.BigDecimal; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -120,7 +119,7 @@ public DataType dataType() { } @Override - public final ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public final ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var fieldEval = toEvaluator.apply(field); var percentileEval = Cast.cast(source(), percentile.dataType(), DOUBLE, toEvaluator.apply(percentile)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java index c332e94b20049..9846ebe4111c0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java @@ -37,7 +37,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -178,9 +177,7 @@ public boolean foldable() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (start.foldable() && end.foldable()) { int startOffset = stringToInt(String.valueOf(start.fold())); int endOffset = stringToInt(String.valueOf(end.fold())); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java index 4ed9a01c29797..d9e41233952de 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java @@ -48,7 +48,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -146,9 +145,7 @@ public boolean foldable() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { boolean ordering = true; if (isValidOrder() == false) { throw new IllegalArgumentException( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java index fd3b9e7664dff..d6a30c6ca151c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java @@ -32,7 +32,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -130,9 +129,7 @@ public Nullability nullable() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new MvZipEvaluator.Factory( source(), toEvaluator.apply(mvLeft), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java index 30c6abc5398e3..575bb085c41f7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java @@ -35,7 +35,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -192,8 +191,8 @@ public boolean foldable() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { - List childEvaluators = children().stream().map(toEvaluator).toList(); + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + List childEvaluators = children().stream().map(toEvaluator::apply).toList(); return new ExpressionEvaluator.Factory() { @Override public ExpressionEvaluator get(DriverContext context) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java index 6fd4f79125a21..1a51af8dfeeb4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java @@ -13,9 +13,9 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import java.util.Map; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asLuceneComponent2D; import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asLuceneComponent2Ds; @@ -34,15 +34,12 @@ abstract class SpatialEvaluatorFactory { this.factoryCreator = factoryCreator; } - public abstract EvalOperator.ExpressionEvaluator.Factory get( - SpatialSourceSupplier function, - Function toEvaluator - ); + public abstract EvalOperator.ExpressionEvaluator.Factory get(SpatialSourceSupplier function, EvaluatorMapper.ToEvaluator toEvaluator); static EvalOperator.ExpressionEvaluator.Factory makeSpatialEvaluator( SpatialSourceSupplier s, Map> evaluatorRules, - Function toEvaluator + EvaluatorMapper.ToEvaluator toEvaluator ) { var evaluatorKey = new SpatialEvaluatorKey( s.crsType(), @@ -149,10 +146,7 @@ protected static class SpatialEvaluatorFactoryWithFields extends SpatialEvaluato } @Override - public EvalOperator.ExpressionEvaluator.Factory get( - SpatialSourceSupplier s, - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory get(SpatialSourceSupplier s, EvaluatorMapper.ToEvaluator toEvaluator) { return factoryCreator.apply(s.source(), toEvaluator.apply(s.left()), toEvaluator.apply(s.right())); } } @@ -176,10 +170,7 @@ protected static class SpatialEvaluatorWithConstantFactory extends SpatialEvalua } @Override - public EvalOperator.ExpressionEvaluator.Factory get( - SpatialSourceSupplier s, - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory get(SpatialSourceSupplier s, EvaluatorMapper.ToEvaluator toEvaluator) { return factoryCreator.apply(s.source(), toEvaluator.apply(s.left()), asLuceneComponent2D(s.crsType(), s.right())); } } @@ -205,10 +196,7 @@ protected static class SpatialEvaluatorWithConstantArrayFactory extends SpatialE } @Override - public EvalOperator.ExpressionEvaluator.Factory get( - SpatialSourceSupplier s, - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory get(SpatialSourceSupplier s, EvaluatorMapper.ToEvaluator toEvaluator) { return factoryCreator.apply(s.source(), toEvaluator.apply(s.left()), asLuceneComponent2Ds(s.crsType(), s.right())); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java index 68ca793089499..ee2b4450a64ff 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java @@ -32,7 +32,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.Function; import java.util.function.Predicate; import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.asGeometryDocValueReader; @@ -112,9 +111,7 @@ public SpatialRelatesFunction surrogate() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return SpatialEvaluatorFactory.makeSpatialEvaluator(this, evaluatorRules(), toEvaluator); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java index 14bded51aa55f..17bcc68004bff 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java @@ -30,7 +30,6 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import java.io.IOException; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils.makeGeometryFromLiteral; @@ -177,9 +176,7 @@ public Object fold() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (right().foldable()) { return toEvaluator(toEvaluator, left(), makeGeometryFromLiteral(right()), leftDocValues); } else if (left().foldable()) { @@ -209,7 +206,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator( } private EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator, + ToEvaluator toEvaluator, Expression field, Geometry geometry, boolean docValues @@ -222,7 +219,7 @@ private EvalOperator.ExpressionEvaluator.Factory toEvaluator( } private EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator, + ToEvaluator toEvaluator, Expression field, Point point, boolean docValues diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StX.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StX.java index 18046135933b0..d1d85b03eb18a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StX.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StX.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; @@ -72,9 +71,7 @@ protected Expression.TypeResolution resolveType() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new StXFromWKBEvaluator.Factory(toEvaluator.apply(field()), source()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StY.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StY.java index bf97c3e2a3547..2056dcaed87a7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StY.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StY.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.UNSPECIFIED; @@ -72,9 +71,7 @@ protected TypeResolution resolveType() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new StYFromWKBEvaluator.Factory(toEvaluator.apply(field()), source()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java index 23ee942bcf53a..46ecc9e026d3d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import java.util.stream.Stream; import static org.elasticsearch.common.unit.ByteSizeUnit.MB; @@ -106,8 +105,8 @@ public boolean foldable() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { - var values = children().stream().map(toEvaluator).toArray(ExpressionEvaluator.Factory[]::new); + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + var values = children().stream().map(toEvaluator::apply).toArray(ExpressionEvaluator.Factory[]::new); return new ConcatEvaluator.Factory(source(), context -> new BreakingBytesRefBuilder(context.breaker(), "concat"), values); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java index 1d2b743fe5a7a..e97e65a3e60fc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -126,7 +125,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new EndsWithEvaluator.Factory(source(), toEvaluator.apply(str), toEvaluator.apply(suffix)); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrim.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrim.java index ece70da51ef19..8a4a5f4d841a5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrim.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LTrim.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -69,7 +68,7 @@ protected TypeResolution resolveType() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new LTrimEvaluator.Factory(source(), toEvaluator.apply(field())); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java index b0e5b41f971e1..e7572caafd8f5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Left.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -95,7 +94,7 @@ static BytesRef process( } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new LeftEvaluator.Factory( source(), context -> new BytesRef(), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java index 241eab6d5b904..f4bb7f35cb466 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Length.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -87,7 +86,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new LengthEvaluator.Factory(source(), toEvaluator.apply(field())); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Locate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Locate.java index f6eff2fcbd6b3..528baa613cc02 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Locate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Locate.java @@ -28,7 +28,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -161,7 +160,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { ExpressionEvaluator.Factory strExpr = toEvaluator.apply(str); ExpressionEvaluator.Factory substrExpr = toEvaluator.apply(substr); if (start == null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java index bb923ec924d31..b46c46c89deba 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java @@ -22,7 +22,6 @@ import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -79,9 +78,7 @@ protected TypeResolution resolveType() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return AutomataMatch.toEvaluator(source(), toEvaluator.apply(field()), pattern().createAutomaton()); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrim.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrim.java index 4c210607cfbe0..b79e1adf99a20 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrim.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RTrim.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -69,7 +68,7 @@ protected TypeResolution resolveType() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new RTrimEvaluator.Factory(source(), toEvaluator.apply(field())); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java index 3ff28e08f4ce1..2cc14399df2ae 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.common.unit.ByteSizeUnit.MB; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; @@ -143,7 +142,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { ExpressionEvaluator.Factory strExpr = toEvaluator.apply(str); if (number.foldable()) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java index 30c8793fe371a..4fa191244cb42 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -146,7 +145,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var strEval = toEvaluator.apply(str); var newStrEval = toEvaluator.apply(newStr); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java index ab6d3bf6cef99..b069b984ea81e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Right.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -99,7 +98,7 @@ static BytesRef process( } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new RightEvaluator.Factory( source(), context -> new BytesRef(), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java index e6225a008fceb..6481ce5764e1f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.common.unit.ByteSizeUnit.MB; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; @@ -111,7 +110,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (field.foldable()) { Object folded = field.fold(); if (folded instanceof Integer num) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java index 79ff23ac6737a..b1f5da56d011b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java @@ -28,7 +28,6 @@ import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -158,7 +157,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var str = toEvaluator.apply(left()); if (right().foldable() == false) { return new SplitVariableEvaluator.Factory(source(), str, toEvaluator.apply(right()), context -> new BytesRef()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java index fc40a73471194..2256ec2179adf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java @@ -26,7 +26,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -123,7 +122,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new StartsWithEvaluator.Factory(source(), toEvaluator.apply(str), toEvaluator.apply(prefix)); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java index 7c2ecd0c60e49..73ea409676fbd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Substring.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; @@ -180,7 +179,7 @@ protected NodeInfo info() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var strFactory = toEvaluator.apply(str); var startFactory = toEvaluator.apply(start); if (length == null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java index 62255a0a31ea6..c475469488d7b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.List; import java.util.Locale; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -96,7 +95,7 @@ static BytesRef process(BytesRef val, @Fixed Locale locale) { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var fieldEvaluator = toEvaluator.apply(field); return new ToLowerEvaluator.Factory(source(), fieldEvaluator, configuration().locale()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java index e6eba0d01e4da..1b5084a7916ef 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.util.List; import java.util.Locale; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -96,7 +95,7 @@ static BytesRef process(BytesRef val, @Fixed Locale locale) { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var fieldEvaluator = toEvaluator.apply(field); return new ToUpperEvaluator.Factory(source(), fieldEvaluator, configuration().locale()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Trim.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Trim.java index 36dc3d97992ab..1fe7529caa2da 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Trim.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Trim.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -69,7 +68,7 @@ protected TypeResolution resolveType() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var field = toEvaluator.apply(field()); return new TrimEvaluator.Factory(source(), field); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java index 15470bb56b29f..714c4ca04a862 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java @@ -23,7 +23,6 @@ import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -85,9 +84,7 @@ protected TypeResolution resolveType() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return AutomataMatch.toEvaluator( source(), toEvaluator.apply(field()), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java index 5b7cc74faed86..d407dd8bf7de1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java @@ -20,7 +20,6 @@ import java.time.Period; import java.time.temporal.TemporalAmount; import java.util.Collection; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD; @@ -159,7 +158,7 @@ public final Object fold() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (dataType() == DATETIME) { // One of the arguments has to be a datetime and the other a temporal amount. Expression datetimeArgument; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java index 400e70b641111..62201bcfa858d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; @@ -170,7 +169,7 @@ public static String formatIncompatibleTypesMessage(String symbol, DataType left } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var commonType = dataType(); var leftType = left().dataType(); if (leftType.isNumeric()) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java index 67b770d14339e..fb32282005f02 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java @@ -25,7 +25,6 @@ import java.time.Duration; import java.time.Period; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; @@ -61,7 +60,7 @@ public String getWriteableName() { } @Override - public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { DataType type = dataType(); if (type.isNumeric()) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java index b50d70e69819d..db771a6354883 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java @@ -27,7 +27,6 @@ import java.time.ZoneId; import java.util.List; import java.util.Map; -import java.util.function.Function; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; @@ -168,9 +167,7 @@ public BinaryComparisonOperation getFunctionType() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { // Our type is always boolean, so figure out the evaluator type from the inputs DataType commonType = commonType(left().dataType(), right().dataType()); EvalOperator.ExpressionEvaluator.Factory lhs; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java index 333f32e82c579..eda6aadccc86a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java @@ -33,7 +33,6 @@ import java.util.BitSet; import java.util.Collections; import java.util.List; -import java.util.function.Function; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; @@ -213,9 +212,7 @@ protected Expression canonicalize() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var commonType = commonType(); EvalOperator.ExpressionEvaluator.Factory lhs; EvalOperator.ExpressionEvaluator.Factory[] factories; @@ -226,7 +223,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator( .toArray(EvalOperator.ExpressionEvaluator.Factory[]::new); } else { lhs = toEvaluator.apply(value); - factories = list.stream().map(e -> toEvaluator.apply(e)).toArray(EvalOperator.ExpressionEvaluator.Factory[]::new); + factories = list.stream().map(toEvaluator::apply).toArray(EvalOperator.ExpressionEvaluator.Factory[]::new); } if (commonType == BOOLEAN) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DeepCopy.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DeepCopy.java index d25305a9ea190..593a444eceec2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DeepCopy.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DeepCopy.java @@ -20,7 +20,6 @@ import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import java.io.IOException; -import java.util.function.Function; /** * Expression that makes a deep copy of the block it receives. @@ -41,9 +40,7 @@ public String getWriteableName() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { EvalOperator.ExpressionEvaluator.Factory childEval = toEvaluator.apply(child()); return ctx -> new EvalOperator.ExpressionEvaluator() { private final EvalOperator.ExpressionEvaluator child = childEval.get(ctx); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistryTests.java index 7f19419b21816..801bd8700d014 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistryTests.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; -import java.util.function.Function; import static java.util.Collections.emptyList; import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration; @@ -258,9 +257,7 @@ protected NodeInfo info() { } @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return null; } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java index f760694391ee4..c9b6de64e079d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.evaluator.EvalMapper; +import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.scalar.VaragsTestCaseBuilder; @@ -33,7 +34,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.function.Function; import java.util.function.Supplier; import static org.elasticsearch.compute.data.BlockUtils.toJavaObject; @@ -174,7 +174,7 @@ public void testCoalesceIsLazy() { Layout.Builder builder = new Layout.Builder(); buildLayout(builder, exp); Layout layout = builder.build(); - Function map = child -> { + EvaluatorMapper.ToEvaluator toEvaluator = child -> { if (child == evil) { return dvrCtx -> new EvalOperator.ExpressionEvaluator() { @Override @@ -189,7 +189,7 @@ public void close() {} return EvalMapper.toEvaluator(child, layout); }; try ( - EvalOperator.ExpressionEvaluator eval = exp.toEvaluator(map).get(driverContext()); + EvalOperator.ExpressionEvaluator eval = exp.toEvaluator(toEvaluator).get(driverContext()); Block block = eval.eval(row(testCase.getDataValues())) ) { assertThat(toJavaObject(block, 0), testCase.getMatcher()); From 60ae7463a8c239f811c30cca42009081bac8940a Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Fri, 4 Oct 2024 09:06:39 -0400 Subject: [PATCH 085/194] [ESQL] Support datetime data type in Least and Greatest functions (#113961) While working on Date Nanos, I noticed that Least and Greatest didn't have support for datetime. This PR corrects that and adds tests for it. It seems to me that resolveType() is doing the wrong thing for these functions, as it accepts types that then do not have evaluator mappings, but refactoring that seems out of scope right now. --------- Co-authored-by: Elastic Machine --- docs/changelog/113961.yaml | 5 +++++ .../functions/kibana/definition/greatest.json | 18 ++++++++++++++++++ .../functions/kibana/definition/least.json | 18 ++++++++++++++++++ .../esql/functions/types/greatest.asciidoc | 1 + .../esql/functions/types/least.asciidoc | 1 + .../src/main/resources/date.csv-spec | 16 ++++++++++++++++ .../src/main/resources/meta.csv-spec | 12 ++++++------ .../xpack/esql/action/EsqlCapabilities.java | 5 +++++ .../function/scalar/conditional/Greatest.java | 8 ++++---- .../function/scalar/conditional/Least.java | 8 ++++---- .../scalar/conditional/GreatestTests.java | 15 +++++++++++++++ .../scalar/conditional/LeastTests.java | 15 +++++++++++++++ 12 files changed, 108 insertions(+), 14 deletions(-) create mode 100644 docs/changelog/113961.yaml diff --git a/docs/changelog/113961.yaml b/docs/changelog/113961.yaml new file mode 100644 index 0000000000000..24cb1f45f029e --- /dev/null +++ b/docs/changelog/113961.yaml @@ -0,0 +1,5 @@ +pr: 113961 +summary: "[ESQL] Support datetime data type in Least and Greatest functions" +area: ES|QL +type: bug +issues: [] diff --git a/docs/reference/esql/functions/kibana/definition/greatest.json b/docs/reference/esql/functions/kibana/definition/greatest.json index 0e32fca5b4ca1..2818a5ac56339 100644 --- a/docs/reference/esql/functions/kibana/definition/greatest.json +++ b/docs/reference/esql/functions/kibana/definition/greatest.json @@ -35,6 +35,24 @@ "variadic" : true, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "first", + "type" : "date", + "optional" : false, + "description" : "First of the columns to evaluate." + }, + { + "name" : "rest", + "type" : "date", + "optional" : true, + "description" : "The rest of the columns to evaluate." + } + ], + "variadic" : true, + "returnType" : "date" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/kibana/definition/least.json b/docs/reference/esql/functions/kibana/definition/least.json index 0ba34cf3cc9a2..7b545896f4ddc 100644 --- a/docs/reference/esql/functions/kibana/definition/least.json +++ b/docs/reference/esql/functions/kibana/definition/least.json @@ -34,6 +34,24 @@ "variadic" : true, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "first", + "type" : "date", + "optional" : false, + "description" : "First of the columns to evaluate." + }, + { + "name" : "rest", + "type" : "date", + "optional" : true, + "description" : "The rest of the columns to evaluate." + } + ], + "variadic" : true, + "returnType" : "date" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/types/greatest.asciidoc b/docs/reference/esql/functions/types/greatest.asciidoc index 537be55cd17ef..1454bbb6f81c1 100644 --- a/docs/reference/esql/functions/types/greatest.asciidoc +++ b/docs/reference/esql/functions/types/greatest.asciidoc @@ -7,6 +7,7 @@ first | rest | result boolean | boolean | boolean boolean | | boolean +date | date | date double | double | double integer | integer | integer integer | | integer diff --git a/docs/reference/esql/functions/types/least.asciidoc b/docs/reference/esql/functions/types/least.asciidoc index 537be55cd17ef..1454bbb6f81c1 100644 --- a/docs/reference/esql/functions/types/least.asciidoc +++ b/docs/reference/esql/functions/types/least.asciidoc @@ -7,6 +7,7 @@ first | rest | result boolean | boolean | boolean boolean | | boolean +date | date | date double | double | double integer | integer | integer integer | | integer diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index f52829741ed6e..1fdb6150a0e81 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -1270,3 +1270,19 @@ emp_no:integer | birth_date:datetime 10007 | 1957-05-23T00:00:00Z 10008 | 1958-02-19T00:00:00Z ; + +Least for dates +required_capability: least_greatest_for_dates +ROW a = LEAST(TO_DATETIME("1957-05-23T00:00:00Z"), TO_DATETIME("1958-02-19T00:00:00Z")); + +a:datetime +1957-05-23T00:00:00 +; + +GREATEST for dates +required_capability: least_greatest_for_dates +ROW a = GREATEST(TO_DATETIME("1957-05-23T00:00:00Z"), TO_DATETIME("1958-02-19T00:00:00Z")); + +a:datetime +1958-02-19T00:00:00 +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec index 13c3857a5c497..68d780022b0e2 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec @@ -33,9 +33,9 @@ double e() "double exp(number:double|integer|long|unsigned_long)" "double|integer|long|unsigned_long floor(number:double|integer|long|unsigned_long)" "keyword from_base64(string:keyword|text)" -"boolean|double|integer|ip|keyword|long|text|version greatest(first:boolean|double|integer|ip|keyword|long|text|version, ?rest...:boolean|double|integer|ip|keyword|long|text|version)" +"boolean|date|double|integer|ip|keyword|long|text|version greatest(first:boolean|date|double|integer|ip|keyword|long|text|version, ?rest...:boolean|date|double|integer|ip|keyword|long|text|version)" "ip ip_prefix(ip:ip, prefixLengthV4:integer, prefixLengthV6:integer)" -"boolean|double|integer|ip|keyword|long|text|version least(first:boolean|double|integer|ip|keyword|long|text|version, ?rest...:boolean|double|integer|ip|keyword|long|text|version)" +"boolean|date|double|integer|ip|keyword|long|text|version least(first:boolean|date|double|integer|ip|keyword|long|text|version, ?rest...:boolean|date|double|integer|ip|keyword|long|text|version)" "keyword left(string:keyword|text, length:integer)" "integer length(string:keyword|text)" "integer locate(string:keyword|text, substring:keyword|text, ?start:integer)" @@ -165,9 +165,9 @@ ends_with |[str, suffix] |["keyword|text", "keyword|te exp |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. floor |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. from_base64 |string |"keyword|text" |A base64 string. -greatest |first |"boolean|double|integer|ip|keyword|long|text|version" |First of the columns to evaluate. +greatest |first |"boolean|date|double|integer|ip|keyword|long|text|version" |First of the columns to evaluate. ip_prefix |[ip, prefixLengthV4, prefixLengthV6]|[ip, integer, integer] |[IP address of type `ip` (both IPv4 and IPv6 are supported)., Prefix length for IPv4 addresses., Prefix length for IPv6 addresses.] -least |first |"boolean|double|integer|ip|keyword|long|text|version" |First of the columns to evaluate. +least |first |"boolean|date|double|integer|ip|keyword|long|text|version" |First of the columns to evaluate. left |[string, length] |["keyword|text", integer] |[The string from which to return a substring., The number of characters to return.] length |string |"keyword|text" |String expression. If `null`, the function returns `null`. locate |[string, substring, start] |["keyword|text", "keyword|text", "integer"] |[An input string, A substring to locate in the input string, The start index] @@ -431,9 +431,9 @@ ends_with |boolean exp |double |false |false |false floor |"double|integer|long|unsigned_long" |false |false |false from_base64 |keyword |false |false |false -greatest |"boolean|double|integer|ip|keyword|long|text|version" |false |true |false +greatest |"boolean|date|double|integer|ip|keyword|long|text|version" |false |true |false ip_prefix |ip |[false, false, false] |false |false -least |"boolean|double|integer|ip|keyword|long|text|version" |false |true |false +least |"boolean|date|double|integer|ip|keyword|long|text|version" |false |true |false left |keyword |[false, false] |false |false length |integer |false |false |false locate |integer |[false, false, true] |false |false diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index f0fa89dedd9ab..e7a1599d48bd3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -278,6 +278,11 @@ public enum Cap { */ TO_DATE_NANOS(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG), + /** + * Support for datetime in least and greatest functions + */ + LEAST_GREATEST_FOR_DATES, + /** * Support CIDRMatch in CombineDisjunctions rule. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java index c374eac32acc0..9d815d15accdc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java @@ -43,7 +43,7 @@ public class Greatest extends EsqlScalarFunction implements OptionalArgument { private DataType dataType; @FunctionInfo( - returnType = { "boolean", "double", "integer", "ip", "keyword", "long", "text", "version" }, + returnType = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" }, description = "Returns the maximum value from multiple columns. This is similar to <>\n" + "except it is intended to run on multiple columns at once.", note = "When run on `keyword` or `text` fields, this returns the last string in alphabetical order. " @@ -54,12 +54,12 @@ public Greatest( Source source, @Param( name = "first", - type = { "boolean", "double", "integer", "ip", "keyword", "long", "text", "version" }, + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" }, description = "First of the columns to evaluate." ) Expression first, @Param( name = "rest", - type = { "boolean", "double", "integer", "ip", "keyword", "long", "text", "version" }, + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" }, description = "The rest of the columns to evaluate.", optional = true ) List rest @@ -152,7 +152,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (dataType == DataType.INTEGER) { return new GreatestIntEvaluator.Factory(source(), factories); } - if (dataType == DataType.LONG) { + if (dataType == DataType.LONG || dataType == DataType.DATETIME) { return new GreatestLongEvaluator.Factory(source(), factories); } if (dataType == DataType.KEYWORD diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java index 90d0d33995cb0..435a14d0fef33 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java @@ -43,7 +43,7 @@ public class Least extends EsqlScalarFunction implements OptionalArgument { private DataType dataType; @FunctionInfo( - returnType = { "boolean", "double", "integer", "ip", "keyword", "long", "text", "version" }, + returnType = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" }, description = "Returns the minimum value from multiple columns. " + "This is similar to <> except it is intended to run on multiple columns at once.", examples = @Example(file = "math", tag = "least") @@ -52,12 +52,12 @@ public Least( Source source, @Param( name = "first", - type = { "boolean", "double", "integer", "ip", "keyword", "long", "text", "version" }, + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" }, description = "First of the columns to evaluate." ) Expression first, @Param( name = "rest", - type = { "boolean", "double", "integer", "ip", "keyword", "long", "text", "version" }, + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" }, description = "The rest of the columns to evaluate.", optional = true ) List rest @@ -151,7 +151,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (dataType == DataType.INTEGER) { return new LeastIntEvaluator.Factory(source(), factories); } - if (dataType == DataType.LONG) { + if (dataType == DataType.LONG || dataType == DataType.DATETIME) { return new LeastLongEvaluator.Factory(source(), factories); } if (dataType == DataType.KEYWORD diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java index 7cc03be7d6273..311e3e3d89149 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java @@ -100,6 +100,21 @@ public static Iterable parameters() { ) ) ); + suppliers.add( + new TestCaseSupplier( + "(a, b)", + List.of(DataType.DATETIME, DataType.DATETIME), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(1727877348000L, DataType.DATETIME, "a"), + new TestCaseSupplier.TypedData(1727790948000L, DataType.DATETIME, "b") + ), + "GreatestLongEvaluator[values=[MvMax[field=Attribute[channel=0]], MvMax[field=Attribute[channel=1]]]]", + DataType.DATETIME, + equalTo(1727877348000L) + ) + ) + ); return parameterSuppliersFromTypedData(anyNullIsNull(false, suppliers)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java index aa475f05ebe69..69842fde90312 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java @@ -99,6 +99,21 @@ public static Iterable parameters() { ) ) ); + suppliers.add( + new TestCaseSupplier( + "(a, b)", + List.of(DataType.DATETIME, DataType.DATETIME), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(1727877348000L, DataType.DATETIME, "a"), + new TestCaseSupplier.TypedData(1727790948000L, DataType.DATETIME, "b") + ), + "LeastLongEvaluator[values=[MvMin[field=Attribute[channel=0]], MvMin[field=Attribute[channel=1]]]]", + DataType.DATETIME, + equalTo(1727790948000L) + ) + ) + ); return parameterSuppliersFromTypedData(anyNullIsNull(false, suppliers)); } From f58b6b426d4a860681ce88e3532e041fdbd5122a Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Fri, 4 Oct 2024 15:07:04 +0200 Subject: [PATCH 086/194] ES|QL: Ensure minimum capacity for PlanStreamInput caches (#114116) --- docs/changelog/114116.yaml | 5 +++++ .../elasticsearch/xpack/esql/io/stream/PlanStreamInput.java | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 docs/changelog/114116.yaml diff --git a/docs/changelog/114116.yaml b/docs/changelog/114116.yaml new file mode 100644 index 0000000000000..8d1c9e162ae23 --- /dev/null +++ b/docs/changelog/114116.yaml @@ -0,0 +1,5 @@ +pr: 114116 +summary: "ES|QL: Ensure minimum capacity for `PlanStreamInput` caches" +area: ES|QL +type: bug +issues: [] diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java index ef4417a1c7a02..2b09a395c4a3d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamInput.java @@ -209,7 +209,7 @@ private Attribute attributeFromCache(int id) throws IOException { private void cacheAttribute(int id, Attribute attr) { assert id >= 0; if (id >= attributesCache.length) { - attributesCache = ArrayUtil.grow(attributesCache); + attributesCache = ArrayUtil.grow(attributesCache, id + 1); } attributesCache[id] = attr; } @@ -252,7 +252,7 @@ private EsField esFieldFromCache(int id) throws IOException { private void cacheEsField(int id, EsField field) { assert id >= 0; if (id >= esFieldsCache.length) { - esFieldsCache = ArrayUtil.grow(esFieldsCache); + esFieldsCache = ArrayUtil.grow(esFieldsCache, id + 1); } esFieldsCache[id] = field; } From 16fb3ccabe09e6fad9f3bf58ea96d6c9c7f89122 Mon Sep 17 00:00:00 2001 From: Panagiotis Bailis Date: Fri, 4 Oct 2024 16:36:07 +0300 Subject: [PATCH 087/194] Add PIT information to collapse requests (#114046) --- .../action/search/ExpandSearchPhase.java | 10 ++- .../action/search/ExpandSearchPhaseTests.java | 83 ++++++++++++++++--- 2 files changed, 79 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index 7ee9be25b3d59..5457ca60d0da4 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -10,6 +10,7 @@ package org.elasticsearch.action.search; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.Maps; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.InnerHitBuilder; @@ -82,8 +83,15 @@ private void doRun() { CollapseBuilder innerCollapseBuilder = innerHitBuilder.getInnerCollapseBuilder(); SearchSourceBuilder sourceBuilder = buildExpandSearchSourceBuilder(innerHitBuilder, innerCollapseBuilder).query(groupQuery) .postFilter(searchRequest.source().postFilter()) - .runtimeMappings(searchRequest.source().runtimeMappings()); + .runtimeMappings(searchRequest.source().runtimeMappings()) + .pointInTimeBuilder(searchRequest.source().pointInTimeBuilder()); SearchRequest groupRequest = new SearchRequest(searchRequest); + if (searchRequest.pointInTimeBuilder() != null) { + // if the original request has a point in time, we propagate it to the inner search request + // and clear the indices and preference from the inner search request + groupRequest.indices(Strings.EMPTY_ARRAY); + groupRequest.preference(null); + } groupRequest.source(sourceBuilder); multiRequest.add(groupRequest); } diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index 45c4b3bd2d7c7..5240d704dea3b 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -11,7 +11,10 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -19,6 +22,7 @@ import org.elasticsearch.search.AbstractSearchTestCase; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.test.ESTestCase; @@ -317,14 +321,12 @@ public void testExpandRequestOptions() throws IOException { @Override void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener listener) { final QueryBuilder postFilter = QueryBuilders.existsQuery("foo"); - assertTrue(request.requests().stream().allMatch((r) -> "foo".equals(r.preference()))); + assertTrue(request.requests().stream().allMatch((r) -> "foobar".equals(r.preference()))); assertTrue(request.requests().stream().allMatch((r) -> "baz".equals(r.routing()))); assertTrue(request.requests().stream().allMatch((r) -> version == r.source().version())); assertTrue(request.requests().stream().allMatch((r) -> seqNoAndTerm == r.source().seqNoAndPrimaryTerm())); assertTrue(request.requests().stream().allMatch((r) -> postFilter.equals(r.source().postFilter()))); - assertTrue(request.requests().stream().allMatch((r) -> r.source().fetchSource().fetchSource() == false)); - assertTrue(request.requests().stream().allMatch((r) -> r.source().fetchSource().includes().length == 0)); - assertTrue(request.requests().stream().allMatch((r) -> r.source().fetchSource().excludes().length == 0)); + assertTrue(request.requests().stream().allMatch((r) -> r.source().fetchSource() == null)); } }; mockSearchPhaseContext.getRequest() @@ -338,17 +340,72 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL .preference("foobar") .routing("baz"); - SearchHits hits = SearchHits.empty(new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0f); - ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, hits, () -> new SearchPhase("test") { + SearchHit hit = new SearchHit(1, "ID"); + hit.setDocumentField("someField", new DocumentField("someField", Collections.singletonList("foo"))); + SearchHits hits = new SearchHits(new SearchHit[] { hit }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F); + try { + ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, hits, () -> new SearchPhase("test") { + @Override + public void run() { + mockSearchPhaseContext.sendSearchResponse(new SearchResponseSections(hits, null, null, false, null, null, 1), null); + } + }); + phase.run(); + mockSearchPhaseContext.assertNoFailure(); + } finally { + hits.decRef(); + } + } finally { + var resp = mockSearchPhaseContext.searchResponse.get(); + if (resp != null) { + resp.decRef(); + } + } + } + + public void testExpandSearchRespectsOriginalPIT() { + MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1); + final PointInTimeBuilder pit = new PointInTimeBuilder(new BytesArray("foo")); + try { + boolean version = randomBoolean(); + final boolean seqNoAndTerm = randomBoolean(); + + mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null, null) { @Override - public void run() { - mockSearchPhaseContext.sendSearchResponse(new SearchResponseSections(hits, null, null, false, null, null, 1), null); + void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener listener) { + assertTrue(request.requests().stream().allMatch((r) -> r.preference() == null)); + assertTrue(request.requests().stream().allMatch((r) -> r.indices() == Strings.EMPTY_ARRAY)); + assertTrue(request.requests().stream().allMatch((r) -> r.source().pointInTimeBuilder().equals(pit))); } - }); - phase.run(); - mockSearchPhaseContext.assertNoFailure(); - assertNotNull(mockSearchPhaseContext.searchResponse.get()); - mockSearchPhaseContext.execute(() -> {}); + }; + mockSearchPhaseContext.getRequest() + .source( + new SearchSourceBuilder().collapse( + new CollapseBuilder("someField").setInnerHits( + new InnerHitBuilder().setName("foobarbaz").setVersion(version).setSeqNoAndPrimaryTerm(seqNoAndTerm) + ) + ).fetchSource(false).postFilter(QueryBuilders.existsQuery("foo")).pointInTimeBuilder(pit) + ) + .routing("baz"); + + SearchHit hit = new SearchHit(1, "ID"); + hit.setDocumentField("someField", new DocumentField("someField", Collections.singletonList("foo"))); + SearchHits hits = new SearchHits(new SearchHit[] { hit }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0F); + try { + ExpandSearchPhase phase = new ExpandSearchPhase(mockSearchPhaseContext, hits, () -> new SearchPhase("test") { + @Override + public void run() { + mockSearchPhaseContext.sendSearchResponse( + new SearchResponseSections(hits, null, null, false, null, null, 1), + new AtomicArray<>(0) + ); + } + }); + phase.run(); + mockSearchPhaseContext.assertNoFailure(); + } finally { + hits.decRef(); + } } finally { var resp = mockSearchPhaseContext.searchResponse.get(); if (resp != null) { From e32a70a40803fa75dea0999df171d337eb122567 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 4 Oct 2024 14:40:27 +0100 Subject: [PATCH 088/194] Unmute `S3BlobStoreRepositoryTests#testMetrics` (#114129) We're awaiting more information about failures of this test, so we need to actually run it occasionally... Relates #101608 --- .../repositories/s3/S3BlobStoreRepositoryTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index 429a81b02bd5e..6b4dd5ed86e2d 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -235,7 +235,6 @@ public void testAbortRequestStats() throws Exception { } @TestIssueLogging(issueUrl = "https://github.com/elastic/elasticsearch/issues/101608", value = "com.amazonaws.request:DEBUG") - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/101608") public void testMetrics() throws Exception { // Create the repository and perform some activities final String repository = createRepository(randomRepositoryName(), false); From 0ef5219902b03751871e50232f88887d13c6e4de Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Fri, 4 Oct 2024 15:46:31 +0200 Subject: [PATCH 089/194] Don't validate internal stats if they are empty (#113846) Fixes a validation step that might prevent creating empty aggregations iif the output format does not allows negative numbers. --- docs/changelog/113846.yaml | 6 ++++ .../aggregations/metrics/InternalStats.java | 2 +- .../metrics/ExtendedStatsAggregatorTests.java | 35 +++++++++++++++++++ .../metrics/StatsAggregatorTests.java | 26 ++++++++++++++ 4 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/113846.yaml diff --git a/docs/changelog/113846.yaml b/docs/changelog/113846.yaml new file mode 100644 index 0000000000000..5fdd56e98d706 --- /dev/null +++ b/docs/changelog/113846.yaml @@ -0,0 +1,6 @@ +pr: 113846 +summary: Don't validate internal stats if they are empty +area: Aggregations +type: bug +issues: + - 113811 diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java index cd4a0bac7f429..6fdd41d374d0e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalStats.java @@ -76,7 +76,7 @@ public InternalStats( } private void verifyFormattingStats() { - if (format != DocValueFormat.RAW) { + if (format != DocValueFormat.RAW && count != 0) { verifyFormattingStat(Fields.MIN, format, min); verifyFormattingStat(Fields.MAX, format, max); verifyFormattingStat(Fields.AVG, format, getAvg()); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java index 8b8a4f97d540e..ae4ed3568683a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregatorTests.java @@ -14,16 +14,20 @@ import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import java.io.IOException; +import java.util.Map; import java.util.function.Consumer; import static java.util.Collections.singleton; +import static org.elasticsearch.search.aggregations.AggregationBuilders.stats; public class ExtendedStatsAggregatorTests extends AggregatorTestCase { private static final double TOLERANCE = 1e-5; @@ -49,6 +53,37 @@ public void testEmpty() throws IOException { }); } + public void testEmptyDate() throws IOException { + DateFormatter.forPattern("epoch_millis"); + final MappedFieldType ft = new DateFieldMapper.DateFieldType( + "field", + true, + true, + false, + true, + DateFormatter.forPattern("epoch_millis"), + DateFieldMapper.Resolution.MILLISECONDS, + null, + null, + Map.of() + ); + testCase(ft, iw -> {}, stats -> { + assertEquals(0d, stats.getCount(), 0); + assertEquals(0d, stats.getSum(), 0); + assertEquals(Float.NaN, stats.getAvg(), 0); + assertEquals(Double.POSITIVE_INFINITY, stats.getMin(), 0); + assertEquals(Double.NEGATIVE_INFINITY, stats.getMax(), 0); + assertEquals(Double.NaN, stats.getVariance(), 0); + assertEquals(Double.NaN, stats.getVariancePopulation(), 0); + assertEquals(Double.NaN, stats.getVarianceSampling(), 0); + assertEquals(Double.NaN, stats.getStdDeviation(), 0); + assertEquals(Double.NaN, stats.getStdDeviationPopulation(), 0); + assertEquals(Double.NaN, stats.getStdDeviationSampling(), 0); + assertEquals(0d, stats.getSumOfSquares(), 0); + assertFalse(AggregationInspectionHelper.hasValue(stats)); + }); + } + public void testRandomDoubles() throws IOException { MappedFieldType ft = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.DOUBLE); final ExtendedSimpleStatsAggregator expected = new ExtendedSimpleStatsAggregator(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java index 9f48cb2279320..ddd1fa987ad2b 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsAggregatorTests.java @@ -18,7 +18,9 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; @@ -73,6 +75,30 @@ public void testEmpty() throws IOException { }, ft); } + public void testEmptyDate() throws IOException { + DateFormatter.forPattern("epoch_millis"); + final MappedFieldType ft = new DateFieldMapper.DateFieldType( + "field", + true, + true, + false, + true, + DateFormatter.forPattern("epoch_millis"), + DateFieldMapper.Resolution.MILLISECONDS, + null, + null, + Map.of() + ); + testCase(stats("_name").field(ft.name()), iw -> {}, stats -> { + assertEquals(0d, stats.getCount(), 0); + assertEquals(0d, stats.getSum(), 0); + assertEquals(Float.NaN, stats.getAvg(), 0); + assertEquals(Double.POSITIVE_INFINITY, stats.getMin(), 0); + assertEquals(Double.NEGATIVE_INFINITY, stats.getMax(), 0); + assertFalse(AggregationInspectionHelper.hasValue(stats)); + }, ft); + } + public void testRandomDoubles() throws IOException { final MappedFieldType ft = new NumberFieldMapper.NumberFieldType("field", NumberType.DOUBLE); final SimpleStatsAggregator expected = new SimpleStatsAggregator(); From 70f4a0ebd15da280d30527b184170e0974da635e Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 4 Oct 2024 23:46:34 +1000 Subject: [PATCH 090/194] Mute org.elasticsearch.xpack.inference.InferenceCrudIT testGet #114135 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index e83f15d445c93..2ea9aa3a6647d 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -351,6 +351,9 @@ tests: - class: org.elasticsearch.xpack.inference.services.openai.OpenAiServiceTests method: testInfer_StreamRequest_ErrorResponse issue: https://github.com/elastic/elasticsearch/issues/114105 +- class: org.elasticsearch.xpack.inference.InferenceCrudIT + method: testGet + issue: https://github.com/elastic/elasticsearch/issues/114135 # Examples: # From 663ab04d7e44de11d9f554390ed2a9cdd8a18c48 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Fri, 4 Oct 2024 07:02:47 -0700 Subject: [PATCH 091/194] Add query fetch and indexing metrics per index mode (#113978) This change exposes query, fetch and indexing metrics for each index mode. --- .../monitor/metrics/IndicesMetricsIT.java | 270 +++++++++++++++++- .../monitor/metrics/IndicesMetrics.java | 90 +++++- 2 files changed, 348 insertions(+), 12 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java index b72257b884f08..4a060eadc735b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java @@ -9,23 +9,38 @@ package org.elasticsearch.monitor.metrics; +import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.mapper.OnScriptError; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.LongFieldScript; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.telemetry.Measurement; import org.elasticsearch.telemetry.TestTelemetryPlugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.json.JsonXContent; import org.hamcrest.Matcher; +import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; import static org.elasticsearch.index.mapper.DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0) @@ -42,7 +57,7 @@ public List> getSettings() { @Override protected Collection> nodePlugins() { - return List.of(TestTelemetryPlugin.class, TestAPMInternalSettings.class); + return List.of(TestTelemetryPlugin.class, TestAPMInternalSettings.class, FailingFieldPlugin.class); } @Override @@ -54,27 +69,57 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { } static final String STANDARD_INDEX_COUNT = "es.indices.standard.total"; + static final String STANDARD_BYTES_SIZE = "es.indices.standard.size"; static final String STANDARD_DOCS_COUNT = "es.indices.standard.docs.total"; - static final String STANDARD_BYTES_SIZE = "es.indices.standard.bytes.total"; + static final String STANDARD_QUERY_COUNT = "es.indices.standard.query.total"; + static final String STANDARD_QUERY_TIME = "es.indices.standard.query.time"; + static final String STANDARD_QUERY_FAILURE = "es.indices.standard.query.failure.total"; + static final String STANDARD_FETCH_COUNT = "es.indices.standard.fetch.total"; + static final String STANDARD_FETCH_TIME = "es.indices.standard.fetch.time"; + static final String STANDARD_FETCH_FAILURE = "es.indices.standard.fetch.failure.total"; + static final String STANDARD_INDEXING_COUNT = "es.indices.standard.indexing.total"; + static final String STANDARD_INDEXING_TIME = "es.indices.standard.indexing.time"; + static final String STANDARD_INDEXING_FAILURE = "es.indices.standard.indexing.failure.total"; static final String TIME_SERIES_INDEX_COUNT = "es.indices.time_series.total"; + static final String TIME_SERIES_BYTES_SIZE = "es.indices.time_series.size"; static final String TIME_SERIES_DOCS_COUNT = "es.indices.time_series.docs.total"; - static final String TIME_SERIES_BYTES_SIZE = "es.indices.time_series.bytes.total"; + static final String TIME_SERIES_QUERY_COUNT = "es.indices.time_series.query.total"; + static final String TIME_SERIES_QUERY_TIME = "es.indices.time_series.query.time"; + static final String TIME_SERIES_QUERY_FAILURE = "es.indices.time_series.query.failure.total"; + static final String TIME_SERIES_FETCH_COUNT = "es.indices.time_series.fetch.total"; + static final String TIME_SERIES_FETCH_TIME = "es.indices.time_series.fetch.time"; + static final String TIME_SERIES_FETCH_FAILURE = "es.indices.time_series.fetch.failure.total"; + static final String TIME_SERIES_INDEXING_COUNT = "es.indices.time_series.indexing.total"; + static final String TIME_SERIES_INDEXING_TIME = "es.indices.time_series.indexing.time"; + static final String TIME_SERIES_INDEXING_FAILURE = "es.indices.time_series.indexing.failure.total"; static final String LOGSDB_INDEX_COUNT = "es.indices.logsdb.total"; + static final String LOGSDB_BYTES_SIZE = "es.indices.logsdb.size"; static final String LOGSDB_DOCS_COUNT = "es.indices.logsdb.docs.total"; - static final String LOGSDB_BYTES_SIZE = "es.indices.logsdb.bytes.total"; + static final String LOGSDB_QUERY_COUNT = "es.indices.logsdb.query.total"; + static final String LOGSDB_QUERY_TIME = "es.indices.logsdb.query.time"; + static final String LOGSDB_QUERY_FAILURE = "es.indices.logsdb.query.failure.total"; + static final String LOGSDB_FETCH_COUNT = "es.indices.logsdb.fetch.total"; + static final String LOGSDB_FETCH_TIME = "es.indices.logsdb.fetch.time"; + static final String LOGSDB_FETCH_FAILURE = "es.indices.logsdb.fetch.failure.total"; + static final String LOGSDB_INDEXING_COUNT = "es.indices.logsdb.indexing.total"; + static final String LOGSDB_INDEXING_TIME = "es.indices.logsdb.indexing.time"; + static final String LOGSDB_INDEXING_FAILURE = "es.indices.logsdb.indexing.failure.total"; - public void testIndicesMetrics() { + public void testIndicesMetrics() throws Exception { String node = internalCluster().startNode(); ensureStableCluster(1); final TestTelemetryPlugin telemetry = internalCluster().getInstance(PluginsService.class, node) .filterPlugins(TestTelemetryPlugin.class) .findFirst() .orElseThrow(); + final IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); + var indexing0 = indicesService.stats(CommonStatsFlags.ALL, false).getIndexing().getTotal(); telemetry.resetMeter(); long numStandardIndices = randomIntBetween(1, 5); long numStandardDocs = populateStandardIndices(numStandardIndices); + var indexing1 = indicesService.stats(CommonStatsFlags.ALL, false).getIndexing().getTotal(); collectThenAssertMetrics( telemetry, 1, @@ -104,6 +149,7 @@ public void testIndicesMetrics() { long numTimeSeriesIndices = randomIntBetween(1, 5); long numTimeSeriesDocs = populateTimeSeriesIndices(numTimeSeriesIndices); + var indexing2 = indicesService.stats(CommonStatsFlags.ALL, false).getIndexing().getTotal(); collectThenAssertMetrics( telemetry, 2, @@ -133,6 +179,7 @@ public void testIndicesMetrics() { long numLogsdbIndices = randomIntBetween(1, 5); long numLogsdbDocs = populateLogsdbIndices(numLogsdbIndices); + var indexing3 = indicesService.stats(CommonStatsFlags.ALL, false).getIndexing().getTotal(); collectThenAssertMetrics( telemetry, 3, @@ -159,6 +206,142 @@ public void testIndicesMetrics() { greaterThan(0L) ) ); + // indexing stats + collectThenAssertMetrics( + telemetry, + 4, + Map.of( + STANDARD_INDEXING_COUNT, + equalTo(numStandardDocs), + STANDARD_INDEXING_TIME, + greaterThanOrEqualTo(0L), + STANDARD_INDEXING_FAILURE, + equalTo(indexing1.getIndexFailedCount() - indexing0.getIndexCount()), + + TIME_SERIES_INDEXING_COUNT, + equalTo(numTimeSeriesDocs), + TIME_SERIES_INDEXING_TIME, + greaterThanOrEqualTo(0L), + TIME_SERIES_INDEXING_FAILURE, + equalTo(indexing2.getIndexFailedCount() - indexing1.getIndexFailedCount()), + + LOGSDB_INDEXING_COUNT, + equalTo(numLogsdbDocs), + LOGSDB_INDEXING_TIME, + greaterThanOrEqualTo(0L), + LOGSDB_INDEXING_FAILURE, + equalTo(indexing3.getIndexFailedCount() - indexing2.getIndexFailedCount()) + ) + ); + telemetry.resetMeter(); + + // search and fetch + client().prepareSearch("standard*").setSize(100).get().decRef(); + var nodeStats1 = indicesService.stats(CommonStatsFlags.ALL, false).getSearch().getTotal(); + collectThenAssertMetrics( + telemetry, + 1, + Map.of( + STANDARD_QUERY_COUNT, + equalTo(numStandardIndices), + STANDARD_QUERY_TIME, + equalTo(nodeStats1.getQueryTimeInMillis()), + STANDARD_FETCH_COUNT, + equalTo(nodeStats1.getFetchCount()), + STANDARD_FETCH_TIME, + equalTo(nodeStats1.getFetchTimeInMillis()), + + TIME_SERIES_QUERY_COUNT, + equalTo(0L), + TIME_SERIES_QUERY_TIME, + equalTo(0L), + + LOGSDB_QUERY_COUNT, + equalTo(0L), + LOGSDB_QUERY_TIME, + equalTo(0L) + ) + ); + + client().prepareSearch("time*").setSize(100).get().decRef(); + var nodeStats2 = indicesService.stats(CommonStatsFlags.ALL, false).getSearch().getTotal(); + collectThenAssertMetrics( + telemetry, + 2, + Map.of( + STANDARD_QUERY_COUNT, + equalTo(numStandardIndices), + STANDARD_QUERY_TIME, + equalTo(nodeStats1.getQueryTimeInMillis()), + + TIME_SERIES_QUERY_COUNT, + equalTo(numTimeSeriesIndices), + TIME_SERIES_QUERY_TIME, + equalTo(nodeStats2.getQueryTimeInMillis() - nodeStats1.getQueryTimeInMillis()), + TIME_SERIES_FETCH_COUNT, + equalTo(nodeStats2.getFetchCount() - nodeStats1.getFetchCount()), + TIME_SERIES_FETCH_TIME, + equalTo(nodeStats2.getFetchTimeInMillis() - nodeStats1.getFetchTimeInMillis()), + + LOGSDB_QUERY_COUNT, + equalTo(0L), + LOGSDB_QUERY_TIME, + equalTo(0L) + ) + ); + client().prepareSearch("logs*").setSize(100).get().decRef(); + var nodeStats3 = indicesService.stats(CommonStatsFlags.ALL, false).getSearch().getTotal(); + collectThenAssertMetrics( + telemetry, + 3, + Map.of( + STANDARD_QUERY_COUNT, + equalTo(numStandardIndices), + STANDARD_QUERY_TIME, + equalTo(nodeStats1.getQueryTimeInMillis()), + + TIME_SERIES_QUERY_COUNT, + equalTo(numTimeSeriesIndices), + TIME_SERIES_QUERY_TIME, + equalTo(nodeStats2.getQueryTimeInMillis() - nodeStats1.getQueryTimeInMillis()), + + LOGSDB_QUERY_COUNT, + equalTo(numLogsdbIndices), + LOGSDB_QUERY_TIME, + equalTo(nodeStats3.getQueryTimeInMillis() - nodeStats2.getQueryTimeInMillis()), + LOGSDB_FETCH_COUNT, + equalTo(nodeStats3.getFetchCount() - nodeStats2.getFetchCount()), + LOGSDB_FETCH_TIME, + equalTo(nodeStats3.getFetchTimeInMillis() - nodeStats2.getFetchTimeInMillis()) + ) + ); + // search failures + expectThrows(Exception.class, () -> { client().prepareSearch("logs*").setRuntimeMappings(parseMapping(""" + { + "fail_me": { + "type": "long", + "script": {"source": "<>", "lang": "failing_field"} + } + } + """)).setQuery(new RangeQueryBuilder("fail_me").gte(0)).setAllowPartialSearchResults(true).get(); }); + collectThenAssertMetrics( + telemetry, + 4, + Map.of( + STANDARD_QUERY_FAILURE, + equalTo(0L), + STANDARD_FETCH_FAILURE, + equalTo(0L), + TIME_SERIES_QUERY_FAILURE, + equalTo(0L), + TIME_SERIES_FETCH_FAILURE, + equalTo(0L), + LOGSDB_QUERY_FAILURE, + equalTo(numLogsdbIndices), + LOGSDB_FETCH_FAILURE, + equalTo(0L) + ) + ); } void collectThenAssertMetrics(TestTelemetryPlugin telemetry, int times, Map> matchers) { @@ -175,7 +358,7 @@ int populateStandardIndices(long numIndices) { int totalDocs = 0; for (int i = 0; i < numIndices; i++) { String indexName = "standard-" + i; - createIndex(indexName); + createIndex(indexName, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build()); int numDocs = between(1, 5); for (int d = 0; d < numDocs; d++) { indexDoc(indexName, Integer.toString(d), "f", Integer.toString(d)); @@ -190,7 +373,11 @@ int populateTimeSeriesIndices(long numIndices) { int totalDocs = 0; for (int i = 0; i < numIndices; i++) { String indexName = "time_series-" + i; - Settings settings = Settings.builder().put("mode", "time_series").putList("routing_path", List.of("host")).build(); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put("mode", "time_series") + .putList("routing_path", List.of("host")) + .build(); client().admin() .indices() .prepareCreate(indexName) @@ -214,6 +401,7 @@ int populateTimeSeriesIndices(long numIndices) { } totalDocs += numDocs; flush(indexName); + refresh(indexName); } return totalDocs; } @@ -222,7 +410,7 @@ int populateLogsdbIndices(long numIndices) { int totalDocs = 0; for (int i = 0; i < numIndices; i++) { String indexName = "logsdb-" + i; - Settings settings = Settings.builder().put("mode", "logsdb").build(); + Settings settings = Settings.builder().put("mode", "logsdb").put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build(); client().admin() .indices() .prepareCreate(indexName) @@ -237,9 +425,75 @@ int populateLogsdbIndices(long numIndices) { .setSource("@timestamp", timestamp, "host.name", randomFrom("prod", "qa"), "cpu", randomIntBetween(1, 100)) .get(); } + int numFailures = between(0, 2); + for (int d = 0; d < numFailures; d++) { + expectThrows(Exception.class, () -> { + client().prepareIndex(indexName) + .setSource( + "@timestamp", + "malformed-timestamp", + "host.name", + randomFrom("prod", "qa"), + "cpu", + randomIntBetween(1, 100) + ) + .get(); + }); + } totalDocs += numDocs; flush(indexName); + refresh(indexName); } return totalDocs; } + + private Map parseMapping(String mapping) throws IOException { + try (XContentParser parser = createParser(JsonXContent.jsonXContent, mapping)) { + return parser.map(); + } + } + + public static class FailingFieldPlugin extends Plugin implements ScriptPlugin { + + @Override + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { + return new ScriptEngine() { + @Override + public String getType() { + return "failing_field"; + } + + @Override + @SuppressWarnings("unchecked") + public FactoryType compile( + String name, + String code, + ScriptContext context, + Map params + ) { + return (FactoryType) new LongFieldScript.Factory() { + @Override + public LongFieldScript.LeafFactory newFactory( + String fieldName, + Map params, + SearchLookup searchLookup, + OnScriptError onScriptError + ) { + return ctx -> new LongFieldScript(fieldName, params, searchLookup, onScriptError, ctx) { + @Override + public void execute() { + throw new IllegalStateException("Accessing failing field"); + } + }; + } + }; + } + + @Override + public Set> getSupportedContexts() { + return Set.of(LongFieldScript.CONTEXT); + } + }; + } + } } diff --git a/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java b/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java index 17e290283d5e0..e07f6908330df 100644 --- a/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java +++ b/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java @@ -18,8 +18,10 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.search.stats.SearchStats; import org.elasticsearch.index.shard.IllegalIndexShardStateException; import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.IndexingStats; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.telemetry.metric.LongWithAttributes; import org.elasticsearch.telemetry.metric.MeterRegistry; @@ -50,8 +52,8 @@ public IndicesMetrics(MeterRegistry meterRegistry, IndicesService indicesService } private static List registerAsyncMetrics(MeterRegistry registry, IndicesStatsCache cache) { - List metrics = new ArrayList<>(IndexMode.values().length * 3); - assert IndexMode.values().length == 3 : "index modes have changed"; + final int TOTAL_METRICS = 36; + List metrics = new ArrayList<>(TOTAL_METRICS); for (IndexMode indexMode : IndexMode.values()) { String name = indexMode.getName(); metrics.add( @@ -72,13 +74,89 @@ private static List registerAsyncMetrics(MeterRegistry registry, ); metrics.add( registry.registerLongGauge( - "es.indices." + name + ".bytes.total", + "es.indices." + name + ".size", "total size in bytes of " + name + " indices", - "unit", + "bytes", () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).numBytes) ) ); + // query (count, took, failures) - use gauges as shards can be removed + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".query.total", + "total queries of " + name + " indices", + "unit", + () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).search.getQueryCount()) + ) + ); + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".query.time", + "total query time of " + name + " indices", + "ms", + () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).search.getQueryTimeInMillis()) + ) + ); + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".query.failure.total", + "total query failures of " + name + " indices", + "unit", + () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).search.getQueryFailure()) + ) + ); + // fetch (count, took, failures) - use gauges as shards can be removed + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".fetch.total", + "total fetches of " + name + " indices", + "unit", + () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).search.getFetchCount()) + ) + ); + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".fetch.time", + "total fetch time of " + name + " indices", + "ms", + () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).search.getFetchTimeInMillis()) + ) + ); + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".fetch.failure.total", + "total fetch failures of " + name + " indices", + "unit", + () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).search.getFetchFailure()) + ) + ); + // indexing + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".indexing.total", + "total indexing operations of " + name + " indices", + "unit", + () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).indexing.getIndexCount()) + ) + ); + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".indexing.time", + "total indexing time of " + name + " indices", + "ms", + () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).indexing.getIndexTime().millis()) + ) + ); + metrics.add( + registry.registerLongGauge( + "es.indices." + name + ".indexing.failure.total", + "total indexing failures of " + name + " indices", + "unit", + () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).indexing.getIndexFailedCount()) + ) + ); } + assert metrics.size() == TOTAL_METRICS : "total number of metrics has changed"; return metrics; } @@ -107,6 +185,8 @@ static class IndexStats { int numIndices = 0; long numDocs = 0; long numBytes = 0; + SearchStats.Stats search = new SearchStats().getTotal(); + IndexingStats.Stats indexing = new IndexingStats().getTotal(); } private static class IndicesStatsCache extends SingleObjectCache> { @@ -152,6 +232,8 @@ private Map internalGetIndicesStats() { try { indexStats.numDocs += indexShard.commitStats().getNumDocs(); indexStats.numBytes += indexShard.storeStats().sizeInBytes(); + indexStats.search.add(indexShard.searchStats().getTotal()); + indexStats.indexing.add(indexShard.indexingStats().getTotal()); } catch (IllegalIndexShardStateException | AlreadyClosedException ignored) { // ignored } From 3c5b74bca9fa7834c38e9d1cd6ccb38e258de19e Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Fri, 4 Oct 2024 09:25:31 -0500 Subject: [PATCH 092/194] Better support for RouteBuilder for deprecation removal vs. keep (#113712) This commit is a follow up to #113151 to better clarify how to deprecate HTTP routes. That change introduced RouteBuilder.deprecateAndKeep to enable the ability to deprecate an HTTP API, but not require REST compatibilty headers in the next major version. This commit deprecates the java methnod RouteBuilder.deprecated and introduces RouteBuilder.deprecatedForRemoval. The valid options are now RouteBuilder.deprecateAndKeep vs. RouteBuilder.deprecatedForRemoval where the later will require compatiblity headers to access the route in any newer REST API versions than the lastFullySupportedVersion declared. The javadoc should help to provide some clarification. Additionally, the deprecation level should not be configurable. The deprecation level of WARN, when applied to routes, is informational only (and may require compatibility headers in the next version). The deprecation level of CRITICAL means means no access what so ever in the next major version. Generally, CRTICIAL is used for any cases where the compatibility is required (which means it is the last version for any access), or no compatibility is planned. Some examples: ``` Route.builder(GET, "/foo/bar").build() -> no deprecations Route.builder(GET, "/foo/bar").deprecateAndKeep("my message").build() -> deprecated, but removal is not influenced by REST API compatibilty Route.builder(GET, "/foo/bar").deprecatedForRemoval("my message", V_8).build() -> will be available in V_8, but emit a warn level deprecation, V_9 will require compatiblity headers and emit a CRITICAL deprecation, and V_10 this will no longer be available Route.builder(GET, "/foo/bar").replaces(GET, "/foo/baz", V_8).build() -> /foo/bar will be available in all versions. /foo/baz will be available in V_8 but emit a warn level deprecation, V_9 will require compatibility headers and emit a CRITICAL deprecation, and V_10 this will no longer be available. This is effectively a short cut to register a new route ("/foo/bar") and deprecatedForRemoval the path being replaced. ``` The functionality remains unchanged and this refactoring only provides better contracts and cleans up some of the implementation. --- .../rest/DeprecationRestHandler.java | 28 +++-- .../elasticsearch/rest/RestController.java | 91 +++++---------- .../org/elasticsearch/rest/RestHandler.java | 89 ++++++++++----- .../rest/DeprecationRestHandlerTests.java | 44 +++++++- .../rest/RestControllerTests.java | 66 ++++++++--- .../xpack/deprecation/DeprecationHttpIT.java | 105 ++++++++++++++++++ .../TestDeprecationHeaderRestAction.java | 19 +++- 7 files changed, 318 insertions(+), 124 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/rest/DeprecationRestHandler.java b/server/src/main/java/org/elasticsearch/rest/DeprecationRestHandler.java index 98ab7d53ffbe6..8d363f6e63511 100644 --- a/server/src/main/java/org/elasticsearch/rest/DeprecationRestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/DeprecationRestHandler.java @@ -13,7 +13,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.core.Nullable; import java.util.Objects; @@ -29,7 +28,6 @@ public class DeprecationRestHandler extends FilterRestHandler implements RestHan private final DeprecationLogger deprecationLogger; private final boolean compatibleVersionWarning; private final String deprecationKey; - @Nullable private final Level deprecationLevel; /** @@ -39,6 +37,8 @@ public class DeprecationRestHandler extends FilterRestHandler implements RestHan * @param handler The rest handler to deprecate (it's possible that the handler is reused with a different name!) * @param method a method of a deprecated endpoint * @param path a path of a deprecated endpoint + * @param deprecationLevel The level of the deprecation warning, must be non-null + * and either {@link Level#WARN} or {@link DeprecationLogger#CRITICAL} * @param deprecationMessage The message to warn users with when they use the {@code handler} * @param deprecationLogger The deprecation logger * @param compatibleVersionWarning set to false so that a deprecation warning will be issued for the handled request, @@ -51,7 +51,7 @@ public DeprecationRestHandler( RestHandler handler, RestRequest.Method method, String path, - @Nullable Level deprecationLevel, + Level deprecationLevel, String deprecationMessage, DeprecationLogger deprecationLogger, boolean compatibleVersionWarning @@ -61,7 +61,7 @@ public DeprecationRestHandler( this.deprecationLogger = Objects.requireNonNull(deprecationLogger); this.compatibleVersionWarning = compatibleVersionWarning; this.deprecationKey = DEPRECATED_ROUTE_KEY + "_" + method + "_" + path; - if (deprecationLevel != null && (deprecationLevel != Level.WARN && deprecationLevel != DeprecationLogger.CRITICAL)) { + if (deprecationLevel != Level.WARN && deprecationLevel != DeprecationLogger.CRITICAL) { throw new IllegalArgumentException( "unexpected deprecation logger level: " + deprecationLevel + ", expected either 'CRITICAL' or 'WARN'" ); @@ -77,19 +77,18 @@ public DeprecationRestHandler( @Override public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception { if (compatibleVersionWarning == false) { - // The default value for deprecated requests without a version warning is WARN - if (deprecationLevel == null || deprecationLevel == Level.WARN) { + // emit a standard deprecation warning + if (Level.WARN == deprecationLevel) { deprecationLogger.warn(DeprecationCategory.API, deprecationKey, deprecationMessage); - } else { + } else if (DeprecationLogger.CRITICAL == deprecationLevel) { deprecationLogger.critical(DeprecationCategory.API, deprecationKey, deprecationMessage); } } else { - // The default value for deprecated requests with a version warning is CRITICAL, - // because they have a specific version where the endpoint is removed - if (deprecationLevel == null || deprecationLevel == DeprecationLogger.CRITICAL) { - deprecationLogger.compatibleCritical(deprecationKey, deprecationMessage); - } else { + // emit a compatibility warning + if (Level.WARN == deprecationLevel) { deprecationLogger.compatible(Level.WARN, deprecationKey, deprecationMessage); + } else if (DeprecationLogger.CRITICAL == deprecationLevel) { + deprecationLogger.compatibleCritical(deprecationKey, deprecationMessage); } } @@ -139,4 +138,9 @@ public static String requireValidHeader(String value) { return value; } + + // test only + Level getDeprecationLevel() { + return deprecationLevel; + } } diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java index 924cd361c671d..c2064fdd931de 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestController.java +++ b/server/src/main/java/org/elasticsearch/rest/RestController.java @@ -144,25 +144,6 @@ public ServerlessApiProtections getApiProtections() { return apiProtections; } - /** - * Registers a REST handler to be executed when the provided {@code method} and {@code path} match the request. - * - * @param method GET, POST, etc. - * @param path Path to handle (e.g. "/{index}/{type}/_bulk") - * @param version API version to handle (e.g. RestApiVersion.V_8) - * @param handler The handler to actually execute - * @param deprecationMessage The message to log and send as a header in the response - */ - protected void registerAsDeprecatedHandler( - RestRequest.Method method, - String path, - RestApiVersion version, - RestHandler handler, - String deprecationMessage - ) { - registerAsDeprecatedHandler(method, path, version, handler, deprecationMessage, null); - } - /** * Registers a REST handler to be executed when the provided {@code method} and {@code path} match the request. * @@ -179,40 +160,23 @@ protected void registerAsDeprecatedHandler( RestApiVersion version, RestHandler handler, String deprecationMessage, - @Nullable Level deprecationLevel + Level deprecationLevel ) { assert (handler instanceof DeprecationRestHandler) == false; - if (version == RestApiVersion.current()) { - // e.g. it was marked as deprecated in 8.x, and we're currently running 8.x - registerHandler( - method, - path, - version, - new DeprecationRestHandler(handler, method, path, deprecationLevel, deprecationMessage, deprecationLogger, false) - ); - } else if (version == RestApiVersion.minimumSupported()) { - // e.g. it was marked as deprecated in 7.x, and we're currently running 8.x + if (RestApiVersion.onOrAfter(RestApiVersion.minimumSupported()).test(version)) { registerHandler( method, path, version, - new DeprecationRestHandler(handler, method, path, deprecationLevel, deprecationMessage, deprecationLogger, true) - ); - } else { - // e.g. it was marked as deprecated in 7.x, and we're currently running *9.x* - logger.debug( - "Deprecated route [" - + method - + " " - + path - + "] for handler [" - + handler.getClass() - + "] " - + "with version [" - + version - + "], which is less than the minimum supported version [" - + RestApiVersion.minimumSupported() - + "]" + new DeprecationRestHandler( + handler, + method, + path, + deprecationLevel, + deprecationMessage, + deprecationLogger, + version != RestApiVersion.current() + ) ); } } @@ -250,21 +214,12 @@ protected void registerAsReplacedHandler( RestHandler handler, RestRequest.Method replacedMethod, String replacedPath, - RestApiVersion replacedVersion + RestApiVersion replacedVersion, + String replacedMessage, + Level deprecationLevel ) { - // e.g. [POST /_optimize] is deprecated! Use [POST /_forcemerge] instead. - final String replacedMessage = "[" - + replacedMethod.name() - + " " - + replacedPath - + "] is deprecated! Use [" - + method.name() - + " " - + path - + "] instead."; - registerHandler(method, path, version, handler); - registerAsDeprecatedHandler(replacedMethod, replacedPath, replacedVersion, handler, replacedMessage); + registerAsDeprecatedHandler(replacedMethod, replacedPath, replacedVersion, handler, replacedMessage, deprecationLevel); } /** @@ -284,7 +239,15 @@ protected void registerHandler(RestRequest.Method method, String path, RestApiVe private void registerHandlerNoWrap(RestRequest.Method method, String path, RestApiVersion version, RestHandler handler) { assert RestApiVersion.minimumSupported() == version || RestApiVersion.current() == version - : "REST API compatibility is only supported for version " + RestApiVersion.minimumSupported().major; + : "REST API compatibility is only supported for version " + + RestApiVersion.minimumSupported().major + + " [method=" + + method + + ", path=" + + path + + ", handler=" + + handler.getClass().getCanonicalName() + + "]"; if (RESERVED_PATHS.contains(path)) { throw new IllegalArgumentException("path [" + path + "] is a reserved path and may not be registered"); @@ -299,7 +262,7 @@ private void registerHandlerNoWrap(RestRequest.Method method, String path, RestA } public void registerHandler(final Route route, final RestHandler handler) { - if (route.isReplacement()) { + if (route.hasReplacement()) { Route replaced = route.getReplacedRoute(); registerAsReplacedHandler( route.getMethod(), @@ -308,7 +271,9 @@ public void registerHandler(final Route route, final RestHandler handler) { handler, replaced.getMethod(), replaced.getPath(), - replaced.getRestApiVersion() + replaced.getRestApiVersion(), + replaced.getDeprecationMessage(), + replaced.getDeprecationLevel() ); } else if (route.isDeprecated()) { registerAsDeprecatedHandler( diff --git a/server/src/main/java/org/elasticsearch/rest/RestHandler.java b/server/src/main/java/org/elasticsearch/rest/RestHandler.java index ede295fee9f4d..0e3b8d37dd25c 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/RestHandler.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.Level; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.RestRequest.Method; @@ -136,11 +137,10 @@ class Route { private final Method method; private final String path; private final RestApiVersion restApiVersion; - - private final String deprecationMessage; @Nullable + private final String deprecationMessage; private final Level deprecationLevel; - + @Nullable private final Route replacedRoute; private Route( @@ -153,12 +153,16 @@ private Route( ) { this.method = Objects.requireNonNull(method); this.path = Objects.requireNonNull(path); + // the last version in which this route was fully supported this.restApiVersion = Objects.requireNonNull(restApiVersion); - // a deprecated route will have a deprecation message, and the restApiVersion - // will represent the version when the route was deprecated + // a route marked as deprecated to keep or remove will have a deprecation message and level (warn for keep, critical for remove) this.deprecationMessage = deprecationMessage; - this.deprecationLevel = deprecationLevel; + this.deprecationLevel = Objects.requireNonNull(deprecationLevel); + + if (deprecationMessage == null && deprecationLevel != Level.OFF) { + throw new IllegalArgumentException("deprecationMessage must be set if deprecationLevel is not OFF"); + } // a route that replaces another route will have a reference to the route that was replaced this.replacedRoute = replacedRoute; @@ -173,7 +177,7 @@ private Route( * @param path the path, e.g. "/" */ public Route(Method method, String path) { - this(method, path, RestApiVersion.current(), null, null, null); + this(method, path, RestApiVersion.current(), null, Level.OFF, null); } public static class RouteBuilder { @@ -183,7 +187,6 @@ public static class RouteBuilder { private RestApiVersion restApiVersion; private String deprecationMessage; - @Nullable private Level deprecationLevel; private Route replacedRoute; @@ -194,6 +197,16 @@ private RouteBuilder(Method method, String path) { this.restApiVersion = RestApiVersion.current(); } + /** + * @deprecated Use {@link #deprecatedForRemoval(String, RestApiVersion)} if the intent is deprecate the path and remove in the + * next major version. Use {@link #deprecateAndKeep(String)} if the intent is to deprecate the path but not remove it. + * This method will delegate to {@link #deprecatedForRemoval(String, RestApiVersion)}. + */ + @Deprecated(since = "9.0.0", forRemoval = true) + public RouteBuilder deprecated(String deprecationMessage, RestApiVersion lastFullySupportedVersion) { + return deprecatedForRemoval(deprecationMessage, lastFullySupportedVersion); + } + /** * Marks that the route being built has been deprecated (for some reason -- the deprecationMessage) for removal. Notes the last * major version in which the path is fully supported without compatibility headers. If this path is being replaced by another @@ -202,7 +215,7 @@ private RouteBuilder(Method method, String path) { * For example: * 
 {@code
              * Route.builder(GET, "_upgrade")
-             *  .deprecated("The _upgrade API is no longer useful and will be removed.", RestApiVersion.V_7)
+             *  .deprecatedForRemoval("The _upgrade API is no longer useful and will be removed.", RestApiVersion.V_7)
              *  .build()}
* * @param deprecationMessage the user-visible explanation of this deprecation @@ -211,10 +224,12 @@ private RouteBuilder(Method method, String path) { * The next major version (i.e. 9) will have no support whatsoever for this route. * @return a reference to this object. */ - public RouteBuilder deprecated(String deprecationMessage, RestApiVersion lastFullySupportedVersion) { + public RouteBuilder deprecatedForRemoval(String deprecationMessage, RestApiVersion lastFullySupportedVersion) { assert this.replacedRoute == null; this.restApiVersion = Objects.requireNonNull(lastFullySupportedVersion); this.deprecationMessage = Objects.requireNonNull(deprecationMessage); + // if being deprecated for removal in the current version, then it's a warning, otherwise it's critical + this.deprecationLevel = lastFullySupportedVersion == RestApiVersion.current() ? Level.WARN : DeprecationLogger.CRITICAL; return this; } @@ -227,16 +242,38 @@ public RouteBuilder deprecated(String deprecationMessage, RestApiVersion lastFul * Route.builder(GET, "/_security/user/") * .replaces(GET, "/_xpack/security/user/", RestApiVersion.V_7).build()} * - * @param method the method being replaced - * @param path the path being replaced + * @param replacedMethod the method being replaced + * @param replacedPath the path being replaced * @param lastFullySupportedVersion the last {@link RestApiVersion} (i.e. 7) for which this route is fully supported. * The next major version (i.e. 8) will require compatibility header(s). (;compatible-with=7) * The next major version (i.e. 9) will have no support whatsoever for this route. * @return a reference to this object. */ - public RouteBuilder replaces(Method method, String path, RestApiVersion lastFullySupportedVersion) { + public RouteBuilder replaces(Method replacedMethod, String replacedPath, RestApiVersion lastFullySupportedVersion) { assert this.deprecationMessage == null; - this.replacedRoute = new Route(method, path, lastFullySupportedVersion, null, null, null); + + // if being replaced in the current version, then it's a warning, otherwise it's critical + Level deprecationLevel = lastFullySupportedVersion == RestApiVersion.current() ? Level.WARN : DeprecationLogger.CRITICAL; + + // e.g. [POST /_optimize] is deprecated! Use [POST /_forcemerge] instead. + final String replacedMessage = "[" + + replacedMethod.name() + + " " + + replacedPath + + "] is deprecated! Use [" + + this.method.name() + + " " + + this.path + + "] instead."; + + this.replacedRoute = new Route( + replacedMethod, + replacedPath, + lastFullySupportedVersion, + replacedMessage, + deprecationLevel, + null + ); return this; } @@ -246,7 +283,7 @@ public RouteBuilder replaces(Method method, String path, RestApiVersion lastFull * For example: * 
 {@code
              * Route.builder(GET, "_upgrade")
-             *  .deprecated("The _upgrade API is no longer useful but will not be removed.")
+             *  .deprecateAndKeep("The _upgrade API is no longer useful but will not be removed.")
              *  .build()}
* * @param deprecationMessage the user-visible explanation of this deprecation @@ -261,14 +298,15 @@ public RouteBuilder deprecateAndKeep(String deprecationMessage) { } public Route build() { - if (replacedRoute != null) { - return new Route(method, path, restApiVersion, null, null, replacedRoute); - } else if (deprecationMessage != null) { - return new Route(method, path, restApiVersion, deprecationMessage, deprecationLevel, null); - } else { - // this is a little silly, but perfectly legal - return new Route(method, path, restApiVersion, null, null, null); - } + assert (deprecationMessage != null) == (deprecationLevel != null); // both must be set or neither + return new Route( + method, + path, + restApiVersion, + deprecationMessage, + deprecationLevel == null ? Level.OFF : deprecationLevel, + replacedRoute + ); } } @@ -288,11 +326,11 @@ public RestApiVersion getRestApiVersion() { return restApiVersion; } + @Nullable public String getDeprecationMessage() { return deprecationMessage; } - @Nullable public Level getDeprecationLevel() { return deprecationLevel; } @@ -301,11 +339,12 @@ public boolean isDeprecated() { return deprecationMessage != null; } + @Nullable public Route getReplacedRoute() { return replacedRoute; } - public boolean isReplacement() { + public boolean hasReplacement() { return replacedRoute != null; } } diff --git a/server/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java b/server/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java index 4e0fe14fb1def..b534a6be0dc5f 100644 --- a/server/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java @@ -73,7 +73,7 @@ public void testHandleRequestLogsThenForwards() throws Exception { RestChannel channel = mock(RestChannel.class); NodeClient client = mock(NodeClient.class); - final Level deprecationLevel = randomBoolean() ? null : randomFrom(Level.WARN, DeprecationLogger.CRITICAL); + final Level deprecationLevel = randomFrom(Level.WARN, DeprecationLogger.CRITICAL); DeprecationRestHandler deprecatedHandler = new DeprecationRestHandler( handler, @@ -159,17 +159,55 @@ public void testInvalidHeaderValueEmpty() { public void testSupportsBulkContentTrue() { when(handler.supportsBulkContent()).thenReturn(true); assertTrue( - new DeprecationRestHandler(handler, METHOD, PATH, null, deprecationMessage, deprecationLogger, false).supportsBulkContent() + new DeprecationRestHandler(handler, METHOD, PATH, Level.WARN, deprecationMessage, deprecationLogger, false) + .supportsBulkContent() ); } public void testSupportsBulkContentFalse() { when(handler.supportsBulkContent()).thenReturn(false); assertFalse( - new DeprecationRestHandler(handler, METHOD, PATH, null, deprecationMessage, deprecationLogger, false).supportsBulkContent() + new DeprecationRestHandler(handler, METHOD, PATH, Level.WARN, deprecationMessage, deprecationLogger, false) + .supportsBulkContent() ); } + public void testDeprecationLevel() { + DeprecationRestHandler handler = new DeprecationRestHandler( + this.handler, + METHOD, + PATH, + Level.WARN, + deprecationMessage, + deprecationLogger, + false + ); + assertEquals(Level.WARN, handler.getDeprecationLevel()); + + handler = new DeprecationRestHandler( + this.handler, + METHOD, + PATH, + DeprecationLogger.CRITICAL, + deprecationMessage, + deprecationLogger, + false + ); + assertEquals(DeprecationLogger.CRITICAL, handler.getDeprecationLevel()); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> new DeprecationRestHandler(this.handler, METHOD, PATH, null, deprecationMessage, deprecationLogger, false) + ); + assertEquals(exception.getMessage(), "unexpected deprecation logger level: null, expected either 'CRITICAL' or 'WARN'"); + + exception = expectThrows( + IllegalArgumentException.class, + () -> new DeprecationRestHandler(this.handler, METHOD, PATH, Level.OFF, deprecationMessage, deprecationLogger, false) + ); + assertEquals(exception.getMessage(), "unexpected deprecation logger level: OFF, expected either 'CRITICAL' or 'WARN'"); + } + /** * {@code ASCIIHeaderGenerator} only uses characters expected to be valid in headers (simplified US-ASCII). */ diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java index 1d946681661e7..8f1904ce42438 100644 --- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java +++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.rest; +import org.apache.logging.log4j.Level; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.breaker.CircuitBreaker; @@ -17,6 +18,7 @@ import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.io.stream.BytesStream; import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; @@ -85,6 +87,7 @@ import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -349,18 +352,22 @@ public void testRegisterAsDeprecatedHandler() { String path = "/_" + randomAlphaOfLengthBetween(1, 6); RestHandler handler = (request, channel, client) -> {}; String deprecationMessage = randomAlphaOfLengthBetween(1, 10); - RestApiVersion deprecatedInVersion = RestApiVersion.current(); - Route route = Route.builder(method, path).deprecated(deprecationMessage, deprecatedInVersion).build(); + List replacedInVersions = List.of(RestApiVersion.current(), RestApiVersion.minimumSupported()); + for (RestApiVersion replacedInVersion : replacedInVersions) { + Level level = replacedInVersion == RestApiVersion.current() ? Level.WARN : DeprecationLogger.CRITICAL; + clearInvocations(controller); + Route route = Route.builder(method, path).deprecatedForRemoval(deprecationMessage, replacedInVersion).build(); - // don't want to test everything -- just that it actually wraps the handler - doCallRealMethod().when(controller).registerHandler(route, handler); - doCallRealMethod().when(controller) - .registerAsDeprecatedHandler(method, path, deprecatedInVersion, handler, deprecationMessage, null); + // don't want to test everything -- just that it actually wraps the handler + doCallRealMethod().when(controller).registerHandler(route, handler); + doCallRealMethod().when(controller) + .registerAsDeprecatedHandler(method, path, replacedInVersion, handler, deprecationMessage, level); - controller.registerHandler(route, handler); + controller.registerHandler(route, handler); - verify(controller).registerHandler(eq(method), eq(path), eq(deprecatedInVersion), any(DeprecationRestHandler.class)); + verify(controller).registerHandler(eq(method), eq(path), eq(replacedInVersion), any(DeprecationRestHandler.class)); + } } public void testRegisterAsReplacedHandler() { @@ -383,17 +390,40 @@ public void testRegisterAsReplacedHandler() { + path + "] instead."; - final Route route = Route.builder(method, path).replaces(replacedMethod, replacedPath, previous).build(); - - // don't want to test everything -- just that it actually wraps the handlers - doCallRealMethod().when(controller).registerHandler(route, handler); - doCallRealMethod().when(controller) - .registerAsReplacedHandler(method, path, current, handler, replacedMethod, replacedPath, previous); - - controller.registerHandler(route, handler); + List replacedInVersions = List.of(current, previous); + for (RestApiVersion replacedInVersion : replacedInVersions) { + clearInvocations(controller); + Route route = Route.builder(method, path).replaces(replacedMethod, replacedPath, replacedInVersion).build(); + // don't want to test everything -- just that it actually wraps the handler + doCallRealMethod().when(controller).registerHandler(route, handler); + Level level = replacedInVersion == current ? Level.WARN : DeprecationLogger.CRITICAL; + doCallRealMethod().when(controller) + .registerAsReplacedHandler( + method, + path, + current, + handler, + replacedMethod, + replacedPath, + replacedInVersion, + deprecationMessage, + level + ); - verify(controller).registerHandler(method, path, current, handler); - verify(controller).registerAsDeprecatedHandler(replacedMethod, replacedPath, previous, handler, deprecationMessage); + controller.registerHandler(route, handler); + + // verify we registered the primary handler + verify(controller).registerHandler(method, path, current, handler); + // verify we register the replaced handler with the correct deprecation message and level + verify(controller).registerAsDeprecatedHandler( + replacedMethod, + replacedPath, + replacedInVersion, + handler, + deprecationMessage, + level + ); + } } public void testRegisterSecondMethodWithDifferentNamedWildcard() { diff --git a/x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java b/x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java index f6dd43164e387..3fb9573dd7b62 100644 --- a/x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java +++ b/x-pack/plugin/deprecation/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/DeprecationHttpIT.java @@ -477,6 +477,111 @@ public void testDeprecationWarnMessagesCanBeIndexed() throws Exception { } + public void testDeprecateAndKeep() throws Exception { + final Request request = new Request("GET", "/_test_cluster/deprecated_but_dont_remove"); + request.setEntity(buildSettingsRequest(Collections.singletonList(TEST_NOT_DEPRECATED_SETTING), "settings")); + Response response = performScopedRequest(request); + + final List deprecatedWarnings = getWarningHeaders(response.getHeaders()); + assertThat( + extractWarningValuesFromWarningHeaders(deprecatedWarnings), + containsInAnyOrder("[/_test_cluster/deprecated_but_dont_remove] is deprecated, but no plans to remove quite yet") + ); + + assertBusy(() -> { + List> documents = DeprecationTestUtils.getIndexedDeprecations(client(), xOpaqueId()); + + logger.warn(documents); + + // only assert the relevant fields: level, message, and category + assertThat( + documents, + containsInAnyOrder( + allOf( + hasEntry("elasticsearch.event.category", "api"), + hasEntry("log.level", "WARN"), + hasEntry("message", "[/_test_cluster/deprecated_but_dont_remove] is deprecated, but no plans to remove quite yet") + ) + ) + ); + }, 30, TimeUnit.SECONDS); + } + + public void testReplacesInCurrentVersion() throws Exception { + final Request request = new Request("GET", "/_test_cluster/old_name1"); // deprecated in current version + request.setEntity(buildSettingsRequest(Collections.singletonList(TEST_NOT_DEPRECATED_SETTING), "settings")); + Response response = performScopedRequest(request); + + final List deprecatedWarnings = getWarningHeaders(response.getHeaders()); + assertThat( + extractWarningValuesFromWarningHeaders(deprecatedWarnings), + containsInAnyOrder("[GET /_test_cluster/old_name1] is deprecated! Use [GET /_test_cluster/new_name1] instead.") + ); + + assertBusy(() -> { + List> documents = DeprecationTestUtils.getIndexedDeprecations(client(), xOpaqueId()); + + logger.warn(documents); + + // only assert the relevant fields: level, message, and category + assertThat( + documents, + containsInAnyOrder( + allOf( + hasEntry("elasticsearch.event.category", "api"), + hasEntry("log.level", "WARN"), + hasEntry("message", "[GET /_test_cluster/old_name1] is deprecated! Use [GET /_test_cluster/new_name1] instead.") + ) + ) + ); + }, 30, TimeUnit.SECONDS); + } + + public void testReplacesInCompatibleVersion() throws Exception { + final Request request = new Request("GET", "/_test_cluster/old_name2"); // deprecated in minimum supported version + request.setEntity(buildSettingsRequest(Collections.singletonList(TEST_DEPRECATED_SETTING_TRUE1), "deprecated_settings")); + final RequestOptions compatibleOptions = request.getOptions() + .toBuilder() + .addHeader("Accept", "application/vnd.elasticsearch+json;compatible-with=" + RestApiVersion.minimumSupported().major) + .addHeader("Content-Type", "application/vnd.elasticsearch+json;compatible-with=" + RestApiVersion.minimumSupported().major) + .build(); + request.setOptions(compatibleOptions); + Response response = performScopedRequest(request); + + final List deprecatedWarnings = getWarningHeaders(response.getHeaders()); + assertThat( + extractWarningValuesFromWarningHeaders(deprecatedWarnings), + containsInAnyOrder( + "[GET /_test_cluster/old_name2] is deprecated! Use [GET /_test_cluster/new_name2] instead.", + "You are using a compatible API for this request" + ) + ); + assertBusy(() -> { + List> documents = DeprecationTestUtils.getIndexedDeprecations(client(), xOpaqueId()); + + logger.warn(documents); + + // only assert the relevant fields: level, message, and category + assertThat( + documents, + containsInAnyOrder( + allOf( + + hasEntry("elasticsearch.event.category", "compatible_api"), + hasEntry("log.level", "CRITICAL"), + hasEntry("message", "[GET /_test_cluster/old_name2] is deprecated! Use [GET /_test_cluster/new_name2] instead.") + ), + allOf( + hasEntry("elasticsearch.event.category", "compatible_api"), + hasEntry("log.level", "CRITICAL"), + // this message comes from the test, not production code. this is the message for setting the deprecated setting + hasEntry("message", "You are using a compatible API for this request") + ) + ) + ); + }, 30, TimeUnit.SECONDS); + } + /** * Check that log messages about REST API compatibility are recorded to an index */ diff --git a/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationHeaderRestAction.java b/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationHeaderRestAction.java index 70942b04f85b8..9e5f999d1f825 100644 --- a/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationHeaderRestAction.java +++ b/x-pack/plugin/deprecation/qa/rest/src/main/java/org/elasticsearch/xpack/deprecation/TestDeprecationHeaderRestAction.java @@ -97,10 +97,23 @@ public List routes() { return List.of( // note: RestApiVersion.current() is acceptable here because this is test code -- ordinary callers of `.deprecated(...)` // should use an actual version - Route.builder(GET, "/_test_cluster/deprecated_settings").deprecated(DEPRECATED_ENDPOINT, RestApiVersion.current()).build(), + Route.builder(GET, "/_test_cluster/deprecated_settings") + .deprecatedForRemoval(DEPRECATED_ENDPOINT, RestApiVersion.current()) + .build(), + // TODO: s/deprecated/deprecatedForRemoval when removing `deprecated` method Route.builder(POST, "/_test_cluster/deprecated_settings").deprecated(DEPRECATED_ENDPOINT, RestApiVersion.current()).build(), - Route.builder(GET, "/_test_cluster/compat_only").deprecated(DEPRECATED_ENDPOINT, RestApiVersion.minimumSupported()).build(), - Route.builder(GET, "/_test_cluster/only_deprecated_setting").build() + Route.builder(GET, "/_test_cluster/compat_only") + .deprecatedForRemoval(DEPRECATED_ENDPOINT, RestApiVersion.minimumSupported()) + .build(), + Route.builder(GET, "/_test_cluster/only_deprecated_setting").build(), + Route.builder(GET, "/_test_cluster/deprecated_but_dont_remove") + .deprecateAndKeep("[/_test_cluster/deprecated_but_dont_remove] is deprecated, but no plans to remove quite yet") + .build(), + Route.builder(GET, "/_test_cluster/new_name1").replaces(GET, "/_test_cluster/old_name1", RestApiVersion.current()).build(), + Route.builder(GET, "/_test_cluster/new_name2") + .replaces(GET, "/_test_cluster/old_name2", RestApiVersion.minimumSupported()) + .build() + ); } From fcdfa5b18e620803a2ac9d7f0c9b7da75d9735f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Fri, 4 Oct 2024 17:07:35 +0200 Subject: [PATCH 093/194] Remove some easy/straightforward instances of UpdateForV9 (#114134) --- .../upgrades/FullClusterRestartIT.java | 27 -------- .../upgrades/SystemIndicesUpgradeIT.java | 20 ------ .../test/rest/ESRestTestCase.java | 62 +++---------------- .../test/rest/RestTestLegacyFeatures.java | 49 --------------- .../test/rest/yaml/section/DoSection.java | 12 ++-- .../xpack/restart/FullClusterRestartIT.java | 10 --- .../xpack/restart/FullClusterRestartIT.java | 61 +++--------------- ...nfigIndexMappingsFullClusterRestartIT.java | 11 +--- .../MlHiddenIndicesFullClusterRestartIT.java | 31 ---------- .../xpack/restart/WatcherMappingUpdateIT.java | 11 +--- .../upgrades/AbstractUpgradeTestCase.java | 12 ---- .../upgrades/MLModelDeploymentsUpgradeIT.java | 12 ---- .../MlAssignmentPlannerUpgradeIT.java | 6 -- 13 files changed, 26 insertions(+), 298 deletions(-) diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 9255281e7e5da..8570662f7b523 100644 --- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -1602,10 +1602,6 @@ public void testResize() throws Exception { @SuppressWarnings("unchecked") public void testSystemIndexMetadataIsUpgraded() throws Exception { - - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // assumeTrue can be removed (condition always true) - var originalClusterTaskIndexIsSystemIndex = oldClusterHasFeature(RestTestLegacyFeatures.TASK_INDEX_SYSTEM_INDEX); - assumeTrue(".tasks became a system index in 7.10.0", originalClusterTaskIndexIsSystemIndex); final String systemIndexWarning = "this request accesses system indices: [.tasks], but in a future major version, direct " + "access to system indices will be prevented by default"; if (isRunningAgainstOldCluster()) { @@ -1665,29 +1661,6 @@ public void testSystemIndexMetadataIsUpgraded() throws Exception { throw new AssertionError(".tasks index does not exist yet"); } }); - - // If we are on 7.x create an alias that includes both a system index and a non-system index so we can be sure it gets - // upgraded properly. If we're already on 8.x, skip this part of the test. - if (clusterHasFeature(RestTestLegacyFeatures.SYSTEM_INDICES_REST_ACCESS_ENFORCED) == false) { - // Create an alias to make sure it gets upgraded properly - Request putAliasRequest = newXContentRequest(HttpMethod.POST, "/_aliases", (builder, params) -> { - builder.startArray("actions"); - for (var index : List.of(".tasks", "test_index_reindex")) { - builder.startObject() - .startObject("add") - .field("index", index) - .field("alias", "test-system-alias") - .endObject() - .endObject(); - } - return builder.endArray(); - }); - putAliasRequest.setOptions(expectVersionSpecificWarnings(v -> { - v.current(systemIndexWarning); - v.compatible(systemIndexWarning); - })); - assertThat(client().performRequest(putAliasRequest).getStatusLine().getStatusCode(), is(200)); - } } else { assertBusy(() -> { Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata"); diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java index dd8ecdd82ca7b..6a526a6dbfded 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SystemIndicesUpgradeIT.java @@ -15,7 +15,6 @@ import org.elasticsearch.client.ResponseException; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.XContentTestUtils.JsonMapView; -import org.elasticsearch.test.rest.RestTestLegacyFeatures; import java.util.Map; @@ -87,25 +86,6 @@ public void testSystemIndicesUpgrades() throws Exception { throw new AssertionError(".tasks index does not exist yet"); } }); - - // If we are on 7.x create an alias that includes both a system index and a non-system index so we can be sure it gets - // upgraded properly. If we're already on 8.x, skip this part of the test. - if (clusterHasFeature(RestTestLegacyFeatures.SYSTEM_INDICES_REST_ACCESS_ENFORCED) == false) { - // Create an alias to make sure it gets upgraded properly - Request putAliasRequest = new Request("POST", "/_aliases"); - putAliasRequest.setJsonEntity(""" - { - "actions": [ - {"add": {"index": ".tasks", "alias": "test-system-alias"}}, - {"add": {"index": "test_index_reindex", "alias": "test-system-alias"}} - ] - }"""); - putAliasRequest.setOptions(expectVersionSpecificWarnings(v -> { - v.current(systemIndexWarning); - v.compatible(systemIndexWarning); - })); - assertThat(client().performRequest(putAliasRequest).getStatusLine().getStatusCode(), is(200)); - } } else if (isUpgradedCluster()) { assertBusy(() -> { Request clusterStateRequest = new Request("GET", "/_cluster/state/metadata"); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 215973b5dece2..d17016f850300 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -719,10 +719,6 @@ protected boolean preserveTemplatesUponCompletion() { * all feature states, deleting system indices, system associated indices, and system data streams. */ protected boolean resetFeatureStates() { - if (clusterHasFeature(RestTestLegacyFeatures.FEATURE_STATE_RESET_SUPPORTED) == false) { - return false; - } - // ML reset fails when ML is disabled in versions before 8.7 if (isMlEnabled() == false && clusterHasFeature(RestTestLegacyFeatures.ML_STATE_RESET_FALLBACK_ON_DISABLED) == false) { return false; @@ -917,22 +913,10 @@ private void wipeCluster() throws Exception { .filter(name -> isXPackTemplate(name) == false) .collect(Collectors.toList()); if (names.isEmpty() == false) { - // Ideally we would want to check if the elected master node supports this feature and send the delete request - // directly to that node, but node-specific feature checks is something we want to avoid if possible. - if (clusterHasFeature(RestTestLegacyFeatures.DELETE_TEMPLATE_MULTIPLE_NAMES_SUPPORTED)) { - try { - adminClient().performRequest(new Request("DELETE", "_index_template/" + String.join(",", names))); - } catch (ResponseException e) { - logger.warn(() -> format("unable to remove multiple composable index templates %s", names), e); - } - } else { - for (String name : names) { - try { - adminClient().performRequest(new Request("DELETE", "_index_template/" + name)); - } catch (ResponseException e) { - logger.warn(() -> format("unable to remove composable index template %s", name), e); - } - } + try { + adminClient().performRequest(new Request("DELETE", "_index_template/" + String.join(",", names))); + } catch (ResponseException e) { + logger.warn(() -> format("unable to remove multiple composable index templates %s", names), e); } } } catch (Exception e) { @@ -948,22 +932,10 @@ private void wipeCluster() throws Exception { .filter(name -> isXPackTemplate(name) == false) .collect(Collectors.toList()); if (names.isEmpty() == false) { - // Ideally we would want to check if the elected master node supports this feature and send the delete request - // directly to that node, but node-specific feature checks is something we want to avoid if possible. - if (clusterHasFeature(RestTestLegacyFeatures.DELETE_TEMPLATE_MULTIPLE_NAMES_SUPPORTED)) { - try { - adminClient().performRequest(new Request("DELETE", "_component_template/" + String.join(",", names))); - } catch (ResponseException e) { - logger.warn(() -> format("unable to remove multiple component templates %s", names), e); - } - } else { - for (String componentTemplate : names) { - try { - adminClient().performRequest(new Request("DELETE", "_component_template/" + componentTemplate)); - } catch (ResponseException e) { - logger.warn(() -> format("unable to remove component template %s", componentTemplate), e); - } - } + try { + adminClient().performRequest(new Request("DELETE", "_component_template/" + String.join(",", names))); + } catch (ResponseException e) { + logger.warn(() -> format("unable to remove multiple component templates %s", names), e); } } } catch (Exception e) { @@ -1141,7 +1113,6 @@ protected static void wipeAllIndices() throws IOException { } protected static void wipeAllIndices(boolean preserveSecurityIndices) throws IOException { - boolean includeHidden = clusterHasFeature(RestTestLegacyFeatures.HIDDEN_INDICES_SUPPORTED); try { // remove all indices except some history indices which can pop up after deleting all data streams but shouldn't interfere final List indexPatterns = new ArrayList<>( @@ -1151,7 +1122,7 @@ protected static void wipeAllIndices(boolean preserveSecurityIndices) throws IOE indexPatterns.add("-.security-*"); } final Request deleteRequest = new Request("DELETE", Strings.collectionToCommaDelimitedString(indexPatterns)); - deleteRequest.addParameter("expand_wildcards", "open,closed" + (includeHidden ? ",hidden" : "")); + deleteRequest.addParameter("expand_wildcards", "open,closed,hidden"); final Response response = adminClient().performRequest(deleteRequest); try (InputStream is = response.getEntity().getContent()) { assertTrue((boolean) XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true).get("acknowledged")); @@ -1320,9 +1291,8 @@ private void wipeRollupJobs() throws IOException { } protected void refreshAllIndices() throws IOException { - boolean includeHidden = clusterHasFeature(RestTestLegacyFeatures.HIDDEN_INDICES_SUPPORTED); Request refreshRequest = new Request("POST", "/_refresh"); - refreshRequest.addParameter("expand_wildcards", "open" + (includeHidden ? ",hidden" : "")); + refreshRequest.addParameter("expand_wildcards", "open,hidden"); // Allow system index deprecation warnings refreshRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(warnings -> { if (warnings.isEmpty()) { @@ -2488,18 +2458,6 @@ public static void setIgnoredErrorResponseCodes(Request request, RestStatus... r } private static XContentType randomSupportedContentType() { - if (clusterHasFeature(RestTestLegacyFeatures.SUPPORTS_TRUE_BINARY_RESPONSES) == false) { - // Very old versions encode binary stored fields using base64 in all formats, not just JSON, but we expect to see raw binary - // fields in non-JSON formats, so we stick to JSON in these cases. - return XContentType.JSON; - } - - if (clusterHasFeature(RestTestLegacyFeatures.SUPPORTS_VENDOR_XCONTENT_TYPES) == false) { - // The VND_* formats were introduced part-way through the 7.x series for compatibility with 8.x, but are not supported by older - // 7.x versions. - return randomFrom(XContentType.JSON, XContentType.CBOR, XContentType.YAML, XContentType.SMILE); - } - return randomFrom(XContentType.values()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestLegacyFeatures.java b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestLegacyFeatures.java index 427398b9a8c0e..194dfc057b84f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestLegacyFeatures.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestLegacyFeatures.java @@ -27,23 +27,8 @@ public class RestTestLegacyFeatures implements FeatureSpecification { public static final NodeFeature ML_STATE_RESET_FALLBACK_ON_DISABLED = new NodeFeature("ml.state_reset_fallback_on_disabled"); @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature FEATURE_STATE_RESET_SUPPORTED = new NodeFeature("system_indices.feature_state_reset_supported"); - public static final NodeFeature SYSTEM_INDICES_REST_ACCESS_ENFORCED = new NodeFeature("system_indices.rest_access_enforced"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature SYSTEM_INDICES_REST_ACCESS_DEPRECATED = new NodeFeature("system_indices.rest_access_deprecated"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature HIDDEN_INDICES_SUPPORTED = new NodeFeature("indices.hidden_supported"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) public static final NodeFeature COMPONENT_TEMPLATE_SUPPORTED = new NodeFeature("indices.component_template_supported"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature DELETE_TEMPLATE_MULTIPLE_NAMES_SUPPORTED = new NodeFeature( - "indices.delete_template_multiple_names_supported" - ); public static final NodeFeature ML_NEW_MEMORY_FORMAT = new NodeFeature("ml.new_memory_format"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature SUPPORTS_VENDOR_XCONTENT_TYPES = new NodeFeature("rest.supports_vendor_xcontent_types"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature SUPPORTS_TRUE_BINARY_RESPONSES = new NodeFeature("rest.supports_true_binary_responses"); /** These are "pure test" features: normally we would not need them, and test for TransportVersion/fallback to Version (see for example * {@code ESRestTestCase#minimumTransportVersion()}. However, some tests explicitly check and validate the content of a response, so @@ -61,21 +46,6 @@ public class RestTestLegacyFeatures implements FeatureSpecification { public static final NodeFeature DESIRED_NODE_API_SUPPORTED = new NodeFeature("desired_node_supported"); public static final NodeFeature SECURITY_UPDATE_API_KEY = new NodeFeature("security.api_key_update"); public static final NodeFeature SECURITY_BULK_UPDATE_API_KEY = new NodeFeature("security.api_key_bulk_update"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature WATCHES_VERSION_IN_META = new NodeFeature("watcher.version_in_meta"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature SECURITY_ROLE_DESCRIPTORS_OPTIONAL = new NodeFeature("security.role_descriptors_optional"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature SEARCH_AGGREGATIONS_FORCE_INTERVAL_SELECTION_DATE_HISTOGRAM = new NodeFeature( - "search.aggregations.force_interval_selection_on_date_histogram" - ); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature TRANSFORM_NEW_API_ENDPOINT = new NodeFeature("transform.new_api_endpoint"); - // Ref: https://github.com/elastic/elasticsearch/pull/65205 - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature ML_INDICES_HIDDEN = new NodeFeature("ml.indices_hidden"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature ML_ANALYTICS_MAPPINGS = new NodeFeature("ml.analytics_mappings"); public static final NodeFeature TSDB_NEW_INDEX_FORMAT = new NodeFeature("indices.tsdb_new_format"); public static final NodeFeature TSDB_GENERALLY_AVAILABLE = new NodeFeature("indices.tsdb_supported"); @@ -104,14 +74,10 @@ public class RestTestLegacyFeatures implements FeatureSpecification { @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) public static final NodeFeature REPLICATION_OF_CLOSED_INDICES = new NodeFeature("indices.closed_replication_supported"); @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature TASK_INDEX_SYSTEM_INDEX = new NodeFeature("tasks.moved_to_system_index"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) public static final NodeFeature SOFT_DELETES_ENFORCED = new NodeFeature("indices.soft_deletes_enforced"); @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) public static final NodeFeature NEW_TRANSPORT_COMPRESSED_SETTING = new NodeFeature("transport.new_compressed_setting"); @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - public static final NodeFeature SHUTDOWN_SUPPORTED = new NodeFeature("shutdown.supported"); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) public static final NodeFeature SERVICE_ACCOUNTS_SUPPORTED = new NodeFeature("auth.service_accounts_supported"); @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) public static final NodeFeature TRANSFORM_SUPPORTED = new NodeFeature("transform.supported"); @@ -140,27 +106,14 @@ public class RestTestLegacyFeatures implements FeatureSpecification { @Override public Map getHistoricalFeatures() { return Map.ofEntries( - entry(FEATURE_STATE_RESET_SUPPORTED, Version.V_7_13_0), - entry(SYSTEM_INDICES_REST_ACCESS_ENFORCED, Version.V_8_0_0), - entry(SYSTEM_INDICES_REST_ACCESS_DEPRECATED, Version.V_7_10_0), - entry(HIDDEN_INDICES_SUPPORTED, Version.V_7_7_0), entry(COMPONENT_TEMPLATE_SUPPORTED, Version.V_7_8_0), - entry(DELETE_TEMPLATE_MULTIPLE_NAMES_SUPPORTED, Version.V_7_13_0), entry(ML_STATE_RESET_FALLBACK_ON_DISABLED, Version.V_8_7_0), entry(SECURITY_UPDATE_API_KEY, Version.V_8_4_0), entry(SECURITY_BULK_UPDATE_API_KEY, Version.V_8_5_0), entry(ML_NEW_MEMORY_FORMAT, Version.V_8_11_0), - entry(SUPPORTS_VENDOR_XCONTENT_TYPES, Version.V_7_11_0), - entry(SUPPORTS_TRUE_BINARY_RESPONSES, Version.V_7_7_0), entry(TRANSPORT_VERSION_SUPPORTED, VERSION_INTRODUCING_TRANSPORT_VERSIONS), entry(STATE_REPLACED_TRANSPORT_VERSION_WITH_NODES_VERSION, Version.V_8_11_0), entry(ML_MEMORY_OVERHEAD_FIXED, Version.V_8_2_1), - entry(WATCHES_VERSION_IN_META, Version.V_7_13_0), - entry(SECURITY_ROLE_DESCRIPTORS_OPTIONAL, Version.V_7_3_0), - entry(SEARCH_AGGREGATIONS_FORCE_INTERVAL_SELECTION_DATE_HISTOGRAM, Version.V_7_2_0), - entry(TRANSFORM_NEW_API_ENDPOINT, Version.V_7_5_0), - entry(ML_INDICES_HIDDEN, Version.V_7_7_0), - entry(ML_ANALYTICS_MAPPINGS, Version.V_7_3_0), entry(REST_ELASTIC_PRODUCT_HEADER_PRESENT, Version.V_8_0_1), entry(DESIRED_NODE_API_SUPPORTED, Version.V_8_1_0), entry(TSDB_NEW_INDEX_FORMAT, Version.V_8_2_0), @@ -173,10 +126,8 @@ public Map getHistoricalFeatures() { entry(INDEXING_SLOWLOG_LEVEL_SETTING_REMOVED, Version.V_8_0_0), entry(DEPRECATION_WARNINGS_LEAK_FIXED, Version.V_7_17_9), entry(REPLICATION_OF_CLOSED_INDICES, Version.V_7_2_0), - entry(TASK_INDEX_SYSTEM_INDEX, Version.V_7_10_0), entry(SOFT_DELETES_ENFORCED, Version.V_8_0_0), entry(NEW_TRANSPORT_COMPRESSED_SETTING, Version.V_7_14_0), - entry(SHUTDOWN_SUPPORTED, Version.V_7_15_0), entry(SERVICE_ACCOUNTS_SUPPORTED, Version.V_7_13_0), entry(TRANSFORM_SUPPORTED, Version.V_7_2_0), entry(SLM_SUPPORTED, Version.V_7_4_0), diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java index 7cfbd2b3a57f1..86c3f42a6a8ec 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java @@ -371,14 +371,10 @@ public void execute(ClientYamlTestExecutionContext executionContext) throws IOEx ? executionContext.getClientYamlTestCandidate().getTestPath() : null; - // #84038 and #84089 mean that this assertion fails when running against < 7.17.2 and 8.0.0 released versions - // This is really difficult to express just with features, so I will break it down into 2 parts: version check for v7, - // and feature check for v8. This way the version check can be removed once we move to v9 - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - var fixedInV7 = executionContext.clusterHasFeature("gte_v7.17.2", false) - && executionContext.clusterHasFeature("gte_v8.0.0", false) == false; - var fixedProductionHeader = fixedInV7 - || executionContext.clusterHasFeature(RestTestLegacyFeatures.REST_ELASTIC_PRODUCT_HEADER_PRESENT.id(), false); + var fixedProductionHeader = executionContext.clusterHasFeature( + RestTestLegacyFeatures.REST_ELASTIC_PRODUCT_HEADER_PRESENT.id(), + false + ); if (fixedProductionHeader) { checkElasticProductHeader(response.getHeaders("X-elastic-product")); } diff --git a/x-pack/plugin/shutdown/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/plugin/shutdown/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index 8125c8d9d52ad..fa6a908891400 100644 --- a/x-pack/plugin/shutdown/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/plugin/shutdown/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -14,18 +14,15 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.FeatureFlag; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.cluster.util.resource.Resource; import org.elasticsearch.test.rest.ESRestTestCase; -import org.elasticsearch.test.rest.RestTestLegacyFeatures; import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; import org.elasticsearch.upgrades.ParameterizedFullClusterRestartTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; -import org.junit.Before; import org.junit.ClassRule; import java.io.IOException; @@ -88,13 +85,6 @@ protected Settings restClientSettings() { .build(); } - @Before - public void checkClusterVersion() { - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // always true - var originalClusterSupportsShutdown = oldClusterHasFeature(RestTestLegacyFeatures.SHUTDOWN_SUPPORTED); - assumeTrue("no shutdown in versions before 7.15", originalClusterSupportsShutdown); - } - @SuppressWarnings("unchecked") public void testNodeShutdown() throws Exception { if (isRunningAgainstOldCluster()) { diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index 303f799e0d9cd..c57e5653d1279 100644 --- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -361,29 +361,10 @@ public void testApiKeySuperuser() throws IOException { ) ) ); - if (clusterHasFeature(RestTestLegacyFeatures.SECURITY_ROLE_DESCRIPTORS_OPTIONAL)) { - createApiKeyRequest.setJsonEntity(""" - { - "name": "super_legacy_key" - }"""); - } else { - createApiKeyRequest.setJsonEntity(""" - { - "name": "super_legacy_key", - "role_descriptors": { - "super": { - "cluster": [ "all" ], - "indices": [ - { - "names": [ "*" ], - "privileges": [ "all" ], - "allow_restricted_indices": true - } - ] - } - } - }"""); - } + createApiKeyRequest.setJsonEntity(""" + { + "name": "super_legacy_key" + }"""); final Map createApiKeyResponse = entityAsMap(client().performRequest(createApiKeyRequest)); final byte[] keyBytes = (createApiKeyResponse.get("id") + ":" + createApiKeyResponse.get("api_key")).getBytes( StandardCharsets.UTF_8 @@ -393,20 +374,6 @@ public void testApiKeySuperuser() throws IOException { final Request saveApiKeyRequest = new Request("PUT", "/api_keys/_doc/super_legacy_key"); saveApiKeyRequest.setJsonEntity("{\"auth_header\":\"" + apiKeyAuthHeader + "\"}"); assertOK(client().performRequest(saveApiKeyRequest)); - - if (clusterHasFeature(RestTestLegacyFeatures.SYSTEM_INDICES_REST_ACCESS_ENFORCED) == false) { - final Request indexRequest = new Request("POST", ".security/_doc"); - indexRequest.setJsonEntity(""" - { - "doc_type": "foo" - }"""); - if (clusterHasFeature(RestTestLegacyFeatures.SYSTEM_INDICES_REST_ACCESS_DEPRECATED)) { - indexRequest.setOptions(systemIndexWarningHandlerOptions(".security-7").addHeader("Authorization", apiKeyAuthHeader)); - } else { - indexRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", apiKeyAuthHeader)); - } - assertOK(client().performRequest(indexRequest)); - } } else { final Request getRequest = new Request("GET", "/api_keys/_doc/super_legacy_key"); final Map getResponseMap = responseAsMap(client().performRequest(getRequest)); @@ -472,15 +439,7 @@ public void testRollupAfterRestart() throws Exception { // create the rollup job final Request createRollupJobRequest = new Request("PUT", "/_rollup/job/rollup-job-test"); - - String intervalType; - if (clusterHasFeature(RestTestLegacyFeatures.SEARCH_AGGREGATIONS_FORCE_INTERVAL_SELECTION_DATE_HISTOGRAM)) { - intervalType = "fixed_interval"; - } else { - intervalType = "interval"; - } - - createRollupJobRequest.setJsonEntity(Strings.format(""" + createRollupJobRequest.setJsonEntity(""" { "index_pattern": "rollup-*", "rollup_index": "results-rollup", @@ -489,7 +448,7 @@ public void testRollupAfterRestart() throws Exception { "groups": { "date_histogram": { "field": "timestamp", - "%s": "5m" + "fixed_interval": "5m" } }, "metrics": [ @@ -498,7 +457,7 @@ public void testRollupAfterRestart() throws Exception { "metrics": [ "min", "max", "sum" ] } ] - }""", intervalType)); + }"""); Map createRollupJobResponse = entityAsMap(client().performRequest(createRollupJobRequest)); assertThat(createRollupJobResponse.get("acknowledged"), equalTo(Boolean.TRUE)); @@ -550,11 +509,7 @@ public void testTransformLegacyTemplateCleanup() throws Exception { assertThat(createIndexResponse.get("acknowledged"), equalTo(Boolean.TRUE)); // create a transform - String endpoint = clusterHasFeature(RestTestLegacyFeatures.TRANSFORM_NEW_API_ENDPOINT) - ? "_transform/transform-full-cluster-restart-test" - : "_data_frame/transforms/transform-full-cluster-restart-test"; - final Request createTransformRequest = new Request("PUT", endpoint); - + final Request createTransformRequest = new Request("PUT", "_transform/transform-full-cluster-restart-test"); createTransformRequest.setJsonEntity(""" { "source": { diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java index 3674f811ebb0a..c825de31a7f6e 100644 --- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java @@ -28,8 +28,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractXpackFullClusterRestartTestCase { @@ -62,13 +60,8 @@ public void testMlConfigIndexMappingsAfterMigration() throws Exception { if (isRunningAgainstOldCluster()) { // trigger .ml-config index creation createAnomalyDetectorJob(OLD_CLUSTER_JOB_ID); - if (clusterHasFeature(RestTestLegacyFeatures.ML_ANALYTICS_MAPPINGS)) { - // .ml-config has mappings for analytics as the feature was introduced in 7.3.0 - assertThat(getDataFrameAnalysisMappings().keySet(), hasItem("outlier_detection")); - } else { - // .ml-config does not yet have correct mappings, it will need an update after cluster is upgraded - assertThat(getDataFrameAnalysisMappings(), is(nullValue())); - } + // .ml-config has mappings for analytics as the feature was introduced in 7.3.0 + assertThat(getDataFrameAnalysisMappings().keySet(), hasItem("outlier_detection")); } else { // trigger .ml-config index mappings update createAnomalyDetectorJob(NEW_CLUSTER_JOB_ID); diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java index 16345a19fc950..7dc0a2f48bbc9 100644 --- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java @@ -38,7 +38,6 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; public class MlHiddenIndicesFullClusterRestartIT extends AbstractXpackFullClusterRestartTestCase { @@ -78,36 +77,6 @@ public void testMlIndicesBecomeHidden() throws Exception { // trigger ML indices creation createAnomalyDetectorJob(JOB_ID); openAnomalyDetectorJob(JOB_ID); - - if (clusterHasFeature(RestTestLegacyFeatures.ML_INDICES_HIDDEN) == false) { - Map indexSettingsMap = contentAsMap(getMlIndicesSettings()); - Map aliasesMap = contentAsMap(getMlAliases()); - - assertThat("Index settings map was: " + indexSettingsMap, indexSettingsMap, is(aMapWithSize(greaterThanOrEqualTo(4)))); - for (Map.Entry e : indexSettingsMap.entrySet()) { - String indexName = e.getKey(); - @SuppressWarnings("unchecked") - Map settings = (Map) e.getValue(); - assertThat(settings, is(notNullValue())); - assertThat( - "Index " + indexName + " expected not to be hidden but was, settings = " + settings, - XContentMapValues.extractValue(settings, "settings", "index", "hidden"), - is(nullValue()) - ); - } - - for (Tuple, String> indexAndAlias : EXPECTED_INDEX_ALIAS_PAIRS) { - List indices = indexAndAlias.v1(); - String alias = indexAndAlias.v2(); - for (String index : indices) { - assertThat( - indexAndAlias + " expected not be hidden but was, aliasesMap = " + aliasesMap, - XContentMapValues.extractValue(aliasesMap, index, "aliases", alias, "is_hidden"), - is(nullValue()) - ); - } - } - } } else { // The 5 operations in MlInitializationService.makeMlInternalIndicesHidden() run sequentially, so might // not all be finished when this test runs. The desired state should exist within a few seconds of startup, diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java index 767f27d4e4f93..fee6910fcf6c0 100644 --- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java @@ -76,12 +76,7 @@ public void testMappingsAreUpdated() throws Exception { """); client().performRequest(putWatchRequest); - if (clusterHasFeature(RestTestLegacyFeatures.WATCHES_VERSION_IN_META)) { - assertMappingVersion(".watches", getOldClusterVersion()); - } else { - // watches indices from before 7.10 do not have mapping versions in _meta - assertNoMappingVersion(".watches"); - } + assertMappingVersion(".watches", getOldClusterVersion()); } else { assertMappingVersion(".watches", Build.current().version()); } @@ -101,9 +96,7 @@ private void assertNoMappingVersion(String index) throws Exception { assertBusy(() -> { Request mappingRequest = new Request("GET", index + "/_mappings"); assert isRunningAgainstOldCluster(); - if (clusterHasFeature(RestTestLegacyFeatures.SYSTEM_INDICES_REST_ACCESS_DEPRECATED)) { - mappingRequest.setOptions(getWarningHandlerOptions(index)); - } + mappingRequest.setOptions(getWarningHandlerOptions(index)); Response response = client().performRequest(mappingRequest); String responseBody = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); assertThat(responseBody, not(containsString("\"version\":\""))); diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java index af67ab5751e96..4324aed5fee18 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractUpgradeTestCase.java @@ -7,7 +7,6 @@ package org.elasticsearch.upgrades; import org.elasticsearch.Build; -import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.io.Streams; @@ -15,7 +14,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Booleans; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.test.SecuritySettingsSourceField; import org.junit.Before; @@ -49,16 +47,6 @@ protected static boolean isOriginalClusterCurrent() { return UPGRADE_FROM_VERSION.equals(Build.current().version()); } - @Deprecated(forRemoval = true) - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - // Tests should be reworked to rely on features from the current cluster (old, mixed or upgraded). - // Version test against the original cluster will be removed - protected static boolean isOriginalClusterVersionAtLeast(Version supportedVersion) { - // Always assume non-semantic versions are OK: this method will be removed in V9, we are testing the pre-upgrade cluster version, - // and non-semantic versions are always V8+ - return parseLegacyVersion(UPGRADE_FROM_VERSION).map(x -> x.onOrAfter(supportedVersion)).orElse(true); - } - @Override protected boolean resetFeatureStates() { return false; diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java index 4de2c610e5c48..8c051d03d5f04 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java @@ -8,13 +8,11 @@ package org.elasticsearch.upgrades; import org.apache.http.util.EntityUtils; -import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.Strings; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.xcontent.XContentType; import org.junit.After; import org.junit.Before; @@ -101,11 +99,6 @@ public void removeLogging() throws IOException { } public void testTrainedModelDeployment() throws Exception { - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // upgrade will always be from v8, condition can be removed - var originalClusterAtLeastV8 = isOriginalClusterVersionAtLeast(Version.V_8_0_0); - // These tests assume the original cluster is v8 - testing for features on the _current_ cluster will break for NEW - assumeTrue("NLP model deployments added in 8.0", originalClusterAtLeastV8); - final String modelId = "upgrade-deployment-test"; switch (CLUSTER_TYPE) { @@ -140,11 +133,6 @@ public void testTrainedModelDeployment() throws Exception { } public void testTrainedModelDeploymentStopOnMixedCluster() throws Exception { - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // upgrade will always be from v8, condition can be removed - var originalClusterAtLeastV8 = isOriginalClusterVersionAtLeast(Version.V_8_0_0); - // These tests assume the original cluster is v8 - testing for features on the _current_ cluster will break for NEW - assumeTrue("NLP model deployments added in 8.0", originalClusterAtLeastV8); - final String modelId = "upgrade-deployment-test-stop-mixed-cluster"; switch (CLUSTER_TYPE) { diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlAssignmentPlannerUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlAssignmentPlannerUpgradeIT.java index 7cefaa2edb388..74165eeb07b8a 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlAssignmentPlannerUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlAssignmentPlannerUpgradeIT.java @@ -7,14 +7,12 @@ package org.elasticsearch.upgrades; -import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.Strings; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.test.rest.RestTestLegacyFeatures; @@ -71,10 +69,6 @@ public class MlAssignmentPlannerUpgradeIT extends AbstractUpgradeTestCase { @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/101926") public void testMlAssignmentPlannerUpgrade() throws Exception { - @UpdateForV9(owner = UpdateForV9.Owner.MACHINE_LEARNING) // upgrade will always be from v8, condition can be removed - var originalClusterAtLeastV8 = isOriginalClusterVersionAtLeast(Version.V_8_0_0); - // These tests assume the original cluster is v8 - testing for features on the _current_ cluster will break for NEW - assumeTrue("NLP model deployments added in 8.0", originalClusterAtLeastV8); assumeFalse("This test deploys multiple models which cannot be accommodated on a single processor", IS_SINGLE_PROCESSOR_TEST); logger.info("Starting testMlAssignmentPlannerUpgrade, model size {}", RAW_MODEL_SIZE); From 3b9150dbb064a4afe3eb3602e9478d15eb72e409 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 4 Oct 2024 11:14:44 -0400 Subject: [PATCH 094/194] [CI] Switch to local ssds for DRA workflow pipeline (#114147) --- .buildkite/pipelines/dra-workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipelines/dra-workflow.yml b/.buildkite/pipelines/dra-workflow.yml index 5ef0a2ffb3df9..4a131065e6883 100644 --- a/.buildkite/pipelines/dra-workflow.yml +++ b/.buildkite/pipelines/dra-workflow.yml @@ -6,8 +6,8 @@ steps: provider: gcp image: family/elasticsearch-ubuntu-2204 machineType: custom-32-98304 - buildDirectory: /dev/shm/bk - diskSizeGb: 500 + localSsds: 1 + localSsdInterface: nvme - wait # The hadoop build depends on the ES artifact # So let's trigger the hadoop build any time we build a new staging artifact From 95ea135106038469f4361681d834c6c87bab8314 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 4 Oct 2024 17:21:59 +0100 Subject: [PATCH 095/194] Clarify integer settings for `repository-s3` repos (#114093) Today there are a handful of integer settings for `repository-s3` repositories whose docs link to the page about numeric field types. Yet these settings are not fields, and do not support floating-point values either. The convention throughout the rest of the docs is to just call these things `integer` without linking to anything. This commit aligns the `repository-s3` docs with this convention. --- .../snapshot-restore/repository-s3.asciidoc | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/docs/reference/snapshot-restore/repository-s3.asciidoc b/docs/reference/snapshot-restore/repository-s3.asciidoc index 1f55296139cd3..71a9fd8b87c96 100644 --- a/docs/reference/snapshot-restore/repository-s3.asciidoc +++ b/docs/reference/snapshot-restore/repository-s3.asciidoc @@ -296,9 +296,8 @@ include::repository-shared-settings.asciidoc[] `max_multipart_parts` :: - (<>) The maximum number of parts that {es} will write during a multipart upload - of a single object. Files which are larger than `buffer_size × max_multipart_parts` will be - chunked into several smaller objects. {es} may also split a file across multiple objects to + (integer) The maximum number of parts that {es} will write during a multipart upload of a single object. Files which are larger than + `buffer_size × max_multipart_parts` will be chunked into several smaller objects. {es} may also split a file across multiple objects to satisfy other constraints such as the `chunk_size` limit. Defaults to `10000` which is the https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html[maximum number of parts in a multipart upload in AWS S3]. @@ -321,20 +320,14 @@ include::repository-shared-settings.asciidoc[] `delete_objects_max_size`:: - (<>) Sets the maxmimum batch size, betewen 1 and 1000, used - for `DeleteObjects` requests. Defaults to 1000 which is the maximum number - supported by the - https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html[AWS - DeleteObjects API]. + (integer) Sets the maxmimum batch size, betewen 1 and 1000, used for `DeleteObjects` requests. Defaults to 1000 which is the maximum + number supported by the https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html[AWS DeleteObjects API]. `max_multipart_upload_cleanup_size`:: - (<>) Sets the maximum number of possibly-dangling multipart - uploads to clean up in each batch of snapshot deletions. Defaults to `1000` - which is the maximum number supported by the - https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html[AWS - ListMultipartUploads API]. If set to `0`, {es} will not attempt to clean up - dangling multipart uploads. + (integer) Sets the maximum number of possibly-dangling multipart uploads to clean up in each batch of snapshot deletions. Defaults to + `1000` which is the maximum number supported by the https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html[AWS + ListMultipartUploads API]. If set to `0`, {es} will not attempt to clean up dangling multipart uploads. NOTE: The option of defining client settings in the repository settings as documented below is considered deprecated, and will be removed in a future From e3705a1884432e6335fa002ebf095b12e10648c9 Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Fri, 4 Oct 2024 12:34:48 -0400 Subject: [PATCH 096/194] Took time and cluster details get updated for coordinator only query operations (#114075) * Took time and cluster details get updated for coordinator only query operations The ComputeService.runCompute pathway for coordinator only operations (such as `FROM foo | LIMIT 0` or a ROW command) get updated with overall took time. This also includes support for cross-cluster coordinator only operations, which come about with queries like `FROM foo,remote:foo | LIMIT 0`. The _clusters metadata is now properly updated for those cases as well. Fixes https://github.com/elastic/elasticsearch/issues/114014 --- .../xpack/esql/qa/single_node/RestEsqlIT.java | 6 +- .../xpack/esql/qa/rest/RestEsqlTestCase.java | 3 +- .../esql/action/CrossClustersQueryIT.java | 103 ++++++++++++++++++ .../xpack/esql/plugin/ComputeService.java | 31 +++++- .../xpack/esql/session/EsqlSession.java | 25 +++++ .../xpack/esql/session/EsqlSessionTests.java | 11 +- 6 files changed, 167 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index c5ab20469bf77..974180526d750 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -117,7 +117,8 @@ public void testDoNotLogWithInfo() throws IOException { setLoggingLevel("INFO"); RequestObjectBuilder builder = requestObjectBuilder().query("ROW DO_NOT_LOG_ME = 1"); Map result = runEsql(builder); - assertEquals(2, result.size()); + assertEquals(3, result.size()); + assertThat(((Integer) result.get("took")).intValue(), greaterThanOrEqualTo(0)); Map colA = Map.of("name", "DO_NOT_LOG_ME", "type", "integer"); assertEquals(List.of(colA), result.get("columns")); assertEquals(List.of(List.of(1)), result.get("values")); @@ -136,7 +137,8 @@ public void testDoLogWithDebug() throws IOException { setLoggingLevel("DEBUG"); RequestObjectBuilder builder = requestObjectBuilder().query("ROW DO_LOG_ME = 1"); Map result = runEsql(builder); - assertEquals(2, result.size()); + assertEquals(3, result.size()); + assertThat(((Integer) result.get("took")).intValue(), greaterThanOrEqualTo(0)); Map colA = Map.of("name", "DO_LOG_ME", "type", "integer"); assertEquals(List.of(colA), result.get("columns")); assertEquals(List.of(List.of(1)), result.get("values")); diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java index 39340ab745a4d..c3e9652f51fb4 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java @@ -249,7 +249,8 @@ public static RequestObjectBuilder jsonBuilder() throws IOException { public void testGetAnswer() throws IOException { Map answer = runEsql(requestObjectBuilder().query("row a = 1, b = 2")); - assertEquals(2, answer.size()); + assertEquals(3, answer.size()); + assertThat(((Integer) answer.get("took")).intValue(), greaterThanOrEqualTo(0)); Map colA = Map.of("name", "a", "type", "integer"); Map colB = Map.of("name", "b", "type", "integer"); assertEquals(List.of(colA, colB), answer.get("columns")); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java index 03757d44a9f58..4f4f3d112247e 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java @@ -43,6 +43,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { private static final String REMOTE_CLUSTER = "cluster-a"; @@ -339,6 +340,108 @@ public void testSearchesWhereNonExistentClusterIsSpecifiedWithWildcards() { } } + /** + * Searches with LIMIT 0 are used by Kibana to get a list of columns. After the initial planning + * (which involves cross-cluster field-caps calls), it is a coordinator only operation at query time + * which uses a different pathway compared to queries that require data node (and remote data node) operations + * at query time. + */ + public void testCCSExecutionOnSearchesWithLimit0() { + setupTwoClusters(); + + // Ensure non-cross cluster queries have overall took time + try (EsqlQueryResponse resp = runQuery("FROM logs* | LIMIT 0")) { + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertNotNull(executionInfo); + assertThat(executionInfo.isCrossClusterSearch(), is(false)); + assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); + } + + // ensure cross-cluster searches have overall took time and correct per-cluster details in EsqlExecutionInfo + try (EsqlQueryResponse resp = runQuery("FROM logs*,cluster-a:* | LIMIT 0")) { + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertNotNull(executionInfo); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + long overallTookMillis = executionInfo.overallTook().millis(); + assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); + + EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); + assertThat(remoteCluster.getIndexExpression(), equalTo("*")); + assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertNull(remoteCluster.getTotalShards()); + assertNull(remoteCluster.getSuccessfulShards()); + assertNull(remoteCluster.getSkippedShards()); + assertNull(remoteCluster.getFailedShards()); + + EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER); + assertThat(localCluster.getIndexExpression(), equalTo("logs*")); + assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertNull(localCluster.getTotalShards()); + assertNull(localCluster.getSuccessfulShards()); + assertNull(localCluster.getSkippedShards()); + assertNull(localCluster.getFailedShards()); + } + + try (EsqlQueryResponse resp = runQuery("FROM logs*,cluster-a:nomatch* | LIMIT 0")) { + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertNotNull(executionInfo); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + long overallTookMillis = executionInfo.overallTook().millis(); + assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); + + EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); + assertThat(remoteCluster.getIndexExpression(), equalTo("nomatch*")); + assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + assertThat(remoteCluster.getTook().millis(), equalTo(0L)); + assertThat(remoteCluster.getTotalShards(), equalTo(0)); + assertThat(remoteCluster.getSuccessfulShards(), equalTo(0)); + assertThat(remoteCluster.getSkippedShards(), equalTo(0)); + assertThat(remoteCluster.getFailedShards(), equalTo(0)); + + EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER); + assertThat(localCluster.getIndexExpression(), equalTo("logs*")); + assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertNull(localCluster.getTotalShards()); + assertNull(localCluster.getSuccessfulShards()); + assertNull(localCluster.getSkippedShards()); + assertNull(localCluster.getFailedShards()); + } + + try (EsqlQueryResponse resp = runQuery("FROM nomatch*,cluster-a:* | LIMIT 0")) { + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertNotNull(executionInfo); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + long overallTookMillis = executionInfo.overallTook().millis(); + assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); + + EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); + assertThat(remoteCluster.getIndexExpression(), equalTo("*")); + assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertNull(remoteCluster.getTotalShards()); + assertNull(remoteCluster.getSuccessfulShards()); + assertNull(remoteCluster.getSkippedShards()); + assertNull(remoteCluster.getFailedShards()); + + EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER); + assertThat(localCluster.getIndexExpression(), equalTo("nomatch*")); + // TODO: in https://github.com/elastic/elasticsearch/issues/112886, this will be changed to be SKIPPED + assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); + assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + } + } + public void testMetadataIndex() { Map testClusterInfo = setupTwoClusters(); int localNumShards = (Integer) testClusterInfo.get("local.num_shards"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index d1f2007af2757..3ec39d1b0ac4b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -171,17 +171,21 @@ public void execute( null, null ); + String local = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; try ( var computeListener = ComputeListener.create( - RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, + local, transportService, rootTask, execInfo, configuration.getQueryStartTimeNanos(), - listener.map(r -> new Result(physicalPlan.output(), collectedPages, r.getProfiles(), execInfo)) + listener.map(r -> { + updateExecutionInfoAfterCoordinatorOnlyQuery(configuration.getQueryStartTimeNanos(), execInfo); + return new Result(physicalPlan.output(), collectedPages, r.getProfiles(), execInfo); + }) ) ) { - runCompute(rootTask, computeContext, coordinatorPlan, computeListener.acquireCompute()); + runCompute(rootTask, computeContext, coordinatorPlan, computeListener.acquireCompute(local)); return; } } else { @@ -247,6 +251,27 @@ public void execute( } } + private static void updateExecutionInfoAfterCoordinatorOnlyQuery(long queryStartNanos, EsqlExecutionInfo execInfo) { + long tookTimeNanos = System.nanoTime() - queryStartNanos; + execInfo.overallTook(new TimeValue(tookTimeNanos, TimeUnit.NANOSECONDS)); + if (execInfo.isCrossClusterSearch()) { + for (String clusterAlias : execInfo.clusterAliases()) { + // The local cluster 'took' time gets updated as part of the acquireCompute(local) call in the coordinator, so + // here we only need to update status for remote clusters since there are no remote ComputeListeners in this case. + // This happens in cross cluster searches that use LIMIT 0, e.g, FROM logs*,remote*:logs* | LIMIT 0. + if (clusterAlias.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) == false) { + execInfo.swapCluster(clusterAlias, (k, v) -> { + if (v.getStatus() == EsqlExecutionInfo.Cluster.Status.RUNNING) { + return new EsqlExecutionInfo.Cluster.Builder(v).setStatus(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL).build(); + } else { + return v; + } + }); + } + } + } + } + private List getRemoteClusters( Map clusterToConcreteIndices, Map clusterToOriginalIndices diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 608e45bb2085b..96391c841856f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -72,6 +72,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Predicate; @@ -245,6 +246,7 @@ private void preAnalyze( if (indexResolution.isValid()) { updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); updateExecutionInfoWithUnavailableClusters(executionInfo, indexResolution.getUnavailableClusters()); + updateTookTimeForRemoteClusters(executionInfo); Set newClusters = enrichPolicyResolver.groupIndicesPerCluster( indexResolution.get().concreteIndices().toArray(String[]::new) ).keySet(); @@ -285,6 +287,7 @@ static void updateExecutionInfoWithClustersWithNoMatchingIndices(EsqlExecutionIn } Set clustersRequested = executionInfo.clusterAliases(); Set clustersWithNoMatchingIndices = Sets.difference(clustersRequested, clustersWithResolvedIndices); + clustersWithNoMatchingIndices.removeAll(indexResolution.getUnavailableClusters()); /* * These are clusters in the original request that are not present in the field-caps response. They were * specified with an index or indices that do not exist, so the search on that cluster is done. @@ -304,6 +307,28 @@ static void updateExecutionInfoWithClustersWithNoMatchingIndices(EsqlExecutionIn } } + private void updateTookTimeForRemoteClusters(EsqlExecutionInfo executionInfo) { + if (executionInfo.isCrossClusterSearch()) { + for (String clusterAlias : executionInfo.clusterAliases()) { + if (clusterAlias.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) == false) { + executionInfo.swapCluster(clusterAlias, (k, v) -> { + if (v.getTook() == null && v.getStatus() != EsqlExecutionInfo.Cluster.Status.SKIPPED) { + // set took time in case we are finished with the remote cluster (e.g., FROM foo | LIMIT 0). + // this will be overwritten later if ES|QL operations happen on the remote cluster (the typical scenario) + TimeValue took = new TimeValue( + System.nanoTime() - configuration.getQueryStartTimeNanos(), + TimeUnit.NANOSECONDS + ); + return new EsqlExecutionInfo.Cluster.Builder(v).setTook(took).build(); + } else { + return v; + } + }); + } + } + } + } + private void preAnalyzeIndices( LogicalPlan parsed, EsqlExecutionInfo executionInfo, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java index 8dcad2f354b26..326756ad0b5f4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionTests.java @@ -216,6 +216,7 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { randomMapping(), Map.of("logs-a", IndexMode.STANDARD) ); + // mark remote1 as unavailable IndexResolution indexResolution = IndexResolution.valid(esIndex, Set.of(remote1Alias)); EsqlSession.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); @@ -226,12 +227,10 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias); assertThat(remote1Cluster.getIndexExpression(), equalTo("*")); - assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); - assertThat(remote1Cluster.getTook().millis(), equalTo(0L)); - assertThat(remote1Cluster.getTotalShards(), equalTo(0)); - assertThat(remote1Cluster.getSuccessfulShards(), equalTo(0)); - assertThat(remote1Cluster.getSkippedShards(), equalTo(0)); - assertThat(remote1Cluster.getFailedShards(), equalTo(0)); + // remote1 is left as RUNNING, since another method (updateExecutionInfoWithUnavailableClusters) not under test changes status + assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.RUNNING)); + assertNull(remote1Cluster.getTook()); + assertNull(remote1Cluster.getTotalShards()); EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias); assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*")); From bb2294643ae842f488fb1ea02b76034c9235c806 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 4 Oct 2024 12:59:50 -0400 Subject: [PATCH 097/194] [ci] Increase disk size for DRA workflow job --- .buildkite/pipelines/dra-workflow.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/pipelines/dra-workflow.yml b/.buildkite/pipelines/dra-workflow.yml index 4a131065e6883..25477c8541fa9 100644 --- a/.buildkite/pipelines/dra-workflow.yml +++ b/.buildkite/pipelines/dra-workflow.yml @@ -8,6 +8,7 @@ steps: machineType: custom-32-98304 localSsds: 1 localSsdInterface: nvme + diskSizeGb: 350 - wait # The hadoop build depends on the ES artifact # So let's trigger the hadoop build any time we build a new staging artifact From c59e3509dab743928f25fc11e8e75b4aa2e701c9 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Fri, 4 Oct 2024 18:01:13 +0100 Subject: [PATCH 098/194] [ML] Deploy default on chunked infer (#114141) --- .../ElasticsearchInternalService.java | 41 +++++++++++-------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java index dd14e16412996..9b4c0e50bdebe 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java @@ -608,26 +608,33 @@ public void chunkedInfer( return; } - var configUpdate = chunkingOptions != null - ? new TokenizationConfigUpdate(chunkingOptions.windowSize(), chunkingOptions.span()) - : new TokenizationConfigUpdate(null, null); + if (model instanceof ElasticsearchInternalModel esModel) { - var request = buildInferenceRequest( - model.getConfigurations().getInferenceEntityId(), - configUpdate, - input, - inputType, - timeout, - true - ); + var configUpdate = chunkingOptions != null + ? new TokenizationConfigUpdate(chunkingOptions.windowSize(), chunkingOptions.span()) + : new TokenizationConfigUpdate(null, null); + + var request = buildInferenceRequest( + model.getConfigurations().getInferenceEntityId(), + configUpdate, + input, + inputType, + timeout, + true + ); - client.execute( - InferModelAction.INSTANCE, - request, - listener.delegateFailureAndWrap( + ActionListener mlResultsListener = listener.delegateFailureAndWrap( (l, inferenceResult) -> l.onResponse(translateToChunkedResults(inferenceResult.getInferenceResults())) - ) - ); + ); + + var maybeDeployListener = mlResultsListener.delegateResponse( + (l, exception) -> maybeStartDeployment(esModel, exception, request, mlResultsListener) + ); + + client.execute(InferModelAction.INSTANCE, request, maybeDeployListener); + } else { + listener.onFailure(notElasticsearchModelException(model)); + } } private static List translateToChunkedResults(List inferenceResults) { From 3391f5007e537d1e8eed30ad50ca89b6a301f3a9 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Fri, 4 Oct 2024 10:38:20 -0700 Subject: [PATCH 099/194] Fix flattened ignore_above tests (#114155) This fixes tests so that they can work with multiple shards. --- .../test/search/600_flattened_ignore_above.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml index e2c3006232c53..a4a9b1aaecb22 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml @@ -9,6 +9,8 @@ flattened ignore_above single-value field: body: mappings: properties: + name: + type: keyword keyword: type: keyword ignore_above: 5 @@ -22,6 +24,7 @@ flattened ignore_above single-value field: id: "1" refresh: true body: + name: "A" keyword: "foo" flat: { "value": "foo", "key": "foo key" } @@ -31,12 +34,14 @@ flattened ignore_above single-value field: id: "2" refresh: true body: + name: "B" keyword: "foo bar" flat: { "value": "foo bar", "key": "foo bar key"} - do: search: index: test + sort: name body: fields: - keyword @@ -69,6 +74,8 @@ flattened ignore_above multi-value field: body: mappings: properties: + name: + type: keyword keyword: type: keyword ignore_above: 5 @@ -82,6 +89,7 @@ flattened ignore_above multi-value field: id: "1" refresh: true body: + name: "A" keyword: ["foo","bar"] flat: { "value": ["foo", "bar"], "key": "foo bar array key" } @@ -91,12 +99,14 @@ flattened ignore_above multi-value field: id: "2" refresh: true body: + name: "B" keyword: ["foobar", "foo", "bar"] flat: { "value": ["foobar", "foo"], "key": ["foo key", "bar key"]} - do: search: index: test + sort: name body: fields: - keyword From 147461f5b1cf4efe4cd394a1793e6027049dbe4e Mon Sep 17 00:00:00 2001 From: Drew Tate Date: Fri, 4 Oct 2024 11:57:37 -0600 Subject: [PATCH 100/194] [ES|QL] add reverse function (#113297) Adds a REVERSE string function --- docs/changelog/113297.yaml | 5 + .../functions/description/reverse.asciidoc | 5 + .../esql/functions/examples/reverse.asciidoc | 22 +++ .../functions/kibana/definition/reverse.json | 38 +++++ .../esql/functions/kibana/docs/reverse.md | 10 ++ .../esql/functions/layout/reverse.asciidoc | 15 ++ .../functions/parameters/reverse.asciidoc | 6 + .../esql/functions/signature/reverse.svg | 1 + .../esql/functions/string-functions.asciidoc | 2 + .../esql/functions/types/reverse.asciidoc | 10 ++ .../elasticsearch/common/util/ArrayUtils.java | 16 ++ .../src/main/resources/meta.csv-spec | 6 +- .../src/main/resources/string.csv-spec | 107 +++++++++++++ .../scalar/string/ReverseEvaluator.java | 111 ++++++++++++++ .../xpack/esql/action/EsqlCapabilities.java | 5 + .../function/EsqlFunctionRegistry.java | 18 ++- .../function/scalar/EsqlScalarFunction.java | 2 + .../function/scalar/package-info.java | 7 +- .../function/scalar/string/Reverse.java | 140 ++++++++++++++++++ .../string/ReverseSerializationTests.java | 19 +++ .../function/scalar/string/ReverseTests.java | 65 ++++++++ .../rest-api-spec/test/esql/80_text.yml | 25 +++- 22 files changed, 624 insertions(+), 11 deletions(-) create mode 100644 docs/changelog/113297.yaml create mode 100644 docs/reference/esql/functions/description/reverse.asciidoc create mode 100644 docs/reference/esql/functions/examples/reverse.asciidoc create mode 100644 docs/reference/esql/functions/kibana/definition/reverse.json create mode 100644 docs/reference/esql/functions/kibana/docs/reverse.md create mode 100644 docs/reference/esql/functions/layout/reverse.asciidoc create mode 100644 docs/reference/esql/functions/parameters/reverse.asciidoc create mode 100644 docs/reference/esql/functions/signature/reverse.svg create mode 100644 docs/reference/esql/functions/types/reverse.asciidoc create mode 100644 x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseEvaluator.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java diff --git a/docs/changelog/113297.yaml b/docs/changelog/113297.yaml new file mode 100644 index 0000000000000..476619f432639 --- /dev/null +++ b/docs/changelog/113297.yaml @@ -0,0 +1,5 @@ +pr: 113297 +summary: "[ES|QL] add reverse function" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/reference/esql/functions/description/reverse.asciidoc b/docs/reference/esql/functions/description/reverse.asciidoc new file mode 100644 index 0000000000000..fbb3f3f6b4d54 --- /dev/null +++ b/docs/reference/esql/functions/description/reverse.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Returns a new string representing the input string in reverse order. diff --git a/docs/reference/esql/functions/examples/reverse.asciidoc b/docs/reference/esql/functions/examples/reverse.asciidoc new file mode 100644 index 0000000000000..67c8af077b174 --- /dev/null +++ b/docs/reference/esql/functions/examples/reverse.asciidoc @@ -0,0 +1,22 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Examples* + +[source.merge.styled,esql] +---- +include::{esql-specs}/string.csv-spec[tag=reverse] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/string.csv-spec[tag=reverse-result] +|=== +`REVERSE` works with unicode, too! It keeps unicode grapheme clusters together during reversal. +[source.merge.styled,esql] +---- +include::{esql-specs}/string.csv-spec[tag=reverseEmoji] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/string.csv-spec[tag=reverseEmoji-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/reverse.json b/docs/reference/esql/functions/kibana/definition/reverse.json new file mode 100644 index 0000000000000..1b222691530f2 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/reverse.json @@ -0,0 +1,38 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "reverse", + "description" : "Returns a new string representing the input string in reverse order.", + "signatures" : [ + { + "params" : [ + { + "name" : "str", + "type" : "keyword", + "optional" : false, + "description" : "String expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "str", + "type" : "text", + "optional" : false, + "description" : "String expression. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "text" + } + ], + "examples" : [ + "ROW message = \"Some Text\" | EVAL message_reversed = REVERSE(message);", + "ROW bending_arts = \"💧🪨🔥💨\" | EVAL bending_arts_reversed = REVERSE(bending_arts);" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/esql/functions/kibana/docs/reverse.md b/docs/reference/esql/functions/kibana/docs/reverse.md new file mode 100644 index 0000000000000..cbeade9189d80 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/reverse.md @@ -0,0 +1,10 @@ + + +### REVERSE +Returns a new string representing the input string in reverse order. + +``` +ROW message = "Some Text" | EVAL message_reversed = REVERSE(message); +``` diff --git a/docs/reference/esql/functions/layout/reverse.asciidoc b/docs/reference/esql/functions/layout/reverse.asciidoc new file mode 100644 index 0000000000000..99c236d63492e --- /dev/null +++ b/docs/reference/esql/functions/layout/reverse.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-reverse]] +=== `REVERSE` + +*Syntax* + +[.text-center] +image::esql/functions/signature/reverse.svg[Embedded,opts=inline] + +include::../parameters/reverse.asciidoc[] +include::../description/reverse.asciidoc[] +include::../types/reverse.asciidoc[] +include::../examples/reverse.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/reverse.asciidoc b/docs/reference/esql/functions/parameters/reverse.asciidoc new file mode 100644 index 0000000000000..d56d115662491 --- /dev/null +++ b/docs/reference/esql/functions/parameters/reverse.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`str`:: +String expression. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/signature/reverse.svg b/docs/reference/esql/functions/signature/reverse.svg new file mode 100644 index 0000000000000..c23ce5583a8c0 --- /dev/null +++ b/docs/reference/esql/functions/signature/reverse.svg @@ -0,0 +1 @@ +REVERSE(str) \ No newline at end of file diff --git a/docs/reference/esql/functions/string-functions.asciidoc b/docs/reference/esql/functions/string-functions.asciidoc index ed97769b900e7..f5222330d579d 100644 --- a/docs/reference/esql/functions/string-functions.asciidoc +++ b/docs/reference/esql/functions/string-functions.asciidoc @@ -17,6 +17,7 @@ * <> * <> * <> +* <> * <> * <> * <> @@ -38,6 +39,7 @@ include::layout/locate.asciidoc[] include::layout/ltrim.asciidoc[] include::layout/repeat.asciidoc[] include::layout/replace.asciidoc[] +include::layout/reverse.asciidoc[] include::layout/right.asciidoc[] include::layout/rtrim.asciidoc[] include::layout/space.asciidoc[] diff --git a/docs/reference/esql/functions/types/reverse.asciidoc b/docs/reference/esql/functions/types/reverse.asciidoc new file mode 100644 index 0000000000000..974066d225bca --- /dev/null +++ b/docs/reference/esql/functions/types/reverse.asciidoc @@ -0,0 +1,10 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +str | result +keyword | keyword +text | text +|=== diff --git a/server/src/main/java/org/elasticsearch/common/util/ArrayUtils.java b/server/src/main/java/org/elasticsearch/common/util/ArrayUtils.java index 96b694e04bd5e..be40bf16e20e4 100644 --- a/server/src/main/java/org/elasticsearch/common/util/ArrayUtils.java +++ b/server/src/main/java/org/elasticsearch/common/util/ArrayUtils.java @@ -126,4 +126,20 @@ public static void reverseSubArray(long[] array, int offset, int length) { end--; } } + + /** + * Reverse the {@code length} values on the array starting from {@code offset}. + */ + public static void reverseArray(byte[] array, int offset, int length) { + int start = offset; + int end = offset + length; + while (start < end) { + final byte temp = array[start]; + array[start] = array[end - 1]; + array[end - 1] = temp; + start++; + end--; + } + } + } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec index 68d780022b0e2..6e8d5fba67cee 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec @@ -69,6 +69,7 @@ double pi() "double pow(base:double|integer|long|unsigned_long, exponent:double|integer|long|unsigned_long)" "keyword repeat(string:keyword|text, number:integer)" "keyword replace(string:keyword|text, regex:keyword|text, newString:keyword|text)" +"keyword|text reverse(str:keyword|text)" "keyword right(string:keyword|text, length:integer)" "double|integer|long|unsigned_long round(number:double|integer|long|unsigned_long, ?decimals:integer)" "keyword|text rtrim(string:keyword|text)" @@ -201,6 +202,7 @@ pi |null |null pow |[base, exponent] |["double|integer|long|unsigned_long", "double|integer|long|unsigned_long"] |["Numeric expression for the base. If `null`\, the function returns `null`.", "Numeric expression for the exponent. If `null`\, the function returns `null`."] repeat |[string, number] |["keyword|text", integer] |[String expression., Number times to repeat.] replace |[string, regex, newString] |["keyword|text", "keyword|text", "keyword|text"] |[String expression., Regular expression., Replacement string.] +reverse |str |"keyword|text" |String expression. If `null`, the function returns `null`. right |[string, length] |["keyword|text", integer] |[The string from which to returns a substring., The number of characters to return.] round |[number, decimals] |["double|integer|long|unsigned_long", integer] |["The numeric value to round. If `null`\, the function returns `null`.", "The number of decimal places to round to. Defaults to 0. If `null`\, the function returns `null`."] rtrim |string |"keyword|text" |String expression. If `null`, the function returns `null`. @@ -333,6 +335,7 @@ pi |Returns {wikipedia}/Pi[Pi], the ratio of a circle's circumference pow |Returns the value of `base` raised to the power of `exponent`. repeat |Returns a string constructed by concatenating `string` with itself the specified `number` of times. replace |The function substitutes in the string `str` any match of the regular expression `regex` with the replacement string `newStr`. +reverse |Returns a new string representing the input string in reverse order. right |Return the substring that extracts 'length' chars from 'str' starting from the right. round |Rounds a number to the specified number of decimal places. Defaults to 0, which returns the nearest integer. If the precision is a negative number, rounds to the number of digits left of the decimal point. rtrim |Removes trailing whitespaces from a string. @@ -467,6 +470,7 @@ pi |double pow |double |[false, false] |false |false repeat |keyword |[false, false] |false |false replace |keyword |[false, false, false] |false |false +reverse |"keyword|text" |false |false |false right |keyword |[false, false] |false |false round |"double|integer|long|unsigned_long" |[false, true] |false |false rtrim |"keyword|text" |false |false |false @@ -544,5 +548,5 @@ required_capability: meta meta functions | stats a = count(*), b = count(*), c = count(*) | mv_expand c; a:long | b:long | c:long -121 | 121 | 121 +122 | 122 | 122 ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index ffcceab26bcaf..5313e6630c75d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -1194,6 +1194,113 @@ a:keyword | upper:keyword | lower:keyword π/2 + a + B + Λ ºC | Π/2 + A + B + Λ ºC | π/2 + a + b + λ ºc ; +reverse +required_capability: fn_reverse +from employees | sort emp_no | eval name_reversed = REVERSE(first_name) | keep emp_no, first_name, name_reversed | limit 1; + +emp_no:integer | first_name:keyword | name_reversed:keyword +10001 | Georgi | igroeG +; + +reverseRow +required_capability: fn_reverse +// tag::reverse[] +ROW message = "Some Text" | EVAL message_reversed = REVERSE(message); +// end::reverse[] + +// tag::reverse-result[] +message:keyword | message_reversed:keyword +Some Text | txeT emoS +// end::reverse-result[] +; + +reverseEmoji +required_capability: fn_reverse +// tag::reverseEmoji[] +ROW bending_arts = "💧🪨🔥💨" | EVAL bending_arts_reversed = REVERSE(bending_arts); +// end::reverseEmoji[] + +// tag::reverseEmoji-result[] +bending_arts:keyword | bending_arts_reversed:keyword +💧🪨🔥💨 | 💨🔥🪨💧 +// end::reverseEmoji-result[] +; + +reverseEmoji2 +required_capability: fn_reverse +ROW off_on_holiday = "🏠➡️🚌➡️✈️➡️🏝️" | EVAL back_home_again = REVERSE(off_on_holiday); + +off_on_holiday:keyword | back_home_again:keyword +🏠➡️🚌➡️✈️➡️🏝️ | 🏝️➡️✈️➡️🚌➡️🏠 +; + +reverseGraphemeClusters +required_capability: fn_reverse +ROW message = "áéíóúàèìòùâêîôû😊👍🏽🎉💖कंठाी" | EVAL message_reversed = REVERSE(message); + +message:keyword | message_reversed:keyword +áéíóúàèìòùâêîôû😊👍🏽🎉💖कंठाी | ठाीकं💖🎉👍🏽😊ûôîêâùòìèàúóíéá +; + +reverseMultiValue +required_capability: fn_reverse +FROM employees | SORT emp_no | EVAL jobs_reversed = REVERSE(job_positions) | KEEP job*, emp_no | LIMIT 5; + +warning:Line 1:53: evaluation of [REVERSE(job_positions)] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:53: java.lang.IllegalArgumentException: single-value function encountered multi-value + +job_positions:keyword | jobs_reversed:keyword | emp_no:integer +["Accountant", "Senior Python Developer"] | null | 10001 +Senior Team Lead | daeL maeT roineS | 10002 +null | null | 10003 +[Head Human Resources, Reporting Analyst, Support Engineer, Tech Lead] | null | 10004 +null | null | 10005 +; + +reverseNested +required_capability: fn_reverse +FROM employees | SORT emp_no | EVAL name_reversed = REVERSE(REVERSE(first_name)), eq = name_reversed == first_name | KEEP first_name, name_reversed, eq, emp_no | LIMIT 5; + +first_name:keyword | name_reversed:keyword | eq:boolean | emp_no:integer +Georgi | Georgi | true | 10001 +Bezalel | Bezalel | true | 10002 +Parto | Parto | true | 10003 +Chirstian | Chirstian | true | 10004 +Kyoichi | Kyoichi | true | 10005 +; + +reverseRowNull +required_capability: fn_reverse +ROW x = null | EVAL y = REVERSE(x); + +x:null | y:null +null | null +; + + +reverseRowInlineCastWithNull +required_capability: fn_reverse +ROW x = 1 | EVAL y = REVERSE((null + 1)::string); + +x:integer | y:string +1 | null +; + +reverseWithTextFields +required_capability: fn_reverse +FROM books +| EVAL title_reversed = REVERSE(title), author_reversed_twice = REVERSE(REVERSE(author)), eq = author_reversed_twice == author +| KEEP title, title_reversed, author, author_reversed_twice, eq, book_no +| SORT book_no +| WHERE book_no IN ("1211", "1463") +| LIMIT 2; + +title:text | title_reversed:text | author:text | author_reversed_twice:text | eq:boolean | book_no:keyword +The brothers Karamazov | vozamaraK srehtorb ehT | Fyodor Dostoevsky | Fyodor Dostoevsky | true | 1211 +Realms of Tolkien: Images of Middle-earth | htrae-elddiM fo segamI :neikloT fo smlaeR | J. R. R. Tolkien | J. R. R. Tolkien | true | 1463 +; + + values required_capability: agg_values diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseEvaluator.java new file mode 100644 index 0000000000000..68ea53ad342e1 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseEvaluator.java @@ -0,0 +1,111 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import java.lang.IllegalArgumentException; +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Warnings; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link Reverse}. + * This class is generated. Do not edit it. + */ +public final class ReverseEvaluator implements EvalOperator.ExpressionEvaluator { + private final Warnings warnings; + + private final EvalOperator.ExpressionEvaluator val; + + private final DriverContext driverContext; + + public ReverseEvaluator(Source source, EvalOperator.ExpressionEvaluator val, + DriverContext driverContext) { + this.val = val; + this.driverContext = driverContext; + this.warnings = Warnings.createWarnings(driverContext.warningsMode(), source); + } + + @Override + public Block eval(Page page) { + try (BytesRefBlock valBlock = (BytesRefBlock) val.eval(page)) { + BytesRefVector valVector = valBlock.asVector(); + if (valVector == null) { + return eval(page.getPositionCount(), valBlock); + } + return eval(page.getPositionCount(), valVector).asBlock(); + } + } + + public BytesRefBlock eval(int positionCount, BytesRefBlock valBlock) { + try(BytesRefBlock.Builder result = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + BytesRef valScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + if (valBlock.isNull(p)) { + result.appendNull(); + continue position; + } + if (valBlock.getValueCount(p) != 1) { + if (valBlock.getValueCount(p) > 1) { + warnings.registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue position; + } + result.appendBytesRef(Reverse.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch))); + } + return result.build(); + } + } + + public BytesRefVector eval(int positionCount, BytesRefVector valVector) { + try(BytesRefVector.Builder result = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { + BytesRef valScratch = new BytesRef(); + position: for (int p = 0; p < positionCount; p++) { + result.appendBytesRef(Reverse.process(valVector.getBytesRef(p, valScratch))); + } + return result.build(); + } + } + + @Override + public String toString() { + return "ReverseEvaluator[" + "val=" + val + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(val); + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory val; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory val) { + this.source = source; + this.val = val; + } + + @Override + public ReverseEvaluator get(DriverContext context) { + return new ReverseEvaluator(source, val.get(context), context); + } + + @Override + public String toString() { + return "ReverseEvaluator[" + "val=" + val + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index e7a1599d48bd3..c39a2041a61be 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -27,6 +27,11 @@ */ public class EsqlCapabilities { public enum Cap { + /** + * Support for function {@code REVERSE}. + */ + FN_REVERSE, + /** * Support for function {@code CBRT}. Done in #108574. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 0923db51f19cf..7c0f1fa3a8ad0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -124,6 +124,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.RTrim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Repeat; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Replace; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Reverse; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Right; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Space; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split; @@ -300,22 +301,23 @@ private FunctionDefinition[][] functions() { def(Tau.class, Tau::new, "tau") }, // string new FunctionDefinition[] { - def(Length.class, Length::new, "length"), - def(Substring.class, Substring::new, "substring"), def(Concat.class, Concat::new, "concat"), + def(EndsWith.class, EndsWith::new, "ends_with"), def(LTrim.class, LTrim::new, "ltrim"), - def(RTrim.class, RTrim::new, "rtrim"), - def(Trim.class, Trim::new, "trim"), def(Left.class, Left::new, "left"), + def(Length.class, Length::new, "length"), + def(Locate.class, Locate::new, "locate"), + def(RTrim.class, RTrim::new, "rtrim"), + def(Repeat.class, Repeat::new, "repeat"), def(Replace.class, Replace::new, "replace"), + def(Reverse.class, Reverse::new, "reverse"), def(Right.class, Right::new, "right"), + def(Space.class, Space::new, "space"), def(StartsWith.class, StartsWith::new, "starts_with"), - def(EndsWith.class, EndsWith::new, "ends_with"), + def(Substring.class, Substring::new, "substring"), def(ToLower.class, ToLower::new, "to_lower"), def(ToUpper.class, ToUpper::new, "to_upper"), - def(Locate.class, Locate::new, "locate"), - def(Repeat.class, Repeat::new, "repeat"), - def(Space.class, Space::new, "space") }, + def(Trim.class, Trim::new, "trim") }, // date new FunctionDefinition[] { def(DateDiff.class, DateDiff::new, "date_diff"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java index 14b0c872a3b86..afe9bf6e45eda 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java @@ -43,6 +43,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.Locate; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Repeat; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Replace; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Reverse; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Right; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; @@ -100,6 +101,7 @@ public static List getNamedWriteables() { entries.add(Right.ENTRY); entries.add(Repeat.ENTRY); entries.add(Replace.ENTRY); + entries.add(Reverse.ENTRY); entries.add(Round.ENTRY); entries.add(Split.ENTRY); entries.add(Substring.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java index f8b05aea324dc..46538b77edc74 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java @@ -151,6 +151,8 @@ *
  • {@code docs/reference/esql/functions/parameters/myfunction.asciidoc}
    • *
  • {@code docs/reference/esql/functions/signature/myfunction.svg}
    • *
  • {@code docs/reference/esql/functions/types/myfunction.asciidoc}
    • + *
  • {@code docs/reference/esql/functions/kibana/definition/myfunction.json}
    • + *
  • {@code docs/reference/esql/functions/kibana/docs/myfunction.asciidoc}
    • * * * Make sure to commit them. Add a reference to the @@ -194,6 +196,9 @@ * for your function. Now add something like {@code required_capability: my_function} * to all of your csv-spec tests. Run those csv-spec tests as integration tests to double * check that they run on the main branch. + *

    + * **Note:** you may notice tests gated based on Elasticsearch version. This was the old way + * of doing things. Now, we use specific capabilities for each function. * *
  • * Open the PR. The subject and description of the PR are important because those'll turn @@ -201,7 +206,7 @@ * happy. But functions don't need an essay. *
    • *
  • - * Add the {@code >enhancement} and {@code :Query Languages/ES|QL} tags if you are able. + * Add the {@code >enhancement} and {@code :Analytics/ES|QL} tags if you are able. * Request a review if you can, probably from one of the folks that github proposes to you. *
    • *
  • diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java new file mode 100644 index 0000000000000..bf4e47d8d0de4 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Reverse.java @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; + +import java.io.IOException; +import java.text.BreakIterator; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import static org.elasticsearch.common.util.ArrayUtils.reverseArray; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; + +/** + * Function that reverses a string. + */ +public class Reverse extends UnaryScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Reverse", Reverse::new); + + @FunctionInfo( + returnType = { "keyword", "text" }, + description = "Returns a new string representing the input string in reverse order.", + examples = { + @Example(file = "string", tag = "reverse"), + @Example( + file = "string", + tag = "reverseEmoji", + description = "`REVERSE` works with unicode, too! It keeps unicode grapheme clusters together during reversal." + ) } + ) + public Reverse( + Source source, + @Param( + name = "str", + type = { "keyword", "text" }, + description = "String expression. If `null`, the function returns `null`." + ) Expression field + ) { + super(source, field); + } + + private Reverse(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + + return isString(field, sourceText(), DEFAULT); + } + + /** + * Reverses a unicode string, keeping grapheme clusters together + * @param str + * @return + */ + public static String reverseStringWithUnicodeCharacters(String str) { + BreakIterator boundary = BreakIterator.getCharacterInstance(Locale.ROOT); + boundary.setText(str); + + List characters = new ArrayList<>(); + int start = boundary.first(); + for (int end = boundary.next(); end != BreakIterator.DONE; start = end, end = boundary.next()) { + characters.add(str.substring(start, end)); + } + + StringBuilder reversed = new StringBuilder(str.length()); + for (int i = characters.size() - 1; i >= 0; i--) { + reversed.append(characters.get(i)); + } + + return reversed.toString(); + } + + private static boolean isOneByteUTF8(BytesRef ref) { + int end = ref.offset + ref.length; + for (int i = ref.offset; i < end; i++) { + if (ref.bytes[i] < 0) { + return false; + } + } + return true; + } + + @Evaluator + static BytesRef process(BytesRef val) { + if (isOneByteUTF8(val)) { + // this is the fast path. we know we can just reverse the bytes. + BytesRef reversed = BytesRef.deepCopyOf(val); + reverseArray(reversed.bytes, reversed.offset, reversed.length); + return reversed; + } + return BytesRefs.toBytesRef(reverseStringWithUnicodeCharacters(val.utf8ToString())); + } + + @Override + public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + var fieldEvaluator = toEvaluator.apply(field); + return new ReverseEvaluator.Factory(source(), fieldEvaluator); + } + + @Override + public Expression replaceChildren(List newChildren) { + assert newChildren.size() == 1; + return new Reverse(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Reverse::new, field); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseSerializationTests.java new file mode 100644 index 0000000000000..7b1ad8c9dffd0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseSerializationTests.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractUnaryScalarSerializationTests; + +public class ReverseSerializationTests extends AbstractUnaryScalarSerializationTests { + @Override + protected Reverse create(Source source, Expression child) { + return new Reverse(source, child); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java new file mode 100644 index 0000000000000..2873f18d53957 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; + +public class ReverseTests extends AbstractScalarFunctionTestCase { + public ReverseTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + List suppliers = new ArrayList<>(); + + for (DataType stringType : new DataType[] { DataType.KEYWORD, DataType.TEXT }) { + for (var supplier : TestCaseSupplier.stringCases(stringType)) { + suppliers.add(makeSupplier(supplier)); + } + } + + return parameterSuppliersFromTypedData(suppliers); + } + + @Override + protected Expression build(Source source, List args) { + return new Reverse(source, args.get(0)); + } + + private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier) { + return new TestCaseSupplier(fieldSupplier.name(), List.of(fieldSupplier.type()), () -> { + var fieldTypedData = fieldSupplier.get(); + String expectedToString = "ReverseEvaluator[val=Attribute[channel=0]]"; + String value = BytesRefs.toString(fieldTypedData.data()); + String expectedValue = Reverse.reverseStringWithUnicodeCharacters(value); + + return new TestCaseSupplier.TestCase( + List.of(fieldTypedData), + expectedToString, + fieldSupplier.type(), + equalTo(new BytesRef(expectedValue)) + ); + }); + } +} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml index 9607b64385721..939f153b8b0ea 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml @@ -3,7 +3,7 @@ setup: - requires: cluster_features: ["gte_v8.11.0"] reason: "ESQL is available in 8.11+" - test_runner_features: allowed_warnings_regex + test_runner_features: [allowed_warnings_regex, capabilities] - do: indices.create: @@ -385,8 +385,31 @@ setup: - length: { values: 2 } - match: { values.0: [ [ "foo", "bar" ] ] } - match: { values.1: [ "baz" ] } +--- +"reverse text": + - requires: + capabilities: + - method: POST + path: /_query + parameters: [method, path, parameters, capabilities] + capabilities: [fn_reverse] + reason: "reverse not yet added" + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM test | SORT name | EVAL job_reversed = REVERSE(job), tag_reversed = REVERSE(tag) | KEEP job_reversed, tag_reversed' + + - match: { columns.0.name: "job_reversed" } + - match: { columns.0.type: "text" } + - match: { columns.1.name: "tag_reversed" } + - match: { columns.1.type: "text" } + - length: { values: 2 } + - match: { values.0: [ "rotceriD TI", "rab oof" ] } + - match: { values.1: [ "tsilaicepS lloryaP", "zab" ] } --- "stats text with raw": - do: From 7e3de044e106c172849f0c0e0fd2a4f600d948ad Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Fri, 4 Oct 2024 13:58:35 -0400 Subject: [PATCH 101/194] Remove semantic_text.inner_hits feature (#114156) --- .../org/elasticsearch/xpack/inference/InferenceFeatures.java | 4 +--- .../xpack/inference/queries/SemanticQueryBuilder.java | 3 --- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java index 30ccb48d5c709..fd330a8cf6cc6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java @@ -10,7 +10,6 @@ import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper; -import org.elasticsearch.xpack.inference.queries.SemanticQueryBuilder; import org.elasticsearch.xpack.inference.rank.random.RandomRankRetrieverBuilder; import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankRetrieverBuilder; @@ -26,8 +25,7 @@ public Set getFeatures() { return Set.of( TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_RETRIEVER_SUPPORTED, RandomRankRetrieverBuilder.RANDOM_RERANKER_RETRIEVER_SUPPORTED, - SemanticTextFieldMapper.SEMANTIC_TEXT_SEARCH_INFERENCE_ID, - SemanticQueryBuilder.SEMANTIC_TEXT_INNER_HITS + SemanticTextFieldMapper.SEMANTIC_TEXT_SEARCH_INFERENCE_ID ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java index 00d44b8e7a0e2..9f7fcb1ef407c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java @@ -16,7 +16,6 @@ import org.elasticsearch.cluster.metadata.InferenceFieldMetadata; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.MatchNoneQueryBuilder; @@ -50,8 +49,6 @@ import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; public class SemanticQueryBuilder extends AbstractQueryBuilder { - public static final NodeFeature SEMANTIC_TEXT_INNER_HITS = new NodeFeature("semantic_text.inner_hits"); - public static final String NAME = "semantic"; private static final ParseField FIELD_FIELD = new ParseField("field"); From 93d7f3d84b36569e74017f0a61ecc096296420f7 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 4 Oct 2024 16:26:25 -0400 Subject: [PATCH 102/194] [CI] Only trigger DRA workflow on intake for main branches (#114077) --- .buildkite/pipelines/intake.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index e44a1e67e9d59..1bb13c4c10966 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -76,6 +76,7 @@ steps: - trigger: elasticsearch-dra-workflow label: Trigger DRA snapshot workflow async: true + branches: "main 8.* 7.17" build: branch: "$BUILDKITE_BRANCH" commit: "$BUILDKITE_COMMIT" From 6e2d55472031366be5ff7311a8c3012d0fa8ef3d Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sat, 5 Oct 2024 06:30:52 +1000 Subject: [PATCH 103/194] Mute org.elasticsearch.action.bulk.IncrementalBulkIT testIncrementalBulkHighWatermarkBackOff #114073 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 2ea9aa3a6647d..a1d33df8df904 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -354,6 +354,9 @@ tests: - class: org.elasticsearch.xpack.inference.InferenceCrudIT method: testGet issue: https://github.com/elastic/elasticsearch/issues/114135 +- class: org.elasticsearch.action.bulk.IncrementalBulkIT + method: testIncrementalBulkHighWatermarkBackOff + issue: https://github.com/elastic/elasticsearch/issues/114073 # Examples: # From fb7a36d0a44858e51f19b7e0dbe84f291527ee36 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Fri, 4 Oct 2024 14:45:09 -0600 Subject: [PATCH 104/194] Fix high water mark test which submits data early (#114159) It is possible in the incremental high watermark test that the data is submitted causing a corruption of the bulk request. This commit fixes the issue to ensure we only send new data after it has been requested. Additionally, it adds an assertion to prevent this error from happening again. --- .../org/elasticsearch/action/bulk/IncrementalBulkIT.java | 3 +++ .../action/bulk/IncrementalBulkService.java | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java index 60ea4138e923d..cde8d41b292b7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/IncrementalBulkIT.java @@ -233,6 +233,9 @@ public void testIncrementalBulkHighWatermarkBackOff() throws Exception { handlers.add(handlerThrottled); + // Wait until we are ready for the next page + assertBusy(() -> assertTrue(nextPage.get())); + for (IncrementalBulkService.Handler h : handlers) { refCounted.incRef(); PlainActionFuture future = new PlainActionFuture<>(); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java b/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java index d8c2389dd7d69..d5ad3aa2d29a1 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java @@ -105,6 +105,7 @@ public static class Handler implements Releasable { private boolean closed = false; private boolean globalFailure = false; private boolean incrementalRequestSubmitted = false; + private boolean bulkInProgress = false; private ThreadContext.StoredContext requestContext; private Exception bulkActionLevelFailure = null; private long currentBulkSize = 0L; @@ -130,6 +131,7 @@ protected Handler( public void addItems(List> items, Releasable releasable, Runnable nextItems) { assert closed == false; + assert bulkInProgress == false; if (bulkActionLevelFailure != null) { shortCircuitDueToTopLevelFailure(items, releasable); nextItems.run(); @@ -143,6 +145,7 @@ public void addItems(List> items, Releasable releasable, Runn requestContext.restore(); final ArrayList toRelease = new ArrayList<>(releasables); releasables.clear(); + bulkInProgress = true; client.bulk(bulkRequest, ActionListener.runAfter(new ActionListener<>() { @Override @@ -158,6 +161,7 @@ public void onFailure(Exception e) { handleBulkFailure(isFirstRequest, e); } }, () -> { + bulkInProgress = false; requestContext = threadContext.newStoredContext(); toRelease.forEach(Releasable::close); nextItems.run(); @@ -177,6 +181,7 @@ private boolean shouldBackOff() { } public void lastItems(List> items, Releasable releasable, ActionListener listener) { + assert bulkInProgress == false; if (bulkActionLevelFailure != null) { shortCircuitDueToTopLevelFailure(items, releasable); errorResponse(listener); @@ -187,7 +192,9 @@ public void lastItems(List> items, Releasable releasable, Act requestContext.restore(); final ArrayList toRelease = new ArrayList<>(releasables); releasables.clear(); - client.bulk(bulkRequest, ActionListener.runBefore(new ActionListener<>() { + // We do not need to set this back to false as this will be the last request. + bulkInProgress = true; + client.bulk(bulkRequest, ActionListener.runAfter(new ActionListener<>() { private final boolean isFirstRequest = incrementalRequestSubmitted == false; From 991cff9912da57344501b2dc55c8e70a0238cb7e Mon Sep 17 00:00:00 2001 From: Gergely Kalapos Date: Fri, 4 Oct 2024 23:10:37 +0200 Subject: [PATCH 105/194] [otel-data] Hide 10m and 60m aggregated metrics generated for the APM UI (#114042) * Hide aggregated metrics generated for the APM UI * Update 30_aggregated_metrics_tests.yml * Review feedback - Introduced templates for 1 minute aggregations - Moved dynamic templates `ecs_ip` and `all_strings_to_keywords` into a dedicated file and now I pull the file in instead of repeating them - Introduced `metrics-[x]m.otel@custom` - Added tests with terms aggregation that assert by default 1 bucket (only 1m) with metricsset.interval, and with allowing hidden indices it's 3 buckets (1m, 10m, 60m) * Update 30_aggregated_metrics_tests.yml Simplify - no need to separate tests for the 3bucket queries. * Rename metrics-otel-fixed@mappings to ecs-tsdb@mappings --------- Co-authored-by: Elastic Machine --- .../ecs-tsdb@mappings.yaml | 19 + .../metrics-otel@template.yaml | 1 + ...rics-service_destination.10m@template.yaml | 40 ++ ...trics-service_destination.1m@template.yaml | 39 ++ ...rics-service_destination.60m@template.yaml | 40 ++ ...ics-service_summary.10m.otel@template.yaml | 40 ++ ...rics-service_summary.1m.otel@template.yaml | 39 ++ ...ics-service_summary.60m.otel@template.yaml | 40 ++ ...service_transaction.10m.otel@template.yaml | 40 ++ ...-service_transaction.1m.otel@template.yaml | 39 ++ ...service_transaction.60m.otel@template.yaml | 40 ++ ...metrics-transaction.10m.otel@template.yaml | 40 ++ .../metrics-transaction.1m.otel@template.yaml | 39 ++ ...metrics-transaction.60m.otel@template.yaml | 40 ++ .../src/main/resources/resources.yaml | 15 +- .../test/30_aggregated_metrics_tests.yml | 428 ++++++++++++++++++ 16 files changed, 938 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.10m@template.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.1m@template.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.60m@template.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.10m.otel@template.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.1m.otel@template.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.60m.otel@template.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.10m.otel@template.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.1m.otel@template.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.60m.otel@template.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.10m.otel@template.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.1m.otel@template.yaml create mode 100644 x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.60m.otel@template.yaml create mode 100644 x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_aggregated_metrics_tests.yml diff --git a/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml b/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml new file mode 100644 index 0000000000000..1c9d32a4289b9 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/component-templates/ecs-tsdb@mappings.yaml @@ -0,0 +1,19 @@ +version: ${xpack.oteldata.template.version} +_meta: + description: | + Default mappings that can be changed by users for + the OpenTelemetry metrics index template installed by x-pack + managed: true +template: + mappings: + dynamic_templates: + - ecs_ip: + mapping: + type: ip + path_match: [ "ip", "*.ip", "*_ip" ] + match_mapping_type: string + - all_strings_to_keywords: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml index c2a318f809b7d..3b4c3127bb71c 100644 --- a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-otel@template.yaml @@ -14,6 +14,7 @@ composed_of: - semconv-resource-to-ecs@mappings - metrics@custom - metrics-otel@custom + - ecs-tsdb@mappings ignore_missing_component_templates: - metrics@custom - metrics-otel@custom diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.10m@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.10m@template.yaml new file mode 100644 index 0000000000000..f5033135120bc --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.10m@template.yaml @@ -0,0 +1,40 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-service_destination.10m.otel-*"] +priority: 130 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-10m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-10m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 10m + name: + type: constant_keyword + value: service_destination diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.1m@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.1m@template.yaml new file mode 100644 index 0000000000000..9168062f30bfb --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.1m@template.yaml @@ -0,0 +1,39 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-service_destination.1m.otel-*"] +priority: 130 +data_stream: {} +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-1m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-1m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 1m + name: + type: constant_keyword + value: service_destination diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.60m@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.60m@template.yaml new file mode 100644 index 0000000000000..47c2d7d014322 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_destination.60m@template.yaml @@ -0,0 +1,40 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-service_destination.60m.otel-*"] +priority: 130 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-60m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-60m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 60m + name: + type: constant_keyword + value: service_destination diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.10m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.10m.otel@template.yaml new file mode 100644 index 0000000000000..c9438e8c27402 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.10m.otel@template.yaml @@ -0,0 +1,40 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-service_summary.10m.otel-*"] +priority: 130 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-10m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-10m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 10m + name: + type: constant_keyword + value: service_summary diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.1m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.1m.otel@template.yaml new file mode 100644 index 0000000000000..b29caa3fe34a7 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.1m.otel@template.yaml @@ -0,0 +1,39 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-service_summary.1m.otel-*"] +priority: 130 +data_stream: {} +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-1m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-1m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 1m + name: + type: constant_keyword + value: service_summary diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.60m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.60m.otel@template.yaml new file mode 100644 index 0000000000000..4cab3e41a1dfa --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_summary.60m.otel@template.yaml @@ -0,0 +1,40 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-service_summary.60m.otel-*"] +priority: 130 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-60m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-60m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 60m + name: + type: constant_keyword + value: service_summary diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.10m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.10m.otel@template.yaml new file mode 100644 index 0000000000000..037f3546205d6 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.10m.otel@template.yaml @@ -0,0 +1,40 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-service_transaction.10m.otel-*"] +priority: 130 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-10m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-10m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 10m + name: + type: constant_keyword + value: service_transaction diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.1m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.1m.otel@template.yaml new file mode 100644 index 0000000000000..303ac2c406fd0 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.1m.otel@template.yaml @@ -0,0 +1,39 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-service_transaction.1m.otel-*"] +priority: 130 +data_stream: {} +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-1m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-1m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 1m + name: + type: constant_keyword + value: service_transaction diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.60m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.60m.otel@template.yaml new file mode 100644 index 0000000000000..ea42079ced4dd --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-service_transaction.60m.otel@template.yaml @@ -0,0 +1,40 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-service_transaction.60m.otel-*"] +priority: 130 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-60m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-60m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 60m + name: + type: constant_keyword + value: service_transaction diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.10m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.10m.otel@template.yaml new file mode 100644 index 0000000000000..81e70cc3361fc --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.10m.otel@template.yaml @@ -0,0 +1,40 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-transaction.10m.otel-*"] +priority: 130 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-10m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-10m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 10m + name: + type: constant_keyword + value: transaction diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.1m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.1m.otel@template.yaml new file mode 100644 index 0000000000000..c54b90bf8b683 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.1m.otel@template.yaml @@ -0,0 +1,39 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-transaction.1m.otel-*"] +priority: 130 +data_stream: {} +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-1m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-1m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 1m + name: + type: constant_keyword + value: transaction diff --git a/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.60m.otel@template.yaml b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.60m.otel@template.yaml new file mode 100644 index 0000000000000..8afe8b87951c0 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/resources/index-templates/metrics-transaction.60m.otel@template.yaml @@ -0,0 +1,40 @@ +--- +version: ${xpack.oteldata.template.version} +index_patterns: ["metrics-transaction.60m.otel-*"] +priority: 130 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: aggregated APM metrics template installed by x-pack + managed: true +composed_of: + - metrics@tsdb-settings + - otel@mappings + - metrics-otel@mappings + - semconv-resource-to-ecs@mappings + - metrics@custom + - metrics-otel@custom + - metrics-60m.otel@custom + - ecs-tsdb@mappings +ignore_missing_component_templates: + - metrics@custom + - metrics-otel@custom + - metrics-60m.otel@custom +template: + settings: + index: + mode: time_series + mappings: + properties: + data_stream.type: + type: constant_keyword + value: metrics + metricset: + properties: + interval: + type: constant_keyword + value: 60m + name: + type: constant_keyword + value: transaction diff --git a/x-pack/plugin/otel-data/src/main/resources/resources.yaml b/x-pack/plugin/otel-data/src/main/resources/resources.yaml index ba219b09388fb..e32037901a49c 100644 --- a/x-pack/plugin/otel-data/src/main/resources/resources.yaml +++ b/x-pack/plugin/otel-data/src/main/resources/resources.yaml @@ -1,7 +1,7 @@ # "version" holds the version of the templates and ingest pipelines installed # by xpack-plugin otel-data. This must be increased whenever an existing template is # changed, in order for it to be updated on Elasticsearch upgrade. -version: 3 +version: 4 component-templates: - otel@mappings @@ -9,7 +9,20 @@ component-templates: - semconv-resource-to-ecs@mappings - metrics-otel@mappings - traces-otel@mappings + - ecs-tsdb@mappings index-templates: - logs-otel@template - metrics-otel@template - traces-otel@template + - metrics-transaction.60m.otel@template + - metrics-transaction.10m.otel@template + - metrics-transaction.1m.otel@template + - metrics-service_transaction.60m.otel@template + - metrics-service_transaction.10m.otel@template + - metrics-service_transaction.1m.otel@template + - metrics-service_summary.60m.otel@template + - metrics-service_summary.10m.otel@template + - metrics-service_summary.1m.otel@template + - metrics-service_destination.60m@template + - metrics-service_destination.10m@template + - metrics-service_destination.1m@template diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_aggregated_metrics_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_aggregated_metrics_tests.yml new file mode 100644 index 0000000000000..c26a53d841f59 --- /dev/null +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/30_aggregated_metrics_tests.yml @@ -0,0 +1,428 @@ +--- +setup: + - do: + cluster.health: + wait_for_events: languid + - do: + cluster.put_component_template: + name: metrics-otel@custom + body: + template: + settings: + index: + routing_path: [unit, attributes.*, resource.attributes.*] + mode: time_series + time_series: + start_time: 2024-07-01T13:03:08.138Z +--- +"metrics-service_destination.10m must be hidden": + - do: + bulk: + index: metrics-service_destination.10m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_destination"},"resource":{"attributes":{ "metricset.interval": "10m" } } }' + - is_false: errors + - do: + search: + index: metrics-service_destination.10m.otel-default + body: + fields: ["metricset.name", "metricset.interval"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.metricset\.name: ["service_destination"] } + - match: { hits.hits.0.fields.metricset\.interval: ["10m"] } + - do: + indices.get_data_stream: + name: metrics-service_destination.10m.otel-default + - match: { data_streams.0.hidden: true } +--- +"metrics-service_destination.60m must be hidden": + - do: + bulk: + index: metrics-service_destination.60m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_destination" },"resource":{"attributes":{ "metricset.interval": "60m" } } }' + - is_false: errors + - do: + search: + index: metrics-service_destination.60m.otel-default + body: + fields: ["metricset.name", "metricset.interval"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.metricset\.name: ["service_destination"] } + - match: { hits.hits.0.fields.metricset\.interval: ["60m"] } + - do: + indices.get_data_stream: + name: metrics-service_destination.60m.otel-default + - match: { data_streams.0.hidden: true } +--- +"metrics-service_summary.10m must be hidden": + - do: + bulk: + index: metrics-service_summary.10m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_summary"},"resource":{"attributes":{ "metricset.interval": "10m" } } }' + - is_false: errors + - do: + search: + index: metrics-service_summary.10m.otel-default + body: + fields: ["metricset.name", "metricset.interval"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.metricset\.name: ["service_summary"] } + - match: { hits.hits.0.fields.metricset\.interval: ["10m"] } + - do: + indices.get_data_stream: + name: metrics-service_summary.10m.otel-default + - match: { data_streams.0.hidden: true } +--- +"metrics-service_summary.60m must be hidden": + - do: + bulk: + index: metrics-service_summary.60m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_summary" },"resource":{"attributes":{ "metricset.interval": "60m" } } }' + - is_false: errors + - do: + search: + index: metrics-service_summary.60m.otel-default + body: + fields: ["metricset.name", "metricset.interval"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.metricset\.name: ["service_summary"] } + - match: { hits.hits.0.fields.metricset\.interval: ["60m"] } + - do: + indices.get_data_stream: + name: metrics-service_summary.60m.otel-default + - match: { data_streams.0.hidden: true } +--- +"metrics-service_transaction.10m must be hidden": + - do: + bulk: + index: metrics-service_transaction.10m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_transaction" },"resource":{"attributes":{ "metricset.interval": "10m" } } }' + - is_false: errors + - do: + search: + index: metrics-service_transaction.10m.otel-default + body: + fields: ["metricset.name", "metricset.interval"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.metricset\.name: ["service_transaction"] } + - match: { hits.hits.0.fields.metricset\.interval: ["10m"] } + - do: + indices.get_data_stream: + name: metrics-service_transaction.10m.otel-default + - match: { data_streams.0.hidden: true } +--- +"metrics-service_transaction.60m must be hidden": + - do: + bulk: + index: metrics-service_transaction.60m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_transaction"},"resource":{"attributes":{ "metricset.interval": "60m" } } }' + - is_false: errors + - do: + search: + index: metrics-service_transaction.60m.otel-default + body: + fields: ["metricset.name", "metricset.interval"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.metricset\.name: ["service_transaction"] } + - match: { hits.hits.0.fields.metricset\.interval: ["60m"] } + - do: + indices.get_data_stream: + name: metrics-service_transaction.60m.otel-default + - match: { data_streams.0.hidden: true } +--- +"metrics-transaction.10m must be hidden": + - do: + bulk: + index: metrics-transaction.10m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "transaction"},"resource":{"attributes":{ "metricset.interval": "10m" } } }' + - is_false: errors + - do: + search: + index: metrics-transaction.10m.otel-default + body: + fields: ["metricset.name", "metricset.interval"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.metricset\.name: ["transaction"] } + - match: { hits.hits.0.fields.metricset\.interval: ["10m"] } + - do: + indices.get_data_stream: + name: metrics-transaction.10m.otel-default + - match: { data_streams.0.hidden: true } +--- +"metrics-transaction.60m must be hidden": + - do: + bulk: + index: metrics-transaction.60m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "transaction"},"resource":{"attributes":{ "metricset.interval": "60m" } } }' + - is_false: errors + - do: + search: + index: metrics-transaction.60m.otel-default + body: + fields: ["metricset.name", "metricset.interval"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.metricset\.name: ["transaction"] } + - match: { hits.hits.0.fields.metricset\.interval: ["60m"] } + - do: + indices.get_data_stream: + name: metrics-transaction.60m.otel-default + - match: { data_streams.0.hidden: true } +--- +"Terms aggregation on metricset.interval from metrics-transaction must by default only contain 1m": + - do: + bulk: + index: metrics-transaction.60m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "transaction"},"resource":{"attributes":{ "metricset.interval": "60m" } } }' + - do: + bulk: + index: metrics-transaction.10m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "transaction"},"resource":{"attributes":{ "metricset.interval": "10m" } } }' + - do: + bulk: + index: metrics-transaction.1m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "transaction"},"resource":{"attributes":{ "metricset.interval": "1m" } } }' + - is_false: errors + - do: + search: + index: metrics-*.otel-default + body: > + { + + "size": 0, + "aggs": { + "intervals": { + "terms": { + "field": "metricset.interval" + } + } + } + } + - length: { aggregations.intervals.buckets: 1 } + - match: { aggregations.intervals.buckets.0.key: "1m" } + - match: { aggregations.intervals.buckets.0.doc_count: 1 } + # With including hidden indices, 10m and 60m aggregation also show up + - do: + search: + index: metrics-*.otel-default + expand_wildcards: open,hidden + body: > + { + "size": 0, + "aggs": { + "intervals": { + "terms": { + "field": "metricset.interval" + } + } + } + } + - length: { aggregations.intervals.buckets: 3 } +--- +"Terms aggregation on metricset.interval from metrics-service_transaction must by default only contain 1m": + - do: + bulk: + index: metrics-service_transaction.60m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_transaction"},"resource":{"attributes":{ "metricset.interval": "60m" } } }' + - do: + bulk: + index: metrics-service_transaction.10m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_transaction"},"resource":{"attributes":{ "metricset.interval": "10m" } } }' + - do: + bulk: + index: metrics-service_transaction.1m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_transaction"},"resource":{"attributes":{ "metricset.interval": "1m" } } }' + - is_false: errors + - do: + search: + index: metrics-*.otel-default + body: > + { + + "size": 0, + "aggs": { + "intervals": { + "terms": { + "field": "metricset.interval" + } + } + } + } + - length: { aggregations.intervals.buckets: 1 } + - match: { aggregations.intervals.buckets.0.key: "1m" } + - match: { aggregations.intervals.buckets.0.doc_count: 1 } + # With including hidden indices, 10m and 60m aggregation also show up + - do: + search: + index: metrics-*.otel-default + expand_wildcards: open,hidden + body: > + { + "size": 0, + "aggs": { + "intervals": { + "terms": { + "field": "metricset.interval" + } + } + } + } + - length: { aggregations.intervals.buckets: 3 } +--- +"Terms aggregation on metricset.interval from metrics-service_summary must by default only contain 1m": + - do: + bulk: + index: metrics-service_summary.60m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_summary"},"resource":{"attributes":{ "metricset.interval": "60m" } } }' + - do: + bulk: + index: metrics-service_summary.10m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_summary"},"resource":{"attributes":{ "metricset.interval": "10m" } } }' + - do: + bulk: + index: metrics-service_summary.1m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_summary"},"resource":{"attributes":{ "metricset.interval": "1m" } } }' + - is_false: errors + - do: + search: + index: metrics-*.otel-default + body: > + { + + "size": 0, + "aggs": { + "intervals": { + "terms": { + "field": "metricset.interval" + } + } + } + } + - length: { aggregations.intervals.buckets: 1 } + - match: { aggregations.intervals.buckets.0.key: "1m" } + - match: { aggregations.intervals.buckets.0.doc_count: 1 } + # With including hidden indices, 10m and 60m aggregation also show up + - do: + search: + index: metrics-*.otel-default + expand_wildcards: open,hidden + body: > + { + "size": 0, + "aggs": { + "intervals": { + "terms": { + "field": "metricset.interval" + } + } + } + } + - length: { aggregations.intervals.buckets: 3 } +--- +"Terms aggregation on metricset.interval from metrics-service_destination must by default only contain 1m": + - do: + bulk: + index: metrics-service_destination.60m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_destination"},"resource":{"attributes":{ "metricset.interval": "60m" } } }' + - do: + bulk: + index: metrics-service_destination.10m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_destination"},"resource":{"attributes":{ "metricset.interval": "10m" } } }' + - do: + bulk: + index: metrics-service_destination.1m.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z" ,"attributes":{"processor.event":"metric","transaction.root":false, "metricset.name" : "service_destination"},"resource":{"attributes":{ "metricset.interval": "1m" } } }' + - is_false: errors + - do: + search: + index: metrics-*.otel-default + body: > + { + + "size": 0, + "aggs": { + "intervals": { + "terms": { + "field": "metricset.interval" + } + } + } + } + - length: { aggregations.intervals.buckets: 1 } + - match: { aggregations.intervals.buckets.0.key: "1m" } + - match: { aggregations.intervals.buckets.0.doc_count: 1 } + # With including hidden indices, 10m and 60m aggregation also show up + - do: + search: + index: metrics-*.otel-default + expand_wildcards: open,hidden + body: > + { + "size": 0, + "aggs": { + "intervals": { + "terms": { + "field": "metricset.interval" + } + } + } + } + - length: { aggregations.intervals.buckets: 3 } From 7344da02d8f9e8ea332a5fdf7971268abc0fc69d Mon Sep 17 00:00:00 2001 From: Dan Rubinstein Date: Fri, 4 Oct 2024 17:10:46 -0400 Subject: [PATCH 106/194] Fix HuggingFaceMixedIT test sometimes failing when run on version before 8.16 (#114061) * Fix HuggingFaceMixedIT test sometimes failing when run on version before 8.16 * Fixing typo in expected error message --- .../qa/mixed/HuggingFaceServiceMixedIT.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/HuggingFaceServiceMixedIT.java b/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/HuggingFaceServiceMixedIT.java index 59d3faf6489a6..457ae525b7f4b 100644 --- a/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/HuggingFaceServiceMixedIT.java +++ b/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/HuggingFaceServiceMixedIT.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Map; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.not; @@ -27,6 +28,7 @@ public class HuggingFaceServiceMixedIT extends BaseMixedTestCase { private static final String HF_EMBEDDINGS_ADDED = "8.12.0"; private static final String HF_ELSER_ADDED = "8.12.0"; + private static final String HF_EMBEDDINGS_CHUNKING_SETTINGS_ADDED = "8.16.0"; private static final String MINIMUM_SUPPORTED_VERSION = "8.15.0"; private static MockWebServer embeddingsServer; @@ -59,7 +61,24 @@ public void testHFEmbeddings() throws IOException { final String inferenceId = "mixed-cluster-embeddings"; embeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponse())); - put(inferenceId, embeddingConfig(getUrl(embeddingsServer)), TaskType.TEXT_EMBEDDING); + + try { + put(inferenceId, embeddingConfig(getUrl(embeddingsServer)), TaskType.TEXT_EMBEDDING); + } catch (Exception e) { + if (bwcVersion.before(Version.fromString(HF_EMBEDDINGS_CHUNKING_SETTINGS_ADDED))) { + // Chunking settings were added in 8.16.0. if the version is before that, an exception will be thrown if the index mapping + // was created based on a mapping from an old node + assertThat( + e.getMessage(), + containsString( + "One or more nodes in your cluster does not support chunking_settings. " + + "Please update all nodes in your cluster to the latest version to use chunking_settings." + ) + ); + return; + } + } + var configs = (List>) get(TaskType.TEXT_EMBEDDING, inferenceId).get("endpoints"); assertThat(configs, hasSize(1)); assertEquals("hugging_face", configs.get(0).get("service")); From 0c69de1c6e620c30905ed6a1b8cf42db9c399522 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sat, 5 Oct 2024 07:14:39 +1000 Subject: [PATCH 107/194] Mute org.elasticsearch.xpack.esql.expression.function.aggregate.AvgTests testFold {TestCase= #7} #114175 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index a1d33df8df904..f27e3da11b15f 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -357,6 +357,9 @@ tests: - class: org.elasticsearch.action.bulk.IncrementalBulkIT method: testIncrementalBulkHighWatermarkBackOff issue: https://github.com/elastic/elasticsearch/issues/114073 +- class: org.elasticsearch.xpack.esql.expression.function.aggregate.AvgTests + method: "testFold {TestCase= #7}" + issue: https://github.com/elastic/elasticsearch/issues/114175 # Examples: # From c1f2f80c948590fe550724322adf81164a3df53b Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Fri, 4 Oct 2024 14:54:03 -0700 Subject: [PATCH 108/194] Correctly inject subobjects parameter in logsdb tests (#113643) --- .../logsdb/qa/DataGenerationHelper.java | 51 ++----------------- .../index/mapper/ObjectMapper.java | 2 +- .../datasource/DataSourceRequest.java | 8 +-- .../DefaultMappingParametersHandler.java | 33 +++++++++++- .../logsdb/datageneration/fields/Context.java | 48 +++++++++++++---- .../GenericSubObjectFieldDataGenerator.java | 23 +++++++-- .../fields/NestedFieldDataGenerator.java | 5 +- .../fields/ObjectFieldDataGenerator.java | 5 +- .../TopLevelObjectFieldDataGenerator.java | 16 ++++-- 9 files changed, 114 insertions(+), 77 deletions(-) diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java index 515d07103bff8..8b29b1609711f 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java @@ -12,25 +12,18 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.logsdb.datageneration.DataGenerator; import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; import org.elasticsearch.logsdb.datageneration.fields.PredefinedField; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.function.Consumer; public class DataGenerationHelper { - private final ObjectMapper.Subobjects subobjects; private final boolean keepArraySource; private final DataGenerator dataGenerator; @@ -40,44 +33,10 @@ public DataGenerationHelper() { } public DataGenerationHelper(Consumer builderConfigurator) { - // TODO enable subobjects: auto - // It is disabled because it currently does not have auto flattening and that results in asserts being triggered when using copy_to. - this.subobjects = ESTestCase.randomValueOtherThan( - ObjectMapper.Subobjects.AUTO, - () -> ESTestCase.randomFrom(ObjectMapper.Subobjects.values()) - ); this.keepArraySource = ESTestCase.randomBoolean(); - var specificationBuilder = DataGeneratorSpecification.builder().withFullyDynamicMapping(ESTestCase.randomBoolean()); - if (subobjects != ObjectMapper.Subobjects.ENABLED) { - specificationBuilder = specificationBuilder.withNestedFieldsLimit(0); - } - - specificationBuilder.withDataSourceHandlers(List.of(new DataSourceHandler() { - @Override - public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) { - if (subobjects == ObjectMapper.Subobjects.ENABLED) { - // Use default behavior - return null; - } - - assert request.isNested() == false; - - // "enabled: false" is not compatible with subobjects: false - // "dynamic: false/strict/runtime" is not compatible with subobjects: false - return new DataSourceResponse.ObjectMappingParametersGenerator(() -> { - var parameters = new HashMap(); - parameters.put("subobjects", subobjects.toString()); - if (ESTestCase.randomBoolean()) { - parameters.put("dynamic", "true"); - } - if (ESTestCase.randomBoolean()) { - parameters.put("enabled", "true"); - } - return parameters; - }); - } - })) + var specificationBuilder = DataGeneratorSpecification.builder() + .withFullyDynamicMapping(ESTestCase.randomBoolean()) .withPredefinedFields( List.of( // Customized because it always needs doc_values for aggregations. @@ -136,11 +95,7 @@ void logsDbMapping(XContentBuilder builder) throws IOException { } void standardMapping(XContentBuilder builder) throws IOException { - if (subobjects != ObjectMapper.Subobjects.ENABLED) { - dataGenerator.writeMapping(builder, Map.of("subobjects", subobjects.toString())); - } else { - dataGenerator.writeMapping(builder); - } + dataGenerator.writeMapping(builder); } void logsDbSettings(Settings.Builder builder) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 0b9727aa66c8a..5e63fee8c5adc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -63,7 +63,7 @@ public enum Subobjects { this.printedValue = printedValue; } - static Subobjects from(Object node) { + public static Subobjects from(Object node) { if (node instanceof Boolean value) { return value ? Subobjects.ENABLED : Subobjects.DISABLED; } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java index 067d1b96e965e..8dee5876aa207 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java @@ -9,10 +9,10 @@ package org.elasticsearch.logsdb.datageneration.datasource; +import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping; -import org.elasticsearch.test.ESTestCase; import java.util.Set; @@ -116,15 +116,11 @@ public DataSourceResponse.LeafMappingParametersGenerator accept(DataSourceHandle } } - record ObjectMappingParametersGenerator(boolean isRoot, boolean isNested) + record ObjectMappingParametersGenerator(boolean isRoot, boolean isNested, ObjectMapper.Subobjects parentSubobjects) implements DataSourceRequest { public DataSourceResponse.ObjectMappingParametersGenerator accept(DataSourceHandler handler) { return handler.handle(this); } - - public String syntheticSourceKeepValue() { - return isRoot() ? ESTestCase.randomFrom("none", "arrays") : ESTestCase.randomFrom("none", "arrays", "all"); - } } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java index 69f839d461b40..1046e22e65caa 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -10,6 +10,7 @@ package org.elasticsearch.logsdb.datageneration.datasource; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping; import org.elasticsearch.test.ESTestCase; @@ -78,8 +79,11 @@ private Supplier> scaledFloatMapping(Map inj @Override public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) { if (request.isNested()) { + assert request.parentSubobjects() != ObjectMapper.Subobjects.DISABLED; + return new DataSourceResponse.ObjectMappingParametersGenerator(() -> { var parameters = new HashMap(); + if (ESTestCase.randomBoolean()) { parameters.put("dynamic", ESTestCase.randomFrom("true", "false", "strict")); } @@ -93,14 +97,41 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequ return new DataSourceResponse.ObjectMappingParametersGenerator(() -> { var parameters = new HashMap(); + + if (request.parentSubobjects() == ObjectMapper.Subobjects.DISABLED) { + // "enabled: false" is not compatible with subobjects: false + // changing "dynamic" from parent context is not compatible with subobjects: false + // changing subobjects value is not compatible with subobjects: false + if (ESTestCase.randomBoolean()) { + parameters.put("enabled", "true"); + } + + return parameters; + } + if (ESTestCase.randomBoolean()) { parameters.put("dynamic", ESTestCase.randomFrom("true", "false", "strict", "runtime")); } if (ESTestCase.randomBoolean()) { parameters.put("enabled", ESTestCase.randomFrom("true", "false")); } + // Changing subobjects from subobjects: false is not supported, but we can f.e. go from "true" to "false". + // TODO enable subobjects: auto + // It is disabled because it currently does not have auto flattening and that results in asserts being triggered when using + // copy_to. + if (ESTestCase.randomBoolean()) { + parameters.put( + "subobjects", + ESTestCase.randomValueOtherThan( + ObjectMapper.Subobjects.AUTO, + () -> ESTestCase.randomFrom(ObjectMapper.Subobjects.values()) + ).toString() + ); + } + if (ESTestCase.randomBoolean()) { - parameters.put(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, request.syntheticSourceKeepValue()); + var value = request.isRoot() ? ESTestCase.randomFrom("none", "arrays") : ESTestCase.randomFrom("none", "arrays", "all"); + parameters.put(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, value); } return parameters; diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java index ebf13eb93ff4b..c1ec15a3479b3 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java @@ -9,6 +9,7 @@ package org.elasticsearch.logsdb.datageneration.fields; +import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; @@ -31,9 +32,14 @@ class Context { private final AtomicInteger nestedFieldsCount; private final Set eligibleCopyToDestinations; private final DynamicMapping parentDynamicMapping; + private final ObjectMapper.Subobjects currentSubobjectsConfig; - Context(DataGeneratorSpecification specification, DynamicMapping parentDynamicMapping) { - this(specification, "", 0, new AtomicInteger(0), new HashSet<>(), parentDynamicMapping); + Context( + DataGeneratorSpecification specification, + DynamicMapping parentDynamicMapping, + ObjectMapper.Subobjects currentSubobjectsConfig + ) { + this(specification, "", 0, new AtomicInteger(0), new HashSet<>(), parentDynamicMapping, currentSubobjectsConfig); } private Context( @@ -42,7 +48,8 @@ private Context( int objectDepth, AtomicInteger nestedFieldsCount, Set eligibleCopyToDestinations, - DynamicMapping parentDynamicMapping + DynamicMapping parentDynamicMapping, + ObjectMapper.Subobjects currentSubobjectsConfig ) { this.specification = specification; this.childFieldGenerator = specification.dataSource().get(new DataSourceRequest.ChildFieldGenerator(specification)); @@ -52,6 +59,7 @@ private Context( this.nestedFieldsCount = nestedFieldsCount; this.eligibleCopyToDestinations = eligibleCopyToDestinations; this.parentDynamicMapping = parentDynamicMapping; + this.currentSubobjectsConfig = currentSubobjectsConfig; } public DataGeneratorSpecification specification() { @@ -66,21 +74,30 @@ public DataSourceResponse.FieldTypeGenerator fieldTypeGenerator(DynamicMapping d return specification.dataSource().get(new DataSourceRequest.FieldTypeGenerator(dynamicMapping)); } - public Context subObject(String name, DynamicMapping dynamicMapping) { + public Context subObject(String name, DynamicMapping dynamicMapping, ObjectMapper.Subobjects subobjects) { return new Context( specification, pathToField(name), objectDepth + 1, nestedFieldsCount, eligibleCopyToDestinations, - dynamicMapping + dynamicMapping, + subobjects ); } - public Context nestedObject(String name, DynamicMapping dynamicMapping) { + public Context nestedObject(String name, DynamicMapping dynamicMapping, ObjectMapper.Subobjects subobjects) { nestedFieldsCount.incrementAndGet(); // copy_to can't be used across nested documents so all currently eligible fields are not eligible inside nested document. - return new Context(specification, pathToField(name), objectDepth + 1, nestedFieldsCount, new HashSet<>(), dynamicMapping); + return new Context( + specification, + pathToField(name), + objectDepth + 1, + nestedFieldsCount, + new HashSet<>(), + dynamicMapping, + subobjects + ); } public boolean shouldAddDynamicObjectField(DynamicMapping dynamicMapping) { @@ -99,10 +116,11 @@ public boolean shouldAddObjectField() { return childFieldGenerator.generateRegularSubObject(); } - public boolean shouldAddNestedField() { + public boolean shouldAddNestedField(ObjectMapper.Subobjects subobjects) { if (objectDepth >= specification.maxObjectDepth() || nestedFieldsCount.get() >= specification.nestedFieldsLimit() - || parentDynamicMapping == DynamicMapping.FORCED) { + || parentDynamicMapping == DynamicMapping.FORCED + || subobjects == ObjectMapper.Subobjects.DISABLED) { return false; } @@ -131,6 +149,14 @@ public DynamicMapping determineDynamicMapping(Map mappingParamet return dynamicParameter.equals("strict") ? DynamicMapping.FORBIDDEN : DynamicMapping.SUPPORTED; } + public ObjectMapper.Subobjects determineSubobjects(Map mappingParameters) { + if (currentSubobjectsConfig == ObjectMapper.Subobjects.DISABLED) { + return ObjectMapper.Subobjects.DISABLED; + } + + return ObjectMapper.Subobjects.from(mappingParameters.getOrDefault("subobjects", "true")); + } + public Set getEligibleCopyToDestinations() { return eligibleCopyToDestinations; } @@ -142,4 +168,8 @@ public void markFieldAsEligibleForCopyTo(String field) { private String pathToField(String field) { return path.isEmpty() ? field : path + "." + field; } + + public ObjectMapper.Subobjects getCurrentSubobjectsConfig() { + return currentSubobjectsConfig; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java index ba03b2f91c53c..83a68519d5de1 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java @@ -10,6 +10,7 @@ package org.elasticsearch.logsdb.datageneration.fields; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; @@ -31,7 +32,7 @@ public class GenericSubObjectFieldDataGenerator { this.context = context; } - List generateChildFields(DynamicMapping dynamicMapping) { + List generateChildFields(DynamicMapping dynamicMapping, ObjectMapper.Subobjects subobjects) { var existingFieldNames = new HashSet(); // no child fields is legal var childFieldsCount = context.childFieldGenerator().generateChildFieldCount(); @@ -42,12 +43,24 @@ List generateChildFields(DynamicMapping dynamicMapping) { if (context.shouldAddDynamicObjectField(dynamicMapping)) { result.add( - new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject(fieldName, DynamicMapping.FORCED)), true) + new ChildField( + fieldName, + new ObjectFieldDataGenerator(context.subObject(fieldName, DynamicMapping.FORCED, subobjects)), + true + ) ); } else if (context.shouldAddObjectField()) { - result.add(new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject(fieldName, dynamicMapping)), false)); - } else if (context.shouldAddNestedField()) { - result.add(new ChildField(fieldName, new NestedFieldDataGenerator(context.nestedObject(fieldName, dynamicMapping)), false)); + result.add( + new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject(fieldName, dynamicMapping, subobjects)), false) + ); + } else if (context.shouldAddNestedField(subobjects)) { + result.add( + new ChildField( + fieldName, + new NestedFieldDataGenerator(context.nestedObject(fieldName, dynamicMapping, subobjects)), + false + ) + ); } else { var fieldTypeInfo = context.fieldTypeGenerator(dynamicMapping).generator().get(); diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java index ba168b221f572..69853debf9b77 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java @@ -28,13 +28,14 @@ public class NestedFieldDataGenerator implements FieldDataGenerator { this.mappingParameters = context.specification() .dataSource() - .get(new DataSourceRequest.ObjectMappingParametersGenerator(false, true)) + .get(new DataSourceRequest.ObjectMappingParametersGenerator(false, true, context.getCurrentSubobjectsConfig())) .mappingGenerator() .get(); var dynamicMapping = context.determineDynamicMapping(mappingParameters); + var subobjects = context.determineSubobjects(mappingParameters); var genericGenerator = new GenericSubObjectFieldDataGenerator(context); - this.childFields = genericGenerator.generateChildFields(dynamicMapping); + this.childFields = genericGenerator.generateChildFields(dynamicMapping, subobjects); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java index 084310ac967fc..701642c57619b 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java @@ -28,13 +28,14 @@ public class ObjectFieldDataGenerator implements FieldDataGenerator { this.mappingParameters = context.specification() .dataSource() - .get(new DataSourceRequest.ObjectMappingParametersGenerator(false, false)) + .get(new DataSourceRequest.ObjectMappingParametersGenerator(false, false, context.getCurrentSubobjectsConfig())) .mappingGenerator() .get(); var dynamicMapping = context.determineDynamicMapping(mappingParameters); + var subobjects = context.determineSubobjects(mappingParameters); var genericGenerator = new GenericSubObjectFieldDataGenerator(context); - this.childFields = genericGenerator.generateChildFields(dynamicMapping); + this.childFields = genericGenerator.generateChildFields(dynamicMapping, subobjects); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java index 2c7aa65d8c6d1..1374362df7f4a 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/TopLevelObjectFieldDataGenerator.java @@ -10,6 +10,7 @@ package org.elasticsearch.logsdb.datageneration.fields; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.xcontent.XContentBuilder; @@ -37,7 +38,12 @@ public TopLevelObjectFieldDataGenerator(DataGeneratorSpecification specification this.mappingParameters = Map.of(); } else { this.mappingParameters = new HashMap<>( - specification.dataSource().get(new DataSourceRequest.ObjectMappingParametersGenerator(true, false)).mappingGenerator().get() + // Value of subobjects here is for a parent of this object. + // Since there is no parent we pass ENABLED to allow to set subobjects to any value at top level. + specification.dataSource() + .get(new DataSourceRequest.ObjectMappingParametersGenerator(true, false, ObjectMapper.Subobjects.ENABLED)) + .mappingGenerator() + .get() ); // Top-level object can't be disabled because @timestamp is a required field in data streams. this.mappingParameters.remove("enabled"); @@ -46,11 +52,15 @@ public TopLevelObjectFieldDataGenerator(DataGeneratorSpecification specification ? DynamicMapping.FORBIDDEN : DynamicMapping.SUPPORTED; } - this.context = new Context(specification, dynamicMapping); + var subobjects = ObjectMapper.Subobjects.from(mappingParameters.getOrDefault("subobjects", "true")); + + // Value of subobjects here is for a parent of this object. + // Since there is no parent we pass ENABLED to allow to set subobjects to any value at top level. + this.context = new Context(specification, dynamicMapping, ObjectMapper.Subobjects.ENABLED); var genericGenerator = new GenericSubObjectFieldDataGenerator(context); this.predefinedFields = genericGenerator.generateChildFields(specification.predefinedFields()); - this.generatedChildFields = genericGenerator.generateChildFields(dynamicMapping); + this.generatedChildFields = genericGenerator.generateChildFields(dynamicMapping, subobjects); } public CheckedConsumer mappingWriter(Map customMappingParameters) { From b19d923c6d94b8142c4ffc7c863a16869984ab23 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Fri, 4 Oct 2024 19:01:48 -0400 Subject: [PATCH 109/194] Re-add semantic_text.inner_hits cluster feature (#114180) Re-add the `semantic_text.inner_hits` cluster feature to fix serverless test failures --- .../org/elasticsearch/xpack/inference/InferenceFeatures.java | 4 +++- .../xpack/inference/queries/SemanticQueryBuilder.java | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java index fd330a8cf6cc6..30ccb48d5c709 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java @@ -10,6 +10,7 @@ import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper; +import org.elasticsearch.xpack.inference.queries.SemanticQueryBuilder; import org.elasticsearch.xpack.inference.rank.random.RandomRankRetrieverBuilder; import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankRetrieverBuilder; @@ -25,7 +26,8 @@ public Set getFeatures() { return Set.of( TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_RETRIEVER_SUPPORTED, RandomRankRetrieverBuilder.RANDOM_RERANKER_RETRIEVER_SUPPORTED, - SemanticTextFieldMapper.SEMANTIC_TEXT_SEARCH_INFERENCE_ID + SemanticTextFieldMapper.SEMANTIC_TEXT_SEARCH_INFERENCE_ID, + SemanticQueryBuilder.SEMANTIC_TEXT_INNER_HITS ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java index 9f7fcb1ef407c..a33a49cc52d94 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilder.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.InferenceFieldMetadata; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.MatchNoneQueryBuilder; @@ -49,6 +50,9 @@ import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; public class SemanticQueryBuilder extends AbstractQueryBuilder { + // **** THE semantic_text.inner_hits CLUSTER FEATURE IS DEFUNCT, NEVER USE IT **** + public static final NodeFeature SEMANTIC_TEXT_INNER_HITS = new NodeFeature("semantic_text.inner_hits"); + public static final String NAME = "semantic"; private static final ParseField FIELD_FIELD = new ParseField("field"); From 8b501dc5e7119f13418bc5453d04ac80b1a19734 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sat, 5 Oct 2024 14:30:12 +1000 Subject: [PATCH 110/194] Mute org.elasticsearch.action.bulk.IncrementalBulkIT testMultipleBulkPartsWithBackoff #114181 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index f27e3da11b15f..606281b35ed6e 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -360,6 +360,9 @@ tests: - class: org.elasticsearch.xpack.esql.expression.function.aggregate.AvgTests method: "testFold {TestCase= #7}" issue: https://github.com/elastic/elasticsearch/issues/114175 +- class: org.elasticsearch.action.bulk.IncrementalBulkIT + method: testMultipleBulkPartsWithBackoff + issue: https://github.com/elastic/elasticsearch/issues/114181 # Examples: # From 166667f84202fa1a1006881a4f4f509d9df44cfc Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sat, 5 Oct 2024 14:32:02 +1000 Subject: [PATCH 111/194] Mute org.elasticsearch.action.bulk.IncrementalBulkIT testIncrementalBulkLowWatermarkBackOff #114182 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 606281b35ed6e..f4ad6f9aba16b 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -363,6 +363,9 @@ tests: - class: org.elasticsearch.action.bulk.IncrementalBulkIT method: testMultipleBulkPartsWithBackoff issue: https://github.com/elastic/elasticsearch/issues/114181 +- class: org.elasticsearch.action.bulk.IncrementalBulkIT + method: testIncrementalBulkLowWatermarkBackOff + issue: https://github.com/elastic/elasticsearch/issues/114182 # Examples: # From 6535bdae477de97a4e6652ee2fe58fed13c32eff Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sun, 6 Oct 2024 15:26:44 +1100 Subject: [PATCH 112/194] Mute org.elasticsearch.aggregations.AggregationsClientYamlTestSuiteIT test {yaml=aggregations/stats_metric_fail_formatting/fail formatting} #114187 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index f4ad6f9aba16b..992996e811853 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -366,6 +366,9 @@ tests: - class: org.elasticsearch.action.bulk.IncrementalBulkIT method: testIncrementalBulkLowWatermarkBackOff issue: https://github.com/elastic/elasticsearch/issues/114182 +- class: org.elasticsearch.aggregations.AggregationsClientYamlTestSuiteIT + method: test {yaml=aggregations/stats_metric_fail_formatting/fail formatting} + issue: https://github.com/elastic/elasticsearch/issues/114187 # Examples: # From a110d710db028f519a77d8bb73aa33e4365f1168 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Sun, 6 Oct 2024 10:26:30 -0700 Subject: [PATCH 113/194] ingest-geoip: establish boundaries (#109655) --- .../ingest/geoip/DatabaseNodeServiceIT.java | 2 +- ...gDatabasesWhilePerformingGeoLookupsIT.java | 4 +- .../elasticsearch/ingest/geoip/Database.java | 60 +- .../geoip/DatabaseReaderLazyLoader.java | 130 +--- .../ingest/geoip/GeoIpCache.java | 2 +- .../ingest/geoip/GeoIpProcessor.java | 535 +--------------- .../ingest/geoip/IpDataLookup.java | 31 + .../ingest/geoip/IpDataLookupFactories.java | 107 ++++ .../ingest/geoip/IpDatabase.java | 53 +- .../ingest/geoip/MaxmindIpDataLookups.java | 606 ++++++++++++++++++ .../ingest/geoip/ConfigDatabasesTests.java | 4 +- .../ingest/geoip/GeoIpProcessorTests.java | 58 +- .../ingest/geoip/GeoIpTestUtils.java | 32 + .../ingest/geoip/MMDBUtilTests.java | 2 +- .../ingest/geoip/MaxMindSupportTests.java | 49 -- 15 files changed, 840 insertions(+), 835 deletions(-) create mode 100644 modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookup.java create mode 100644 modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java create mode 100644 modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceIT.java index 73d8976c3a4b7..786f091e0c024 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceIT.java @@ -84,7 +84,7 @@ private void assertValidDatabase(DatabaseNodeService databaseNodeService, String IpDatabase database = databaseNodeService.getDatabase(databaseFileName); assertNotNull(database); assertThat(database.getDatabaseType(), equalTo(databaseType)); - CountryResponse countryResponse = database.getCountry("89.160.20.128"); + CountryResponse countryResponse = database.getResponse("89.160.20.128", GeoIpTestUtils::getCountry); assertNotNull(countryResponse); Country country = countryResponse.getCountry(); assertNotNull(country); diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java index 2c7d5fbcc56b7..b28926673069d 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java @@ -205,10 +205,10 @@ private static DatabaseNodeService createRegistry(Path geoIpConfigDir, Path geoI private static void lazyLoadReaders(DatabaseNodeService databaseNodeService) throws IOException { if (databaseNodeService.get("GeoLite2-City.mmdb") != null) { databaseNodeService.get("GeoLite2-City.mmdb").getDatabaseType(); - databaseNodeService.get("GeoLite2-City.mmdb").getCity("2.125.160.216"); + databaseNodeService.get("GeoLite2-City.mmdb").getResponse("2.125.160.216", GeoIpTestUtils::getCity); } databaseNodeService.get("GeoLite2-City-Test.mmdb").getDatabaseType(); - databaseNodeService.get("GeoLite2-City-Test.mmdb").getCity("2.125.160.216"); + databaseNodeService.get("GeoLite2-City-Test.mmdb").getResponse("2.125.160.216", GeoIpTestUtils::getCity); } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java index dccda0d58cfbf..52ca5eea52c1a 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java @@ -9,7 +9,6 @@ package org.elasticsearch.ingest.geoip; -import org.elasticsearch.common.Strings; import org.elasticsearch.core.Nullable; import java.util.Arrays; @@ -19,12 +18,10 @@ import java.util.Set; /** - * A high-level representation of a kind of geoip database that is supported by the {@link GeoIpProcessor}. + * A high-level representation of a kind of ip location database that is supported by the {@link GeoIpProcessor}. *

    * A database has a set of properties that are valid to use with it (see {@link Database#properties()}), * as well as a list of default properties to use if no properties are specified (see {@link Database#defaultProperties()}). - *

    - * See especially {@link Database#getDatabase(String, String)} which is used to obtain instances of this class. */ enum Database { @@ -142,61 +139,6 @@ enum Database { ) ); - private static final String CITY_DB_SUFFIX = "-City"; - private static final String COUNTRY_DB_SUFFIX = "-Country"; - private static final String ASN_DB_SUFFIX = "-ASN"; - private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP"; - private static final String CONNECTION_TYPE_DB_SUFFIX = "-Connection-Type"; - private static final String DOMAIN_DB_SUFFIX = "-Domain"; - private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise"; - private static final String ISP_DB_SUFFIX = "-ISP"; - - @Nullable - private static Database getMaxmindDatabase(final String databaseType) { - if (databaseType.endsWith(Database.CITY_DB_SUFFIX)) { - return Database.City; - } else if (databaseType.endsWith(Database.COUNTRY_DB_SUFFIX)) { - return Database.Country; - } else if (databaseType.endsWith(Database.ASN_DB_SUFFIX)) { - return Database.Asn; - } else if (databaseType.endsWith(Database.ANONYMOUS_IP_DB_SUFFIX)) { - return Database.AnonymousIp; - } else if (databaseType.endsWith(Database.CONNECTION_TYPE_DB_SUFFIX)) { - return Database.ConnectionType; - } else if (databaseType.endsWith(Database.DOMAIN_DB_SUFFIX)) { - return Database.Domain; - } else if (databaseType.endsWith(Database.ENTERPRISE_DB_SUFFIX)) { - return Database.Enterprise; - } else if (databaseType.endsWith(Database.ISP_DB_SUFFIX)) { - return Database.Isp; - } else { - return null; // no match was found - } - } - - /** - * Parses the passed-in databaseType (presumably from the passed-in databaseFile) and return the Database instance that is - * associated with that databaseType. - * - * @param databaseType the database type String from the metadata of the database file - * @param databaseFile the database file from which the database type was obtained - * @throws IllegalArgumentException if the databaseType is not associated with a Database instance - * @return the Database instance that is associated with the databaseType - */ - public static Database getDatabase(final String databaseType, final String databaseFile) { - Database database = null; - - if (Strings.hasText(databaseType)) { - database = getMaxmindDatabase(databaseType); - } - - if (database == null) { - throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]"); - } - - return database; - } - private final Set properties; private final Set defaultProperties; diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java index e160c8ad1543f..120afe0e9e815 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java @@ -9,18 +9,8 @@ package org.elasticsearch.ingest.geoip; -import com.maxmind.db.DatabaseRecord; -import com.maxmind.db.Network; import com.maxmind.db.NoCache; import com.maxmind.db.Reader; -import com.maxmind.geoip2.model.AnonymousIpResponse; -import com.maxmind.geoip2.model.AsnResponse; -import com.maxmind.geoip2.model.CityResponse; -import com.maxmind.geoip2.model.ConnectionTypeResponse; -import com.maxmind.geoip2.model.CountryResponse; -import com.maxmind.geoip2.model.DomainResponse; -import com.maxmind.geoip2.model.EnterpriseResponse; -import com.maxmind.geoip2.model.IspResponse; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -28,8 +18,6 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.common.CheckedSupplier; -import org.elasticsearch.common.network.InetAddresses; -import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.core.Booleans; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Nullable; @@ -37,19 +25,16 @@ import java.io.File; import java.io.IOException; -import java.net.InetAddress; import java.nio.file.Files; import java.nio.file.Path; -import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; /** * Facilitates lazy loading of the database reader, so that when the geoip plugin is installed, but not used, * no memory is being wasted on the database reader. */ -class DatabaseReaderLazyLoader implements IpDatabase { +public class DatabaseReaderLazyLoader implements IpDatabase { private static final boolean LOAD_DATABASE_ON_HEAP = Booleans.parseBoolean(System.getProperty("es.geoip.load_db_on_heap", "false")); @@ -96,94 +81,6 @@ public final String getDatabaseType() throws IOException { return databaseType.get(); } - @Nullable - @Override - public CityResponse getCity(String ipAddress) { - return getResponse(ipAddress, (reader, ip) -> lookup(reader, ip, CityResponse.class, CityResponse::new)); - } - - @Nullable - @Override - public CountryResponse getCountry(String ipAddress) { - return getResponse(ipAddress, (reader, ip) -> lookup(reader, ip, CountryResponse.class, CountryResponse::new)); - } - - @Nullable - @Override - public AsnResponse getAsn(String ipAddress) { - return getResponse( - ipAddress, - (reader, ip) -> lookup( - reader, - ip, - AsnResponse.class, - (response, responseIp, network, locales) -> new AsnResponse(response, responseIp, network) - ) - ); - } - - @Nullable - @Override - public AnonymousIpResponse getAnonymousIp(String ipAddress) { - return getResponse( - ipAddress, - (reader, ip) -> lookup( - reader, - ip, - AnonymousIpResponse.class, - (response, responseIp, network, locales) -> new AnonymousIpResponse(response, responseIp, network) - ) - ); - } - - @Nullable - @Override - public ConnectionTypeResponse getConnectionType(String ipAddress) { - return getResponse( - ipAddress, - (reader, ip) -> lookup( - reader, - ip, - ConnectionTypeResponse.class, - (response, responseIp, network, locales) -> new ConnectionTypeResponse(response, responseIp, network) - ) - ); - } - - @Nullable - @Override - public DomainResponse getDomain(String ipAddress) { - return getResponse( - ipAddress, - (reader, ip) -> lookup( - reader, - ip, - DomainResponse.class, - (response, responseIp, network, locales) -> new DomainResponse(response, responseIp, network) - ) - ); - } - - @Nullable - @Override - public EnterpriseResponse getEnterprise(String ipAddress) { - return getResponse(ipAddress, (reader, ip) -> lookup(reader, ip, EnterpriseResponse.class, EnterpriseResponse::new)); - } - - @Nullable - @Override - public IspResponse getIsp(String ipAddress) { - return getResponse( - ipAddress, - (reader, ip) -> lookup( - reader, - ip, - IspResponse.class, - (response, responseIp, network, locales) -> new IspResponse(response, responseIp, network) - ) - ); - } - boolean preLookup() { return currentUsages.updateAndGet(current -> current < 0 ? current : current + 1) > 0; } @@ -199,14 +96,12 @@ int current() { return currentUsages.get(); } + @Override @Nullable - private RESPONSE getResponse( - String ipAddress, - CheckedBiFunction, Exception> responseProvider - ) { + public RESPONSE getResponse(String ipAddress, CheckedBiFunction responseProvider) { return cache.putIfAbsent(ipAddress, databasePath.toString(), ip -> { try { - return responseProvider.apply(get(), ipAddress).orElse(null); + return responseProvider.apply(get(), ipAddress); } catch (Exception e) { throw ExceptionsHelper.convertToRuntime(e); } @@ -263,23 +158,6 @@ private static File pathToFile(Path databasePath) { return databasePath.toFile(); } - @FunctionalInterface - private interface ResponseBuilder { - RESPONSE build(RESPONSE response, String responseIp, Network network, List locales); - } - - private Optional lookup(Reader reader, String ip, Class clazz, ResponseBuilder builder) - throws IOException { - InetAddress inetAddress = InetAddresses.forString(ip); - DatabaseRecord record = reader.getRecord(inetAddress, clazz); - RESPONSE result = record.getData(); - if (result == null) { - return Optional.empty(); - } else { - return Optional.of(builder.build(result, NetworkAddress.format(inetAddress), record.getNetwork(), List.of("en"))); - } - } - long getBuildDateMillis() throws IOException { if (buildDate.get() == null) { synchronized (buildDate) { diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java index 335331ac0ab9d..d9c9c3aaf3266 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java @@ -26,7 +26,7 @@ * cost of deserialization for each lookup (cached or not). This comes at slight expense of higher memory usage, but significant * reduction of CPU usage. */ -final class GeoIpCache { +public final class GeoIpCache { /** * Internal-only sentinel object for recording that a result from the geoip database was null (i.e. there was no result). By caching diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index ce160b060ae4c..e2b516bf5b943 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -9,23 +9,6 @@ package org.elasticsearch.ingest.geoip; -import com.maxmind.db.Network; -import com.maxmind.geoip2.model.AnonymousIpResponse; -import com.maxmind.geoip2.model.AsnResponse; -import com.maxmind.geoip2.model.CityResponse; -import com.maxmind.geoip2.model.ConnectionTypeResponse; -import com.maxmind.geoip2.model.ConnectionTypeResponse.ConnectionType; -import com.maxmind.geoip2.model.CountryResponse; -import com.maxmind.geoip2.model.DomainResponse; -import com.maxmind.geoip2.model.EnterpriseResponse; -import com.maxmind.geoip2.model.IspResponse; -import com.maxmind.geoip2.record.City; -import com.maxmind.geoip2.record.Continent; -import com.maxmind.geoip2.record.Country; -import com.maxmind.geoip2.record.Location; -import com.maxmind.geoip2.record.Subdivision; - -import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; @@ -34,10 +17,10 @@ import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; import org.elasticsearch.ingest.geoip.Database.Property; +import org.elasticsearch.ingest.geoip.IpDataLookupFactories.IpDataLookupFactory; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -60,7 +43,7 @@ public final class GeoIpProcessor extends AbstractProcessor { private final Supplier isValid; private final String targetField; private final CheckedSupplier supplier; - private final Set properties; + private final IpDataLookup ipDataLookup; private final boolean ignoreMissing; private final boolean firstOnly; private final String databaseFile; @@ -73,7 +56,7 @@ public final class GeoIpProcessor extends AbstractProcessor { * @param supplier a supplier of a geo-IP database reader; ideally this is lazily-loaded once on first use * @param isValid a supplier that determines if the available database files are up-to-date and license compliant * @param targetField the target field - * @param properties the properties; ideally this is lazily-loaded once on first use + * @param ipDataLookup a lookup capable of retrieving a result from an available geo-IP database reader * @param ignoreMissing true if documents with a missing value for the field should be ignored * @param firstOnly true if only first result should be returned in case of array * @param databaseFile the name of the database file being queried; used only for tagging documents if the database is unavailable @@ -85,7 +68,7 @@ public final class GeoIpProcessor extends AbstractProcessor { final CheckedSupplier supplier, final Supplier isValid, final String targetField, - final Set properties, + final IpDataLookup ipDataLookup, final boolean ignoreMissing, final boolean firstOnly, final String databaseFile @@ -95,7 +78,7 @@ public final class GeoIpProcessor extends AbstractProcessor { this.isValid = isValid; this.targetField = targetField; this.supplier = supplier; - this.properties = properties; + this.ipDataLookup = ipDataLookup; this.ignoreMissing = ignoreMissing; this.firstOnly = firstOnly; this.databaseFile = databaseFile; @@ -127,7 +110,7 @@ public IngestDocument execute(IngestDocument document) throws IOException { } if (ip instanceof String ipString) { - Map data = getGeoData(ipDatabase, ipString); + Map data = ipDataLookup.getData(ipDatabase, ipString); if (data.isEmpty() == false) { document.setFieldValue(targetField, data); } @@ -138,7 +121,7 @@ public IngestDocument execute(IngestDocument document) throws IOException { if (ipAddr instanceof String == false) { throw new IllegalArgumentException("array in field [" + field + "] should only contain strings"); } - Map data = getGeoData(ipDatabase, (String) ipAddr); + Map data = ipDataLookup.getData(ipDatabase, (String) ipAddr); if (data.isEmpty()) { dataList.add(null); continue; @@ -161,26 +144,6 @@ public IngestDocument execute(IngestDocument document) throws IOException { return document; } - private Map getGeoData(IpDatabase ipDatabase, String ipAddress) throws IOException { - final String databaseType = ipDatabase.getDatabaseType(); - final Database database; - try { - database = Database.getDatabase(databaseType, databaseFile); - } catch (IllegalArgumentException e) { - throw new ElasticsearchParseException(e.getMessage(), e); - } - return switch (database) { - case City -> retrieveCityGeoData(ipDatabase, ipAddress); - case Country -> retrieveCountryGeoData(ipDatabase, ipAddress); - case Asn -> retrieveAsnGeoData(ipDatabase, ipAddress); - case AnonymousIp -> retrieveAnonymousIpGeoData(ipDatabase, ipAddress); - case ConnectionType -> retrieveConnectionTypeGeoData(ipDatabase, ipAddress); - case Domain -> retrieveDomainGeoData(ipDatabase, ipAddress); - case Enterprise -> retrieveEnterpriseGeoData(ipDatabase, ipAddress); - case Isp -> retrieveIspGeoData(ipDatabase, ipAddress); - }; - } - @Override public String getType() { return TYPE; @@ -199,478 +162,7 @@ String getDatabaseType() throws IOException { } Set getProperties() { - return properties; - } - - private Map retrieveCityGeoData(IpDatabase ipDatabase, String ipAddress) { - CityResponse response = ipDatabase.getCity(ipAddress); - if (response == null) { - return Map.of(); - } - Country country = response.getCountry(); - City city = response.getCity(); - Location location = response.getLocation(); - Continent continent = response.getContinent(); - Subdivision subdivision = response.getMostSpecificSubdivision(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getTraits().getIpAddress()); - case COUNTRY_ISO_CODE -> { - String countryIsoCode = country.getIsoCode(); - if (countryIsoCode != null) { - geoData.put("country_iso_code", countryIsoCode); - } - } - case COUNTRY_NAME -> { - String countryName = country.getName(); - if (countryName != null) { - geoData.put("country_name", countryName); - } - } - case CONTINENT_CODE -> { - String continentCode = continent.getCode(); - if (continentCode != null) { - geoData.put("continent_code", continentCode); - } - } - case CONTINENT_NAME -> { - String continentName = continent.getName(); - if (continentName != null) { - geoData.put("continent_name", continentName); - } - } - case REGION_ISO_CODE -> { - // ISO 3166-2 code for country subdivisions. - // See iso.org/iso-3166-country-codes.html - String countryIso = country.getIsoCode(); - String subdivisionIso = subdivision.getIsoCode(); - if (countryIso != null && subdivisionIso != null) { - String regionIsoCode = countryIso + "-" + subdivisionIso; - geoData.put("region_iso_code", regionIsoCode); - } - } - case REGION_NAME -> { - String subdivisionName = subdivision.getName(); - if (subdivisionName != null) { - geoData.put("region_name", subdivisionName); - } - } - case CITY_NAME -> { - String cityName = city.getName(); - if (cityName != null) { - geoData.put("city_name", cityName); - } - } - case TIMEZONE -> { - String locationTimeZone = location.getTimeZone(); - if (locationTimeZone != null) { - geoData.put("timezone", locationTimeZone); - } - } - case LOCATION -> { - Double latitude = location.getLatitude(); - Double longitude = location.getLongitude(); - if (latitude != null && longitude != null) { - Map locationObject = new HashMap<>(); - locationObject.put("lat", latitude); - locationObject.put("lon", longitude); - geoData.put("location", locationObject); - } - } - } - } - return geoData; - } - - private Map retrieveCountryGeoData(IpDatabase ipDatabase, String ipAddress) { - CountryResponse response = ipDatabase.getCountry(ipAddress); - if (response == null) { - return Map.of(); - } - Country country = response.getCountry(); - Continent continent = response.getContinent(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getTraits().getIpAddress()); - case COUNTRY_ISO_CODE -> { - String countryIsoCode = country.getIsoCode(); - if (countryIsoCode != null) { - geoData.put("country_iso_code", countryIsoCode); - } - } - case COUNTRY_NAME -> { - String countryName = country.getName(); - if (countryName != null) { - geoData.put("country_name", countryName); - } - } - case CONTINENT_CODE -> { - String continentCode = continent.getCode(); - if (continentCode != null) { - geoData.put("continent_code", continentCode); - } - } - case CONTINENT_NAME -> { - String continentName = continent.getName(); - if (continentName != null) { - geoData.put("continent_name", continentName); - } - } - } - } - return geoData; - } - - private Map retrieveAsnGeoData(IpDatabase ipDatabase, String ipAddress) { - AsnResponse response = ipDatabase.getAsn(ipAddress); - if (response == null) { - return Map.of(); - } - Long asn = response.getAutonomousSystemNumber(); - String organizationName = response.getAutonomousSystemOrganization(); - Network network = response.getNetwork(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getIpAddress()); - case ASN -> { - if (asn != null) { - geoData.put("asn", asn); - } - } - case ORGANIZATION_NAME -> { - if (organizationName != null) { - geoData.put("organization_name", organizationName); - } - } - case NETWORK -> { - if (network != null) { - geoData.put("network", network.toString()); - } - } - } - } - return geoData; - } - - private Map retrieveAnonymousIpGeoData(IpDatabase ipDatabase, String ipAddress) { - AnonymousIpResponse response = ipDatabase.getAnonymousIp(ipAddress); - if (response == null) { - return Map.of(); - } - - boolean isHostingProvider = response.isHostingProvider(); - boolean isTorExitNode = response.isTorExitNode(); - boolean isAnonymousVpn = response.isAnonymousVpn(); - boolean isAnonymous = response.isAnonymous(); - boolean isPublicProxy = response.isPublicProxy(); - boolean isResidentialProxy = response.isResidentialProxy(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getIpAddress()); - case HOSTING_PROVIDER -> { - geoData.put("hosting_provider", isHostingProvider); - } - case TOR_EXIT_NODE -> { - geoData.put("tor_exit_node", isTorExitNode); - } - case ANONYMOUS_VPN -> { - geoData.put("anonymous_vpn", isAnonymousVpn); - } - case ANONYMOUS -> { - geoData.put("anonymous", isAnonymous); - } - case PUBLIC_PROXY -> { - geoData.put("public_proxy", isPublicProxy); - } - case RESIDENTIAL_PROXY -> { - geoData.put("residential_proxy", isResidentialProxy); - } - } - } - return geoData; - } - - private Map retrieveConnectionTypeGeoData(IpDatabase ipDatabase, String ipAddress) { - ConnectionTypeResponse response = ipDatabase.getConnectionType(ipAddress); - if (response == null) { - return Map.of(); - } - - ConnectionType connectionType = response.getConnectionType(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getIpAddress()); - case CONNECTION_TYPE -> { - if (connectionType != null) { - geoData.put("connection_type", connectionType.toString()); - } - } - } - } - return geoData; - } - - private Map retrieveDomainGeoData(IpDatabase ipDatabase, String ipAddress) { - DomainResponse response = ipDatabase.getDomain(ipAddress); - if (response == null) { - return Map.of(); - } - - String domain = response.getDomain(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getIpAddress()); - case DOMAIN -> { - if (domain != null) { - geoData.put("domain", domain); - } - } - } - } - return geoData; - } - - private Map retrieveEnterpriseGeoData(IpDatabase ipDatabase, String ipAddress) { - EnterpriseResponse response = ipDatabase.getEnterprise(ipAddress); - if (response == null) { - return Map.of(); - } - - Country country = response.getCountry(); - City city = response.getCity(); - Location location = response.getLocation(); - Continent continent = response.getContinent(); - Subdivision subdivision = response.getMostSpecificSubdivision(); - - Long asn = response.getTraits().getAutonomousSystemNumber(); - String organizationName = response.getTraits().getAutonomousSystemOrganization(); - Network network = response.getTraits().getNetwork(); - - String isp = response.getTraits().getIsp(); - String ispOrganization = response.getTraits().getOrganization(); - String mobileCountryCode = response.getTraits().getMobileCountryCode(); - String mobileNetworkCode = response.getTraits().getMobileNetworkCode(); - - boolean isHostingProvider = response.getTraits().isHostingProvider(); - boolean isTorExitNode = response.getTraits().isTorExitNode(); - boolean isAnonymousVpn = response.getTraits().isAnonymousVpn(); - boolean isAnonymous = response.getTraits().isAnonymous(); - boolean isPublicProxy = response.getTraits().isPublicProxy(); - boolean isResidentialProxy = response.getTraits().isResidentialProxy(); - - String userType = response.getTraits().getUserType(); - - String domain = response.getTraits().getDomain(); - - ConnectionType connectionType = response.getTraits().getConnectionType(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getTraits().getIpAddress()); - case COUNTRY_ISO_CODE -> { - String countryIsoCode = country.getIsoCode(); - if (countryIsoCode != null) { - geoData.put("country_iso_code", countryIsoCode); - } - } - case COUNTRY_NAME -> { - String countryName = country.getName(); - if (countryName != null) { - geoData.put("country_name", countryName); - } - } - case CONTINENT_CODE -> { - String continentCode = continent.getCode(); - if (continentCode != null) { - geoData.put("continent_code", continentCode); - } - } - case CONTINENT_NAME -> { - String continentName = continent.getName(); - if (continentName != null) { - geoData.put("continent_name", continentName); - } - } - case REGION_ISO_CODE -> { - // ISO 3166-2 code for country subdivisions. - // See iso.org/iso-3166-country-codes.html - String countryIso = country.getIsoCode(); - String subdivisionIso = subdivision.getIsoCode(); - if (countryIso != null && subdivisionIso != null) { - String regionIsoCode = countryIso + "-" + subdivisionIso; - geoData.put("region_iso_code", regionIsoCode); - } - } - case REGION_NAME -> { - String subdivisionName = subdivision.getName(); - if (subdivisionName != null) { - geoData.put("region_name", subdivisionName); - } - } - case CITY_NAME -> { - String cityName = city.getName(); - if (cityName != null) { - geoData.put("city_name", cityName); - } - } - case TIMEZONE -> { - String locationTimeZone = location.getTimeZone(); - if (locationTimeZone != null) { - geoData.put("timezone", locationTimeZone); - } - } - case LOCATION -> { - Double latitude = location.getLatitude(); - Double longitude = location.getLongitude(); - if (latitude != null && longitude != null) { - Map locationObject = new HashMap<>(); - locationObject.put("lat", latitude); - locationObject.put("lon", longitude); - geoData.put("location", locationObject); - } - } - case ASN -> { - if (asn != null) { - geoData.put("asn", asn); - } - } - case ORGANIZATION_NAME -> { - if (organizationName != null) { - geoData.put("organization_name", organizationName); - } - } - case NETWORK -> { - if (network != null) { - geoData.put("network", network.toString()); - } - } - case HOSTING_PROVIDER -> { - geoData.put("hosting_provider", isHostingProvider); - } - case TOR_EXIT_NODE -> { - geoData.put("tor_exit_node", isTorExitNode); - } - case ANONYMOUS_VPN -> { - geoData.put("anonymous_vpn", isAnonymousVpn); - } - case ANONYMOUS -> { - geoData.put("anonymous", isAnonymous); - } - case PUBLIC_PROXY -> { - geoData.put("public_proxy", isPublicProxy); - } - case RESIDENTIAL_PROXY -> { - geoData.put("residential_proxy", isResidentialProxy); - } - case DOMAIN -> { - if (domain != null) { - geoData.put("domain", domain); - } - } - case ISP -> { - if (isp != null) { - geoData.put("isp", isp); - } - } - case ISP_ORGANIZATION_NAME -> { - if (ispOrganization != null) { - geoData.put("isp_organization_name", ispOrganization); - } - } - case MOBILE_COUNTRY_CODE -> { - if (mobileCountryCode != null) { - geoData.put("mobile_country_code", mobileCountryCode); - } - } - case MOBILE_NETWORK_CODE -> { - if (mobileNetworkCode != null) { - geoData.put("mobile_network_code", mobileNetworkCode); - } - } - case USER_TYPE -> { - if (userType != null) { - geoData.put("user_type", userType); - } - } - case CONNECTION_TYPE -> { - if (connectionType != null) { - geoData.put("connection_type", connectionType.toString()); - } - } - } - } - return geoData; - } - - private Map retrieveIspGeoData(IpDatabase ipDatabase, String ipAddress) { - IspResponse response = ipDatabase.getIsp(ipAddress); - if (response == null) { - return Map.of(); - } - - String isp = response.getIsp(); - String ispOrganization = response.getOrganization(); - String mobileNetworkCode = response.getMobileNetworkCode(); - String mobileCountryCode = response.getMobileCountryCode(); - Long asn = response.getAutonomousSystemNumber(); - String organizationName = response.getAutonomousSystemOrganization(); - Network network = response.getNetwork(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getIpAddress()); - case ASN -> { - if (asn != null) { - geoData.put("asn", asn); - } - } - case ORGANIZATION_NAME -> { - if (organizationName != null) { - geoData.put("organization_name", organizationName); - } - } - case NETWORK -> { - if (network != null) { - geoData.put("network", network.toString()); - } - } - case ISP -> { - if (isp != null) { - geoData.put("isp", isp); - } - } - case ISP_ORGANIZATION_NAME -> { - if (ispOrganization != null) { - geoData.put("isp_organization_name", ispOrganization); - } - } - case MOBILE_COUNTRY_CODE -> { - if (mobileCountryCode != null) { - geoData.put("mobile_country_code", mobileCountryCode); - } - } - case MOBILE_NETWORK_CODE -> { - if (mobileNetworkCode != null) { - geoData.put("mobile_network_code", mobileNetworkCode); - } - } - } - } - return geoData; + return ipDataLookup.getProperties(); } /** @@ -752,19 +244,20 @@ public Processor create( databaseType = ipDatabase.getDatabaseType(); } - final Database database; + final IpDataLookupFactory factory; try { - database = Database.getDatabase(databaseType, databaseFile); + factory = IpDataLookupFactories.get(databaseType, databaseFile); } catch (IllegalArgumentException e) { throw newConfigurationException(TYPE, processorTag, "database_file", e.getMessage()); } - final Set properties; + final IpDataLookup ipDataLookup; try { - properties = database.parseProperties(propertyNames); + ipDataLookup = factory.create(propertyNames); } catch (IllegalArgumentException e) { throw newConfigurationException(TYPE, processorTag, "properties", e.getMessage()); } + return new GeoIpProcessor( processorTag, description, @@ -772,7 +265,7 @@ public Processor create( new DatabaseVerifyingSupplier(ipDatabaseProvider, databaseFile, databaseType), () -> ipDatabaseProvider.isValid(databaseFile), targetField, - properties, + ipDataLookup, ignoreMissing, firstOnly, databaseFile diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookup.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookup.java new file mode 100644 index 0000000000000..7442c8e930886 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookup.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +interface IpDataLookup { + /** + * Gets data from the provided {@code ipDatabase} for the provided {@code ip} + * + * @param ipDatabase the database from which to lookup a result + * @param ip the ip address + * @return a map of data corresponding to the configured properties + * @throws IOException if the implementation encounters any problem while retrieving the response + */ + Map getData(IpDatabase ipDatabase, String ip) throws IOException; + + /** + * @return the set of properties this lookup will provide + */ + Set getProperties(); +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java new file mode 100644 index 0000000000000..990788978a0ca --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.core.Nullable; + +import java.util.List; +import java.util.Set; +import java.util.function.Function; + +final class IpDataLookupFactories { + + private IpDataLookupFactories() { + // utility class + } + + interface IpDataLookupFactory { + IpDataLookup create(List properties); + } + + private static final String CITY_DB_SUFFIX = "-City"; + private static final String COUNTRY_DB_SUFFIX = "-Country"; + private static final String ASN_DB_SUFFIX = "-ASN"; + private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP"; + private static final String CONNECTION_TYPE_DB_SUFFIX = "-Connection-Type"; + private static final String DOMAIN_DB_SUFFIX = "-Domain"; + private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise"; + private static final String ISP_DB_SUFFIX = "-ISP"; + + @Nullable + private static Database getMaxmindDatabase(final String databaseType) { + if (databaseType.endsWith(CITY_DB_SUFFIX)) { + return Database.City; + } else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) { + return Database.Country; + } else if (databaseType.endsWith(ASN_DB_SUFFIX)) { + return Database.Asn; + } else if (databaseType.endsWith(ANONYMOUS_IP_DB_SUFFIX)) { + return Database.AnonymousIp; + } else if (databaseType.endsWith(CONNECTION_TYPE_DB_SUFFIX)) { + return Database.ConnectionType; + } else if (databaseType.endsWith(DOMAIN_DB_SUFFIX)) { + return Database.Domain; + } else if (databaseType.endsWith(ENTERPRISE_DB_SUFFIX)) { + return Database.Enterprise; + } else if (databaseType.endsWith(ISP_DB_SUFFIX)) { + return Database.Isp; + } else { + return null; // no match was found + } + } + + /** + * Parses the passed-in databaseType and return the Database instance that is + * associated with that databaseType. + * + * @param databaseType the database type String from the metadata of the database file + * @return the Database instance that is associated with the databaseType + */ + @Nullable + static Database getDatabase(final String databaseType) { + Database database = null; + + if (Strings.hasText(databaseType)) { + database = getMaxmindDatabase(databaseType); + } + + return database; + } + + static Function, IpDataLookup> getMaxmindLookup(final Database database) { + return switch (database) { + case City -> MaxmindIpDataLookups.City::new; + case Country -> MaxmindIpDataLookups.Country::new; + case Asn -> MaxmindIpDataLookups.Asn::new; + case AnonymousIp -> MaxmindIpDataLookups.AnonymousIp::new; + case ConnectionType -> MaxmindIpDataLookups.ConnectionType::new; + case Domain -> MaxmindIpDataLookups.Domain::new; + case Enterprise -> MaxmindIpDataLookups.Enterprise::new; + case Isp -> MaxmindIpDataLookups.Isp::new; + }; + } + + static IpDataLookupFactory get(final String databaseType, final String databaseFile) { + final Database database = getDatabase(databaseType); + if (database == null) { + throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]"); + } + + final Function, IpDataLookup> factoryMethod = getMaxmindLookup(database); + + // note: this can't presently be null, but keep this check -- it will be useful in the near future + if (factoryMethod == null) { + throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]"); + } + + return (properties) -> factoryMethod.apply(database.parseProperties(properties)); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDatabase.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDatabase.java index f416259a87d27..db1ffc1c682b8 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDatabase.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDatabase.java @@ -9,15 +9,9 @@ package org.elasticsearch.ingest.geoip; -import com.maxmind.geoip2.model.AnonymousIpResponse; -import com.maxmind.geoip2.model.AsnResponse; -import com.maxmind.geoip2.model.CityResponse; -import com.maxmind.geoip2.model.ConnectionTypeResponse; -import com.maxmind.geoip2.model.CountryResponse; -import com.maxmind.geoip2.model.DomainResponse; -import com.maxmind.geoip2.model.EnterpriseResponse; -import com.maxmind.geoip2.model.IspResponse; +import com.maxmind.db.Reader; +import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.core.Nullable; import java.io.IOException; @@ -34,44 +28,15 @@ public interface IpDatabase extends AutoCloseable { String getDatabaseType() throws IOException; /** - * @param ipAddress the IP address to look up - * @return a response containing the city data for the given address if it exists, or null if it could not be found - * @throws UnsupportedOperationException may be thrown if the implementation does not support retrieving city data - */ - @Nullable - CityResponse getCity(String ipAddress); - - /** - * @param ipAddress the IP address to look up - * @return a response containing the country data for the given address if it exists, or null if it could not be found - * @throws UnsupportedOperationException may be thrown if the implementation does not support retrieving country data - */ - @Nullable - CountryResponse getCountry(String ipAddress); - - /** - * @param ipAddress the IP address to look up - * @return a response containing the Autonomous System Number for the given address if it exists, or null if it could not - * be found - * @throws UnsupportedOperationException may be thrown if the implementation does not support retrieving ASN data + * Returns a response from this database's reader for the given IP address. + * + * @param ipAddress the address to lookup + * @param responseProvider a method for extracting a response from a {@link Reader}, usually this will be a method reference + * @return a possibly-null response + * @param the type of response that will be returned */ @Nullable - AsnResponse getAsn(String ipAddress); - - @Nullable - AnonymousIpResponse getAnonymousIp(String ipAddress); - - @Nullable - ConnectionTypeResponse getConnectionType(String ipAddress); - - @Nullable - DomainResponse getDomain(String ipAddress); - - @Nullable - EnterpriseResponse getEnterprise(String ipAddress); - - @Nullable - IspResponse getIsp(String ipAddress); + RESPONSE getResponse(String ipAddress, CheckedBiFunction responseProvider); /** * Releases the current database object. Called after processing a single document. Databases should be closed or returned to a diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java new file mode 100644 index 0000000000000..5b22b3f4005a9 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java @@ -0,0 +1,606 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import com.maxmind.db.DatabaseRecord; +import com.maxmind.db.Network; +import com.maxmind.db.Reader; +import com.maxmind.geoip2.model.AbstractResponse; +import com.maxmind.geoip2.model.AnonymousIpResponse; +import com.maxmind.geoip2.model.AsnResponse; +import com.maxmind.geoip2.model.CityResponse; +import com.maxmind.geoip2.model.ConnectionTypeResponse; +import com.maxmind.geoip2.model.CountryResponse; +import com.maxmind.geoip2.model.DomainResponse; +import com.maxmind.geoip2.model.EnterpriseResponse; +import com.maxmind.geoip2.model.IspResponse; +import com.maxmind.geoip2.record.Continent; +import com.maxmind.geoip2.record.Location; +import com.maxmind.geoip2.record.Subdivision; + +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.core.Nullable; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * A collection of {@link IpDataLookup} implementations for MaxMind databases + */ +final class MaxmindIpDataLookups { + + private MaxmindIpDataLookups() { + // utility class + } + + static class AnonymousIp extends AbstractBase { + AnonymousIp(final Set properties) { + super( + properties, + AnonymousIpResponse.class, + (response, ipAddress, network, locales) -> new AnonymousIpResponse(response, ipAddress, network) + ); + } + + @Override + protected Map transform(final AnonymousIpResponse response) { + boolean isHostingProvider = response.isHostingProvider(); + boolean isTorExitNode = response.isTorExitNode(); + boolean isAnonymousVpn = response.isAnonymousVpn(); + boolean isAnonymous = response.isAnonymous(); + boolean isPublicProxy = response.isPublicProxy(); + boolean isResidentialProxy = response.isResidentialProxy(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case HOSTING_PROVIDER -> { + data.put("hosting_provider", isHostingProvider); + } + case TOR_EXIT_NODE -> { + data.put("tor_exit_node", isTorExitNode); + } + case ANONYMOUS_VPN -> { + data.put("anonymous_vpn", isAnonymousVpn); + } + case ANONYMOUS -> { + data.put("anonymous", isAnonymous); + } + case PUBLIC_PROXY -> { + data.put("public_proxy", isPublicProxy); + } + case RESIDENTIAL_PROXY -> { + data.put("residential_proxy", isResidentialProxy); + } + } + } + return data; + } + } + + static class Asn extends AbstractBase { + Asn(Set properties) { + super(properties, AsnResponse.class, (response, ipAddress, network, locales) -> new AsnResponse(response, ipAddress, network)); + } + + @Override + protected Map transform(final AsnResponse response) { + Long asn = response.getAutonomousSystemNumber(); + String organizationName = response.getAutonomousSystemOrganization(); + Network network = response.getNetwork(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case ASN -> { + if (asn != null) { + data.put("asn", asn); + } + } + case ORGANIZATION_NAME -> { + if (organizationName != null) { + data.put("organization_name", organizationName); + } + } + case NETWORK -> { + if (network != null) { + data.put("network", network.toString()); + } + } + } + } + return data; + } + } + + static class City extends AbstractBase { + City(final Set properties) { + super(properties, CityResponse.class, CityResponse::new); + } + + @Override + protected Map transform(final CityResponse response) { + com.maxmind.geoip2.record.Country country = response.getCountry(); + com.maxmind.geoip2.record.City city = response.getCity(); + Location location = response.getLocation(); + Continent continent = response.getContinent(); + Subdivision subdivision = response.getMostSpecificSubdivision(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getTraits().getIpAddress()); + case COUNTRY_ISO_CODE -> { + String countryIsoCode = country.getIsoCode(); + if (countryIsoCode != null) { + data.put("country_iso_code", countryIsoCode); + } + } + case COUNTRY_NAME -> { + String countryName = country.getName(); + if (countryName != null) { + data.put("country_name", countryName); + } + } + case CONTINENT_CODE -> { + String continentCode = continent.getCode(); + if (continentCode != null) { + data.put("continent_code", continentCode); + } + } + case CONTINENT_NAME -> { + String continentName = continent.getName(); + if (continentName != null) { + data.put("continent_name", continentName); + } + } + case REGION_ISO_CODE -> { + // ISO 3166-2 code for country subdivisions. + // See iso.org/iso-3166-country-codes.html + String countryIso = country.getIsoCode(); + String subdivisionIso = subdivision.getIsoCode(); + if (countryIso != null && subdivisionIso != null) { + String regionIsoCode = countryIso + "-" + subdivisionIso; + data.put("region_iso_code", regionIsoCode); + } + } + case REGION_NAME -> { + String subdivisionName = subdivision.getName(); + if (subdivisionName != null) { + data.put("region_name", subdivisionName); + } + } + case CITY_NAME -> { + String cityName = city.getName(); + if (cityName != null) { + data.put("city_name", cityName); + } + } + case TIMEZONE -> { + String locationTimeZone = location.getTimeZone(); + if (locationTimeZone != null) { + data.put("timezone", locationTimeZone); + } + } + case LOCATION -> { + Double latitude = location.getLatitude(); + Double longitude = location.getLongitude(); + if (latitude != null && longitude != null) { + Map locationObject = new HashMap<>(); + locationObject.put("lat", latitude); + locationObject.put("lon", longitude); + data.put("location", locationObject); + } + } + } + } + return data; + } + } + + static class ConnectionType extends AbstractBase { + ConnectionType(final Set properties) { + super( + properties, + ConnectionTypeResponse.class, + (response, ipAddress, network, locales) -> new ConnectionTypeResponse(response, ipAddress, network) + ); + } + + @Override + protected Map transform(final ConnectionTypeResponse response) { + ConnectionTypeResponse.ConnectionType connectionType = response.getConnectionType(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case CONNECTION_TYPE -> { + if (connectionType != null) { + data.put("connection_type", connectionType.toString()); + } + } + } + } + return data; + } + } + + static class Country extends AbstractBase { + Country(final Set properties) { + super(properties, CountryResponse.class, CountryResponse::new); + } + + @Override + protected Map transform(final CountryResponse response) { + com.maxmind.geoip2.record.Country country = response.getCountry(); + Continent continent = response.getContinent(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getTraits().getIpAddress()); + case COUNTRY_ISO_CODE -> { + String countryIsoCode = country.getIsoCode(); + if (countryIsoCode != null) { + data.put("country_iso_code", countryIsoCode); + } + } + case COUNTRY_NAME -> { + String countryName = country.getName(); + if (countryName != null) { + data.put("country_name", countryName); + } + } + case CONTINENT_CODE -> { + String continentCode = continent.getCode(); + if (continentCode != null) { + data.put("continent_code", continentCode); + } + } + case CONTINENT_NAME -> { + String continentName = continent.getName(); + if (continentName != null) { + data.put("continent_name", continentName); + } + } + } + } + return data; + } + } + + static class Domain extends AbstractBase { + Domain(final Set properties) { + super( + properties, + DomainResponse.class, + (response, ipAddress, network, locales) -> new DomainResponse(response, ipAddress, network) + ); + } + + @Override + protected Map transform(final DomainResponse response) { + String domain = response.getDomain(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case DOMAIN -> { + if (domain != null) { + data.put("domain", domain); + } + } + } + } + return data; + } + } + + static class Enterprise extends AbstractBase { + Enterprise(final Set properties) { + super(properties, EnterpriseResponse.class, EnterpriseResponse::new); + } + + @Override + protected Map transform(final EnterpriseResponse response) { + com.maxmind.geoip2.record.Country country = response.getCountry(); + com.maxmind.geoip2.record.City city = response.getCity(); + Location location = response.getLocation(); + Continent continent = response.getContinent(); + Subdivision subdivision = response.getMostSpecificSubdivision(); + + Long asn = response.getTraits().getAutonomousSystemNumber(); + String organizationName = response.getTraits().getAutonomousSystemOrganization(); + Network network = response.getTraits().getNetwork(); + + String isp = response.getTraits().getIsp(); + String ispOrganization = response.getTraits().getOrganization(); + String mobileCountryCode = response.getTraits().getMobileCountryCode(); + String mobileNetworkCode = response.getTraits().getMobileNetworkCode(); + + boolean isHostingProvider = response.getTraits().isHostingProvider(); + boolean isTorExitNode = response.getTraits().isTorExitNode(); + boolean isAnonymousVpn = response.getTraits().isAnonymousVpn(); + boolean isAnonymous = response.getTraits().isAnonymous(); + boolean isPublicProxy = response.getTraits().isPublicProxy(); + boolean isResidentialProxy = response.getTraits().isResidentialProxy(); + + String userType = response.getTraits().getUserType(); + + String domain = response.getTraits().getDomain(); + + ConnectionTypeResponse.ConnectionType connectionType = response.getTraits().getConnectionType(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getTraits().getIpAddress()); + case COUNTRY_ISO_CODE -> { + String countryIsoCode = country.getIsoCode(); + if (countryIsoCode != null) { + data.put("country_iso_code", countryIsoCode); + } + } + case COUNTRY_NAME -> { + String countryName = country.getName(); + if (countryName != null) { + data.put("country_name", countryName); + } + } + case CONTINENT_CODE -> { + String continentCode = continent.getCode(); + if (continentCode != null) { + data.put("continent_code", continentCode); + } + } + case CONTINENT_NAME -> { + String continentName = continent.getName(); + if (continentName != null) { + data.put("continent_name", continentName); + } + } + case REGION_ISO_CODE -> { + // ISO 3166-2 code for country subdivisions. + // See iso.org/iso-3166-country-codes.html + String countryIso = country.getIsoCode(); + String subdivisionIso = subdivision.getIsoCode(); + if (countryIso != null && subdivisionIso != null) { + String regionIsoCode = countryIso + "-" + subdivisionIso; + data.put("region_iso_code", regionIsoCode); + } + } + case REGION_NAME -> { + String subdivisionName = subdivision.getName(); + if (subdivisionName != null) { + data.put("region_name", subdivisionName); + } + } + case CITY_NAME -> { + String cityName = city.getName(); + if (cityName != null) { + data.put("city_name", cityName); + } + } + case TIMEZONE -> { + String locationTimeZone = location.getTimeZone(); + if (locationTimeZone != null) { + data.put("timezone", locationTimeZone); + } + } + case LOCATION -> { + Double latitude = location.getLatitude(); + Double longitude = location.getLongitude(); + if (latitude != null && longitude != null) { + Map locationObject = new HashMap<>(); + locationObject.put("lat", latitude); + locationObject.put("lon", longitude); + data.put("location", locationObject); + } + } + case ASN -> { + if (asn != null) { + data.put("asn", asn); + } + } + case ORGANIZATION_NAME -> { + if (organizationName != null) { + data.put("organization_name", organizationName); + } + } + case NETWORK -> { + if (network != null) { + data.put("network", network.toString()); + } + } + case HOSTING_PROVIDER -> { + data.put("hosting_provider", isHostingProvider); + } + case TOR_EXIT_NODE -> { + data.put("tor_exit_node", isTorExitNode); + } + case ANONYMOUS_VPN -> { + data.put("anonymous_vpn", isAnonymousVpn); + } + case ANONYMOUS -> { + data.put("anonymous", isAnonymous); + } + case PUBLIC_PROXY -> { + data.put("public_proxy", isPublicProxy); + } + case RESIDENTIAL_PROXY -> { + data.put("residential_proxy", isResidentialProxy); + } + case DOMAIN -> { + if (domain != null) { + data.put("domain", domain); + } + } + case ISP -> { + if (isp != null) { + data.put("isp", isp); + } + } + case ISP_ORGANIZATION_NAME -> { + if (ispOrganization != null) { + data.put("isp_organization_name", ispOrganization); + } + } + case MOBILE_COUNTRY_CODE -> { + if (mobileCountryCode != null) { + data.put("mobile_country_code", mobileCountryCode); + } + } + case MOBILE_NETWORK_CODE -> { + if (mobileNetworkCode != null) { + data.put("mobile_network_code", mobileNetworkCode); + } + } + case USER_TYPE -> { + if (userType != null) { + data.put("user_type", userType); + } + } + case CONNECTION_TYPE -> { + if (connectionType != null) { + data.put("connection_type", connectionType.toString()); + } + } + } + } + return data; + } + } + + static class Isp extends AbstractBase { + Isp(final Set properties) { + super(properties, IspResponse.class, (response, ipAddress, network, locales) -> new IspResponse(response, ipAddress, network)); + } + + @Override + protected Map transform(final IspResponse response) { + String isp = response.getIsp(); + String ispOrganization = response.getOrganization(); + String mobileNetworkCode = response.getMobileNetworkCode(); + String mobileCountryCode = response.getMobileCountryCode(); + Long asn = response.getAutonomousSystemNumber(); + String organizationName = response.getAutonomousSystemOrganization(); + Network network = response.getNetwork(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case ASN -> { + if (asn != null) { + data.put("asn", asn); + } + } + case ORGANIZATION_NAME -> { + if (organizationName != null) { + data.put("organization_name", organizationName); + } + } + case NETWORK -> { + if (network != null) { + data.put("network", network.toString()); + } + } + case ISP -> { + if (isp != null) { + data.put("isp", isp); + } + } + case ISP_ORGANIZATION_NAME -> { + if (ispOrganization != null) { + data.put("isp_organization_name", ispOrganization); + } + } + case MOBILE_COUNTRY_CODE -> { + if (mobileCountryCode != null) { + data.put("mobile_country_code", mobileCountryCode); + } + } + case MOBILE_NETWORK_CODE -> { + if (mobileNetworkCode != null) { + data.put("mobile_network_code", mobileNetworkCode); + } + } + } + } + return data; + } + } + + /** + * As an internal detail, the {@code com.maxmind.geoip2.model } classes that are populated by + * {@link Reader#getRecord(InetAddress, Class)} are kinda half-populated and need to go through a second round of construction + * with context from the querying caller. This method gives us a place do that additional binding. Cleverly, the signature + * here matches the constructor for many of these model classes exactly, so an appropriate implementation can 'just' be a method + * reference in some cases (in other cases it needs to be a lambda). + */ + @FunctionalInterface + private interface ResponseBuilder { + RESPONSE build(RESPONSE resp, String address, Network network, List locales); + } + + /** + * The {@link MaxmindIpDataLookups.AbstractBase} is an abstract base implementation of {@link IpDataLookup} that + * provides common functionality for getting a specific kind of {@link AbstractResponse} from a {@link IpDatabase}. + * + * @param the intermediate type of {@link AbstractResponse} + */ + private abstract static class AbstractBase implements IpDataLookup { + + protected final Set properties; + protected final Class clazz; + protected final ResponseBuilder builder; + + AbstractBase(final Set properties, final Class clazz, final ResponseBuilder builder) { + this.properties = Set.copyOf(properties); + this.clazz = clazz; + this.builder = builder; + } + + @Override + public Set getProperties() { + return this.properties; + } + + @Override + public final Map getData(final IpDatabase ipDatabase, final String ipAddress) { + final RESPONSE response = ipDatabase.getResponse(ipAddress, this::lookup); + return (response == null) ? Map.of() : transform(response); + } + + @Nullable + private RESPONSE lookup(final Reader reader, final String ipAddress) throws IOException { + final InetAddress ip = InetAddresses.forString(ipAddress); + final DatabaseRecord record = reader.getRecord(ip, clazz); + final RESPONSE data = record.getData(); + return (data == null) ? null : builder.build(data, NetworkAddress.format(ip), record.getNetwork(), List.of("en")); + } + + /** + * Extract the configured properties from the retrieved response + * @param response the non-null response that was retrieved + * @return a mapping of properties for the ip from the response + */ + protected abstract Map transform(RESPONSE response); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java index 83b3d2cfbbc27..7f38a37b43edf 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java @@ -126,7 +126,7 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception { DatabaseReaderLazyLoader loader = configDatabases.getDatabase("GeoLite2-City.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); - CityResponse cityResponse = loader.getCity("89.160.20.128"); + CityResponse cityResponse = loader.getResponse("89.160.20.128", GeoIpTestUtils::getCity); assertThat(cityResponse.getCity().getName(), equalTo("Tumba")); assertThat(cache.count(), equalTo(1)); } @@ -138,7 +138,7 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception { DatabaseReaderLazyLoader loader = configDatabases.getDatabase("GeoLite2-City.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); - CityResponse cityResponse = loader.getCity("89.160.20.128"); + CityResponse cityResponse = loader.getResponse("89.160.20.128", GeoIpTestUtils::getCity); assertThat(cityResponse.getCity().getName(), equalTo("Linköping")); assertThat(cache.count(), equalTo(1)); }); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index f5c3c08579855..793754ec316b2 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.core.IOUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.RandomDocumentPicks; -import org.elasticsearch.ingest.geoip.Database.Property; import org.elasticsearch.test.ESTestCase; import org.junit.After; import org.junit.Before; @@ -25,7 +24,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; @@ -40,7 +38,9 @@ public class GeoIpProcessorTests extends ESTestCase { - private static final Set ALL_PROPERTIES = Set.of(Property.values()); + private static IpDataLookup ipDataLookupAll(final Database database) { + return IpDataLookupFactories.getMaxmindLookup(database).apply(database.properties()); + } // a temporary directory that mmdb files can be copied to and read from private Path tmpDir; @@ -82,7 +82,7 @@ public void testCity() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -115,7 +115,7 @@ public void testNullValueWithIgnoreMissing() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), true, false, "filename" @@ -137,7 +137,7 @@ public void testNonExistentWithIgnoreMissing() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), true, false, "filename" @@ -156,7 +156,7 @@ public void testNullWithoutIgnoreMissing() { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -178,7 +178,7 @@ public void testNonExistentWithoutIgnoreMissing() { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -198,7 +198,7 @@ public void testCity_withIpV6() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -235,7 +235,7 @@ public void testCityWithMissingLocation() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -263,7 +263,7 @@ public void testCountry() throws Exception { loader("GeoLite2-Country.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.Country), false, false, "filename" @@ -295,7 +295,7 @@ public void testCountryWithMissingLocation() throws Exception { loader("GeoLite2-Country.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.Country), false, false, "filename" @@ -323,7 +323,7 @@ public void testAsn() throws Exception { loader("GeoLite2-ASN.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.Asn), false, false, "filename" @@ -354,7 +354,7 @@ public void testAnonymmousIp() throws Exception { loader("GeoIP2-Anonymous-IP-Test.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.AnonymousIp), false, false, "filename" @@ -388,7 +388,7 @@ public void testConnectionType() throws Exception { loader("GeoIP2-Connection-Type-Test.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.ConnectionType), false, false, "filename" @@ -417,7 +417,7 @@ public void testDomain() throws Exception { loader("GeoIP2-Domain-Test.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.Domain), false, false, "filename" @@ -446,7 +446,7 @@ public void testEnterprise() throws Exception { loader("GeoIP2-Enterprise-Test.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.Enterprise), false, false, "filename" @@ -497,7 +497,7 @@ public void testIsp() throws Exception { loader("GeoIP2-ISP-Test.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.Isp), false, false, "filename" @@ -531,7 +531,7 @@ public void testAddressIsNotInTheDatabase() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -555,7 +555,7 @@ public void testInvalid() { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -576,7 +576,7 @@ public void testListAllValid() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -603,7 +603,7 @@ public void testListPartiallyValid() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -630,7 +630,7 @@ public void testListNoMatches() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -650,7 +650,7 @@ public void testListDatabaseReferenceCounting() throws Exception { GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", () -> { loader.preLookup(); return loader; - }, () -> true, "target_field", ALL_PROPERTIES, false, false, "filename"); + }, () -> true, "target_field", ipDataLookupAll(Database.City), false, false, "filename"); Map document = new HashMap<>(); document.put("source_field", List.of("8.8.8.8", "82.171.64.0")); @@ -678,7 +678,7 @@ public void testListFirstOnly() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, true, "filename" @@ -703,7 +703,7 @@ public void testListFirstOnlyNoMatches() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, true, "filename" @@ -725,7 +725,7 @@ public void testInvalidDatabase() throws Exception { loader("GeoLite2-City.mmdb"), () -> false, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, true, "filename" @@ -748,7 +748,7 @@ public void testNoDatabase() throws Exception { () -> null, () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "GeoLite2-City" @@ -771,7 +771,7 @@ public void testNoDatabase_ignoreMissing() throws Exception { () -> null, () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), true, false, "GeoLite2-City" diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java index 461983bb24488..160671fd39001 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java @@ -9,6 +9,13 @@ package org.elasticsearch.ingest.geoip; +import com.maxmind.db.DatabaseRecord; +import com.maxmind.db.Reader; +import com.maxmind.geoip2.model.CityResponse; +import com.maxmind.geoip2.model.CountryResponse; + +import org.elasticsearch.common.CheckedBiFunction; +import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.core.SuppressForbidden; import java.io.FileNotFoundException; @@ -17,6 +24,7 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import java.util.Set; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; @@ -58,4 +66,28 @@ public static void copyDefaultDatabases(final Path directory, ConfigDatabases co configDatabases.updateDatabase(directory.resolve(database), true); } } + + /** + * A static city-specific responseProvider for use with {@link IpDatabase#getResponse(String, CheckedBiFunction)} in + * tests. + *

    + * Like this: {@code CityResponse city = loader.getResponse("some.ip.address", GeoIpTestUtils::getCity);} + */ + public static CityResponse getCity(Reader reader, String ip) throws IOException { + DatabaseRecord record = reader.getRecord(InetAddresses.forString(ip), CityResponse.class); + CityResponse data = record.getData(); + return data == null ? null : new CityResponse(data, ip, record.getNetwork(), List.of("en")); + } + + /** + * A static country-specific responseProvider for use with {@link IpDatabase#getResponse(String, CheckedBiFunction)} in + * tests. + *

    + * Like this: {@code CountryResponse country = loader.getResponse("some.ip.address", GeoIpTestUtils::getCountry);} + */ + public static CountryResponse getCountry(Reader reader, String ip) throws IOException { + DatabaseRecord record = reader.getRecord(InetAddresses.forString(ip), CountryResponse.class); + CountryResponse data = record.getData(); + return data == null ? null : new CountryResponse(data, ip, record.getNetwork(), List.of("en")); + } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java index f1c7d809b98fe..46a34c2cdad56 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java @@ -116,6 +116,6 @@ public void testDatabaseTypeParsing() throws IOException { } private Database parseDatabaseFromType(String databaseFile) throws IOException { - return Database.getDatabase(MMDBUtil.getDatabaseType(tmpDir.resolve(databaseFile)), null); + return IpDataLookupFactories.getDatabase(MMDBUtil.getDatabaseType(tmpDir.resolve(databaseFile))); } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java index ec05054615bd8..84ea5fd584352 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java @@ -468,36 +468,6 @@ public void testUnknownMaxMindResponseClassess() { ); } - /* - * This tests that this test has a mapping in TYPE_TO_MAX_MIND_CLASS for all MaxMind classes exposed through IpDatabase. - */ - public void testUsedMaxMindResponseClassesAreAccountedFor() { - Set> usedMaxMindResponseClasses = getUsedMaxMindResponseClasses(); - Set> supportedMaxMindClasses = new HashSet<>(TYPE_TO_MAX_MIND_CLASS.values()); - Set> usedButNotSupportedMaxMindResponseClasses = Sets.difference( - usedMaxMindResponseClasses, - supportedMaxMindClasses - ); - assertThat( - "IpDatabase exposes MaxMind response classes that this test does not know what to do with. Add mappings to " - + "TYPE_TO_MAX_MIND_CLASS for the following: " - + usedButNotSupportedMaxMindResponseClasses, - usedButNotSupportedMaxMindResponseClasses, - empty() - ); - Set> supportedButNotUsedMaxMindClasses = Sets.difference( - supportedMaxMindClasses, - usedMaxMindResponseClasses - ); - assertThat( - "This test claims to support MaxMind response classes that are not exposed in IpDatabase. Remove the following from " - + "TYPE_TO_MAX_MIND_CLASS: " - + supportedButNotUsedMaxMindClasses, - supportedButNotUsedMaxMindClasses, - empty() - ); - } - /* * This is the list of field types that causes us to stop recursing. That is, fields of these types are the lowest-level fields that * we care about. @@ -616,23 +586,4 @@ private static String getFormattedList(Set fields) { } return result.toString(); } - - /* - * This returns all AbstractResponse classes that are returned from getter methods on IpDatabase. - */ - private static Set> getUsedMaxMindResponseClasses() { - Set> result = new HashSet<>(); - Method[] methods = IpDatabase.class.getMethods(); - for (Method method : methods) { - if (method.getName().startsWith("get")) { - Class returnType = method.getReturnType(); - try { - result.add(returnType.asSubclass(AbstractResponse.class)); - } catch (ClassCastException ignore) { - // This is not what we were looking for, move on - } - } - } - return result; - } } From 8b09e9119d17dcf82a67aaefdcd5ce224a5c8598 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Mon, 7 Oct 2024 08:32:37 +1100 Subject: [PATCH 114/194] Mute org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT #114194 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 992996e811853..317a4a41d5756 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -369,6 +369,8 @@ tests: - class: org.elasticsearch.aggregations.AggregationsClientYamlTestSuiteIT method: test {yaml=aggregations/stats_metric_fail_formatting/fail formatting} issue: https://github.com/elastic/elasticsearch/issues/114187 +- class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT + issue: https://github.com/elastic/elasticsearch/issues/114194 # Examples: # From 763764c7fac0d5738534e632d7da327711a272d0 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Mon, 7 Oct 2024 02:24:12 -0400 Subject: [PATCH 115/194] Fix BWC for file-settings based role mappings (#113900) * Fix BWC for file settings based role mappings --------- Co-authored-by: Johannes Freden Jansson --- docs/changelog/113900.yaml | 5 + .../rolemapping/GetRoleMappingsResponse.java | 5 + .../security/authz/RoleMappingMetadata.java | 2 + .../RoleMappingFileSettingsIT.java | 158 +++++++++++++----- .../xpack/security/Security.java | 3 +- .../TransportDeleteRoleMappingAction.java | 18 +- .../TransportGetRoleMappingsAction.java | 35 +++- .../TransportPutRoleMappingAction.java | 28 +++- .../mapper/ClusterStateRoleMapper.java | 31 +++- .../test/SecuritySettingsSource.java | 2 +- .../TransportGetRoleMappingsActionTests.java | 8 +- .../TransportPutRoleMappingActionTests.java | 23 ++- .../mapper/ClusterStateRoleMapperTests.java | 11 +- 13 files changed, 265 insertions(+), 64 deletions(-) create mode 100644 docs/changelog/113900.yaml diff --git a/docs/changelog/113900.yaml b/docs/changelog/113900.yaml new file mode 100644 index 0000000000000..25f833d251784 --- /dev/null +++ b/docs/changelog/113900.yaml @@ -0,0 +1,5 @@ +pr: 113900 +summary: Fix BWC for file-settings based role mappings +area: Authentication +type: bug +issues: [] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/GetRoleMappingsResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/GetRoleMappingsResponse.java index 13a751829797f..4f18411ac3af6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/GetRoleMappingsResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/rolemapping/GetRoleMappingsResponse.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; import java.io.IOException; +import java.util.Collection; /** * Response to {@link GetRoleMappingsAction get role-mappings API}. @@ -21,6 +22,10 @@ public class GetRoleMappingsResponse extends ActionResponse { private final ExpressionRoleMapping[] mappings; + public GetRoleMappingsResponse(Collection mappings) { + this(mappings.toArray(new ExpressionRoleMapping[0])); + } + public GetRoleMappingsResponse(ExpressionRoleMapping... mappings) { this.mappings = mappings; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java index da6ff6ad24c34..8f78fdbccd923 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.AbstractNamedDiffable; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.NamedDiff; +import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; @@ -57,6 +58,7 @@ public final class RoleMappingMetadata extends AbstractNamedDiffable cleanup cluster settings..."); + { + // Deleting non-existent native role mappings returns not found even if they exist in config file + var response = client().execute(DeleteRoleMappingAction.INSTANCE, deleteRequest("everyone_kibana")).get(); + assertFalse(response.isFound()); + } + savedClusterState = setupClusterStateListenerForCleanup(internalCluster().getMasterName()); writeJSONFile(internalCluster().getMasterName(), emptyJSON, logger, versionCounter); @@ -307,48 +320,85 @@ public void testRoleMappingsApplied() throws Exception { clusterStateResponse.getState().metadata().persistentSettings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()) ); - // native role mappings are not affected by the removal of the cluster-state based ones + // cluster-state role mapping was removed and is not returned in the API anymore { var request = new GetRoleMappingsRequest(); request.setNames("everyone_kibana", "everyone_fleet"); var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get(); - assertTrue(response.hasMappings()); - assertThat( - Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(), - containsInAnyOrder("everyone_kibana", "everyone_fleet") - ); + assertFalse(response.hasMappings()); } - // and roles are resolved based on the native role mappings + // no role mappings means no roles are resolved for (UserRoleMapper userRoleMapper : internalCluster().getInstances(UserRoleMapper.class)) { PlainActionFuture> resolveRolesFuture = new PlainActionFuture<>(); userRoleMapper.resolveRoles( new UserRoleMapper.UserData("anyUsername", null, List.of(), Map.of(), mock(RealmConfig.class)), resolveRolesFuture ); - assertThat(resolveRolesFuture.get(), contains("kibana_user_native")); + assertThat(resolveRolesFuture.get(), empty()); } + } - { - var request = new DeleteRoleMappingRequest(); - request.setName("everyone_kibana"); - var response = client().execute(DeleteRoleMappingAction.INSTANCE, request).get(); - assertTrue(response.isFound()); - request = new DeleteRoleMappingRequest(); - request.setName("everyone_fleet"); - response = client().execute(DeleteRoleMappingAction.INSTANCE, request).get(); - assertTrue(response.isFound()); + public void testGetRoleMappings() throws Exception { + ensureGreen(); + + final List nativeMappings = List.of("everyone_kibana", "_everyone_kibana", "zzz_mapping", "123_mapping"); + for (var mapping : nativeMappings) { + client().execute(PutRoleMappingAction.INSTANCE, sampleRestRequest(mapping)).actionGet(); } - // no roles are resolved now, because both native and cluster-state based stores have been cleared - for (UserRoleMapper userRoleMapper : internalCluster().getInstances(UserRoleMapper.class)) { - PlainActionFuture> resolveRolesFuture = new PlainActionFuture<>(); - userRoleMapper.resolveRoles( - new UserRoleMapper.UserData("anyUsername", null, List.of(), Map.of(), mock(RealmConfig.class)), - resolveRolesFuture - ); - assertThat(resolveRolesFuture.get(), empty()); + var savedClusterState = setupClusterStateListener(internalCluster().getMasterName(), "everyone_kibana"); + writeJSONFile(internalCluster().getMasterName(), testJSON, logger, versionCounter); + boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS); + assertTrue(awaitSuccessful); + + var request = new GetRoleMappingsRequest(); + var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get(); + assertTrue(response.hasMappings()); + assertThat( + Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(), + containsInAnyOrder( + "everyone_kibana", + "everyone_kibana " + RESERVED_ROLE_MAPPING_SUFFIX, + "_everyone_kibana", + "everyone_fleet " + RESERVED_ROLE_MAPPING_SUFFIX, + "zzz_mapping", + "123_mapping" + ) + ); + + int readOnlyCount = 0; + // assert that cluster-state role mappings come last + for (ExpressionRoleMapping mapping : response.mappings()) { + readOnlyCount = mapping.getName().endsWith(RESERVED_ROLE_MAPPING_SUFFIX) ? readOnlyCount + 1 : readOnlyCount; } + // Two sourced from cluster-state + assertEquals(readOnlyCount, 2); + + // it's possible to delete overlapping native role mapping + assertTrue(client().execute(DeleteRoleMappingAction.INSTANCE, deleteRequest("everyone_kibana")).actionGet().isFound()); + + savedClusterState = setupClusterStateListenerForCleanup(internalCluster().getMasterName()); + writeJSONFile(internalCluster().getMasterName(), emptyJSON, logger, versionCounter); + awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS); + assertTrue(awaitSuccessful); + + final ClusterStateResponse clusterStateResponse = clusterAdmin().state( + new ClusterStateRequest(TEST_REQUEST_TIMEOUT).waitForMetadataVersion(savedClusterState.v2().get()) + ).get(); + + assertNull( + clusterStateResponse.getState().metadata().persistentSettings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()) + ); + + // Make sure remaining native mappings can still be fetched + request = new GetRoleMappingsRequest(); + response = client().execute(GetRoleMappingsAction.INSTANCE, request).get(); + assertTrue(response.hasMappings()); + assertThat( + Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(), + containsInAnyOrder("_everyone_kibana", "zzz_mapping", "123_mapping") + ); } public static Tuple setupClusterStateListenerForError( @@ -433,11 +483,8 @@ public void testRoleMappingApplyWithSecurityIndexClosed() throws Exception { boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS); assertTrue(awaitSuccessful); - // no native role mappings exist - var request = new GetRoleMappingsRequest(); - request.setNames("everyone_kibana", "everyone_fleet"); - var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get(); - assertFalse(response.hasMappings()); + // even if index is closed, cluster-state role mappings are still returned + assertGetResponseHasMappings(true, "everyone_kibana", "everyone_fleet"); // cluster state settings are also applied var clusterStateResponse = clusterAdmin().state( @@ -476,6 +523,12 @@ public void testRoleMappingApplyWithSecurityIndexClosed() throws Exception { } } + private DeleteRoleMappingRequest deleteRequest(String name) { + var request = new DeleteRoleMappingRequest(); + request.setName(name); + return request; + } + private PutRoleMappingRequest sampleRestRequest(String name) throws Exception { var json = """ { @@ -494,4 +547,19 @@ private PutRoleMappingRequest sampleRestRequest(String name) throws Exception { return new PutRoleMappingRequestBuilder(null).source(name, parser).request(); } } + + private static void assertGetResponseHasMappings(boolean readOnly, String... mappings) throws InterruptedException, ExecutionException { + var request = new GetRoleMappingsRequest(); + request.setNames(mappings); + var response = client().execute(GetRoleMappingsAction.INSTANCE, request).get(); + assertTrue(response.hasMappings()); + assertThat( + Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(), + containsInAnyOrder( + Arrays.stream(mappings) + .map(mapping -> mapping + (readOnly ? " " + RESERVED_ROLE_MAPPING_SUFFIX : "")) + .toArray(String[]::new) + ) + ); + } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 79a00fa1293bd..f4d9360d1ed84 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -897,7 +897,8 @@ Collection createComponents( reservedRealm ); components.add(nativeUsersStore); - components.add(new PluginComponentBinding<>(NativeRoleMappingStore.class, nativeRoleMappingStore)); + components.add(clusterStateRoleMapper); + components.add(nativeRoleMappingStore); components.add(new PluginComponentBinding<>(UserRoleMapper.class, userRoleMapper)); components.add(reservedRealm); components.add(realms); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportDeleteRoleMappingAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportDeleteRoleMappingAction.java index 74129facae70a..467cc1c8a9027 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportDeleteRoleMappingAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportDeleteRoleMappingAction.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; @@ -16,17 +17,20 @@ import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingAction; import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequest; import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingResponse; +import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper; import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore; public class TransportDeleteRoleMappingAction extends HandledTransportAction { private final NativeRoleMappingStore roleMappingStore; + private final ClusterStateRoleMapper clusterStateRoleMapper; @Inject public TransportDeleteRoleMappingAction( ActionFilters actionFilters, TransportService transportService, - NativeRoleMappingStore roleMappingStore + NativeRoleMappingStore roleMappingStore, + ClusterStateRoleMapper clusterStateRoleMapper ) { super( DeleteRoleMappingAction.NAME, @@ -36,10 +40,22 @@ public TransportDeleteRoleMappingAction( EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.roleMappingStore = roleMappingStore; + this.clusterStateRoleMapper = clusterStateRoleMapper; } @Override protected void doExecute(Task task, DeleteRoleMappingRequest request, ActionListener listener) { + if (clusterStateRoleMapper.hasMapping(request.getName())) { + // Since it's allowed to add a mapping with the same name in the native role mapping store as the file_settings namespace, + // a warning header is added to signal to the caller that this could be a problem. + HeaderWarning.addWarning( + "A read only role mapping with the same name [" + + request.getName() + + "] has been previously been defined in a configuration file. The role mapping [" + + request.getName() + + "] defined in the configuration file is read only, will not be deleted, and will remain active." + ); + } roleMappingStore.deleteRoleMapping(request, listener.safeMap(DeleteRoleMappingResponse::new)); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java index ac0d3177cca09..d5bc923ab6dca 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java @@ -17,21 +17,29 @@ import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequest; import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsResponse; import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; +import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper; import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore; import java.util.Arrays; +import java.util.Comparator; import java.util.HashSet; +import java.util.List; import java.util.Set; +import java.util.stream.Stream; + +import static org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper.RESERVED_ROLE_MAPPING_SUFFIX; public class TransportGetRoleMappingsAction extends HandledTransportAction { private final NativeRoleMappingStore roleMappingStore; + private final ClusterStateRoleMapper clusterStateRoleMapper; @Inject public TransportGetRoleMappingsAction( ActionFilters actionFilters, TransportService transportService, - NativeRoleMappingStore nativeRoleMappingStore + NativeRoleMappingStore nativeRoleMappingStore, + ClusterStateRoleMapper clusterStateRoleMapper ) { super( GetRoleMappingsAction.NAME, @@ -41,6 +49,7 @@ public TransportGetRoleMappingsAction( EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.roleMappingStore = nativeRoleMappingStore; + this.clusterStateRoleMapper = clusterStateRoleMapper; } @Override @@ -51,9 +60,27 @@ protected void doExecute(Task task, final GetRoleMappingsRequest request, final } else { names = new HashSet<>(Arrays.asList(request.getNames())); } - this.roleMappingStore.getRoleMappings(names, ActionListener.wrap(mappings -> { - ExpressionRoleMapping[] array = mappings.toArray(new ExpressionRoleMapping[mappings.size()]); - listener.onResponse(new GetRoleMappingsResponse(array)); + roleMappingStore.getRoleMappings(names, ActionListener.wrap(mappings -> { + List combinedRoleMappings = Stream.concat( + mappings.stream(), + clusterStateRoleMapper.getMappings(names) + .stream() + .map(this::cloneAndMarkAsReadOnly) + .sorted(Comparator.comparing(ExpressionRoleMapping::getName)) + ).toList(); + listener.onResponse(new GetRoleMappingsResponse(combinedRoleMappings)); }, listener::onFailure)); } + + private ExpressionRoleMapping cloneAndMarkAsReadOnly(ExpressionRoleMapping mapping) { + // Mark role mappings from cluster state as "read only" by adding a suffix to their name + return new ExpressionRoleMapping( + mapping.getName() + " " + RESERVED_ROLE_MAPPING_SUFFIX, + mapping.getExpression(), + mapping.getRoles(), + mapping.getRoleTemplates(), + mapping.getMetadata(), + mapping.isEnabled() + ); + } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingAction.java index 82a3b4f000064..76f520bed517e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingAction.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; @@ -16,27 +17,52 @@ import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingAction; import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingRequest; import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse; +import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper; import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore; +import static org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper.RESERVED_ROLE_MAPPING_SUFFIX; + public class TransportPutRoleMappingAction extends HandledTransportAction { private final NativeRoleMappingStore roleMappingStore; + private final ClusterStateRoleMapper clusterStateRoleMapper; @Inject public TransportPutRoleMappingAction( ActionFilters actionFilters, TransportService transportService, - NativeRoleMappingStore roleMappingStore + NativeRoleMappingStore roleMappingStore, + ClusterStateRoleMapper clusterStateRoleMapper ) { super(PutRoleMappingAction.NAME, transportService, actionFilters, PutRoleMappingRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); this.roleMappingStore = roleMappingStore; + this.clusterStateRoleMapper = clusterStateRoleMapper; } @Override protected void doExecute(Task task, final PutRoleMappingRequest request, final ActionListener listener) { + validateMappingName(request.getName()); + if (clusterStateRoleMapper.hasMapping(request.getName())) { + // Allow to define a mapping with the same name in the native role mapping store as the file_settings namespace, but add a + // warning header to signal to the caller that this could be a problem. + HeaderWarning.addWarning( + "A read only role mapping with the same name [" + + request.getName() + + "] has been previously been defined in a configuration file. " + + "Both role mappings will be used to determine role assignments." + ); + } roleMappingStore.putRoleMapping( request, ActionListener.wrap(created -> listener.onResponse(new PutRoleMappingResponse(created)), listener::onFailure) ); } + + private static void validateMappingName(String mappingName) { + if (mappingName.endsWith(RESERVED_ROLE_MAPPING_SUFFIX)) { + throw new IllegalArgumentException( + "Invalid mapping name [" + mappingName + "]. [" + RESERVED_ROLE_MAPPING_SUFFIX + "] is not an allowed suffix" + ); + } + } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapper.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapper.java index 9a6e9e75c4685..df8d46f133ac7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapper.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapper.java @@ -14,13 +14,16 @@ import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Nullable; import org.elasticsearch.script.ScriptService; import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper; import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata; +import java.util.Arrays; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import static org.elasticsearch.xpack.core.security.SecurityExtension.SecurityComponents; @@ -28,8 +31,7 @@ * A role mapper the reads the role mapping rules (i.e. {@link ExpressionRoleMapping}s) from the cluster state * (i.e. {@link RoleMappingMetadata}). This is not enabled by default. */ -public final class ClusterStateRoleMapper extends AbstractRoleMapperClearRealmCache implements ClusterStateListener { - +public class ClusterStateRoleMapper extends AbstractRoleMapperClearRealmCache implements ClusterStateListener { /** * This setting is never registered by the xpack security plugin - in order to enable the * cluster-state based role mapper another plugin must register it as a boolean setting @@ -45,6 +47,7 @@ public final class ClusterStateRoleMapper extends AbstractRoleMapperClearRealmCa * */ public static final String CLUSTER_STATE_ROLE_MAPPINGS_ENABLED = "xpack.security.authc.cluster_state_role_mappings.enabled"; + public static final String RESERVED_ROLE_MAPPING_SUFFIX = "(read only)"; private static final Logger logger = LogManager.getLogger(ClusterStateRoleMapper.class); private final ScriptService scriptService; @@ -54,8 +57,8 @@ public final class ClusterStateRoleMapper extends AbstractRoleMapperClearRealmCa public ClusterStateRoleMapper(Settings settings, ScriptService scriptService, ClusterService clusterService) { this.scriptService = scriptService; this.clusterService = clusterService; - // this role mapper is disabled by default and only code in other plugins can enable it - this.enabled = settings.getAsBoolean(CLUSTER_STATE_ROLE_MAPPINGS_ENABLED, false); + // this role mapper is enabled by default and only code in other plugins can disable it + this.enabled = settings.getAsBoolean(CLUSTER_STATE_ROLE_MAPPINGS_ENABLED, true); if (this.enabled) { clusterService.addListener(this); } @@ -81,12 +84,30 @@ public void clusterChanged(ClusterChangedEvent event) { } } + public boolean hasMapping(String name) { + return getMappings().stream().map(ExpressionRoleMapping::getName).anyMatch(name::equals); + } + + public Set getMappings(@Nullable Set names) { + if (enabled == false) { + return Set.of(); + } + final Set mappings = getMappings(); + if (names == null || names.isEmpty()) { + return mappings; + } + return mappings.stream().filter(it -> names.contains(it.getName())).collect(Collectors.toSet()); + } + private Set getMappings() { if (enabled == false) { return Set.of(); } else { final Set mappings = RoleMappingMetadata.getFromClusterState(clusterService.state()).getRoleMappings(); - logger.trace("Retrieved [{}] mapping(s) from cluster state", mappings.size()); + logger.trace( + "Retrieved mapping(s) {} from cluster state", + Arrays.toString(mappings.stream().map(ExpressionRoleMapping::getName).toArray(String[]::new)) + ); return mappings; } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java index 6d7817db8ec05..ce5aaacdb92b9 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySettingsSource.java @@ -403,7 +403,7 @@ public static class UnregisteredSecuritySettingsPlugin extends Plugin { ); public static final Setting CLUSTER_STATE_ROLE_MAPPINGS_ENABLED = Setting.boolSetting( "xpack.security.authc.cluster_state_role_mappings.enabled", - false, + true, Setting.Property.NodeScope ); public static final Setting NATIVE_ROLES_ENABLED = Setting.boolSetting( diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsActionTests.java index 6e8698f095d32..799e0c334172c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequest; import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsResponse; import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; +import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper; import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore; import org.hamcrest.Matchers; import org.junit.Before; @@ -34,13 +35,16 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.notNullValue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TransportGetRoleMappingsActionTests extends ESTestCase { private NativeRoleMappingStore store; + private ClusterStateRoleMapper clusterStateRoleMapper; private TransportGetRoleMappingsAction action; private AtomicReference> namesRef; private List result; @@ -49,6 +53,8 @@ public class TransportGetRoleMappingsActionTests extends ESTestCase { @Before public void setupMocks() { store = mock(NativeRoleMappingStore.class); + clusterStateRoleMapper = mock(ClusterStateRoleMapper.class); + when(clusterStateRoleMapper.getMappings(anySet())).thenReturn(Set.of()); TransportService transportService = new TransportService( Settings.EMPTY, mock(Transport.class), @@ -58,7 +64,7 @@ public void setupMocks() { null, Collections.emptySet() ); - action = new TransportGetRoleMappingsAction(mock(ActionFilters.class), transportService, store); + action = new TransportGetRoleMappingsAction(mock(ActionFilters.class), transportService, store, clusterStateRoleMapper); namesRef = new AtomicReference<>(null); result = Collections.emptyList(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingActionTests.java index 6f789a10a3a6c..91ac8b46e5a31 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingActionTests.java @@ -19,12 +19,14 @@ import org.elasticsearch.xpack.core.security.action.rolemapping.PutRoleMappingResponse; import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression; +import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper; import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore; import org.junit.Before; import java.util.Arrays; import java.util.Collections; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import static org.hamcrest.Matchers.aMapWithSize; @@ -33,12 +35,15 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.iterableWithSize; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TransportPutRoleMappingActionTests extends ESTestCase { private NativeRoleMappingStore store; + private ClusterStateRoleMapper clusterStateRoleMapper; private TransportPutRoleMappingAction action; private AtomicReference requestRef; @@ -46,6 +51,9 @@ public class TransportPutRoleMappingActionTests extends ESTestCase { @Before public void setupMocks() { store = mock(NativeRoleMappingStore.class); + clusterStateRoleMapper = mock(ClusterStateRoleMapper.class); + when(clusterStateRoleMapper.getMappings(anySet())).thenReturn(Set.of()); + when(clusterStateRoleMapper.hasMapping(any())).thenReturn(false); TransportService transportService = new TransportService( Settings.EMPTY, mock(Transport.class), @@ -55,7 +63,7 @@ public void setupMocks() { null, Collections.emptySet() ); - action = new TransportPutRoleMappingAction(mock(ActionFilters.class), transportService, store); + action = new TransportPutRoleMappingAction(mock(ActionFilters.class), transportService, store, clusterStateRoleMapper); requestRef = new AtomicReference<>(null); @@ -85,6 +93,19 @@ public void testPutValidMapping() throws Exception { assertThat(mapping.getMetadata().get("dumb"), equalTo(true)); } + public void testPutMappingWithInvalidName() { + final FieldExpression expression = new FieldExpression("username", Collections.singletonList(new FieldExpression.FieldValue("*"))); + IllegalArgumentException illegalArgumentException = expectThrows( + IllegalArgumentException.class, + () -> put("anarchy (read only)", expression, "superuser", Collections.singletonMap("dumb", true)) + ); + + assertThat( + illegalArgumentException.getMessage(), + equalTo("Invalid mapping name [anarchy (read only)]. [(read only)] is not an allowed suffix") + ); + } + private PutRoleMappingResponse put(String name, FieldExpression expression, String role, Map metadata) throws Exception { final PutRoleMappingRequest request = new PutRoleMappingRequest(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapperTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapperTests.java index 7a9dd65f84c67..515b5ef741a00 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapperTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapperTests.java @@ -56,12 +56,12 @@ public void setup() { () -> 1L ); clusterService = mock(ClusterService.class); - enabledSettings = Settings.builder().put("xpack.security.authc.cluster_state_role_mappings.enabled", true).build(); + disabledSettings = Settings.builder().put("xpack.security.authc.cluster_state_role_mappings.enabled", false).build(); if (randomBoolean()) { - disabledSettings = Settings.builder().put("xpack.security.authc.cluster_state_role_mappings.enabled", false).build(); + enabledSettings = Settings.builder().put("xpack.security.authc.cluster_state_role_mappings.enabled", true).build(); } else { - // the cluster state role mapper is disabled by default - disabledSettings = Settings.EMPTY; + // the cluster state role mapper is enabled by default + enabledSettings = Settings.EMPTY; } } @@ -95,6 +95,9 @@ public void testRoleResolving() throws Exception { verify(mapping1).isEnabled(); verify(mapping2).isEnabled(); verify(mapping3).isEnabled(); + verify(mapping1).getName(); + verify(mapping2).getName(); + verify(mapping3).getName(); verify(mapping2).getExpression(); verify(mapping3).getExpression(); verify(mapping3).getRoleNames(same(scriptService), same(expressionModel)); From 57955cb8d48b7adc3b616961279bbb43d274b8b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?= Date: Mon, 7 Oct 2024 10:23:46 +0200 Subject: [PATCH 116/194] [DOCS] Adds DeBERTA v2 to the tokenizers list in API docs (#112752) Co-authored-by: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com> --- .../ingest/processors/inference.asciidoc | 81 +++++++++++++++++++ docs/reference/ml/ml-shared.asciidoc | 27 +++++++ .../apis/infer-trained-model.asciidoc | 12 +++ .../apis/put-trained-models.asciidoc | 31 +++++++ 4 files changed, 151 insertions(+) diff --git a/docs/reference/ingest/processors/inference.asciidoc b/docs/reference/ingest/processors/inference.asciidoc index fa4f246cdd7c8..4699f634afe37 100644 --- a/docs/reference/ingest/processors/inference.asciidoc +++ b/docs/reference/ingest/processors/inference.asciidoc @@ -169,6 +169,18 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -224,6 +236,18 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -304,6 +328,23 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`span`:::: +(Optional, integer) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] + +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -363,6 +404,18 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -424,6 +477,22 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`span`:::: +(Optional, integer) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] + +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -515,6 +584,18 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index 97122141d7558..ef19fbf4e267d 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -988,6 +988,7 @@ values are + -- * `bert`: Use for BERT-style models +* `deberta_v2`: Use for DeBERTa v2 and v3-style models * `mpnet`: Use for MPNet-style models * `roberta`: Use for RoBERTa-style and BART-style models * experimental:[] `xlm_roberta`: Use for XLMRoBERTa-style models @@ -1037,6 +1038,19 @@ sequence. Therefore, do not use `second` in this case. end::inference-config-nlp-tokenization-truncate[] +tag::inference-config-nlp-tokenization-truncate-deberta-v2[] +Indicates how tokens are truncated when they exceed `max_sequence_length`. +The default value is `first`. ++ +-- +* `balanced`: One or both of the first and second sequences may be truncated so as to balance the tokens included from both sequences. +* `none`: No truncation occurs; the inference request receives an error. +* `first`: Only the first sequence is truncated. +* `second`: Only the second sequence is truncated. If there is just one sequence, that sequence is truncated. +-- + +end::inference-config-nlp-tokenization-truncate-deberta-v2[] + tag::inference-config-nlp-tokenization-bert-with-special-tokens[] Tokenize with special tokens. The tokens typically included in BERT-style tokenization are: + @@ -1050,10 +1064,23 @@ tag::inference-config-nlp-tokenization-bert-ja-with-special-tokens[] Tokenize with special tokens if `true`. end::inference-config-nlp-tokenization-bert-ja-with-special-tokens[] +tag::inference-config-nlp-tokenization-deberta-v2[] +DeBERTa-style tokenization is to be performed with the enclosed settings. +end::inference-config-nlp-tokenization-deberta-v2[] + tag::inference-config-nlp-tokenization-max-sequence-length[] Specifies the maximum number of tokens allowed to be output by the tokenizer. end::inference-config-nlp-tokenization-max-sequence-length[] +tag::inference-config-nlp-tokenization-deberta-v2-with-special-tokens[] +Tokenize with special tokens. The tokens typically included in DeBERTa-style tokenization are: ++ +-- +* `[CLS]`: The first token of the sequence being classified. +* `[SEP]`: Indicates sequence separation and sequence end. +-- +end::inference-config-nlp-tokenization-deberta-v2-with-special-tokens[] + tag::inference-config-nlp-tokenization-roberta[] RoBERTa-style tokenization is to be performed with the enclosed settings. end::inference-config-nlp-tokenization-roberta[] diff --git a/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc b/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc index 9aac913e7559f..99c3ecad03a9d 100644 --- a/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc +++ b/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc @@ -137,6 +137,18 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, string) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= + `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] diff --git a/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc b/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc index e29bc8823ab29..32265af5f795b 100644 --- a/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc +++ b/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc @@ -773,6 +773,37 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, boolean) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] ==== +`deberta_v2`:: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +==== +`do_lower_case`::: +(Optional, boolean) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] ++ +-- +Defaults to `false`. +-- + +`max_sequence_length`::: +(Optional, integer) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] + +`span`::: +(Optional, integer) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] + +`truncate`::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] + +`with_special_tokens`::: +(Optional, boolean) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2-with-special-tokens] +==== `roberta`:: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] From 837fd2ed28b0aa47486d8bb6a5297abf4496dbd9 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Mon, 7 Oct 2024 10:37:00 +0200 Subject: [PATCH 117/194] Make H3 CellBoundary immutable (#113792) Small refactor that makes CellBoundary immutable. --- .../org/elasticsearch/h3/CellBoundary.java | 40 +++++++++++++------ .../java/org/elasticsearch/h3/Constants.java | 4 -- .../java/org/elasticsearch/h3/FaceIJK.java | 22 +++++----- .../main/java/org/elasticsearch/h3/H3.java | 12 +++--- .../java/org/elasticsearch/h3/H3Index.java | 4 +- .../elasticsearch/h3/CellBoundaryTests.java | 18 +++++++++ .../org/elasticsearch/h3/GeoToH3Tests.java | 2 +- .../org/elasticsearch/h3/HexRingTests.java | 2 +- 8 files changed, 67 insertions(+), 37 deletions(-) diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/CellBoundary.java b/libs/h3/src/main/java/org/elasticsearch/h3/CellBoundary.java index 74115d5a002d6..e0f9df174c2b5 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/CellBoundary.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/CellBoundary.java @@ -22,36 +22,52 @@ */ package org.elasticsearch.h3; +import java.util.Arrays; +import java.util.Objects; + /** * cell boundary points as {@link LatLng} */ public final class CellBoundary { - /** Maximum number of cell boundary vertices; worst case is pentagon: * 5 original verts + 5 edge crossings */ - private static final int MAX_CELL_BNDRY_VERTS = 10; + static final int MAX_CELL_BNDRY_VERTS = 10; /** How many points it holds */ - private int numVertext; + private final int numPoints; /** The actual points */ - private final LatLng[] points = new LatLng[MAX_CELL_BNDRY_VERTS]; - - CellBoundary() {} + private final LatLng[] points; - void add(LatLng point) { - points[numVertext++] = point; + CellBoundary(LatLng[] points, int numPoints) { + this.points = points; + this.numPoints = numPoints; } /** Number of points in this boundary */ public int numPoints() { - return numVertext; + return numPoints; } /** Return the point at the given position*/ public LatLng getLatLon(int i) { - if (i >= numVertext) { - throw new IndexOutOfBoundsException(); - } + assert i >= 0 && i < numPoints; return points[i]; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final CellBoundary that = (CellBoundary) o; + return numPoints == that.numPoints && Arrays.equals(points, that.points); + } + + @Override + public int hashCode() { + return Objects.hash(numPoints, Arrays.hashCode(points)); + } } diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java b/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java index 570052700615f..3b3f760c0534f 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/Constants.java @@ -34,10 +34,6 @@ final class Constants { * 2.0 * PI */ public static final double M_2PI = 2.0 * Math.PI; - /** - * max H3 resolution; H3 version 1 has 16 resolutions, numbered 0 through 15 - */ - public static int MAX_H3_RES = 15; /** * The number of H3 base cells */ diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java b/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java index ae59ff359d1f8..866fdfe8a7f8b 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/FaceIJK.java @@ -439,7 +439,8 @@ public CellBoundary faceIjkPentToCellBoundary(int res, int start, int length) { // convert each vertex to lat/lng // adjust the face of each vertex as appropriate and introduce // edge-crossing vertices as needed - final CellBoundary boundary = new CellBoundary(); + final LatLng[] points = new LatLng[CellBoundary.MAX_CELL_BNDRY_VERTS]; + int numPoints = 0; final CoordIJK scratch = new CoordIJK(0, 0, 0); final FaceIJK fijk = new FaceIJK(this.face, scratch); final int[][] coord = isResolutionClassIII ? VERTEX_CLASSIII : VERTEX_CLASSII; @@ -501,21 +502,19 @@ public CellBoundary faceIjkPentToCellBoundary(int res, int start, int length) { // find the intersection and add the lat/lng point to the result final Vec2d inter = Vec2d.v2dIntersect(orig2d0, orig2d1, edge0, edge1); - final LatLng point = inter.hex2dToGeo(fijkOrient.face, adjRes, true); - boundary.add(point); + points[numPoints++] = inter.hex2dToGeo(fijkOrient.face, adjRes, true); } // convert vertex to lat/lng and add to the result // vert == start + NUM_PENT_VERTS is only used to test for possible // intersection on last edge if (vert < start + Constants.NUM_PENT_VERTS) { - final LatLng point = fijk.coord.ijkToGeo(fijk.face, adjRes, true); - boundary.add(point); + points[numPoints++] = fijk.coord.ijkToGeo(fijk.face, adjRes, true); } lastFace = fijk.face; lastCoord.reset(fijk.coord.i, fijk.coord.j, fijk.coord.k); } - return boundary; + return new CellBoundary(points, numPoints); } /** @@ -547,7 +546,8 @@ public CellBoundary faceIjkToCellBoundary(final int res, final int start, final // convert each vertex to lat/lng // adjust the face of each vertex as appropriate and introduce // edge-crossing vertices as needed - final CellBoundary boundary = new CellBoundary(); + final LatLng[] points = new LatLng[CellBoundary.MAX_CELL_BNDRY_VERTS]; + int numPoints = 0; final CoordIJK scratch1 = new CoordIJK(0, 0, 0); final FaceIJK fijk = new FaceIJK(this.face, scratch1); final CoordIJK scratch2 = isResolutionClassIII ? new CoordIJK(0, 0, 0) : null; @@ -616,8 +616,7 @@ public CellBoundary faceIjkToCellBoundary(final int res, final int start, final */ final boolean isIntersectionAtVertex = orig2d0.numericallyIdentical(inter) || orig2d1.numericallyIdentical(inter); if (isIntersectionAtVertex == false) { - final LatLng point = inter.hex2dToGeo(this.face, adjRes, true); - boundary.add(point); + points[numPoints++] = inter.hex2dToGeo(this.face, adjRes, true); } } @@ -625,13 +624,12 @@ public CellBoundary faceIjkToCellBoundary(final int res, final int start, final // vert == start + NUM_HEX_VERTS is only used to test for possible // intersection on last edge if (vert < start + Constants.NUM_HEX_VERTS) { - final LatLng point = fijk.coord.ijkToGeo(fijk.face, adjRes, true); - boundary.add(point); + points[numPoints++] = fijk.coord.ijkToGeo(fijk.face, adjRes, true); } lastFace = fijk.face; lastOverage = overage; } - return boundary; + return new CellBoundary(points, numPoints); } /** diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/H3.java b/libs/h3/src/main/java/org/elasticsearch/h3/H3.java index 46bcc3f141dde..8c0bba62cecdb 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/H3.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/H3.java @@ -30,8 +30,10 @@ * Defines the public API of the H3 library. */ public final class H3 { - - public static int MAX_H3_RES = Constants.MAX_H3_RES; + /** + * max H3 resolution; H3 version 1 has 16 resolutions, numbered 0 through 15 + */ + public static int MAX_H3_RES = 15; private static final long[] NORTH = new long[MAX_H3_RES + 1]; private static final long[] SOUTH = new long[MAX_H3_RES + 1]; @@ -97,7 +99,7 @@ public static boolean h3IsValid(long h3) { } int res = H3Index.H3_get_resolution(h3); - if (res < 0 || res > Constants.MAX_H3_RES) { // LCOV_EXCL_BR_LINE + if (res < 0 || res > MAX_H3_RES) { // LCOV_EXCL_BR_LINE // Resolutions less than zero can not be represented in an index return false; } @@ -118,7 +120,7 @@ public static boolean h3IsValid(long h3) { } } - for (int r = res + 1; r <= Constants.MAX_H3_RES; r++) { + for (int r = res + 1; r <= MAX_H3_RES; r++) { int digit = H3Index.H3_get_index_digit(h3, r); if (digit != CoordIJK.Direction.INVALID_DIGIT.digit()) { return false; @@ -601,7 +603,7 @@ private static String[] h3ToStringList(long[] h3s) { * @throws IllegalArgumentException res is not a valid H3 resolution. */ private static void checkResolution(int res) { - if (res < 0 || res > Constants.MAX_H3_RES) { + if (res < 0 || res > MAX_H3_RES) { throw new IllegalArgumentException("resolution [" + res + "] is out of range (must be 0 <= res <= 15)"); } } diff --git a/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java b/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java index 7babedc55eb0e..2b1b9cade21a4 100644 --- a/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java +++ b/libs/h3/src/main/java/org/elasticsearch/h3/H3Index.java @@ -160,14 +160,14 @@ public static int H3_get_resolution(long h3) { * Gets the resolution res integer digit (0-7) of h3. */ public static int H3_get_index_digit(long h3, int res) { - return ((int) ((((h3) >> ((Constants.MAX_H3_RES - (res)) * H3_PER_DIGIT_OFFSET)) & H3_DIGIT_MASK))); + return ((int) ((((h3) >> ((H3.MAX_H3_RES - (res)) * H3_PER_DIGIT_OFFSET)) & H3_DIGIT_MASK))); } /** * Sets the resolution res digit of h3 to the integer digit (0-7) */ public static long H3_set_index_digit(long h3, int res, long digit) { - int x = (Constants.MAX_H3_RES - res) * H3_PER_DIGIT_OFFSET; + int x = (H3.MAX_H3_RES - res) * H3_PER_DIGIT_OFFSET; return (((h3) & ~((H3_DIGIT_MASK << (x)))) | (((digit)) << x)); } diff --git a/libs/h3/src/test/java/org/elasticsearch/h3/CellBoundaryTests.java b/libs/h3/src/test/java/org/elasticsearch/h3/CellBoundaryTests.java index 903e4ed40ec16..00ca6f7021e3d 100644 --- a/libs/h3/src/test/java/org/elasticsearch/h3/CellBoundaryTests.java +++ b/libs/h3/src/test/java/org/elasticsearch/h3/CellBoundaryTests.java @@ -218,4 +218,22 @@ private boolean isSharedBoundary(int clon1, int clat1, int clon2, int clat2, Cel } return false; } + + public void testEqualsAndHashCode() { + final long h3 = H3.geoToH3(GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude(), randomIntBetween(0, 15)); + final CellBoundary boundary1 = H3.h3ToGeoBoundary(h3); + final CellBoundary boundary2 = H3.h3ToGeoBoundary(h3); + assertEquals(boundary1, boundary2); + assertEquals(boundary1.hashCode(), boundary2.hashCode()); + + final long otherH3 = H3.geoToH3(GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude(), randomIntBetween(0, 15)); + final CellBoundary otherCellBoundary = H3.h3ToGeoBoundary(otherH3); + if (otherH3 != h3) { + assertNotEquals(boundary1, otherCellBoundary); + assertNotEquals(boundary1.hashCode(), otherCellBoundary.hashCode()); + } else { + assertEquals(boundary1, otherCellBoundary); + assertEquals(boundary1.hashCode(), otherCellBoundary.hashCode()); + } + } } diff --git a/libs/h3/src/test/java/org/elasticsearch/h3/GeoToH3Tests.java b/libs/h3/src/test/java/org/elasticsearch/h3/GeoToH3Tests.java index cb7d416a5a9d3..3f2c329d9ff3c 100644 --- a/libs/h3/src/test/java/org/elasticsearch/h3/GeoToH3Tests.java +++ b/libs/h3/src/test/java/org/elasticsearch/h3/GeoToH3Tests.java @@ -38,7 +38,7 @@ public void testRandomPoints() { private void testPoint(double lat, double lon) { GeoPoint point = new GeoPoint(PlanetModel.SPHERE, Math.toRadians(lat), Math.toRadians(lon)); - for (int res = 0; res < Constants.MAX_H3_RES; res++) { + for (int res = 0; res < H3.MAX_H3_RES; res++) { String h3Address = H3.geoToH3Address(lat, lon, res); assertEquals(res, H3.getResolution(h3Address)); GeoPolygon polygon = getGeoPolygon(h3Address); diff --git a/libs/h3/src/test/java/org/elasticsearch/h3/HexRingTests.java b/libs/h3/src/test/java/org/elasticsearch/h3/HexRingTests.java index 8fe5c6206fff8..864c0322cac90 100644 --- a/libs/h3/src/test/java/org/elasticsearch/h3/HexRingTests.java +++ b/libs/h3/src/test/java/org/elasticsearch/h3/HexRingTests.java @@ -38,7 +38,7 @@ public void testHexRing() { for (int i = 0; i < 500; i++) { double lat = GeoTestUtil.nextLatitude(); double lon = GeoTestUtil.nextLongitude(); - for (int res = 0; res <= Constants.MAX_H3_RES; res++) { + for (int res = 0; res <= H3.MAX_H3_RES; res++) { String origin = H3.geoToH3Address(lat, lon, res); assertFalse(H3.areNeighborCells(origin, origin)); String[] ring = H3.hexRing(origin); From 6d3abe51a3227f9e44e4408ddb0a01a434cf2817 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 7 Oct 2024 10:12:24 +0100 Subject: [PATCH 118/194] Clarify support for bundled JDK (#113993) Spells out that third-party EOL schedules don't affect our support. Also reorders the information to talk about the benefits of the bundled JDK before mentioning alternatives, and clarifies the division of responsibilities for "supported" JDKs other than the bundled one. --- docs/reference/setup/install.asciidoc | 44 ++++++++++++++------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/docs/reference/setup/install.asciidoc b/docs/reference/setup/install.asciidoc index 89373d0ce8d44..a38fdcfc36fd5 100644 --- a/docs/reference/setup/install.asciidoc +++ b/docs/reference/setup/install.asciidoc @@ -76,27 +76,29 @@ Docker container images may be downloaded from the Elastic Docker Registry. [[jvm-version]] === Java (JVM) Version -{es} is built using Java, and includes a bundled version of -https://openjdk.java.net[OpenJDK] from the JDK maintainers (GPLv2+CE) within -each distribution. The bundled JVM is the recommended JVM. - -To use your own version of Java, set the `ES_JAVA_HOME` environment variable. -If you must use a version of Java that is different from the bundled JVM, it is -best to use the latest release of a link:/support/matrix[supported] -https://www.oracle.com/technetwork/java/eol-135779.html[LTS version of Java]. -{es} is closely coupled to certain OpenJDK-specific features, so it may not -work correctly with other JVMs. {es} will refuse to start if a known-bad -version of Java is used. - -If you use a JVM other than the bundled one, you are responsible for reacting -to announcements related to its security issues and bug fixes, and must -yourself determine whether each update is necessary or not. In contrast, the -bundled JVM is treated as an integral part of {es}, which means that Elastic -takes responsibility for keeping it up to date. Security issues and bugs within -the bundled JVM are treated as if they were within {es} itself. - -The bundled JVM is located within the `jdk` subdirectory of the {es} home -directory. You may remove this directory if using your own JVM. +{es} is built using Java, and includes a bundled version of https://openjdk.java.net[OpenJDK] within each distribution. We strongly +recommend using the bundled JVM in all installations of {es}. + +The bundled JVM is treated the same as any other dependency of {es} in terms of support and maintenance. This means that Elastic takes +responsibility for keeping it up to date, and reacts to security issues and bug reports as needed to address vulnerabilities and other bugs +in {es}. Elastic's support of the bundled JVM is subject to Elastic's https://www.elastic.co/support_policy[support policy] and +https://www.elastic.co/support/eol[end-of-life schedule] and is independent of the support policy and end-of-life schedule offered by the +original supplier of the JVM. Elastic does not support using the bundled JVM for purposes other than running {es}. + +TIP: {es} uses only a subset of the features offered by the JVM. Bugs and security issues in the bundled JVM often relate to features that +{es} does not use. Such issues do not apply to {es}. Elastic analyzes reports of security vulnerabilities in all its dependencies, including +in the bundled JVM, and will issue an https://www.elastic.co/community/security[Elastic Security Advisory] if such an advisory is needed. + +If you decide to run {es} using a version of Java that is different from the bundled one, prefer to use the latest release of a +https://www.oracle.com/technetwork/java/eol-135779.html[LTS version of Java] which is link:/support/matrix[listed in the support matrix]. +Although such a configuration is supported, if you encounter a security issue or other bug in your chosen JVM then Elastic may not be able +to help unless the issue is also present in the bundled JVM. Instead, you must seek assistance directly from the supplier of your chosen +JVM. You must also take responsibility for reacting to security and bug announcements from the supplier of your chosen JVM. {es} may not +perform optimally if using a JVM other than the bundled one. {es} is closely coupled to certain OpenJDK-specific features, so it may not +work correctly with JVMs that are not OpenJDK. {es} will refuse to start if you attempt to use a known-bad JVM version. + +To use your own version of Java, set the `ES_JAVA_HOME` environment variable to the path to your own JVM installation. The bundled JVM is +located within the `jdk` subdirectory of the {es} home directory. You may remove this directory if using your own JVM. [discrete] [[jvm-agents]] From 4ef5ea6d1cc3ae951bc4226f6506b97ee9e62e5e Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Mon, 7 Oct 2024 10:22:38 +0100 Subject: [PATCH 119/194] Change default locale of date mappers to ENGLISH (#112799) English is not changing between COMPAT and CLDR locale databases, whereas ROOT is --- docs/reference/mapping/types/date.asciidoc | 3 +- modules/aggregations/build.gradle | 1 + .../InternalAutoDateHistogramTests.java | 29 ------------------- .../stats_metric_fail_formatting.yml | 2 +- .../org/elasticsearch/TransportVersions.java | 1 + .../common/util/LocaleUtils.java | 6 ++-- .../DataStreamTimestampFieldMapper.java | 3 +- .../index/mapper/DateFieldMapper.java | 13 ++++++--- .../index/mapper/DateScriptFieldType.java | 4 +-- .../index/mapper/RangeFieldMapper.java | 3 +- .../index/mapper/TypeParsers.java | 2 +- .../elasticsearch/search/DocValueFormat.java | 9 +++++- 12 files changed, 31 insertions(+), 45 deletions(-) diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc index 44e1c2949775e..ca2c23f932fc3 100644 --- a/docs/reference/mapping/types/date.asciidoc +++ b/docs/reference/mapping/types/date.asciidoc @@ -125,8 +125,7 @@ The following parameters are accepted by `date` fields: `locale`:: The locale to use when parsing dates since months do not have the same names - and/or abbreviations in all languages. The default is the - https://docs.oracle.com/javase/8/docs/api/java/util/Locale.html#ROOT[`ROOT` locale]. + and/or abbreviations in all languages. The default is ENGLISH. <>:: diff --git a/modules/aggregations/build.gradle b/modules/aggregations/build.gradle index 1b3aac13b3608..dc38364af16cb 100644 --- a/modules/aggregations/build.gradle +++ b/modules/aggregations/build.gradle @@ -48,4 +48,5 @@ dependencies { tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("aggregations/date_agg_per_day_of_week/Date aggregartion per day of week", "week-date behaviour has changed") + task.skipTest("aggregations/stats_metric_fail_formatting/fail formatting", "locale has changed") }) diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java index 5455daf0a79ec..227557590731e 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.aggregations.bucket.histogram; -import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.aggregations.bucket.AggregationMultiBucketAggregationTestCase; import org.elasticsearch.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder.RoundingInfo; @@ -28,7 +27,6 @@ import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.test.InternalAggregationTestCase; -import org.elasticsearch.test.TransportVersionUtils; import java.io.IOException; import java.time.Instant; @@ -459,33 +457,6 @@ public void testCreateWithReplacementBuckets() { assertThat(copy.getInterval(), equalTo(orig.getInterval())); } - public void testSerializationPre830() throws IOException { - // we need to test without sub-aggregations, otherwise we need to also update the interval within the inner aggs - InternalAutoDateHistogram instance = createTestInstance( - randomAlphaOfLengthBetween(3, 7), - createTestMetadata(), - InternalAggregations.EMPTY - ); - TransportVersion version = TransportVersionUtils.randomVersionBetween( - random(), - TransportVersions.MINIMUM_COMPATIBLE, - TransportVersionUtils.getPreviousVersion(TransportVersions.V_8_3_0) - ); - InternalAutoDateHistogram deserialized = copyInstance(instance, version); - assertEquals(1, deserialized.getBucketInnerInterval()); - - InternalAutoDateHistogram modified = new InternalAutoDateHistogram( - deserialized.getName(), - deserialized.getBuckets(), - deserialized.getTargetBuckets(), - deserialized.getBucketInfo(), - deserialized.getFormatter(), - deserialized.getMetadata(), - instance.getBucketInnerInterval() - ); - assertEqualInstances(instance, modified); - } - public void testReadFromPre830() throws IOException { byte[] bytes = Base64.getDecoder() .decode( diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml index d9298a832e650..78811a1c46430 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml @@ -30,7 +30,7 @@ setup: cluster_features: "gte_v8.15.0" reason: fixed in 8.15.0 - do: - catch: /Cannot format stat \[sum\] with format \[DocValueFormat.DateTime\(format\[date_hour_minute_second_millis\] locale\[\], Z, MILLISECONDS\)\]/ + catch: /Cannot format stat \[sum\] with format \[DocValueFormat.DateTime\(format\[date_hour_minute_second_millis\] locale\[en\], Z, MILLISECONDS\)\]/ search: index: test_date body: diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index c55436f85a6e3..f6e4649aa4807 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -234,6 +234,7 @@ static TransportVersion def(int id) { public static final TransportVersion RRF_QUERY_REWRITE = def(8_758_00_0); public static final TransportVersion SEARCH_FAILURE_STATS = def(8_759_00_0); public static final TransportVersion INGEST_GEO_DATABASE_PROVIDERS = def(8_760_00_0); + public static final TransportVersion DATE_TIME_DOC_VALUES_LOCALES = def(8_761_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/common/util/LocaleUtils.java b/server/src/main/java/org/elasticsearch/common/util/LocaleUtils.java index 86e82886ed263..bdbf843ad2b53 100644 --- a/server/src/main/java/org/elasticsearch/common/util/LocaleUtils.java +++ b/server/src/main/java/org/elasticsearch/common/util/LocaleUtils.java @@ -68,16 +68,16 @@ private static Locale parseParts(String[] parts) { switch (parts.length) { case 3: // lang, country, variant - return new Locale(parts[0], parts[1], parts[2]); + return Locale.of(parts[0], parts[1], parts[2]); case 2: // lang, country - return new Locale(parts[0], parts[1]); + return Locale.of(parts[0], parts[1]); case 1: if ("ROOT".equalsIgnoreCase(parts[0])) { return Locale.ROOT; } // lang - return new Locale(parts[0]); + return Locale.of(parts[0]); default: throw new IllegalArgumentException( "Locales can have at most 3 parts but got " + parts.length + ": " + Arrays.asList(parts) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java index 651d9e76e84a2..481901f7c03ce 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java @@ -165,10 +165,11 @@ public void doValidate(MappingLookup lookup) { Map configuredSettings = XContentHelper.convertToMap(BytesReference.bytes(builder), false, XContentType.JSON).v2(); configuredSettings = (Map) configuredSettings.values().iterator().next(); - // Only type, meta and format attributes are allowed: + // Only type, meta, format, and locale attributes are allowed: configuredSettings.remove("type"); configuredSettings.remove("meta"); configuredSettings.remove("format"); + configuredSettings.remove("locale"); // ignoring malformed values is disallowed (see previous check), // however if `index.mapping.ignore_malformed` has been set to true then diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 2e602033442c7..7be5ee2200b5c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -79,11 +79,16 @@ public final class DateFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "date"; public static final String DATE_NANOS_CONTENT_TYPE = "date_nanos"; - public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time||epoch_millis"); + public static final Locale DEFAULT_LOCALE = Locale.ENGLISH; + // although the locale doesn't affect the results, tests still check formatter equality, which does include locale + public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time||epoch_millis") + .withLocale(DEFAULT_LOCALE); public static final DateFormatter DEFAULT_DATE_TIME_NANOS_FORMATTER = DateFormatter.forPattern( "strict_date_optional_time_nanos||epoch_millis" - ); - private static final DateMathParser EPOCH_MILLIS_PARSER = DateFormatter.forPattern("epoch_millis").toDateMathParser(); + ).withLocale(DEFAULT_LOCALE); + private static final DateMathParser EPOCH_MILLIS_PARSER = DateFormatter.forPattern("epoch_millis") + .withLocale(DEFAULT_LOCALE) + .toDateMathParser(); public enum Resolution { MILLISECONDS(CONTENT_TYPE, NumericType.DATE, DateMillisDocValuesField::new) { @@ -232,7 +237,7 @@ public static final class Builder extends FieldMapper.Builder { private final Parameter locale = new Parameter<>( "locale", false, - () -> Locale.ROOT, + () -> DEFAULT_LOCALE, (n, c, o) -> LocaleUtils.parse(o.toString()), m -> toType(m).locale, (xContentBuilder, n, v) -> xContentBuilder.field(n, v.toString()), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java index e519fec09ce78..341944c3d687a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java @@ -69,7 +69,7 @@ private static class Builder extends AbstractScriptFieldType.Builder o == null ? null : LocaleUtils.parse(o.toString()), RuntimeField.initializerNotSupported(), (b, n, v) -> { - if (v != null && false == v.equals(Locale.ROOT)) { + if (v != null && false == v.equals(DateFieldMapper.DEFAULT_LOCALE)) { b.field(n, v.toString()); } }, @@ -97,7 +97,7 @@ protected AbstractScriptFieldType createFieldType( OnScriptError onScriptError ) { String pattern = format.getValue() == null ? DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.pattern() : format.getValue(); - Locale locale = this.locale.getValue() == null ? Locale.ROOT : this.locale.getValue(); + Locale locale = this.locale.getValue() == null ? DateFieldMapper.DEFAULT_LOCALE : this.locale.getValue(); DateFormatter dateTimeFormatter = DateFormatter.forPattern(pattern, supportedVersion).withLocale(locale); return new DateScriptFieldType(name, factory, dateTimeFormatter, script, meta, onScriptError); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index af0dc0c0ad7fe..6ca30304201b2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -59,6 +59,7 @@ public class RangeFieldMapper extends FieldMapper { public static class Defaults { public static final DateFormatter DATE_FORMATTER = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; + public static final Locale LOCALE = DateFieldMapper.DEFAULT_LOCALE; } // this is private since it has a different default @@ -83,7 +84,7 @@ public static class Builder extends FieldMapper.Builder { private final Parameter locale = new Parameter<>( "locale", false, - () -> Locale.ROOT, + () -> Defaults.LOCALE, (n, c, o) -> LocaleUtils.parse(o.toString()), m -> toType(m).locale, (xContentBuilder, n, v) -> xContentBuilder.field(n, v.toString()), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java index f086526eec78e..b60feb4d5746e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java @@ -179,7 +179,7 @@ public static boolean parseMultiField( public static DateFormatter parseDateTimeFormatter(Object node) { if (node instanceof String) { - return DateFormatter.forPattern((String) node); + return DateFormatter.forPattern((String) node).withLocale(DateFieldMapper.DEFAULT_LOCALE); } throw new IllegalArgumentException("Invalid format: [" + node.toString() + "]: expected string value"); } diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index be9ad0ed0a9cd..f1d4f678c5fb9 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateMathParser; +import org.elasticsearch.common.util.LocaleUtils; import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; @@ -236,9 +237,12 @@ private DateTime(DateFormatter formatter, ZoneId timeZone, DateFieldMapper.Resol public DateTime(StreamInput in) throws IOException { String formatterPattern = in.readString(); + Locale locale = in.getTransportVersion().onOrAfter(TransportVersions.DATE_TIME_DOC_VALUES_LOCALES) + ? LocaleUtils.parse(in.readString()) + : DateFieldMapper.DEFAULT_LOCALE; String zoneId = in.readString(); this.timeZone = ZoneId.of(zoneId); - this.formatter = DateFormatter.forPattern(formatterPattern).withZone(this.timeZone); + this.formatter = DateFormatter.forPattern(formatterPattern).withZone(this.timeZone).withLocale(locale); this.parser = formatter.toDateMathParser(); this.resolution = DateFieldMapper.Resolution.ofOrdinal(in.readVInt()); if (in.getTransportVersion().between(TransportVersions.V_7_7_0, TransportVersions.V_8_0_0)) { @@ -259,6 +263,9 @@ public String getWriteableName() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(formatter.pattern()); + if (out.getTransportVersion().onOrAfter(TransportVersions.DATE_TIME_DOC_VALUES_LOCALES)) { + out.writeString(formatter.locale().toString()); + } out.writeString(timeZone.getId()); out.writeVInt(resolution.ordinal()); if (out.getTransportVersion().between(TransportVersions.V_7_7_0, TransportVersions.V_8_0_0)) { From 6b2cc599cef9c3e075add187cf4b76e1c9977dd0 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 7 Oct 2024 12:07:49 +0200 Subject: [PATCH 120/194] Replace some test usages of search(Query, Collector) (#113818) The leftover usages of the deprecated search(Query, TotalHitCountCollector) have been replaced with search(Query, TotalHitCountCollectorManager) --- .../index/engine/InternalEngineTests.java | 51 ++++++++++--------- .../search/SearchCancellationTests.java | 14 +++-- .../index/engine/EngineTestCase.java | 17 +++---- ...ityIndexReaderWrapperIntegrationTests.java | 8 ++- 4 files changed, 42 insertions(+), 48 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 883723de31d46..c8ca3d17de797 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -55,7 +55,7 @@ import org.apache.lucene.search.SortedSetSortField; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TotalHitCountCollector; +import org.apache.lucene.search.TotalHitCountCollectorManager; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.Lock; @@ -640,9 +640,8 @@ public void testTranslogMultipleOperationsSameDocument() throws IOException { recoverFromTranslog(recoveringEngine, translogHandler, Long.MAX_VALUE); recoveringEngine.refresh("test"); try (Engine.Searcher searcher = recoveringEngine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new MatchAllDocsQuery(), collector); - assertThat(collector.getTotalHits(), equalTo(operations.get(operations.size() - 1) instanceof Engine.Delete ? 0 : 1)); + Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager()); + assertThat(totalHits, equalTo(operations.get(operations.size() - 1) instanceof Engine.Delete ? 0 : 1)); } } } @@ -2009,16 +2008,20 @@ public void testConcurrentOutOfOrderDocsOnReplica() throws IOException, Interrup if (lastFieldValueDoc1 != null) { try (Engine.Searcher searcher = engine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new TermQuery(new Term("value", lastFieldValueDoc1)), collector); - assertThat(collector.getTotalHits(), equalTo(1)); + Integer totalHits = searcher.search( + new TermQuery(new Term("value", lastFieldValueDoc1)), + new TotalHitCountCollectorManager() + ); + assertThat(totalHits, equalTo(1)); } } if (lastFieldValueDoc2 != null) { try (Engine.Searcher searcher = engine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new TermQuery(new Term("value", lastFieldValueDoc2)), collector); - assertThat(collector.getTotalHits(), equalTo(1)); + Integer totalHits = searcher.search( + new TermQuery(new Term("value", lastFieldValueDoc2)), + new TotalHitCountCollectorManager() + ); + assertThat(totalHits, equalTo(1)); } } @@ -2244,9 +2247,11 @@ private int assertOpsOnPrimary(List ops, long currentOpVersion // first op and it failed. if (docDeleted == false && lastFieldValue != null) { try (Engine.Searcher searcher = engine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new TermQuery(new Term("value", lastFieldValue)), collector); - assertThat(collector.getTotalHits(), equalTo(1)); + Integer totalHits = searcher.search( + new TermQuery(new Term("value", lastFieldValue)), + new TotalHitCountCollectorManager() + ); + assertThat(totalHits, equalTo(1)); } } } @@ -2270,9 +2275,8 @@ private int assertOpsOnPrimary(List ops, long currentOpVersion assertVisibleCount(engine, docDeleted ? 0 : 1); if (docDeleted == false) { try (Engine.Searcher searcher = engine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new TermQuery(new Term("value", lastFieldValue)), collector); - assertThat(collector.getTotalHits(), equalTo(1)); + Integer totalHits = searcher.search(new TermQuery(new Term("value", lastFieldValue)), new TotalHitCountCollectorManager()); + assertThat(totalHits, equalTo(1)); } } return opsPerformed; @@ -2357,9 +2361,8 @@ public void testNonInternalVersioningOnPrimary() throws IOException { if (docDeleted == false) { logger.info("searching for [{}]", lastFieldValue); try (Engine.Searcher searcher = engine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new TermQuery(new Term("value", lastFieldValue)), collector); - assertThat(collector.getTotalHits(), equalTo(1)); + Integer totalHits = searcher.search(new TermQuery(new Term("value", lastFieldValue)), new TotalHitCountCollectorManager()); + assertThat(totalHits, equalTo(1)); } } } @@ -2375,9 +2378,8 @@ public void testVersioningPromotedReplica() throws IOException { final int opsOnPrimary = assertOpsOnPrimary(primaryOps, finalReplicaVersion, deletedOnReplica, replicaEngine); final long currentSeqNo = getSequenceID(replicaEngine, new Engine.Get(false, false, Term.toString(lastReplicaOp.uid()))).v1(); try (Engine.Searcher searcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new MatchAllDocsQuery(), collector); - if (collector.getTotalHits() > 0) { + Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager()); + if (totalHits > 0) { // last op wasn't delete assertThat(currentSeqNo, equalTo(finalReplicaSeqNo + opsOnPrimary)); } @@ -2400,9 +2402,8 @@ public void testConcurrentExternalVersioningOnPrimary() throws IOException, Inte assertVisibleCount(engine, lastFieldValue == null ? 0 : 1); if (lastFieldValue != null) { try (Engine.Searcher searcher = engine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new TermQuery(new Term("value", lastFieldValue)), collector); - assertThat(collector.getTotalHits(), equalTo(1)); + Integer totalHits = searcher.search(new TermQuery(new Term("value", lastFieldValue)), new TotalHitCountCollectorManager()); + assertThat(totalHits, equalTo(1)); } } } diff --git a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java index fff5dcb4bb80b..5e1296c354015 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java @@ -22,7 +22,7 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.TotalHitCountCollector; +import org.apache.lucene.search.TotalHitCountCollectorManager; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.tests.util.TestUtil; @@ -96,7 +96,6 @@ public void testAddingCancellationActions() throws IOException { } public void testCancellableCollector() throws IOException { - TotalHitCountCollector collector1 = new TotalHitCountCollector(); Runnable cancellation = () -> { throw new TaskCancelledException("cancelled"); }; ContextIndexSearcher searcher = new ContextIndexSearcher( reader, @@ -106,16 +105,15 @@ public void testCancellableCollector() throws IOException { true ); - searcher.search(new MatchAllDocsQuery(), collector1); - assertThat(collector1.getTotalHits(), equalTo(reader.numDocs())); + Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager()); + assertThat(totalHits, equalTo(reader.numDocs())); searcher.addQueryCancellation(cancellation); - expectThrows(TaskCancelledException.class, () -> searcher.search(new MatchAllDocsQuery(), collector1)); + expectThrows(TaskCancelledException.class, () -> searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager())); searcher.removeQueryCancellation(cancellation); - TotalHitCountCollector collector2 = new TotalHitCountCollector(); - searcher.search(new MatchAllDocsQuery(), collector2); - assertThat(collector2.getTotalHits(), equalTo(reader.numDocs())); + Integer totalHits2 = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager()); + assertThat(totalHits2, equalTo(reader.numDocs())); } public void testExitableDirectoryReader() throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index 3e4925bb97efd..0b5803e9887d6 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -42,7 +42,7 @@ import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Sort; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TotalHitCountCollector; +import org.apache.lucene.search.TotalHitCountCollectorManager; import org.apache.lucene.search.Weight; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; @@ -178,9 +178,8 @@ protected static void assertVisibleCount(Engine engine, int numDocs, boolean ref engine.refresh("test"); } try (Engine.Searcher searcher = engine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new MatchAllDocsQuery(), collector); - assertThat(collector.getTotalHits(), equalTo(numDocs)); + Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager()); + assertThat(totalHits, equalTo(numDocs)); } } @@ -971,9 +970,8 @@ protected static void assertVisibleCount(InternalEngine engine, int numDocs, boo engine.refresh("test"); } try (Engine.Searcher searcher = engine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new MatchAllDocsQuery(), collector); - assertThat(collector.getTotalHits(), equalTo(numDocs)); + Integer totalHits = searcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager()); + assertThat(totalHits, equalTo(numDocs)); } } @@ -1170,9 +1168,8 @@ public static void assertOpsOnReplica( assertVisibleCount(replicaEngine, lastFieldValue == null ? 0 : 1); if (lastFieldValue != null) { try (Engine.Searcher searcher = replicaEngine.acquireSearcher("test")) { - final TotalHitCountCollector collector = new TotalHitCountCollector(); - searcher.search(new TermQuery(new Term("value", lastFieldValue)), collector); - assertThat(collector.getTotalHits(), equalTo(1)); + Integer totalHits = searcher.search(new TermQuery(new Term("value", lastFieldValue)), new TotalHitCountCollectorManager()); + assertThat(totalHits, equalTo(1)); } } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java index ccf5d4e6de9f8..df64c4f87410a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java @@ -20,7 +20,7 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TotalHitCountCollector; +import org.apache.lucene.search.TotalHitCountCollectorManager; import org.apache.lucene.store.Directory; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.bytes.BytesArray; @@ -193,10 +193,8 @@ protected IndicesAccessControl getIndicesAccessControl() { int expectedHitCount = valuesHitCount[i]; logger.info("Going to verify hit count with query [{}] with expected total hits [{}]", parsedQuery.query(), expectedHitCount); - TotalHitCountCollector countCollector = new TotalHitCountCollector(); - indexSearcher.search(new MatchAllDocsQuery(), countCollector); - - assertThat(countCollector.getTotalHits(), equalTo(expectedHitCount)); + Integer totalHits = indexSearcher.search(new MatchAllDocsQuery(), new TotalHitCountCollectorManager()); + assertThat(totalHits, equalTo(expectedHitCount)); assertThat(wrappedDirectoryReader.numDocs(), equalTo(expectedHitCount)); } From d1644b37ca64f3a646032895895d5900eee805af Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Mon, 7 Oct 2024 13:09:54 +0200 Subject: [PATCH 121/194] Span term query to convert to match no docs when unmapped field is targeted (#113251) SpanTermQueryBuilder currently creates a valid SpanTermQuery against unmapped fields. In practice, if the field is unmapped, there won't be a match. This commit changes the toQuery impl to return a MatchNoDocsQuery instead like we do in similar scenarios. --- docs/changelog/113251.yaml | 5 ++ .../index/query/SpanTermQueryBuilder.java | 56 +++++++++---------- .../FieldMaskingSpanQueryBuilderTests.java | 5 +- .../query/SpanTermQueryBuilderTests.java | 28 +++------- 4 files changed, 41 insertions(+), 53 deletions(-) create mode 100644 docs/changelog/113251.yaml diff --git a/docs/changelog/113251.yaml b/docs/changelog/113251.yaml new file mode 100644 index 0000000000000..49167e6e4c915 --- /dev/null +++ b/docs/changelog/113251.yaml @@ -0,0 +1,5 @@ +pr: 113251 +summary: Span term query to convert to match no docs when unmapped field is targeted +area: Search +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java index aeffff28269dd..20874a736b1ec 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java @@ -10,7 +10,6 @@ package org.elasticsearch.index.query; import org.apache.lucene.index.Term; -import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.Query; @@ -19,8 +18,8 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.lucene.queries.SpanMatchNoDocsQuery; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentParser; @@ -76,40 +75,37 @@ public SpanTermQueryBuilder(StreamInput in) throws IOException { } @Override - protected SpanQuery doToQuery(SearchExecutionContext context) throws IOException { + protected Query doToQuery(SearchExecutionContext context) throws IOException { MappedFieldType mapper = context.getFieldType(fieldName); - Term term; if (mapper == null) { - term = new Term(fieldName, BytesRefs.toBytesRef(value)); - } else { - if (mapper.getTextSearchInfo().hasPositions() == false) { - throw new IllegalArgumentException( - "Span term query requires position data, but field " + fieldName + " was indexed without position data" - ); - } - Query termQuery = mapper.termQuery(value, context); - List termsList = new ArrayList<>(); - termQuery.visit(new QueryVisitor() { - @Override - public QueryVisitor getSubVisitor(BooleanClause.Occur occur, Query parent) { - if (occur == BooleanClause.Occur.MUST || occur == BooleanClause.Occur.FILTER) { - return this; - } - return EMPTY_VISITOR; + return new SpanMatchNoDocsQuery(fieldName, "unmapped field: " + fieldName); + } + if (mapper.getTextSearchInfo().hasPositions() == false) { + throw new IllegalArgumentException( + "Span term query requires position data, but field " + fieldName + " was indexed without position data" + ); + } + Query termQuery = mapper.termQuery(value, context); + List termsList = new ArrayList<>(); + termQuery.visit(new QueryVisitor() { + @Override + public QueryVisitor getSubVisitor(BooleanClause.Occur occur, Query parent) { + if (occur == BooleanClause.Occur.MUST || occur == BooleanClause.Occur.FILTER) { + return this; } + return EMPTY_VISITOR; + } - @Override - public void consumeTerms(Query query, Term... terms) { - termsList.addAll(Arrays.asList(terms)); - } - }); - if (termsList.size() != 1) { - // This is for safety, but we have called mapper.termQuery above: we really should get one and only one term from the query? - throw new IllegalArgumentException("Cannot extract a term from a query of type " + termQuery.getClass() + ": " + termQuery); + @Override + public void consumeTerms(Query query, Term... terms) { + termsList.addAll(Arrays.asList(terms)); } - term = termsList.get(0); + }); + if (termsList.size() != 1) { + // This is for safety, but we have called mapper.termQuery above: we really should get one and only one term from the query? + throw new IllegalArgumentException("Cannot extract a term from a query of type " + termQuery.getClass() + ": " + termQuery); } - return new SpanTermQuery(term); + return new SpanTermQuery(termsList.get(0)); } public static SpanTermQueryBuilder fromXContent(XContentParser parser) throws IOException, ParsingException { diff --git a/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java index bc2ba833536ff..3890b53e29ffb 100644 --- a/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java @@ -9,13 +9,12 @@ package org.elasticsearch.index.query; -import org.apache.lucene.index.Term; import org.apache.lucene.queries.spans.FieldMaskingSpanQuery; -import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; import org.elasticsearch.common.ParsingException; import org.elasticsearch.core.Strings; +import org.elasticsearch.lucene.queries.SpanMatchNoDocsQuery; import org.elasticsearch.test.AbstractQueryTestCase; import java.io.IOException; @@ -105,7 +104,7 @@ public void testJsonWithTopLevelBoost() throws IOException { } }""", NAME.getPreferredName()); Query q = parseQuery(json).toQuery(createSearchExecutionContext()); - assertEquals(new BoostQuery(new FieldMaskingSpanQuery(new SpanTermQuery(new Term("value", "foo")), "mapped_geo_shape"), 42.0f), q); + assertEquals(new BoostQuery(new FieldMaskingSpanQuery(new SpanMatchNoDocsQuery("value", null), "mapped_geo_shape"), 42.0f), q); } public void testJsonWithDeprecatedName() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java index f0f23d8539e13..c4a9267ff68a0 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanTermQueryBuilderTests.java @@ -15,9 +15,9 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.lucene.queries.SpanMatchNoDocsQuery; import org.elasticsearch.xcontent.json.JsonStringEncoder; import java.io.IOException; @@ -49,18 +49,16 @@ protected SpanTermQueryBuilder createQueryBuilder(String fieldName, Object value @Override protected void doAssertLuceneQuery(SpanTermQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException { - assertThat(query, instanceOf(SpanTermQuery.class)); - SpanTermQuery spanTermQuery = (SpanTermQuery) query; - - String expectedFieldName = expectedFieldName(queryBuilder.fieldName); - assertThat(spanTermQuery.getTerm().field(), equalTo(expectedFieldName)); - MappedFieldType mapper = context.getFieldType(queryBuilder.fieldName()); if (mapper != null) { + String expectedFieldName = expectedFieldName(queryBuilder.fieldName); + assertThat(query, instanceOf(SpanTermQuery.class)); + SpanTermQuery spanTermQuery = (SpanTermQuery) query; + assertThat(spanTermQuery.getTerm().field(), equalTo(expectedFieldName)); Term term = ((TermQuery) mapper.termQuery(queryBuilder.value(), null)).getTerm(); assertThat(spanTermQuery.getTerm(), equalTo(term)); } else { - assertThat(spanTermQuery.getTerm().bytes(), equalTo(BytesRefs.toBytesRef(queryBuilder.value()))); + assertThat(query, instanceOf(SpanMatchNoDocsQuery.class)); } } @@ -117,23 +115,13 @@ public void testParseFailsWithMultipleFields() throws IOException { assertEquals("[span_term] query doesn't support multiple fields, found [message1] and [message2]", e.getMessage()); } - public void testWithMetadataField() throws IOException { - SearchExecutionContext context = createSearchExecutionContext(); - for (String field : new String[] { "field1", "field2" }) { - SpanTermQueryBuilder spanTermQueryBuilder = new SpanTermQueryBuilder(field, "toto"); - Query query = spanTermQueryBuilder.toQuery(context); - Query expected = new SpanTermQuery(new Term(field, "toto")); - assertEquals(expected, query); - } - } - public void testWithBoost() throws IOException { SearchExecutionContext context = createSearchExecutionContext(); - for (String field : new String[] { "field1", "field2" }) { + for (String field : new String[] { TEXT_FIELD_NAME, TEXT_ALIAS_FIELD_NAME }) { SpanTermQueryBuilder spanTermQueryBuilder = new SpanTermQueryBuilder(field, "toto"); spanTermQueryBuilder.boost(10); Query query = spanTermQueryBuilder.toQuery(context); - Query expected = new BoostQuery(new SpanTermQuery(new Term(field, "toto")), 10); + Query expected = new BoostQuery(new SpanTermQuery(new Term(TEXT_FIELD_NAME, "toto")), 10); assertEquals(expected, query); } } From 71faa01b7fd34aa1d04e73e9138b579a2b3b06d2 Mon Sep 17 00:00:00 2001 From: Sean Story Date: Mon, 7 Oct 2024 06:28:25 -0500 Subject: [PATCH 122/194] fix typo - "english" is not a valid language code (#114166) This example request will succeed, but follow-up requests to run a sync on a connector with this language value will fail. --- docs/reference/connector/apis/create-connector-api.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/connector/apis/create-connector-api.asciidoc b/docs/reference/connector/apis/create-connector-api.asciidoc index a115eab8853c0..3ecef6d302732 100644 --- a/docs/reference/connector/apis/create-connector-api.asciidoc +++ b/docs/reference/connector/apis/create-connector-api.asciidoc @@ -116,7 +116,7 @@ PUT _connector/my-connector "name": "My Connector", "description": "My Connector to sync data to Elastic index from Google Drive", "service_type": "google_drive", - "language": "english" + "language": "en" } ---- From c65fc630bdbe70b1387e18ab136a5d9091614d58 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Mon, 7 Oct 2024 22:45:34 +1100 Subject: [PATCH 123/194] Mute org.elasticsearch.index.query.SpanGapQueryBuilderTests testToQuery #114218 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 317a4a41d5756..3f9e1b69ed876 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -371,6 +371,9 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/114187 - class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT issue: https://github.com/elastic/elasticsearch/issues/114194 +- class: org.elasticsearch.index.query.SpanGapQueryBuilderTests + method: testToQuery + issue: https://github.com/elastic/elasticsearch/issues/114218 # Examples: # From 86f21d190cf4551ae54bf55cb31fe4c8084ccc1b Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Mon, 7 Oct 2024 13:01:10 +0100 Subject: [PATCH 124/194] Fix stats test that checks for formatter tostring (#114215) Locale may or may not be there depending on version --- modules/aggregations/build.gradle | 1 - .../test/aggregations/stats_metric_fail_formatting.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/aggregations/build.gradle b/modules/aggregations/build.gradle index dc38364af16cb..1b3aac13b3608 100644 --- a/modules/aggregations/build.gradle +++ b/modules/aggregations/build.gradle @@ -48,5 +48,4 @@ dependencies { tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("aggregations/date_agg_per_day_of_week/Date aggregartion per day of week", "week-date behaviour has changed") - task.skipTest("aggregations/stats_metric_fail_formatting/fail formatting", "locale has changed") }) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml index 78811a1c46430..82371c973407c 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml @@ -30,7 +30,7 @@ setup: cluster_features: "gte_v8.15.0" reason: fixed in 8.15.0 - do: - catch: /Cannot format stat \[sum\] with format \[DocValueFormat.DateTime\(format\[date_hour_minute_second_millis\] locale\[en\], Z, MILLISECONDS\)\]/ + catch: /Cannot format stat \[sum\] with format \[DocValueFormat.DateTime\(format\[date_hour_minute_second_millis\] locale\[(en)?\], Z, MILLISECONDS\)\]/ search: index: test_date body: From bc8f9dc7f3882a461d4b89d69c7554a4cb3858ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Fred=C3=A9n?= <109296772+jfreden@users.noreply.github.com> Date: Mon, 7 Oct 2024 14:21:56 +0200 Subject: [PATCH 125/194] Change read-only suffix for file based role mappings (#114205) This changes the suffix for a role mapping to be `-read-only` instead of ` (read only)` since the name can be used (and will be by Kibana) to get a specific operator defined role mapping. This also removes the suffix when checking cluster state for the mapping. There is a slight risk that `-read-only` exists both in the native store and as `` in the file based mapping. I think that's very low risk, so didn't add any code to cover that case. --- .../RoleMappingFileSettingsIT.java | 18 ++++++++++----- .../role/TransportDeleteRoleAction.java | 22 +++++++++++++++++-- .../TransportGetRoleMappingsAction.java | 10 +++++++-- .../mapper/ClusterStateRoleMapper.java | 2 +- .../role/TransportDeleteRoleActionTests.java | 10 ++++++--- .../TransportPutRoleMappingActionTests.java | 11 ++++++++-- 6 files changed, 58 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java index 9d75baf10376c..19c18bf855b4e 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/RoleMappingFileSettingsIT.java @@ -359,9 +359,9 @@ public void testGetRoleMappings() throws Exception { Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(), containsInAnyOrder( "everyone_kibana", - "everyone_kibana " + RESERVED_ROLE_MAPPING_SUFFIX, + "everyone_kibana" + RESERVED_ROLE_MAPPING_SUFFIX, "_everyone_kibana", - "everyone_fleet " + RESERVED_ROLE_MAPPING_SUFFIX, + "everyone_fleet" + RESERVED_ROLE_MAPPING_SUFFIX, "zzz_mapping", "123_mapping" ) @@ -378,6 +378,16 @@ public void testGetRoleMappings() throws Exception { // it's possible to delete overlapping native role mapping assertTrue(client().execute(DeleteRoleMappingAction.INSTANCE, deleteRequest("everyone_kibana")).actionGet().isFound()); + // Fetch a specific file based role + request = new GetRoleMappingsRequest(); + request.setNames("everyone_kibana" + RESERVED_ROLE_MAPPING_SUFFIX); + response = client().execute(GetRoleMappingsAction.INSTANCE, request).get(); + assertTrue(response.hasMappings()); + assertThat( + Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(), + containsInAnyOrder("everyone_kibana" + RESERVED_ROLE_MAPPING_SUFFIX) + ); + savedClusterState = setupClusterStateListenerForCleanup(internalCluster().getMasterName()); writeJSONFile(internalCluster().getMasterName(), emptyJSON, logger, versionCounter); awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS); @@ -556,9 +566,7 @@ private static void assertGetResponseHasMappings(boolean readOnly, String... map assertThat( Arrays.stream(response.mappings()).map(ExpressionRoleMapping::getName).toList(), containsInAnyOrder( - Arrays.stream(mappings) - .map(mapping -> mapping + (readOnly ? " " + RESERVED_ROLE_MAPPING_SUFFIX : "")) - .toArray(String[]::new) + Arrays.stream(mappings).map(mapping -> mapping + (readOnly ? RESERVED_ROLE_MAPPING_SUFFIX : "")).toArray(String[]::new) ) ); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleAction.java index e8d248233415c..569cdc1a79fd9 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleAction.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; @@ -17,6 +18,7 @@ import org.elasticsearch.xpack.core.security.action.role.DeleteRoleAction; import org.elasticsearch.xpack.core.security.action.role.DeleteRoleRequest; import org.elasticsearch.xpack.core.security.action.role.DeleteRoleResponse; +import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper; import org.elasticsearch.xpack.security.authz.ReservedRoleNameChecker; import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; @@ -25,16 +27,20 @@ public class TransportDeleteRoleAction extends TransportAction { + if (clusterStateRoleMapper.hasMapping(request.name())) { + // Allow to delete a mapping with the same name in the native role mapping store as the file_settings namespace, but + // add a warning header to signal to the caller that this could be a problem. + HeaderWarning.addWarning( + "A read only role mapping with the same name [" + + request.name() + + "] has been previously been defined in a configuration file. " + + "The read only role mapping will still be active." + ); + } + return new DeleteRoleResponse(found); + })); } catch (Exception e) { logger.error((Supplier) () -> "failed to delete role [" + request.name() + "]", e); listener.onFailure(e); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java index d5bc923ab6dca..db0ee01af70e4 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/rolemapping/TransportGetRoleMappingsAction.java @@ -25,6 +25,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import static org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper.RESERVED_ROLE_MAPPING_SUFFIX; @@ -63,7 +64,12 @@ protected void doExecute(Task task, final GetRoleMappingsRequest request, final roleMappingStore.getRoleMappings(names, ActionListener.wrap(mappings -> { List combinedRoleMappings = Stream.concat( mappings.stream(), - clusterStateRoleMapper.getMappings(names) + clusterStateRoleMapper.getMappings(names == null ? null : names.stream().map(name -> { + // If a read-only role is fetched by name including suffix, remove suffix + return name.endsWith(RESERVED_ROLE_MAPPING_SUFFIX) + ? name.substring(0, name.length() - RESERVED_ROLE_MAPPING_SUFFIX.length()) + : name; + }).collect(Collectors.toSet())) .stream() .map(this::cloneAndMarkAsReadOnly) .sorted(Comparator.comparing(ExpressionRoleMapping::getName)) @@ -75,7 +81,7 @@ protected void doExecute(Task task, final GetRoleMappingsRequest request, final private ExpressionRoleMapping cloneAndMarkAsReadOnly(ExpressionRoleMapping mapping) { // Mark role mappings from cluster state as "read only" by adding a suffix to their name return new ExpressionRoleMapping( - mapping.getName() + " " + RESERVED_ROLE_MAPPING_SUFFIX, + mapping.getName() + RESERVED_ROLE_MAPPING_SUFFIX, mapping.getExpression(), mapping.getRoles(), mapping.getRoleTemplates(), diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapper.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapper.java index df8d46f133ac7..baea5970b4637 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapper.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapper.java @@ -47,7 +47,7 @@ public class ClusterStateRoleMapper extends AbstractRoleMapperClearRealmCache im * */ public static final String CLUSTER_STATE_ROLE_MAPPINGS_ENABLED = "xpack.security.authc.cluster_state_role_mappings.enabled"; - public static final String RESERVED_ROLE_MAPPING_SUFFIX = "(read only)"; + public static final String RESERVED_ROLE_MAPPING_SUFFIX = "-read-only-operator-config"; private static final Logger logger = LogManager.getLogger(ClusterStateRoleMapper.class); private final ScriptService scriptService; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleActionTests.java index 84e4dc402c767..d647088017dc1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/role/TransportDeleteRoleActionTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.core.security.action.role.DeleteRoleResponse; import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; +import org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper; import org.elasticsearch.xpack.security.authz.ReservedRoleNameChecker; import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; import org.junit.BeforeClass; @@ -66,7 +67,8 @@ public void testReservedRole() { mock(ActionFilters.class), rolesStore, transportService, - new ReservedRoleNameChecker.Default() + new ReservedRoleNameChecker.Default(), + mock(ClusterStateRoleMapper.class) ); DeleteRoleRequest request = new DeleteRoleRequest(); @@ -115,7 +117,8 @@ private void testValidRole(String roleName) { mock(ActionFilters.class), rolesStore, transportService, - new ReservedRoleNameChecker.Default() + new ReservedRoleNameChecker.Default(), + mock(ClusterStateRoleMapper.class) ); DeleteRoleRequest request = new DeleteRoleRequest(); @@ -168,7 +171,8 @@ public void testException() { mock(ActionFilters.class), rolesStore, transportService, - new ReservedRoleNameChecker.Default() + new ReservedRoleNameChecker.Default(), + mock(ClusterStateRoleMapper.class) ); DeleteRoleRequest request = new DeleteRoleRequest(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingActionTests.java index 91ac8b46e5a31..0bb3e7dd4ac3e 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/rolemapping/TransportPutRoleMappingActionTests.java @@ -29,6 +29,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import static org.elasticsearch.xpack.security.authc.support.mapper.ClusterStateRoleMapper.RESERVED_ROLE_MAPPING_SUFFIX; import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; @@ -97,12 +98,18 @@ public void testPutMappingWithInvalidName() { final FieldExpression expression = new FieldExpression("username", Collections.singletonList(new FieldExpression.FieldValue("*"))); IllegalArgumentException illegalArgumentException = expectThrows( IllegalArgumentException.class, - () -> put("anarchy (read only)", expression, "superuser", Collections.singletonMap("dumb", true)) + () -> put("anarchy" + RESERVED_ROLE_MAPPING_SUFFIX, expression, "superuser", Collections.singletonMap("dumb", true)) ); assertThat( illegalArgumentException.getMessage(), - equalTo("Invalid mapping name [anarchy (read only)]. [(read only)] is not an allowed suffix") + equalTo( + "Invalid mapping name [anarchy" + + RESERVED_ROLE_MAPPING_SUFFIX + + "]. [" + + RESERVED_ROLE_MAPPING_SUFFIX + + "] is not an allowed suffix" + ) ); } From 316b47dfafbe6364e52a9eda121b3e5107478087 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Mon, 7 Oct 2024 23:31:46 +1100 Subject: [PATCH 126/194] Mute org.elasticsearch.xpack.ilm.ExplainLifecycleIT testStepInfoPreservedOnAutoRetry #114220 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 3f9e1b69ed876..5754e6f6f415b 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -374,6 +374,9 @@ tests: - class: org.elasticsearch.index.query.SpanGapQueryBuilderTests method: testToQuery issue: https://github.com/elastic/elasticsearch/issues/114218 +- class: org.elasticsearch.xpack.ilm.ExplainLifecycleIT + method: testStepInfoPreservedOnAutoRetry + issue: https://github.com/elastic/elasticsearch/issues/114220 # Examples: # From c4731aaf08949174e00a29fffc3ada326b0df14d Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Mon, 7 Oct 2024 14:47:09 +0200 Subject: [PATCH 127/194] Improve performance of LongHash#removeAndAdd (#114199) Remove unnecessary remove and later add of the key. --- .../org/elasticsearch/common/util/AbstractHash.java | 12 +++++++++++- .../org/elasticsearch/common/util/BytesRefHash.java | 6 +++--- .../org/elasticsearch/common/util/Int3Hash.java | 6 +++--- .../org/elasticsearch/common/util/LongHash.java | 13 ++++++------- .../org/elasticsearch/common/util/LongLongHash.java | 6 +++--- 5 files changed, 26 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/util/AbstractHash.java b/server/src/main/java/org/elasticsearch/common/util/AbstractHash.java index e975f138e8b7d..687e9cb3fd9dc 100644 --- a/server/src/main/java/org/elasticsearch/common/util/AbstractHash.java +++ b/server/src/main/java/org/elasticsearch/common/util/AbstractHash.java @@ -32,7 +32,17 @@ public long id(long index) { return ids.get(index) - 1; } - protected final long id(long index, long id) { + /** + * Set the id provided key at 0 <= index <= capacity() . + */ + protected final void setId(long index, long id) { + ids.set(index, id + 1); + } + + /** + * Set the id provided key at 0 <= index <= capacity() and get the previous value or -1 if this slot is unused. + */ + protected final long getAndSetId(long index, long id) { return ids.getAndSet(index, id + 1) - 1; } diff --git a/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java b/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java index 48a810789308f..208d29edad71d 100644 --- a/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java +++ b/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java @@ -169,7 +169,7 @@ private long set(BytesRef key, int code, long id) { for (long index = slot;; index = nextSlot(index, mask)) { final long curId = id(index); if (curId == -1) { // means unset - id(index, id); + setId(index, id); append(id, key, code); ++size; return id; @@ -197,7 +197,7 @@ private void reset(int code, long id) { for (long index = slot;; index = nextSlot(index, mask)) { final long curId = id(index); if (curId == -1) { // means unset - id(index, id); + setId(index, id); break; } } @@ -223,7 +223,7 @@ public long add(BytesRef key) { @Override protected void removeAndAdd(long index) { - final long id = id(index, -1); + final long id = getAndSetId(index, -1); assert id >= 0; final int code = hashes.get(id); reset(code, id); diff --git a/server/src/main/java/org/elasticsearch/common/util/Int3Hash.java b/server/src/main/java/org/elasticsearch/common/util/Int3Hash.java index f2a4288bf7c9b..dc49b39a031a1 100644 --- a/server/src/main/java/org/elasticsearch/common/util/Int3Hash.java +++ b/server/src/main/java/org/elasticsearch/common/util/Int3Hash.java @@ -79,7 +79,7 @@ private long set(int key1, int key2, int key3, long id) { while (true) { final long curId = id(index); if (curId == -1) { // means unset - id(index, id); + setId(index, id); append(id, key1, key2, key3); ++size; return id; @@ -106,7 +106,7 @@ private void reset(int key1, int key2, int key3, long id) { while (true) { final long curId = id(index); if (curId == -1) { // means unset - id(index, id); + setId(index, id); append(id, key1, key2, key3); break; } @@ -130,7 +130,7 @@ public long add(int key1, int key2, int key3) { @Override protected void removeAndAdd(long index) { - final long id = id(index, -1); + final long id = getAndSetId(index, -1); assert id >= 0; long keyOffset = id * 3; final int key1 = keys.getAndSet(keyOffset, 0); diff --git a/server/src/main/java/org/elasticsearch/common/util/LongHash.java b/server/src/main/java/org/elasticsearch/common/util/LongHash.java index 4de6772d22447..0c681063c50b0 100644 --- a/server/src/main/java/org/elasticsearch/common/util/LongHash.java +++ b/server/src/main/java/org/elasticsearch/common/util/LongHash.java @@ -67,7 +67,7 @@ private long set(long key, long id) { for (long index = slot;; index = nextSlot(index, mask)) { final long curId = id(index); if (curId == -1) { // means unset - id(index, id); + setId(index, id); append(id, key); ++size; return id; @@ -82,13 +82,13 @@ private void append(long id, long key) { keys.set(id, key); } - private void reset(long key, long id) { + private void reset(long id) { + final long key = keys.get(id); final long slot = slot(hash(key), mask); for (long index = slot;; index = nextSlot(index, mask)) { final long curId = id(index); if (curId == -1) { // means unset - id(index, id); - append(id, key); + setId(index, id); break; } } @@ -109,10 +109,9 @@ public long add(long key) { @Override protected void removeAndAdd(long index) { - final long id = id(index, -1); + final long id = getAndSetId(index, -1); assert id >= 0; - final long key = keys.getAndSet(id, 0); - reset(key, id); + reset(id); } @Override diff --git a/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java b/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java index f160ecdaa7079..4a35cdaa9ab99 100644 --- a/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java +++ b/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java @@ -84,7 +84,7 @@ private long set(long key1, long key2, long id) { for (long index = slot;; index = nextSlot(index, mask)) { final long curId = id(index); if (curId == -1) { // means unset - id(index, id); + setId(index, id); append(id, key1, key2); ++size; return id; @@ -109,7 +109,7 @@ private void reset(long key1, long key2, long id) { for (long index = slot;; index = nextSlot(index, mask)) { final long curId = id(index); if (curId == -1) { // means unset - id(index, id); + setId(index, id); append(id, key1, key2); break; } @@ -132,7 +132,7 @@ public long add(long key1, long key2) { @Override protected void removeAndAdd(long index) { - final long id = id(index, -1); + final long id = getAndSetId(index, -1); assert id >= 0; long keyOffset = id * 2; final long key1 = keys.getAndSet(keyOffset, 0); From c1580639d4dac2d107794fd5c0d40b74b3e935e4 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Mon, 7 Oct 2024 14:48:00 +0200 Subject: [PATCH 128/194] Add getAndSet to Objectarray (#114200) This commit adds a getAndSet implementation to the ObjectArray API and changes the set method to return void. --- .../java/org/elasticsearch/common/util/BigArrays.java | 8 +++++++- .../org/elasticsearch/common/util/BigObjectArray.java | 10 +++++++++- .../common/util/LongObjectPagedHashMap.java | 6 +++--- .../org/elasticsearch/common/util/ObjectArray.java | 7 ++++++- .../common/util/ObjectObjectPagedHashMap.java | 6 +++--- .../org/elasticsearch/common/util/BigArraysTests.java | 1 + .../org/elasticsearch/common/util/MockBigArrays.java | 9 +++++++-- 7 files changed, 36 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/util/BigArrays.java b/server/src/main/java/org/elasticsearch/common/util/BigArrays.java index a33ee4c2edeac..4f0aae9380a01 100644 --- a/server/src/main/java/org/elasticsearch/common/util/BigArrays.java +++ b/server/src/main/java/org/elasticsearch/common/util/BigArrays.java @@ -452,7 +452,13 @@ public T get(long index) { } @Override - public T set(long index, T value) { + public void set(long index, T value) { + assert index >= 0 && index < size(); + array[(int) index] = value; + } + + @Override + public T getAndSet(long index, T value) { assert index >= 0 && index < size(); @SuppressWarnings("unchecked") T ret = (T) array[(int) index]; diff --git a/server/src/main/java/org/elasticsearch/common/util/BigObjectArray.java b/server/src/main/java/org/elasticsearch/common/util/BigObjectArray.java index 019ef341de8dc..95707b64b9a1e 100644 --- a/server/src/main/java/org/elasticsearch/common/util/BigObjectArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/BigObjectArray.java @@ -46,7 +46,15 @@ public T get(long index) { } @Override - public T set(long index, T value) { + public void set(long index, T value) { + final int pageIndex = pageIndex(index); + final int indexInPage = indexInPage(index); + final Object[] page = pages[pageIndex]; + page[indexInPage] = value; + } + + @Override + public T getAndSet(long index, T value) { final int pageIndex = pageIndex(index); final int indexInPage = indexInPage(index); final Object[] page = pages[pageIndex]; diff --git a/server/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java b/server/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java index 8ef3b568b0396..d955863caa091 100644 --- a/server/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java +++ b/server/src/main/java/org/elasticsearch/common/util/LongObjectPagedHashMap.java @@ -77,7 +77,7 @@ public T put(long key, T value) { */ public T remove(long key) { for (long i = slot(hash(key), mask);; i = nextSlot(i, mask)) { - final T previous = values.set(i, null); + final T previous = values.getAndSet(i, null); if (previous == null) { return null; } else if (keys.get(i) == key) { @@ -98,7 +98,7 @@ private T set(long key, T value) { throw new IllegalArgumentException("Null values are not supported"); } for (long i = slot(hash(key), mask);; i = nextSlot(i, mask)) { - final T previous = values.set(i, value); + final T previous = values.getAndSet(i, value); if (previous == null) { // slot was free keys.set(i, key); @@ -180,7 +180,7 @@ protected boolean used(long bucket) { @Override protected void removeAndAdd(long index) { final long key = keys.get(index); - final T value = values.set(index, null); + final T value = values.getAndSet(index, null); --size; final T removed = set(key, value); assert removed == null; diff --git a/server/src/main/java/org/elasticsearch/common/util/ObjectArray.java b/server/src/main/java/org/elasticsearch/common/util/ObjectArray.java index 24b010eb62aad..034b7b3c85692 100644 --- a/server/src/main/java/org/elasticsearch/common/util/ObjectArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/ObjectArray.java @@ -19,9 +19,14 @@ public interface ObjectArray extends BigArray { */ T get(long index); + /** + * Set a value at the given index. + */ + void set(long index, T value); + /** * Set a value at the given index and return the previous value. */ - T set(long index, T value); + T getAndSet(long index, T value); } diff --git a/server/src/main/java/org/elasticsearch/common/util/ObjectObjectPagedHashMap.java b/server/src/main/java/org/elasticsearch/common/util/ObjectObjectPagedHashMap.java index 58722c18b2434..298f910d65a9f 100644 --- a/server/src/main/java/org/elasticsearch/common/util/ObjectObjectPagedHashMap.java +++ b/server/src/main/java/org/elasticsearch/common/util/ObjectObjectPagedHashMap.java @@ -82,7 +82,7 @@ public V put(K key, V value) { public V remove(K key) { final long slot = slot(key.hashCode(), mask); for (long index = slot;; index = nextSlot(index, mask)) { - final V previous = values.set(index, null); + final V previous = values.getAndSet(index, null); if (previous == null) { return null; } else if (keys.get(index).equals(key)) { @@ -104,7 +104,7 @@ private V set(K key, int code, V value) { assert size < maxSize; final long slot = slot(code, mask); for (long index = slot;; index = nextSlot(index, mask)) { - final V previous = values.set(index, value); + final V previous = values.getAndSet(index, value); if (previous == null) { // slot was free keys.set(index, key); @@ -186,7 +186,7 @@ protected boolean used(long bucket) { @Override protected void removeAndAdd(long index) { final K key = keys.get(index); - final V value = values.set(index, null); + final V value = values.getAndSet(index, null); --size; final V removed = set(key, key.hashCode(), value); assert removed == null; diff --git a/server/src/test/java/org/elasticsearch/common/util/BigArraysTests.java b/server/src/test/java/org/elasticsearch/common/util/BigArraysTests.java index fc3096de8b8c0..3ef010f760ab6 100644 --- a/server/src/test/java/org/elasticsearch/common/util/BigArraysTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/BigArraysTests.java @@ -135,6 +135,7 @@ public void testObjectArrayGrowth() { ref[i] = randomFrom(pool); array = bigArrays.grow(array, i + 1); array.set(i, ref[i]); + assertEquals(ref[i], array.getAndSet(i, ref[i])); } for (int i = 0; i < totalLen; ++i) { assertSame(ref[i], array.get(i)); diff --git a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java index 1cc6043e0be17..de87772d5ae82 100644 --- a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java +++ b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java @@ -695,8 +695,13 @@ public T get(long index) { } @Override - public T set(long index, T value) { - return in.set(index, value); + public void set(long index, T value) { + in.set(index, value); + } + + @Override + public T getAndSet(long index, T value) { + return in.getAndSet(index, value); } @Override From 1292580c0333a4f7402cb0d133f2a991a5302d1b Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Mon, 7 Oct 2024 14:52:42 +0200 Subject: [PATCH 129/194] [DOCS] Lookup runtime fields are now GA (#114221) --- docs/reference/mapping/runtime.asciidoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/reference/mapping/runtime.asciidoc b/docs/reference/mapping/runtime.asciidoc index 190081fa801b4..1ee1194279061 100644 --- a/docs/reference/mapping/runtime.asciidoc +++ b/docs/reference/mapping/runtime.asciidoc @@ -821,8 +821,6 @@ address. [[lookup-runtime-fields]] ==== Retrieve fields from related indices -experimental[] - The <> parameter on the `_search` API can also be used to retrieve fields from the related indices via runtime fields with a type of `lookup`. From dcbbbabd3d1b78bd4658ab69e85e7c55bcc8fced Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 8 Oct 2024 00:10:38 +1100 Subject: [PATCH 130/194] Mute org.elasticsearch.xpack.inference.services.openai.OpenAiServiceTests testInfer_StreamRequest #114232 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 5754e6f6f415b..7764c0f8865d4 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -377,6 +377,9 @@ tests: - class: org.elasticsearch.xpack.ilm.ExplainLifecycleIT method: testStepInfoPreservedOnAutoRetry issue: https://github.com/elastic/elasticsearch/issues/114220 +- class: org.elasticsearch.xpack.inference.services.openai.OpenAiServiceTests + method: testInfer_StreamRequest + issue: https://github.com/elastic/elasticsearch/issues/114232 # Examples: # From 9368bbeb193b1d2e7f33af90909311c86fc9a948 Mon Sep 17 00:00:00 2001 From: Larry Gregory Date: Mon, 7 Oct 2024 09:29:58 -0400 Subject: [PATCH 131/194] Adds manage_inference cluster privilege to kibana_system role (#114051) * Adds manage_inference cluster privilege to kibana_system role * Fix * this is what I get for not using a real IDE * Remove whitespace --------- Co-authored-by: Elastic Machine --- .../authz/store/KibanaOwnedReservedRoleDescriptors.java | 2 ++ .../core/security/authz/store/ReservedRolesStoreTests.java | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java index 91a8ea4d368ff..6c28c6f3053ab 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @@ -65,6 +65,8 @@ static RoleDescriptor kibanaSystem(String name) { new String[] { "monitor", "manage_index_templates", + // manage_inference required for Kibana's inference plugin to setup an ELSER endpoint. + "manage_inference", MonitoringBulkAction.NAME, "manage_saml", "manage_token", diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index acf530fb7c5cc..26b306d6f1334 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -436,6 +436,11 @@ public void testKibanaSystemRole() { assertThat(kibanaRole.cluster().check(ClusterUpdateSettingsAction.NAME, request, authentication), is(false)); assertThat(kibanaRole.cluster().check(MonitoringBulkAction.NAME, request, authentication), is(true)); + // Inference + assertTrue(kibanaRole.cluster().check("cluster:admin/xpack/inference/get", request, authentication)); + assertTrue(kibanaRole.cluster().check("cluster:admin/xpack/inference/put", request, authentication)); + assertTrue(kibanaRole.cluster().check("cluster:admin/xpack/inference/delete", request, authentication)); + // Enrich assertThat(kibanaRole.cluster().check("cluster:admin/xpack/enrich/put", request, authentication), is(true)); assertThat(kibanaRole.cluster().check("cluster:admin/xpack/enrich/execute", request, authentication), is(true)); From fe36a4543d05fff2a74e70a6cfcdb9dacdea5438 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Mon, 7 Oct 2024 07:34:05 -0600 Subject: [PATCH 132/194] Make randomInstantBetween return in range [minInstant, maxInstant] (#114177) randomInstantBetween can produce a result which is not within the [minInstant, maxInstant] range. This occurs when the epoch second picked matches the min bound and the nanos are below the min nanos, or the second picked matches the max bound seconds and nanos are above the max bound nanos. This change fixes the function by setting a bound on which nano values can be picked if the min or max epoch second value is picked. --- docs/changelog/114177.yaml | 5 +++++ .../src/main/java/org/elasticsearch/test/ESTestCase.java | 9 +++++---- 2 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 docs/changelog/114177.yaml diff --git a/docs/changelog/114177.yaml b/docs/changelog/114177.yaml new file mode 100644 index 0000000000000..d68486469d797 --- /dev/null +++ b/docs/changelog/114177.yaml @@ -0,0 +1,5 @@ +pr: 114177 +summary: "Make `randomInstantBetween` always return value in range [minInstant, `maxInstant]`" +area: Infra/Metrics +type: bug +issues: [] diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 068a666d78d74..31c8e5bc3d457 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -900,10 +900,11 @@ public static long randomLongBetween(long min, long max) { * @return a random instant between a min and a max value with a random nanosecond precision */ public static Instant randomInstantBetween(Instant minInstant, Instant maxInstant) { - return Instant.ofEpochSecond( - randomLongBetween(minInstant.getEpochSecond(), maxInstant.getEpochSecond()), - randomLongBetween(0, 999999999) - ); + long epochSecond = randomLongBetween(minInstant.getEpochSecond(), maxInstant.getEpochSecond()); + long minNanos = epochSecond == minInstant.getEpochSecond() ? minInstant.getNano() : 0; + long maxNanos = epochSecond == maxInstant.getEpochSecond() ? maxInstant.getNano() : 999999999; + long nanos = randomLongBetween(minNanos, maxNanos); + return Instant.ofEpochSecond(epochSecond, nanos); } /** From e1bba9b390ada7510d52e9c9460843884df42961 Mon Sep 17 00:00:00 2001 From: moxarth-elastic <96762084+moxarth-elastic@users.noreply.github.com> Date: Mon, 7 Oct 2024 19:18:32 +0530 Subject: [PATCH 133/194] [Zoom] Update existing scopes with granular scopes (#113994) --- .../connector/docs/connectors-zoom.asciidoc | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/docs/reference/connector/docs/connectors-zoom.asciidoc b/docs/reference/connector/docs/connectors-zoom.asciidoc index d01b9c2be0368..d945a0aec3da1 100644 --- a/docs/reference/connector/docs/connectors-zoom.asciidoc +++ b/docs/reference/connector/docs/connectors-zoom.asciidoc @@ -63,18 +63,22 @@ To connect to Zoom you need to https://developers.zoom.us/docs/internal-apps/s2s 6. Click on the "Create" button to create the app registration. 7. After the registration is complete, you will be redirected to the app's overview page. Take note of the "App Credentials" value, as you'll need it later. 8. Navigate to the "Scopes" section and click on the "Add Scopes" button. -9. The following scopes need to be added to the app. +9. The following granular scopes need to be added to the app. + [source,bash] ---- -user:read:admin -meeting:read:admin -chat_channel:read:admin -recording:read:admin -chat_message:read:admin -report:read:admin +user:read:list_users:admin +meeting:read:list_meetings:admin +meeting:read:list_past_participants:admin +cloud_recording:read:list_user_recordings:admin +team_chat:read:list_user_channels:admin +team_chat:read:list_user_messages:admin ---- - +[NOTE] +==== +The connector requires a minimum scope of `user:read:list_users:admin` to ingest data into Elasticsearch. +==== ++ 10. Click on the "Done" button to add the selected scopes to your app. 11. Navigate to the "Activation" section and input the necessary information to activate the app. @@ -220,18 +224,22 @@ To connect to Zoom you need to https://developers.zoom.us/docs/internal-apps/s2s 6. Click on the "Create" button to create the app registration. 7. After the registration is complete, you will be redirected to the app's overview page. Take note of the "App Credentials" value, as you'll need it later. 8. Navigate to the "Scopes" section and click on the "Add Scopes" button. -9. The following scopes need to be added to the app. +9. The following granular scopes need to be added to the app. + [source,bash] ---- -user:read:admin -meeting:read:admin -chat_channel:read:admin -recording:read:admin -chat_message:read:admin -report:read:admin +user:read:list_users:admin +meeting:read:list_meetings:admin +meeting:read:list_past_participants:admin +cloud_recording:read:list_user_recordings:admin +team_chat:read:list_user_channels:admin +team_chat:read:list_user_messages:admin ---- - +[NOTE] +==== +The connector requires a minimum scope of `user:read:list_users:admin` to ingest data into Elasticsearch. +==== ++ 10. Click on the "Done" button to add the selected scopes to your app. 11. Navigate to the "Activation" section and input the necessary information to activate the app. From 7decd52132fbdf8e2e7a8b093255434e63e77a0c Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:08:23 +0100 Subject: [PATCH 134/194] Allow incubating Panama Vector in simdvec, and add vectorized ipByteBin (#112933) Add support for vectorized ipByteBin. The structure of the implementation and loading framework mirror that of Lucene, but is simplified by avoiding reflective loading since ES has support for a MRJar section for 21. For now, we just disable warnings-as-errors in this small sourceset, since -Xlint:-incubating is only support since JDK 22. The number of source files is small here. Will investigate how to assert that just the single incubating warning is emitted by javac, at a later point. --- docs/changelog/112933.yaml | 5 + libs/simdvec/build.gradle | 13 ++ libs/simdvec/src/main/java/module-info.java | 1 + .../elasticsearch/simdvec/ESVectorUtil.java | 27 ++++ .../DefaultESVectorUtilSupport.java | 39 +++++ .../DefaultESVectorizationProvider.java | 23 +++ .../vectorization/ESVectorUtilSupport.java | 17 ++ .../ESVectorizationProvider.java | 38 +++++ .../ESVectorizationProvider.java | 87 ++++++++++ .../PanamaESVectorUtilSupport.java | 153 ++++++++++++++++++ .../PanamaESVectorizationProvider.java | 24 +++ .../simdvec/ESVectorUtilTests.java | 130 +++++++++++++++ .../vectorization/BaseVectorizationTests.java | 29 ++++ 13 files changed, 586 insertions(+) create mode 100644 docs/changelog/112933.yaml create mode 100644 libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java create mode 100644 libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorUtilSupport.java create mode 100644 libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorizationProvider.java create mode 100644 libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorUtilSupport.java create mode 100644 libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java create mode 100644 libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java create mode 100644 libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorUtilSupport.java create mode 100644 libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java create mode 100644 libs/simdvec/src/test/java/org/elasticsearch/simdvec/ESVectorUtilTests.java create mode 100644 libs/simdvec/src/test/java/org/elasticsearch/simdvec/internal/vectorization/BaseVectorizationTests.java diff --git a/docs/changelog/112933.yaml b/docs/changelog/112933.yaml new file mode 100644 index 0000000000000..222cd5aadf739 --- /dev/null +++ b/docs/changelog/112933.yaml @@ -0,0 +1,5 @@ +pr: 112933 +summary: "Allow incubating Panama Vector in simdvec, and add vectorized `ipByteBin`" +area: Search +type: enhancement +issues: [] diff --git a/libs/simdvec/build.gradle b/libs/simdvec/build.gradle index 5a523a19d4b68..8b676a15038c1 100644 --- a/libs/simdvec/build.gradle +++ b/libs/simdvec/build.gradle @@ -23,6 +23,19 @@ dependencies { } } +tasks.named("compileMain21Java").configure { + options.compilerArgs << '--add-modules=jdk.incubator.vector' + // we remove Werror, since incubating suppression (-Xlint:-incubating) + // is only support since JDK 22 + options.compilerArgs -= '-Werror' +} + +test { + if (JavaVersion.current().majorVersion.toInteger() >= 21) { + jvmArgs '--add-modules=jdk.incubator.vector' + } +} + tasks.withType(CheckForbiddenApisTask).configureEach { replaceSignatureFiles 'jdk-signatures' } diff --git a/libs/simdvec/src/main/java/module-info.java b/libs/simdvec/src/main/java/module-info.java index 64e685ba3cbb5..44f6e39d5dbab 100644 --- a/libs/simdvec/src/main/java/module-info.java +++ b/libs/simdvec/src/main/java/module-info.java @@ -10,6 +10,7 @@ module org.elasticsearch.simdvec { requires org.elasticsearch.nativeaccess; requires org.apache.lucene.core; + requires org.elasticsearch.logging; exports org.elasticsearch.simdvec to org.elasticsearch.server; } diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java new file mode 100644 index 0000000000000..91193d5fa6eaf --- /dev/null +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/ESVectorUtil.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec; + +import org.elasticsearch.simdvec.internal.vectorization.ESVectorUtilSupport; +import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider; + +import static org.elasticsearch.simdvec.internal.vectorization.ESVectorUtilSupport.B_QUERY; + +public class ESVectorUtil { + + private static final ESVectorUtilSupport IMPL = ESVectorizationProvider.getInstance().getVectorUtilSupport(); + + public static long ipByteBinByte(byte[] q, byte[] d) { + if (q.length != d.length * B_QUERY) { + throw new IllegalArgumentException("vector dimensions incompatible: " + q.length + "!= " + B_QUERY + " x " + d.length); + } + return IMPL.ipByteBinByte(q, d); + } +} diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorUtilSupport.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorUtilSupport.java new file mode 100644 index 0000000000000..4a08096119d6a --- /dev/null +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorUtilSupport.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +import org.apache.lucene.util.BitUtil; + +final class DefaultESVectorUtilSupport implements ESVectorUtilSupport { + + DefaultESVectorUtilSupport() {} + + @Override + public long ipByteBinByte(byte[] q, byte[] d) { + return ipByteBinByteImpl(q, d); + } + + public static long ipByteBinByteImpl(byte[] q, byte[] d) { + long ret = 0; + int size = d.length; + for (int i = 0; i < B_QUERY; i++) { + int r = 0; + long subRet = 0; + for (final int upperBound = d.length & -Integer.BYTES; r < upperBound; r += Integer.BYTES) { + subRet += Integer.bitCount((int) BitUtil.VH_NATIVE_INT.get(q, i * size + r) & (int) BitUtil.VH_NATIVE_INT.get(d, r)); + } + for (; r < d.length; r++) { + subRet += Integer.bitCount((q[i * size + r] & d[r]) & 0xFF); + } + ret += subRet << i; + } + return ret; + } +} diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorizationProvider.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorizationProvider.java new file mode 100644 index 0000000000000..6c0f7ed146b86 --- /dev/null +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorizationProvider.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +final class DefaultESVectorizationProvider extends ESVectorizationProvider { + private final ESVectorUtilSupport vectorUtilSupport; + + DefaultESVectorizationProvider() { + vectorUtilSupport = new DefaultESVectorUtilSupport(); + } + + @Override + public ESVectorUtilSupport getVectorUtilSupport() { + return vectorUtilSupport; + } +} diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorUtilSupport.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorUtilSupport.java new file mode 100644 index 0000000000000..d7611173ca693 --- /dev/null +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorUtilSupport.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +public interface ESVectorUtilSupport { + + short B_QUERY = 4; + + long ipByteBinByte(byte[] q, byte[] d); +} diff --git a/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java new file mode 100644 index 0000000000000..e541c10e145bf --- /dev/null +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +import java.util.Objects; + +public abstract class ESVectorizationProvider { + + public static ESVectorizationProvider getInstance() { + return Objects.requireNonNull( + ESVectorizationProvider.Holder.INSTANCE, + "call to getInstance() from subclass of VectorizationProvider" + ); + } + + ESVectorizationProvider() {} + + public abstract ESVectorUtilSupport getVectorUtilSupport(); + + // visible for tests + static ESVectorizationProvider lookup(boolean testMode) { + return new DefaultESVectorizationProvider(); + } + + /** This static holder class prevents classloading deadlock. */ + private static final class Holder { + private Holder() {} + + static final ESVectorizationProvider INSTANCE = lookup(false); + } +} diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java new file mode 100644 index 0000000000000..5b7aab7ddfa48 --- /dev/null +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/ESVectorizationProvider.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +import org.apache.lucene.util.Constants; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; + +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; + +public abstract class ESVectorizationProvider { + + protected static final Logger logger = LogManager.getLogger(ESVectorizationProvider.class); + + public static ESVectorizationProvider getInstance() { + return Objects.requireNonNull( + ESVectorizationProvider.Holder.INSTANCE, + "call to getInstance() from subclass of VectorizationProvider" + ); + } + + ESVectorizationProvider() {} + + public abstract ESVectorUtilSupport getVectorUtilSupport(); + + // visible for tests + static ESVectorizationProvider lookup(boolean testMode) { + final int runtimeVersion = Runtime.version().feature(); + assert runtimeVersion >= 21; + if (runtimeVersion <= 23) { + // only use vector module with Hotspot VM + if (Constants.IS_HOTSPOT_VM == false) { + logger.warn("Java runtime is not using Hotspot VM; Java vector incubator API can't be enabled."); + return new DefaultESVectorizationProvider(); + } + // is the incubator module present and readable (JVM providers may to exclude them or it is + // build with jlink) + final var vectorMod = lookupVectorModule(); + if (vectorMod.isEmpty()) { + logger.warn( + "Java vector incubator module is not readable. " + + "For optimal vector performance, pass '--add-modules jdk.incubator.vector' to enable Vector API." + ); + return new DefaultESVectorizationProvider(); + } + vectorMod.ifPresent(ESVectorizationProvider.class.getModule()::addReads); + var impl = new PanamaESVectorizationProvider(); + logger.info( + String.format( + Locale.ENGLISH, + "Java vector incubator API enabled; uses preferredBitSize=%d", + PanamaESVectorUtilSupport.VECTOR_BITSIZE + ) + ); + return impl; + } else { + logger.warn( + "You are running with unsupported Java " + + runtimeVersion + + ". To make full use of the Vector API, please update Elasticsearch." + ); + } + return new DefaultESVectorizationProvider(); + } + + private static Optional lookupVectorModule() { + return Optional.ofNullable(ESVectorizationProvider.class.getModule().getLayer()) + .orElse(ModuleLayer.boot()) + .findModule("jdk.incubator.vector"); + } + + /** This static holder class prevents classloading deadlock. */ + private static final class Holder { + private Holder() {} + + static final ESVectorizationProvider INSTANCE = lookup(false); + } +} diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorUtilSupport.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorUtilSupport.java new file mode 100644 index 0000000000000..0e5827d046736 --- /dev/null +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorUtilSupport.java @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +import jdk.incubator.vector.ByteVector; +import jdk.incubator.vector.IntVector; +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.VectorOperators; +import jdk.incubator.vector.VectorShape; +import jdk.incubator.vector.VectorSpecies; + +import org.apache.lucene.util.Constants; + +public final class PanamaESVectorUtilSupport implements ESVectorUtilSupport { + + static final int VECTOR_BITSIZE; + + /** Whether integer vectors can be trusted to actually be fast. */ + static final boolean HAS_FAST_INTEGER_VECTORS; + + static { + // default to platform supported bitsize + VECTOR_BITSIZE = VectorShape.preferredShape().vectorBitSize(); + + // hotspot misses some SSE intrinsics, workaround it + // to be fair, they do document this thing only works well with AVX2/AVX3 and Neon + boolean isAMD64withoutAVX2 = Constants.OS_ARCH.equals("amd64") && VECTOR_BITSIZE < 256; + HAS_FAST_INTEGER_VECTORS = isAMD64withoutAVX2 == false; + } + + @Override + public long ipByteBinByte(byte[] q, byte[] d) { + // 128 / 8 == 16 + if (d.length >= 16 && HAS_FAST_INTEGER_VECTORS) { + if (VECTOR_BITSIZE >= 256) { + return ipByteBin256(q, d); + } else if (VECTOR_BITSIZE == 128) { + return ipByteBin128(q, d); + } + } + return DefaultESVectorUtilSupport.ipByteBinByteImpl(q, d); + } + + private static final VectorSpecies BYTE_SPECIES_128 = ByteVector.SPECIES_128; + private static final VectorSpecies BYTE_SPECIES_256 = ByteVector.SPECIES_256; + + static long ipByteBin256(byte[] q, byte[] d) { + long subRet0 = 0; + long subRet1 = 0; + long subRet2 = 0; + long subRet3 = 0; + int i = 0; + + if (d.length >= ByteVector.SPECIES_256.vectorByteSize() * 2) { + int limit = ByteVector.SPECIES_256.loopBound(d.length); + var sum0 = LongVector.zero(LongVector.SPECIES_256); + var sum1 = LongVector.zero(LongVector.SPECIES_256); + var sum2 = LongVector.zero(LongVector.SPECIES_256); + var sum3 = LongVector.zero(LongVector.SPECIES_256); + for (; i < limit; i += ByteVector.SPECIES_256.length()) { + var vq0 = ByteVector.fromArray(BYTE_SPECIES_256, q, i).reinterpretAsLongs(); + var vq1 = ByteVector.fromArray(BYTE_SPECIES_256, q, i + d.length).reinterpretAsLongs(); + var vq2 = ByteVector.fromArray(BYTE_SPECIES_256, q, i + d.length * 2).reinterpretAsLongs(); + var vq3 = ByteVector.fromArray(BYTE_SPECIES_256, q, i + d.length * 3).reinterpretAsLongs(); + var vd = ByteVector.fromArray(BYTE_SPECIES_256, d, i).reinterpretAsLongs(); + sum0 = sum0.add(vq0.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum1 = sum1.add(vq1.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum2 = sum2.add(vq2.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum3 = sum3.add(vq3.and(vd).lanewise(VectorOperators.BIT_COUNT)); + } + subRet0 += sum0.reduceLanes(VectorOperators.ADD); + subRet1 += sum1.reduceLanes(VectorOperators.ADD); + subRet2 += sum2.reduceLanes(VectorOperators.ADD); + subRet3 += sum3.reduceLanes(VectorOperators.ADD); + } + + if (d.length - i >= ByteVector.SPECIES_128.vectorByteSize()) { + var sum0 = LongVector.zero(LongVector.SPECIES_128); + var sum1 = LongVector.zero(LongVector.SPECIES_128); + var sum2 = LongVector.zero(LongVector.SPECIES_128); + var sum3 = LongVector.zero(LongVector.SPECIES_128); + int limit = ByteVector.SPECIES_128.loopBound(d.length); + for (; i < limit; i += ByteVector.SPECIES_128.length()) { + var vq0 = ByteVector.fromArray(BYTE_SPECIES_128, q, i).reinterpretAsLongs(); + var vq1 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + d.length).reinterpretAsLongs(); + var vq2 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + d.length * 2).reinterpretAsLongs(); + var vq3 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + d.length * 3).reinterpretAsLongs(); + var vd = ByteVector.fromArray(BYTE_SPECIES_128, d, i).reinterpretAsLongs(); + sum0 = sum0.add(vq0.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum1 = sum1.add(vq1.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum2 = sum2.add(vq2.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum3 = sum3.add(vq3.and(vd).lanewise(VectorOperators.BIT_COUNT)); + } + subRet0 += sum0.reduceLanes(VectorOperators.ADD); + subRet1 += sum1.reduceLanes(VectorOperators.ADD); + subRet2 += sum2.reduceLanes(VectorOperators.ADD); + subRet3 += sum3.reduceLanes(VectorOperators.ADD); + } + // tail as bytes + for (; i < d.length; i++) { + subRet0 += Integer.bitCount((q[i] & d[i]) & 0xFF); + subRet1 += Integer.bitCount((q[i + d.length] & d[i]) & 0xFF); + subRet2 += Integer.bitCount((q[i + 2 * d.length] & d[i]) & 0xFF); + subRet3 += Integer.bitCount((q[i + 3 * d.length] & d[i]) & 0xFF); + } + return subRet0 + (subRet1 << 1) + (subRet2 << 2) + (subRet3 << 3); + } + + public static long ipByteBin128(byte[] q, byte[] d) { + long subRet0 = 0; + long subRet1 = 0; + long subRet2 = 0; + long subRet3 = 0; + int i = 0; + + var sum0 = IntVector.zero(IntVector.SPECIES_128); + var sum1 = IntVector.zero(IntVector.SPECIES_128); + var sum2 = IntVector.zero(IntVector.SPECIES_128); + var sum3 = IntVector.zero(IntVector.SPECIES_128); + int limit = ByteVector.SPECIES_128.loopBound(d.length); + for (; i < limit; i += ByteVector.SPECIES_128.length()) { + var vd = ByteVector.fromArray(BYTE_SPECIES_128, d, i).reinterpretAsInts(); + var vq0 = ByteVector.fromArray(BYTE_SPECIES_128, q, i).reinterpretAsInts(); + var vq1 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + d.length).reinterpretAsInts(); + var vq2 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + d.length * 2).reinterpretAsInts(); + var vq3 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + d.length * 3).reinterpretAsInts(); + sum0 = sum0.add(vd.and(vq0).lanewise(VectorOperators.BIT_COUNT)); + sum1 = sum1.add(vd.and(vq1).lanewise(VectorOperators.BIT_COUNT)); + sum2 = sum2.add(vd.and(vq2).lanewise(VectorOperators.BIT_COUNT)); + sum3 = sum3.add(vd.and(vq3).lanewise(VectorOperators.BIT_COUNT)); + } + subRet0 += sum0.reduceLanes(VectorOperators.ADD); + subRet1 += sum1.reduceLanes(VectorOperators.ADD); + subRet2 += sum2.reduceLanes(VectorOperators.ADD); + subRet3 += sum3.reduceLanes(VectorOperators.ADD); + // tail as bytes + for (; i < d.length; i++) { + int dValue = d[i]; + subRet0 += Integer.bitCount((dValue & q[i]) & 0xFF); + subRet1 += Integer.bitCount((dValue & q[i + d.length]) & 0xFF); + subRet2 += Integer.bitCount((dValue & q[i + 2 * d.length]) & 0xFF); + subRet3 += Integer.bitCount((dValue & q[i + 3 * d.length]) & 0xFF); + } + return subRet0 + (subRet1 << 1) + (subRet2 << 2) + (subRet3 << 3); + } +} diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java new file mode 100644 index 0000000000000..62d25d79487ed --- /dev/null +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +final class PanamaESVectorizationProvider extends ESVectorizationProvider { + + private final ESVectorUtilSupport vectorUtilSupport; + + PanamaESVectorizationProvider() { + vectorUtilSupport = new PanamaESVectorUtilSupport(); + } + + @Override + public ESVectorUtilSupport getVectorUtilSupport() { + return vectorUtilSupport; + } +} diff --git a/libs/simdvec/src/test/java/org/elasticsearch/simdvec/ESVectorUtilTests.java b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/ESVectorUtilTests.java new file mode 100644 index 0000000000000..0dbc41c0c1055 --- /dev/null +++ b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/ESVectorUtilTests.java @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec; + +import org.elasticsearch.simdvec.internal.vectorization.BaseVectorizationTests; +import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider; + +import java.util.Arrays; + +import static org.elasticsearch.simdvec.internal.vectorization.ESVectorUtilSupport.B_QUERY; + +public class ESVectorUtilTests extends BaseVectorizationTests { + + static final ESVectorizationProvider defaultedProvider = BaseVectorizationTests.defaultProvider(); + static final ESVectorizationProvider defOrPanamaProvider = BaseVectorizationTests.maybePanamaProvider(); + + public void testIpByteBinInvariants() { + int iterations = atLeast(10); + for (int i = 0; i < iterations; i++) { + int size = randomIntBetween(1, 10); + var d = new byte[size]; + var q = new byte[size * B_QUERY - 1]; + expectThrows(IllegalArgumentException.class, () -> ESVectorUtil.ipByteBinByte(q, d)); + } + } + + public void testBasicIpByteBin() { + testBasicIpByteBinImpl(ESVectorUtil::ipByteBinByte); + testBasicIpByteBinImpl(defaultedProvider.getVectorUtilSupport()::ipByteBinByte); + testBasicIpByteBinImpl(defOrPanamaProvider.getVectorUtilSupport()::ipByteBinByte); + } + + interface IpByteBin { + long apply(byte[] q, byte[] d); + } + + void testBasicIpByteBinImpl(IpByteBin ipByteBinFunc) { + assertEquals(15L, ipByteBinFunc.apply(new byte[] { 1, 1, 1, 1 }, new byte[] { 1 })); + assertEquals(30L, ipByteBinFunc.apply(new byte[] { 1, 2, 1, 2, 1, 2, 1, 2 }, new byte[] { 1, 2 })); + + var d = new byte[] { 1, 2, 3 }; + var q = new byte[] { 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3 }; + assert scalarIpByteBin(q, d) == 60L; // 4 + 8 + 16 + 32 + assertEquals(60L, ipByteBinFunc.apply(q, d)); + + d = new byte[] { 1, 2, 3, 4 }; + q = new byte[] { 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4 }; + assert scalarIpByteBin(q, d) == 75L; // 5 + 10 + 20 + 40 + assertEquals(75L, ipByteBinFunc.apply(q, d)); + + d = new byte[] { 1, 2, 3, 4, 5 }; + q = new byte[] { 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5 }; + assert scalarIpByteBin(q, d) == 105L; // 7 + 14 + 28 + 56 + assertEquals(105L, ipByteBinFunc.apply(q, d)); + + d = new byte[] { 1, 2, 3, 4, 5, 6 }; + q = new byte[] { 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6 }; + assert scalarIpByteBin(q, d) == 135L; // 9 + 18 + 36 + 72 + assertEquals(135L, ipByteBinFunc.apply(q, d)); + + d = new byte[] { 1, 2, 3, 4, 5, 6, 7 }; + q = new byte[] { 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7 }; + assert scalarIpByteBin(q, d) == 180L; // 12 + 24 + 48 + 96 + assertEquals(180L, ipByteBinFunc.apply(q, d)); + + d = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }; + q = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 }; + assert scalarIpByteBin(q, d) == 195L; // 13 + 26 + 52 + 104 + assertEquals(195L, ipByteBinFunc.apply(q, d)); + + d = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + q = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + assert scalarIpByteBin(q, d) == 225L; // 15 + 30 + 60 + 120 + assertEquals(225L, ipByteBinFunc.apply(q, d)); + } + + public void testIpByteBin() { + testIpByteBinImpl(ESVectorUtil::ipByteBinByte); + testIpByteBinImpl(defaultedProvider.getVectorUtilSupport()::ipByteBinByte); + testIpByteBinImpl(defOrPanamaProvider.getVectorUtilSupport()::ipByteBinByte); + } + + void testIpByteBinImpl(IpByteBin ipByteBinFunc) { + int iterations = atLeast(50); + for (int i = 0; i < iterations; i++) { + int size = random().nextInt(5000); + var d = new byte[size]; + var q = new byte[size * B_QUERY]; + random().nextBytes(d); + random().nextBytes(q); + assertEquals(scalarIpByteBin(q, d), ipByteBinFunc.apply(q, d)); + + Arrays.fill(d, Byte.MAX_VALUE); + Arrays.fill(q, Byte.MAX_VALUE); + assertEquals(scalarIpByteBin(q, d), ipByteBinFunc.apply(q, d)); + + Arrays.fill(d, Byte.MIN_VALUE); + Arrays.fill(q, Byte.MIN_VALUE); + assertEquals(scalarIpByteBin(q, d), ipByteBinFunc.apply(q, d)); + } + } + + static int scalarIpByteBin(byte[] q, byte[] d) { + int res = 0; + for (int i = 0; i < B_QUERY; i++) { + res += (popcount(q, i * d.length, d, d.length) << i); + } + return res; + } + + public static int popcount(byte[] a, int aOffset, byte[] b, int length) { + int res = 0; + for (int j = 0; j < length; j++) { + int value = (a[aOffset + j] & b[j]) & 0xFF; + for (int k = 0; k < Byte.SIZE; k++) { + if ((value & (1 << k)) != 0) { + ++res; + } + } + } + return res; + } +} diff --git a/libs/simdvec/src/test/java/org/elasticsearch/simdvec/internal/vectorization/BaseVectorizationTests.java b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/internal/vectorization/BaseVectorizationTests.java new file mode 100644 index 0000000000000..f2bc8a11b04aa --- /dev/null +++ b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/internal/vectorization/BaseVectorizationTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.simdvec.internal.vectorization; + +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +public class BaseVectorizationTests extends ESTestCase { + + @Before + public void sanity() { + assert Runtime.version().feature() < 21 || ModuleLayer.boot().findModule("jdk.incubator.vector").isPresent(); + } + + public static ESVectorizationProvider defaultProvider() { + return new DefaultESVectorizationProvider(); + } + + public static ESVectorizationProvider maybePanamaProvider() { + return ESVectorizationProvider.lookup(true); + } +} From 58cc37922c159714c5c384a44cdf444686021e51 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Mon, 7 Oct 2024 16:23:54 +0200 Subject: [PATCH 135/194] Improve performance of LongLongHash#removeAndAdd (#114230) remove some unnecessary manipulation of the keys in the method removeAndAdd. --- .../org/elasticsearch/common/util/LongLongHash.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java b/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java index 4a35cdaa9ab99..f7708af59dde2 100644 --- a/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java +++ b/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java @@ -104,13 +104,16 @@ private void append(long id, long key1, long key2) { keys.set(keyOffset + 1, key2); } - private void reset(long key1, long key2, long id) { + private void reset(long id) { + final LongArray keys = this.keys; + final long keyOffset = id * 2; + final long key1 = keys.get(keyOffset); + final long key2 = keys.get(keyOffset + 1); final long slot = slot(hash(key1, key2), mask); for (long index = slot;; index = nextSlot(index, mask)) { final long curId = id(index); if (curId == -1) { // means unset setId(index, id); - append(id, key1, key2); break; } } @@ -134,10 +137,7 @@ public long add(long key1, long key2) { protected void removeAndAdd(long index) { final long id = getAndSetId(index, -1); assert id >= 0; - long keyOffset = id * 2; - final long key1 = keys.getAndSet(keyOffset, 0); - final long key2 = keys.getAndSet(keyOffset + 1, 0); - reset(key1, key2, id); + reset(id); } @Override From 67f1b02d56d8ad77921059dc34a9c4d5d945e471 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Mon, 7 Oct 2024 15:50:02 +0100 Subject: [PATCH 136/194] Add an UpdateForV9/10 to reroute cluster state (#114213) --- .../action/admin/cluster/reroute/ClusterRerouteResponse.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java index 9581279201be2..4aa6ed60afe43 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java @@ -21,6 +21,8 @@ import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.core.UpdateForV10; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.xcontent.ToXContent; @@ -43,6 +45,8 @@ public class ClusterRerouteResponse extends ActionResponse implements IsAcknowle /** * To be removed when REST compatibility with {@link org.elasticsearch.Version#V_8_6_0} / {@link RestApiVersion#V_8} no longer needed */ + @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) // to remove from the v9 API only + @UpdateForV10(owner = UpdateForV10.Owner.DISTRIBUTED_COORDINATION) // to remove entirely private final ClusterState state; private final RoutingExplanations explanations; private final boolean acknowledged; From a83046a85ad1e35de2dda40ec6f9123923753ef5 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Mon, 7 Oct 2024 12:05:09 -0400 Subject: [PATCH 137/194] Fixing span gap builder tests (#114218) (#114219) With #113251 having a SpanMatchNoDocsQuery is a valid response to the rewrite. closes #114218 --- muted-tests.yml | 3 --- .../elasticsearch/index/query/SpanGapQueryBuilderTests.java | 5 ++++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 7764c0f8865d4..8b756adce5457 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -371,9 +371,6 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/114187 - class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT issue: https://github.com/elastic/elasticsearch/issues/114194 -- class: org.elasticsearch.index.query.SpanGapQueryBuilderTests - method: testToQuery - issue: https://github.com/elastic/elasticsearch/issues/114218 - class: org.elasticsearch.xpack.ilm.ExplainLifecycleIT method: testStepInfoPreservedOnAutoRetry issue: https://github.com/elastic/elasticsearch/issues/114220 diff --git a/server/src/test/java/org/elasticsearch/index/query/SpanGapQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/SpanGapQueryBuilderTests.java index cef43a635541e..5adca6d562dca 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SpanGapQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SpanGapQueryBuilderTests.java @@ -13,6 +13,7 @@ import org.apache.lucene.queries.spans.SpanQuery; import org.apache.lucene.queries.spans.SpanTermQuery; import org.apache.lucene.search.Query; +import org.elasticsearch.lucene.queries.SpanMatchNoDocsQuery; import org.elasticsearch.test.AbstractQueryTestCase; import java.io.IOException; @@ -50,7 +51,9 @@ protected SpanNearQueryBuilder doCreateTestQueryBuilder() { protected void doAssertLuceneQuery(SpanNearQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException { assertThat( query, - either(instanceOf(SpanNearQuery.class)).or(instanceOf(SpanTermQuery.class)).or(instanceOf(MatchAllQueryBuilder.class)) + either(instanceOf(SpanNearQuery.class)).or(instanceOf(SpanTermQuery.class)) + .or(instanceOf(MatchAllQueryBuilder.class)) + .or(instanceOf(SpanMatchNoDocsQuery.class)) ); if (query instanceof SpanNearQuery spanNearQuery) { assertThat(spanNearQuery.getSlop(), equalTo(queryBuilder.slop())); From 9cfe679173a5d3b6b73447161afa8b246a081b56 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Mon, 7 Oct 2024 19:06:23 +0300 Subject: [PATCH 138/194] Avoid using `dynamic:strict` with `subobjects:false` at root (#114247) --- .../DefaultMappingParametersHandler.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java index 1046e22e65caa..81bd80f464525 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -98,7 +98,22 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequ return new DataSourceResponse.ObjectMappingParametersGenerator(() -> { var parameters = new HashMap(); - if (request.parentSubobjects() == ObjectMapper.Subobjects.DISABLED) { + // Changing subobjects from subobjects: false is not supported, but we can f.e. go from "true" to "false". + // TODO enable subobjects: auto + // It is disabled because it currently does not have auto flattening and that results in asserts being triggered when using + // copy_to. + if (ESTestCase.randomBoolean()) { + parameters.put( + "subobjects", + ESTestCase.randomValueOtherThan( + ObjectMapper.Subobjects.AUTO, + () -> ESTestCase.randomFrom(ObjectMapper.Subobjects.values()) + ).toString() + ); + } + + if (request.parentSubobjects() == ObjectMapper.Subobjects.DISABLED + || parameters.getOrDefault("subobjects", "true").equals("false")) { // "enabled: false" is not compatible with subobjects: false // changing "dynamic" from parent context is not compatible with subobjects: false // changing subobjects value is not compatible with subobjects: false @@ -115,19 +130,6 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequ if (ESTestCase.randomBoolean()) { parameters.put("enabled", ESTestCase.randomFrom("true", "false")); } - // Changing subobjects from subobjects: false is not supported, but we can f.e. go from "true" to "false". - // TODO enable subobjects: auto - // It is disabled because it currently does not have auto flattening and that results in asserts being triggered when using - // copy_to. - if (ESTestCase.randomBoolean()) { - parameters.put( - "subobjects", - ESTestCase.randomValueOtherThan( - ObjectMapper.Subobjects.AUTO, - () -> ESTestCase.randomFrom(ObjectMapper.Subobjects.values()) - ).toString() - ); - } if (ESTestCase.randomBoolean()) { var value = request.isRoot() ? ESTestCase.randomFrom("none", "arrays") : ESTestCase.randomFrom("none", "arrays", "all"); From d47ca34b16243ac34e565a1623a6ce826bf039b8 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Mon, 7 Oct 2024 18:51:15 +0200 Subject: [PATCH 139/194] Fix Gradle configuration in idea for :libs:simdvec (#114251) --- libs/simdvec/build.gradle | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/simdvec/build.gradle b/libs/simdvec/build.gradle index 8b676a15038c1..dab5c25b34679 100644 --- a/libs/simdvec/build.gradle +++ b/libs/simdvec/build.gradle @@ -23,14 +23,15 @@ dependencies { } } -tasks.named("compileMain21Java").configure { +// compileMain21Java does not exist within idea (see MrJarPlugin) so we cannot reference directly by name +tasks.matching { it.name == "compileMain21Java" }.configureEach { options.compilerArgs << '--add-modules=jdk.incubator.vector' // we remove Werror, since incubating suppression (-Xlint:-incubating) // is only support since JDK 22 options.compilerArgs -= '-Werror' } -test { +tasks.named('test').configure { if (JavaVersion.current().majorVersion.toInteger() >= 21) { jvmArgs '--add-modules=jdk.incubator.vector' } From bafdd81d3d0d6793cb819616d675bb0f053f4b22 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 7 Oct 2024 13:02:14 -0400 Subject: [PATCH 140/194] ESQL: Reenable part of heap attack test (#114252) This reenables a test and adds more debugging to another one. We'll use this to collect more information the next time it fails. --- .../elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java index e45ac8a9e0f70..a24bd91206ac0 100644 --- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java +++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java @@ -355,7 +355,6 @@ public void testManyEval() throws IOException { assertMap(map, mapMatcher.entry("columns", columns).entry("values", hasSize(10_000)).entry("took", greaterThanOrEqualTo(0))); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-serverless/issues/1874") public void testTooManyEval() throws IOException { initManyLongs(); assertCircuitBreaks(() -> manyEval(490)); @@ -616,14 +615,13 @@ private void initMvLongsIndex(int docs, int fields, int fieldValues) throws IOEx private void bulk(String name, String bulk) throws IOException { Request request = new Request("POST", "/" + name + "/_bulk"); - request.addParameter("filter_path", "errors"); request.setJsonEntity(bulk); request.setOptions( RequestOptions.DEFAULT.toBuilder() .setRequestConfig(RequestConfig.custom().setSocketTimeout(Math.toIntExact(TimeValue.timeValueMinutes(5).millis())).build()) ); Response response = client().performRequest(request); - assertThat(EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8), equalTo("{\"errors\":false}")); + assertThat(entityAsMap(response), matchesMap().entry("errors", false).extraOk()); } private void initIndex(String name, String bulk) throws IOException { From d8cc7d3d2aa0138025ae6dd10a3929f480f527e5 Mon Sep 17 00:00:00 2001 From: Panagiotis Bailis Date: Mon, 7 Oct 2024 20:15:44 +0300 Subject: [PATCH 141/194] Updating RRF-related test cases to work with multiple shards and/or replicas (#114189) --- muted-tests.yml | 9 - .../retriever/RankDocRetrieverBuilderIT.java | 756 ------------ .../xpack/rank/rrf/RRFRetrieverBuilderIT.java | 36 +- .../rrf/RRFRetrieverBuilderNestedDocsIT.java | 23 +- .../test/rrf/350_rrf_retriever_pagination.yml | 1009 ++++++++++------- ...rrf_retriever_search_api_compatibility.yml | 506 ++++++++- 6 files changed, 1098 insertions(+), 1241 deletions(-) delete mode 100644 server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java diff --git a/muted-tests.yml b/muted-tests.yml index 8b756adce5457..4e1d6bbf44614 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -109,9 +109,6 @@ tests: - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testDeleteJobAsync issue: https://github.com/elastic/elasticsearch/issues/112212 -- class: org.elasticsearch.search.retriever.RankDocRetrieverBuilderIT - method: testRankDocsRetrieverWithCollapse - issue: https://github.com/elastic/elasticsearch/issues/112254 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/rest-api/watcher/put-watch/line_120} issue: https://github.com/elastic/elasticsearch/issues/99517 @@ -336,15 +333,9 @@ tests: - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT method: testPutE5Small_withPlatformAgnosticVariant issue: https://github.com/elastic/elasticsearch/issues/113983 -- class: org.elasticsearch.xpack.rank.rrf.RRFRankClientYamlTestSuiteIT - method: test {yaml=rrf/700_rrf_retriever_search_api_compatibility/rrf retriever with top-level collapse} - issue: https://github.com/elastic/elasticsearch/issues/114019 - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT method: testPutE5WithTrainedModelAndInference issue: https://github.com/elastic/elasticsearch/issues/114023 -- class: org.elasticsearch.xpack.rank.rrf.RRFRetrieverBuilderIT - method: testRRFWithCollapse - issue: https://github.com/elastic/elasticsearch/issues/114074 - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT method: testPutE5Small_withPlatformSpecificVariant issue: https://github.com/elastic/elasticsearch/issues/113950 diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java deleted file mode 100644 index b78448bfd873f..0000000000000 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java +++ /dev/null @@ -1,756 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.search.retriever; - -import org.apache.lucene.search.TotalHits; -import org.apache.lucene.search.join.ScoreMode; -import org.apache.lucene.util.SetOnce; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.search.MultiSearchRequest; -import org.elasticsearch.action.search.MultiSearchResponse; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.TransportMultiSearchAction; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.Maps; -import org.elasticsearch.index.query.InnerHitBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.QueryRewriteContext; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.MockSearchService; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.builder.PointInTimeBuilder; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.collapse.CollapseBuilder; -import org.elasticsearch.search.rank.RankDoc; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.NestedSortBuilder; -import org.elasticsearch.search.sort.ScoreSortBuilder; -import org.elasticsearch.search.sort.ShardDocSortField; -import org.elasticsearch.search.sort.SortBuilder; -import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentType; -import org.junit.Before; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; -import static org.hamcrest.Matchers.equalTo; - -public class RankDocRetrieverBuilderIT extends ESIntegTestCase { - - @Override - protected Collection> nodePlugins() { - return List.of(MockSearchService.TestPlugin.class); - } - - public record RetrieverSource(RetrieverBuilder retriever, SearchSourceBuilder source) {} - - private static String INDEX = "test_index"; - private static final String ID_FIELD = "_id"; - private static final String DOC_FIELD = "doc"; - private static final String TEXT_FIELD = "text"; - private static final String VECTOR_FIELD = "vector"; - private static final String TOPIC_FIELD = "topic"; - private static final String LAST_30D_FIELD = "views.last30d"; - private static final String ALL_TIME_FIELD = "views.all"; - - @Before - public void setup() throws Exception { - String mapping = """ - { - "properties": { - "vector": { - "type": "dense_vector", - "dims": 3, - "element_type": "float", - "index": true, - "similarity": "l2_norm", - "index_options": { - "type": "hnsw" - } - }, - "text": { - "type": "text" - }, - "doc": { - "type": "keyword" - }, - "topic": { - "type": "keyword" - }, - "views": { - "type": "nested", - "properties": { - "last30d": { - "type": "integer" - }, - "all": { - "type": "integer" - } - } - } - } - } - """; - createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).build()); - admin().indices().preparePutMapping(INDEX).setSource(mapping, XContentType.JSON).get(); - indexDoc( - INDEX, - "doc_1", - DOC_FIELD, - "doc_1", - TOPIC_FIELD, - "technology", - TEXT_FIELD, - "the quick brown fox jumps over the lazy dog", - LAST_30D_FIELD, - 100 - ); - indexDoc( - INDEX, - "doc_2", - DOC_FIELD, - "doc_2", - TOPIC_FIELD, - "astronomy", - TEXT_FIELD, - "you know, for Search!", - VECTOR_FIELD, - new float[] { 1.0f, 2.0f, 3.0f }, - LAST_30D_FIELD, - 3 - ); - indexDoc(INDEX, "doc_3", DOC_FIELD, "doc_3", TOPIC_FIELD, "technology", VECTOR_FIELD, new float[] { 6.0f, 6.0f, 6.0f }); - indexDoc( - INDEX, - "doc_4", - DOC_FIELD, - "doc_4", - TOPIC_FIELD, - "technology", - TEXT_FIELD, - "aardvark is a really awesome animal, but not very quick", - ALL_TIME_FIELD, - 100, - LAST_30D_FIELD, - 40 - ); - indexDoc(INDEX, "doc_5", DOC_FIELD, "doc_5", TOPIC_FIELD, "science", TEXT_FIELD, "irrelevant stuff"); - indexDoc( - INDEX, - "doc_6", - DOC_FIELD, - "doc_6", - TEXT_FIELD, - "quick quick quick quick search", - VECTOR_FIELD, - new float[] { 10.0f, 30.0f, 100.0f }, - LAST_30D_FIELD, - 15 - ); - indexDoc( - INDEX, - "doc_7", - DOC_FIELD, - "doc_7", - TOPIC_FIELD, - "biology", - TEXT_FIELD, - "dog", - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - ALL_TIME_FIELD, - 1000 - ); - refresh(INDEX); - } - - public void testRankDocsRetrieverBasicWithPagination() { - final int rankWindowSize = 100; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.boolQuery() - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(9L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(8L)); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 2 and 6 due to prefilter - standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); - standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 7, 2, 3, and 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - null, - 10, - 100, - null - ); - // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 3, 4, 7 and with pagination, we'd just omit the first result - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList( - new RetrieverSource(standard0, null), - new RetrieverSource(standard1, null), - new RetrieverSource(knnRetrieverBuilder, null) - ) - ) - ); - // include some pagination as well - source.from(1); - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2")); - assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_3")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_4")); - assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_7")); - }); - } - - public void testRankDocsRetrieverWithAggs() { - // same as above, but we only want to bring back the top result from each subsearch - // so that would be 1, 2, and 7 - // and final rank would be (based on score): 2, 1, 7 - // aggs should still account for the same docs as the testRankDocsRetriever test, i.e. all but doc_5 - final int rankWindowSize = 1; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.boolQuery() - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(9L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(8L)); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 2 and 6 due to prefilter - standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); - standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 7, 2, 3, and 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - null, - 10, - 100, - null - ); - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList( - new RetrieverSource(standard0, null), - new RetrieverSource(standard1, null), - new RetrieverSource(knnRetrieverBuilder, null) - ) - ) - ); - source.size(1); - source.aggregation(new TermsAggregationBuilder("topic").field(TOPIC_FIELD)); - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(5L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getHits().length, equalTo(1)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2")); - assertNotNull(resp.getAggregations()); - assertNotNull(resp.getAggregations().get("topic")); - Terms terms = resp.getAggregations().get("topic"); - // doc_3 is not part of the final aggs computation as it is only retrieved through the knn retriever - // and is outside of the rank window - assertThat(terms.getBucketByKey("technology").getDocCount(), equalTo(2L)); - assertThat(terms.getBucketByKey("astronomy").getDocCount(), equalTo(1L)); - assertThat(terms.getBucketByKey("biology").getDocCount(), equalTo(1L)); - }); - } - - public void testRankDocsRetrieverWithCollapse() { - final int rankWindowSize = 100; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.boolQuery() - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(9L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(8L)); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 2 and 6 due to prefilter - standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); - standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 7, 2, 3, and 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - null, - 10, - 100, - null - ); - // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 3, 4, 7 - // with collapsing on topic field we would have 6, 2, 1, 7 - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList( - new RetrieverSource(standard0, null), - new RetrieverSource(standard1, null), - new RetrieverSource(knnRetrieverBuilder, null) - ) - ) - ); - source.collapse( - new CollapseBuilder(TOPIC_FIELD).setInnerHits( - new InnerHitBuilder("a").addSort(new FieldSortBuilder(DOC_FIELD).order(SortOrder.DESC)).setSize(10) - ) - ); - source.fetchField(TOPIC_FIELD); - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getHits().length, equalTo(4)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6")); - assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_2")); - assertThat(resp.getHits().getAt(1).field(TOPIC_FIELD).getValue().toString(), equalTo("astronomy")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(2).field(TOPIC_FIELD).getValue().toString(), equalTo("technology")); - assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(0).getId(), equalTo("doc_4")); - assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(1).getId(), equalTo("doc_3")); - assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(2).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_7")); - assertThat(resp.getHits().getAt(3).field(TOPIC_FIELD).getValue().toString(), equalTo("biology")); - }); - } - - public void testRankDocsRetrieverWithNestedCollapseAndAggs() { - final int rankWindowSize = 10; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1 and 6 as doc_4 is collapsed to doc_1 - standard0.queryBuilder = QueryBuilders.boolQuery() - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(9L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(8L)); - standard0.collapseBuilder = new CollapseBuilder(TOPIC_FIELD).setInnerHits( - new InnerHitBuilder("a").addSort(new FieldSortBuilder(DOC_FIELD).order(SortOrder.DESC)).setSize(10) - ); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 2 and 6 due to prefilter - standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); - standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 7, 2, 3, and 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - null, - 10, - 100, - null - ); - // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 3, 4, 7 - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList( - new RetrieverSource(standard0, null), - new RetrieverSource(standard1, null), - new RetrieverSource(knnRetrieverBuilder, null) - ) - ) - ); - source.aggregation(new TermsAggregationBuilder("topic").field(TOPIC_FIELD)); - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6")); - assertNotNull(resp.getAggregations()); - assertNotNull(resp.getAggregations().get("topic")); - Terms terms = resp.getAggregations().get("topic"); - // doc_3 is not part of the final aggs computation as it is only retrieved through the knn retriever - // and is outside of the rank window - assertThat(terms.getBucketByKey("technology").getDocCount(), equalTo(3L)); - assertThat(terms.getBucketByKey("astronomy").getDocCount(), equalTo(1L)); - assertThat(terms.getBucketByKey("biology").getDocCount(), equalTo(1L)); - }); - } - - public void testRankDocsRetrieverWithNestedQuery() { - final int rankWindowSize = 100; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(LAST_30D_FIELD).gt(10L), ScoreMode.Avg); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 2 and 6 due to prefilter - standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); - standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 7, 2, 3, and 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - null, - 10, - 100, - null - ); - // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 3, 4, 7 - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList( - new RetrieverSource(standard0, null), - new RetrieverSource(standard1, null), - new RetrieverSource(knnRetrieverBuilder, null) - ) - ) - ); - source.fetchField(TOPIC_FIELD); - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6")); - assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_2")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_3")); - assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_4")); - assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_7")); - }); - } - - public void testRankDocsRetrieverMultipleCompoundRetrievers() { - final int rankWindowSize = 100; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1, 4, and 6 - standard0.queryBuilder = QueryBuilders.boolQuery() - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_1")).boost(10L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_4")).boost(9L)) - .should(QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_6")).boost(8L)); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 2 and 6 due to prefilter - standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); - standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 7, 2, 3, and 6 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( - VECTOR_FIELD, - new float[] { 3.0f, 3.0f, 3.0f }, - null, - 10, - 100, - null - ); - // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and - // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results) - // so ideal rank would be: 6, 2, 1, 4, 7, 3 - CompoundRetrieverWithRankDocs compoundRetriever1 = new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList( - new RetrieverSource(standard0, null), - new RetrieverSource(standard1, null), - new RetrieverSource(knnRetrieverBuilder, null) - ) - ); - // simple standard retriever that would have the doc_4 as its first (and only) result - StandardRetrieverBuilder standard2 = new StandardRetrieverBuilder(); - standard2.queryBuilder = QueryBuilders.queryStringQuery("aardvark").defaultField(TEXT_FIELD); - - // combining the two retrievers would bring doc_4 at the top as it would be the only one present in both doc sets - // the rest of the docs would be sorted based on their ranks as they have the same score (1/2) - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList(new RetrieverSource(compoundRetriever1, null), new RetrieverSource(standard2, null)) - ) - ); - - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_4")); - assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_2")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_3")); - assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_6")); - assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_7")); - }); - } - - public void testRankDocsRetrieverDifferentNestedSorting() { - final int rankWindowSize = 100; - SearchSourceBuilder source = new SearchSourceBuilder(); - StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); - // this one retrieves docs 1, 4, 6, 2 - standard0.queryBuilder = QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(LAST_30D_FIELD).gt(0), ScoreMode.Avg); - standard0.sortBuilders = List.of( - new FieldSortBuilder(LAST_30D_FIELD).setNestedSort(new NestedSortBuilder("views")).order(SortOrder.DESC) - ); - StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); - // this one retrieves docs 4, 7 - standard1.queryBuilder = QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(ALL_TIME_FIELD).gt(0), ScoreMode.Avg); - standard1.sortBuilders = List.of( - new FieldSortBuilder(ALL_TIME_FIELD).setNestedSort(new NestedSortBuilder("views")).order(SortOrder.ASC) - ); - - source.retriever( - new CompoundRetrieverWithRankDocs( - rankWindowSize, - Arrays.asList(new RetrieverSource(standard0, null), new RetrieverSource(standard1, null)) - ) - ); - - SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); - ElasticsearchAssertions.assertResponse(req, resp -> { - assertNull(resp.pointInTimeId()); - assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(5L)); - assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); - assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_4")); - assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_2")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_6")); - assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_7")); - }); - } - - class CompoundRetrieverWithRankDocs extends RetrieverBuilder { - - private final List sources; - private final int rankWindowSize; - - private CompoundRetrieverWithRankDocs(int rankWindowSize, List sources) { - this.rankWindowSize = rankWindowSize; - this.sources = Collections.unmodifiableList(sources); - } - - @Override - public boolean isCompound() { - return true; - } - - @Override - public QueryBuilder topDocsQuery() { - throw new UnsupportedOperationException("should not be called"); - } - - @Override - public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException { - if (ctx.getPointInTimeBuilder() == null) { - throw new IllegalStateException("PIT is required"); - } - - // Rewrite prefilters - boolean hasChanged = false; - var newPreFilters = rewritePreFilters(ctx); - hasChanged |= newPreFilters != preFilterQueryBuilders; - - // Rewrite retriever sources - List newRetrievers = new ArrayList<>(); - for (var entry : sources) { - RetrieverBuilder newRetriever = entry.retriever.rewrite(ctx); - if (newRetriever != entry.retriever) { - newRetrievers.add(new RetrieverSource(newRetriever, null)); - hasChanged |= newRetriever != entry.retriever; - } else if (newRetriever == entry.retriever) { - var sourceBuilder = entry.source != null - ? entry.source - : createSearchSourceBuilder(ctx.getPointInTimeBuilder(), newRetriever); - var rewrittenSource = sourceBuilder.rewrite(ctx); - newRetrievers.add(new RetrieverSource(newRetriever, rewrittenSource)); - hasChanged |= rewrittenSource != entry.source; - } - } - if (hasChanged) { - return new CompoundRetrieverWithRankDocs(rankWindowSize, newRetrievers); - } - - // execute searches - final SetOnce results = new SetOnce<>(); - final MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); - for (var entry : sources) { - SearchRequest searchRequest = new SearchRequest().source(entry.source); - // The can match phase can reorder shards, so we disable it to ensure the stable ordering - searchRequest.setPreFilterShardSize(Integer.MAX_VALUE); - multiSearchRequest.add(searchRequest); - } - ctx.registerAsyncAction((client, listener) -> { - client.execute(TransportMultiSearchAction.TYPE, multiSearchRequest, new ActionListener<>() { - @Override - public void onResponse(MultiSearchResponse items) { - List topDocs = new ArrayList<>(); - for (int i = 0; i < items.getResponses().length; i++) { - var item = items.getResponses()[i]; - var rankDocs = getRankDocs(item.getResponse()); - sources.get(i).retriever().setRankDocs(rankDocs); - topDocs.add(rankDocs); - } - results.set(combineResults(topDocs)); - listener.onResponse(null); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); - }); - - return new RankDocsRetrieverBuilder( - rankWindowSize, - newRetrievers.stream().map(s -> s.retriever).toList(), - results::get, - newPreFilters - ); - } - - @Override - public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) { - throw new UnsupportedOperationException("should not be called"); - } - - @Override - public String getName() { - return "compound_retriever"; - } - - @Override - protected void doToXContent(XContentBuilder builder, Params params) throws IOException { - - } - - @Override - protected boolean doEquals(Object o) { - return false; - } - - @Override - protected int doHashCode() { - return 0; - } - - private RankDoc[] getRankDocs(SearchResponse searchResponse) { - assert searchResponse != null; - int size = Math.min(rankWindowSize, searchResponse.getHits().getHits().length); - RankDoc[] docs = new RankDoc[size]; - for (int i = 0; i < size; i++) { - var hit = searchResponse.getHits().getAt(i); - long sortValue = (long) hit.getRawSortValues()[hit.getRawSortValues().length - 1]; - int doc = ShardDocSortField.decodeDoc(sortValue); - int shardRequestIndex = ShardDocSortField.decodeShardRequestIndex(sortValue); - docs[i] = new RankDoc(doc, hit.getScore(), shardRequestIndex); - docs[i].rank = i + 1; - } - return docs; - } - - record RankDocAndHitRatio(RankDoc rankDoc, float hitRatio) {} - - /** - * Combines the provided {@code rankResults} to return the final top documents. - */ - public RankDoc[] combineResults(List rankResults) { - int totalQueries = rankResults.size(); - final float step = 1.0f / totalQueries; - Map docsToRankResults = Maps.newMapWithExpectedSize(rankWindowSize); - for (var rankResult : rankResults) { - for (RankDoc scoreDoc : rankResult) { - docsToRankResults.compute(new RankDoc.RankKey(scoreDoc.doc, scoreDoc.shardIndex), (key, value) -> { - if (value == null) { - RankDoc res = new RankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex); - res.rank = scoreDoc.rank; - return new RankDocAndHitRatio(res, step); - } else { - RankDoc res = new RankDoc(scoreDoc.doc, Math.max(scoreDoc.score, value.rankDoc.score), scoreDoc.shardIndex); - res.rank = Math.min(scoreDoc.rank, value.rankDoc.rank); - return new RankDocAndHitRatio(res, value.hitRatio + step); - } - }); - } - } - // sort the results based on hit ratio, then doc, then rank, and final tiebreaker is based on smaller doc id - RankDocAndHitRatio[] sortedResults = docsToRankResults.values().toArray(RankDocAndHitRatio[]::new); - Arrays.sort(sortedResults, (RankDocAndHitRatio doc1, RankDocAndHitRatio doc2) -> { - if (doc1.hitRatio != doc2.hitRatio) { - return doc1.hitRatio < doc2.hitRatio ? 1 : -1; - } - if (false == (Float.isNaN(doc1.rankDoc.score) || Float.isNaN(doc2.rankDoc.score)) - && (doc1.rankDoc.score != doc2.rankDoc.score)) { - return doc1.rankDoc.score < doc2.rankDoc.score ? 1 : -1; - } - if (doc1.rankDoc.rank != doc2.rankDoc.rank) { - return doc1.rankDoc.rank < doc2.rankDoc.rank ? -1 : 1; - } - return doc1.rankDoc.doc < doc2.rankDoc.doc ? -1 : 1; - }); - // trim the results if needed, otherwise each shard will always return `rank_window_size` results. - // pagination and all else will happen on the coordinator when combining the shard responses - RankDoc[] topResults = new RankDoc[Math.min(rankWindowSize, sortedResults.length)]; - for (int rank = 0; rank < topResults.length; ++rank) { - topResults[rank] = sortedResults[rank].rankDoc; - topResults[rank].rank = rank + 1; - topResults[rank].score = sortedResults[rank].hitRatio; - } - return topResults; - } - } - - private SearchSourceBuilder createSearchSourceBuilder(PointInTimeBuilder pit, RetrieverBuilder retrieverBuilder) { - var sourceBuilder = new SearchSourceBuilder().pointInTimeBuilder(pit).trackTotalHits(false).size(100); - retrieverBuilder.extractToSearchSourceBuilder(sourceBuilder, false); - - // Record the shard id in the sort result - List> sortBuilders = sourceBuilder.sorts() != null ? new ArrayList<>(sourceBuilder.sorts()) : new ArrayList<>(); - if (sortBuilders.isEmpty()) { - sortBuilders.add(new ScoreSortBuilder()); - } - sortBuilders.add(new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME)); - sourceBuilder.sort(sortBuilders); - return sourceBuilder; - } -} diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java index 2e7bc44811bf6..be64d34dc8765 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderIT.java @@ -33,7 +33,6 @@ import java.util.Collection; import java.util.List; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -98,7 +97,7 @@ protected void setupIndex() { } } """; - createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).build()); + createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 5)).build()); admin().indices().preparePutMapping(INDEX).setSource(mapping, XContentType.JSON).get(); indexDoc(INDEX, "doc_1", DOC_FIELD, "doc_1", TOPIC_FIELD, "technology", TEXT_FIELD, "term"); indexDoc( @@ -167,8 +166,8 @@ public void testRRFPagination() { QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 3, 2, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + // this one retrieves docs 2, 3, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -214,8 +213,8 @@ public void testRRFWithAggs() { QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 3, 2, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + // this one retrieves docs 2, 3, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -266,8 +265,8 @@ public void testRRFWithCollapse() { QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 3, 2, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + // this one retrieves docs 2, 3, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -320,8 +319,8 @@ public void testRRFRetrieverWithCollapseAndAggs() { QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 3, 2, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + // this one retrieves docs 2, 3, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -383,8 +382,8 @@ public void testMultipleRRFRetrievers() { QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 3, 2, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + // this one retrieves docs 2, 3, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -446,8 +445,8 @@ public void testRRFExplainWithNamedRetrievers() { QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 3, 2, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + // this one retrieves docs 2, 3, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null); source.retriever( new RRFRetrieverBuilder( Arrays.asList( @@ -474,13 +473,12 @@ public void testRRFExplainWithNamedRetrievers() { assertThat(resp.getHits().getAt(0).getExplanation().getDetails().length, equalTo(2)); var rrfDetails = resp.getHits().getAt(0).getExplanation().getDetails()[0]; assertThat(rrfDetails.getDetails().length, equalTo(3)); - assertThat(rrfDetails.getDescription(), containsString("computed for initial ranks [2, 1, 2]")); + assertThat(rrfDetails.getDescription(), containsString("computed for initial ranks [2, 1, 1]")); - assertThat(rrfDetails.getDetails()[0].getDescription(), containsString("for rank [2] in query at index [0]")); assertThat(rrfDetails.getDetails()[0].getDescription(), containsString("for rank [2] in query at index [0]")); assertThat(rrfDetails.getDetails()[0].getDescription(), containsString("[my_custom_retriever]")); assertThat(rrfDetails.getDetails()[1].getDescription(), containsString("for rank [1] in query at index [1]")); - assertThat(rrfDetails.getDetails()[2].getDescription(), containsString("for rank [2] in query at index [2]")); + assertThat(rrfDetails.getDetails()[2].getDescription(), containsString("for rank [1] in query at index [2]")); }); } @@ -503,8 +501,8 @@ public void testRRFExplainWithAnotherNestedRRF() { QueryBuilders.constantScoreQuery(QueryBuilders.idsQuery().addIds("doc_2", "doc_3", "doc_6")).boost(20L) ); standard1.getPreFilterQueryBuilders().add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); - // this one retrieves docs 3, 2, 6, and 7 - KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 4.0f }, null, 10, 100, null); + // this one retrieves docs 2, 3, 6, and 7 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder(VECTOR_FIELD, new float[] { 2.0f }, null, 10, 100, null); RRFRetrieverBuilder nestedRRF = new RRFRetrieverBuilder( Arrays.asList( diff --git a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java index 512874e5009f3..ea251917cfae2 100644 --- a/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java +++ b/x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderNestedDocsIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.retriever.CompoundRetrieverBuilder; import org.elasticsearch.search.retriever.KnnRetrieverBuilder; @@ -21,8 +22,9 @@ import java.util.Arrays; -import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; public class RRFRetrieverBuilderNestedDocsIT extends RRFRetrieverBuilderIT { @@ -68,7 +70,7 @@ protected void setupIndex() { } } """; - createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0).build()); + createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 5)).build()); admin().indices().preparePutMapping(INDEX).setSource(mapping, XContentType.JSON).get(); indexDoc(INDEX, "doc_1", DOC_FIELD, "doc_1", TOPIC_FIELD, "technology", TEXT_FIELD, "term", LAST_30D_FIELD, 100); indexDoc( @@ -134,9 +136,9 @@ public void testRRFRetrieverWithNestedQuery() { final int rankWindowSize = 100; final int rankConstant = 10; SearchSourceBuilder source = new SearchSourceBuilder(); - // this one retrieves docs 1, 4 + // this one retrieves docs 1 StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder( - QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(LAST_30D_FIELD).gte(30L), ScoreMode.Avg) + QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(LAST_30D_FIELD).gte(50L), ScoreMode.Avg) ); // this one retrieves docs 2 and 6 due to prefilter StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder( @@ -157,16 +159,21 @@ public void testRRFRetrieverWithNestedQuery() { ) ); source.fetchField(TOPIC_FIELD); + source.explain(true); SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); ElasticsearchAssertions.assertResponse(req, resp -> { assertNull(resp.pointInTimeId()); assertNotNull(resp.getHits().getTotalHits()); - assertThat(resp.getHits().getTotalHits().value, equalTo(4L)); + assertThat(resp.getHits().getTotalHits().value, equalTo(3L)); assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6")); - assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1")); - assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_2")); - assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_4")); + assertThat((double) resp.getHits().getAt(0).getScore(), closeTo(0.1742, 1e-4)); + assertThat( + Arrays.stream(resp.getHits().getHits()).skip(1).map(SearchHit::getId).toList(), + containsInAnyOrder("doc_1", "doc_2") + ); + assertThat((double) resp.getHits().getAt(1).getScore(), closeTo(0.0909, 1e-4)); + assertThat((double) resp.getHits().getAt(2).getScore(), closeTo(0.0909, 1e-4)); }); } } diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/350_rrf_retriever_pagination.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/350_rrf_retriever_pagination.yml index 47ba3658bb38d..d5d7a5de1dc71 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/350_rrf_retriever_pagination.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/350_rrf_retriever_pagination.yml @@ -1,6 +1,8 @@ setup: - skip: - features: close_to + features: + - close_to + - contains - requires: cluster_features: 'rrf_retriever_composition_supported' @@ -10,8 +12,6 @@ setup: indices.create: index: test body: - settings: - number_of_shards: 1 mappings: properties: number_val: @@ -81,35 +81,49 @@ setup: bool: { should: [ { - term: { - number_val: { - value: "1", - boost: 10.0 - } - } - }, - { - term: { - number_val: { - value: "2", - boost: 9.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "1" + } + } + }, + boost: 10.0 } - }, - { - term: { - number_val: { - value: "3", - boost: 8.0 - } + },{ + constant_score: { + filter: { + term: { + number_val: { + value: "2" + } + } + }, + boost: 9.0 + } }, + { + constant_score: { + filter: { + term: { + number_val: { + value: "3" + } + } + }, + boost: 8.0 } }, { - term: { - number_val: { - value: "4", - boost: 7.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "4" + } + } + }, + boost: 7.0 } } ] @@ -124,35 +138,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "A", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "B", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "C", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "D", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 7.0 } } ] @@ -198,35 +228,49 @@ setup: bool: { should: [ { - term: { - number_val: { - value: "1", - boost: 10.0 - } - } - }, - { - term: { - number_val: { - value: "2", - boost: 9.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "1" + } + } + }, + boost: 10.0 } - }, - { - term: { - number_val: { - value: "3", - boost: 8.0 - } + },{ + constant_score: { + filter: { + term: { + number_val: { + value: "2" + } + } + }, + boost: 9.0 + } }, + { + constant_score: { + filter: { + term: { + number_val: { + value: "3" + } + } + }, + boost: 8.0 } }, { - term: { - number_val: { - value: "4", - boost: 7.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "4" + } + } + }, + boost: 7.0 } } ] @@ -241,35 +285,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "A", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "B", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "C", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "D", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 7.0 } } ] @@ -306,35 +366,49 @@ setup: bool: { should: [ { - term: { - number_val: { - value: "1", - boost: 10.0 - } - } - }, - { - term: { - number_val: { - value: "2", - boost: 9.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "1" + } + } + }, + boost: 10.0 } - }, - { - term: { - number_val: { - value: "3", - boost: 8.0 - } + },{ + constant_score: { + filter: { + term: { + number_val: { + value: "2" + } + } + }, + boost: 9.0 + } }, + { + constant_score: { + filter: { + term: { + number_val: { + value: "3" + } + } + }, + boost: 8.0 } }, { - term: { - number_val: { - value: "4", - boost: 7.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "4" + } + } + }, + boost: 7.0 } } ] @@ -349,35 +423,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "A", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "B", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "C", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "D", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 7.0 } } ] @@ -422,35 +512,49 @@ setup: bool: { should: [ { - term: { - number_val: { - value: "1", - boost: 10.0 - } - } - }, - { - term: { - number_val: { - value: "2", - boost: 9.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "1" + } + } + }, + boost: 10.0 } - }, - { - term: { - number_val: { - value: "3", - boost: 8.0 - } + },{ + constant_score: { + filter: { + term: { + number_val: { + value: "2" + } + } + }, + boost: 9.0 + } }, + { + constant_score: { + filter: { + term: { + number_val: { + value: "3" + } + } + }, + boost: 8.0 } }, { - term: { - number_val: { - value: "4", - boost: 7.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "4" + } + } + }, + boost: 7.0 } } ] @@ -465,35 +569,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "D", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "C", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "A", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "B", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 7.0 } } ] @@ -533,35 +653,49 @@ setup: bool: { should: [ { - term: { - number_val: { - value: "1", - boost: 10.0 - } - } - }, - { - term: { - number_val: { - value: "2", - boost: 9.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "1" + } + } + }, + boost: 10.0 } - }, - { - term: { - number_val: { - value: "3", - boost: 8.0 - } + },{ + constant_score: { + filter: { + term: { + number_val: { + value: "2" + } + } + }, + boost: 9.0 + } }, + { + constant_score: { + filter: { + term: { + number_val: { + value: "3" + } + } + }, + boost: 8.0 } }, { - term: { - number_val: { - value: "4", - boost: 7.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "4" + } + } + }, + boost: 7.0 } } ] @@ -576,35 +710,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "D", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "C", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "A", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "B", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 7.0 } } ] @@ -632,9 +782,9 @@ setup: "Pagination within interleaved results, different result set sizes, rank_window_size covering all results": # perform multiple searches with different "from" parameter, ensuring that results are consistent # rank_window_size covers the entire result set for both queries, so pagination should be consistent - # queryA has a result set of [5, 1] and + # queryA has a result set of [1] and # queryB has a result set of [4, 3, 1, 2] - # so for rank_constant=10, the expected order is [1, 4, 5, 3, 2] + # so for rank_constant=10, the expected order is [1, 4, 3, 2] - do: search: index: test @@ -645,19 +795,11 @@ setup: { retrievers: [ { - # this should clause would generate the result set [5, 1] + # this should clause would generate the result set [1] standard: { query: { bool: { should: [ - { - term: { - number_val: { - value: "5", - boost: 10.0 - } - } - }, { term: { number_val: { @@ -678,35 +820,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "D", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "C", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "A", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "B", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 7.0 } } ] @@ -721,11 +879,11 @@ setup: from : 0 size : 2 - - match: { hits.total.value : 5 } + - match: { hits.total.value : 4 } - length: { hits.hits : 2 } - match: { hits.hits.0._id: "1" } # score for doc 1 is (1/12 + 1/13) - - close_to: {hits.hits.0._score: {value: 0.1602, error: 0.001}} + - close_to: {hits.hits.0._score: {value: 0.1678, error: 0.001}} - match: { hits.hits.1._id: "4" } # score for doc 4 is (1/11) - close_to: {hits.hits.1._score: {value: 0.0909, error: 0.001}} @@ -740,19 +898,11 @@ setup: { retrievers: [ { - # this should clause would generate the result set [5, 1] + # this should clause would generate the result set [1] standard: { query: { bool: { should: [ - { - term: { - number_val: { - value: "5", - boost: 10.0 - } - } - }, { term: { number_val: { @@ -773,35 +923,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "D", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "C", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "A", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "B", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 7.0 } } ] @@ -816,14 +982,15 @@ setup: from : 2 size : 2 - - match: { hits.total.value : 5 } + - match: { hits.total.value : 4 } - length: { hits.hits : 2 } - - match: { hits.hits.0._id: "5" } - # score for doc 5 is (1/11) - - close_to: {hits.hits.0._score: {value: 0.0909, error: 0.001}} - - match: { hits.hits.1._id: "3" } + - match: { hits.hits.0._id: "3" } # score for doc 3 is (1/12) - - close_to: {hits.hits.1._score: {value: 0.0833, error: 0.001}} + - close_to: {hits.hits.0._score: {value: 0.0833, error: 0.001}} + - match: { hits.hits.1._id: "2" } + # score for doc 2 is (1/14) + - close_to: {hits.hits.1._score: {value: 0.0714, error: 0.001}} + - do: search: @@ -835,19 +1002,11 @@ setup: { retrievers: [ { - # this should clause would generate the result set [5, 1] + # this should clause would generate the result set [1] standard: { query: { bool: { should: [ - { - term: { - number_val: { - value: "5", - boost: 10.0 - } - } - }, { term: { number_val: { @@ -868,35 +1027,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "D", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "C", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "A", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "B", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 7.0 } } ] @@ -911,12 +1086,8 @@ setup: from: 4 size: 2 - - match: { hits.total.value: 5 } - - length: { hits.hits: 1 } - - match: { hits.hits.0._id: "2" } - # score for doc 2 is (1/14) - - close_to: {hits.hits.0._score: {value: 0.0714, error: 0.001}} - + - match: { hits.total.value: 4 } + - length: { hits.hits: 0 } --- "Pagination within interleaved results, different result set sizes, rank_window_size not covering all results": @@ -943,19 +1114,27 @@ setup: bool: { should: [ { - term: { - number_val: { - value: "5", - boost: 10.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "5" + } + } + }, + boost: 10.0 } }, { - term: { - number_val: { - value: "1", - boost: 9.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "1" + } + } + }, + boost: 9.0 } } ] @@ -970,35 +1149,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "D", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "C", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "A", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "B", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 7.0 } } ] @@ -1015,11 +1210,11 @@ setup: - match: { hits.total.value : 5 } - length: { hits.hits : 2 } - - match: { hits.hits.0._id: "4" } - # score for doc 4 is (1/11) + - contains: { hits.hits: { _id: "4" } } + - contains: { hits.hits: { _id: "5" } } + + # both docs have the same score (1/11) - close_to: {hits.hits.0._score: {value: 0.0909, error: 0.001}} - - match: { hits.hits.1._id: "5" } - # score for doc 5 is (1/11) - close_to: {hits.hits.1._score: {value: 0.0909, error: 0.001}} - do: @@ -1038,19 +1233,27 @@ setup: bool: { should: [ { - term: { - number_val: { - value: "5", - boost: 10.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "5" + } + } + }, + boost: 10.0 } }, { - term: { - number_val: { - value: "1", - boost: 9.0 - } + constant_score: { + filter: { + term: { + number_val: { + value: "1" + } + } + }, + boost: 9.0 } } ] @@ -1065,35 +1268,51 @@ setup: bool: { should: [ { - term: { - char_val: { - value: "D", - boost: 10.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "D" + } + } + }, + boost: 10.0 } }, { - term: { - char_val: { - value: "C", - boost: 9.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "C" + } + } + }, + boost: 9.0 } }, { - term: { - char_val: { - value: "A", - boost: 8.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "A" + } + } + }, + boost: 8.0 } }, { - term: { - char_val: { - value: "B", - boost: 7.0 - } + constant_score: { + filter: { + term: { + char_val: { + value: "B" + } + } + }, + boost: 7.0 } } ] diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml index 1f7125377b892..517c162c33e95 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/700_rrf_retriever_search_api_compatibility.yml @@ -1,7 +1,6 @@ setup: - skip: features: close_to - - requires: cluster_features: 'rrf_retriever_composition_supported' reason: 'test requires rrf retriever composition support' @@ -10,8 +9,6 @@ setup: indices.create: index: test body: - settings: - number_of_shards: 1 mappings: properties: text: @@ -42,7 +39,7 @@ setup: index: test id: "1" body: - text: "term term term term term term term term term" + text: "term1" vector: [1.0] - do: @@ -50,7 +47,7 @@ setup: index: test id: "2" body: - text: "term term term term term term term term" + text: "term2" text_to_highlight: "search for the truth" keyword: "biology" vector: [2.0] @@ -60,8 +57,8 @@ setup: index: test id: "3" body: - text: "term term term term term term term" - text_to_highlight: "nothing related but still a match" + text: "term3" + text_to_highlight: "nothing related" keyword: "technology" vector: [3.0] @@ -70,14 +67,14 @@ setup: index: test id: "4" body: - text: "term term term term term term" + text: "term4" vector: [4.0] - do: index: index: test id: "5" body: - text: "term term term term term" + text: "term5" text_to_highlight: "You know, for Search!" keyword: "technology" integer: 5 @@ -87,7 +84,7 @@ setup: index: test id: "6" body: - text: "term term term term" + text: "term6" keyword: "biology" integer: 6 vector: [6.0] @@ -96,27 +93,26 @@ setup: index: test id: "7" body: - text: "term term term" + text: "term7" keyword: "astronomy" - vector: [7.0] + vector: [77.0] nested: { views: 50} - do: index: index: test id: "8" body: - text: "term term" + text: "term8" keyword: "technology" - vector: [8.0] nested: { views: 100} - do: index: index: test id: "9" body: - text: "term" + text: "term9" + integer: 2 keyword: "technology" - vector: [9.0] nested: { views: 10} - do: indices.refresh: {} @@ -133,6 +129,7 @@ setup: rrf: retrievers: [ { + # this one retrieves docs 6, 5, 4 knn: { field: vector, query_vector: [ 6.0 ], @@ -141,10 +138,72 @@ setup: } }, { + # this one retrieves docs 4, 5, 1, 2, 6 standard: { query: { - term: { - text: term + bool: { + should: [ + { + constant_score: { + filter: { + term: { + text: term4 + } + }, + boost: 10.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term5 + } + }, + boost: 9.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term1 + } + }, + boost: 8.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term2 + } + }, + boost: 7.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term6 + } + }, + boost: 6.0 + } + }, + { + constant_score: { + filter: { + exists: { + field: text + } + }, + boost: 1 + } + } + ] } } } @@ -158,9 +217,13 @@ setup: terms: field: keyword - - match: { hits.hits.0._id: "5" } - - match: { hits.hits.1._id: "1" } + + - match: { hits.hits.0._id: "4" } + - close_to: { hits.hits.0._score: { value: 0.1678, error: 0.001 } } + - match: { hits.hits.1._id: "5" } + - close_to: { hits.hits.1._score: { value: 0.1666, error: 0.001 } } - match: { hits.hits.2._id: "6" } + - close_to: { hits.hits.2._score: { value: 0.1575, error: 0.001 } } - match: { aggregations.keyword_aggs.buckets.0.key: "technology" } - match: { aggregations.keyword_aggs.buckets.0.doc_count: 4 } @@ -181,6 +244,7 @@ setup: rrf: retrievers: [ { + # this one retrieves docs 6, 5, 4 knn: { field: vector, query_vector: [ 6.0 ], @@ -189,10 +253,72 @@ setup: } }, { + # this one retrieves docs 4, 5, 1, 2, 6 standard: { query: { - term: { - text: term + bool: { + should: [ + { + constant_score: { + filter: { + term: { + text: term4 + } + }, + boost: 10.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term5 + } + }, + boost: 9.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term1 + } + }, + boost: 8.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term2 + } + }, + boost: 7.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term6 + } + }, + boost: 6.0 + } + }, + { + constant_score: { + filter: { + exists: { + field: text + } + }, + boost: 1 + } + } + ] } } } @@ -208,12 +334,14 @@ setup: lang: painless source: "_score" - - - match: { hits.hits.0._id: "5" } - - match: { hits.hits.1._id: "1" } + - match: { hits.hits.0._id: "4" } + - close_to: { hits.hits.0._score: { value: 0.1678, error: 0.001 } } + - match: { hits.hits.1._id: "5" } + - close_to: { hits.hits.1._score: { value: 0.1666, error: 0.001 } } - match: { hits.hits.2._id: "6" } + - close_to: { hits.hits.2._score: { value: 0.1575, error: 0.001 } } - - close_to: { aggregations.max_score.value: { value: 0.15, error: 0.001 }} + - close_to: { aggregations.max_score.value: { value: 0.1678, error: 0.001 }} --- "rrf retriever with top-level collapse": @@ -228,6 +356,7 @@ setup: rrf: retrievers: [ { + # this one retrieves docs 6, 5, 4 knn: { field: vector, query_vector: [ 6.0 ], @@ -236,10 +365,72 @@ setup: } }, { + # this one retrieves docs 4, 5, 1, 2, 6 standard: { query: { - term: { - text: term + bool: { + should: [ + { + constant_score: { + filter: { + term: { + text: term4 + } + }, + boost: 10.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term5 + } + }, + boost: 9.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term1 + } + }, + boost: 8.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term2 + } + }, + boost: 7.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term6 + } + }, + boost: 6.0 + } + }, + { + constant_score: { + filter: { + exists: { + field: text + } + }, + boost: 1 + } + } + ] } } } @@ -250,18 +441,23 @@ setup: size: 3 collapse: { field: keyword, inner_hits: { name: sub_hits, size: 2 } } - - match: { hits.hits.0._id: "5" } - - match: { hits.hits.1._id: "1" } + - match: { hits.total : 9 } + + - match: { hits.hits.0._id: "4" } + - close_to: { hits.hits.0._score: { value: 0.1678, error: 0.001 } } + - match: { hits.hits.1._id: "5" } + - close_to: { hits.hits.1._score: { value: 0.1666, error: 0.001 } } - match: { hits.hits.2._id: "6" } + - close_to: { hits.hits.2._score: { value: 0.1575, error: 0.001 } } - - match: { hits.hits.0.inner_hits.sub_hits.hits.total : 4 } - length: { hits.hits.0.inner_hits.sub_hits.hits.hits : 2 } - - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "5" } - - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.1._id: "3" } + - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "4" } + - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.1._id: "1" } + - match: { hits.hits.1.inner_hits.sub_hits.hits.total : 4 } - length: { hits.hits.1.inner_hits.sub_hits.hits.hits : 2 } - - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "1" } - - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "4" } + - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "5" } + - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "3" } - length: { hits.hits.2.inner_hits.sub_hits.hits.hits: 2 } - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "6" } @@ -280,18 +476,132 @@ setup: rrf: retrievers: [ { - knn: { - field: vector, - query_vector: [ 6.0 ], - k: 3, - num_candidates: 10 + # this one retrieves docs 7, 3 + standard: { + query: { + bool: { + should: [ + { + constant_score: { + filter: { + term: { + text: term7 + } + }, + boost: 10.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term3 + } }, + boost: 9.0 + } + } + ] + } + } } }, { + # this one retrieves docs 1, 2, 3, 7 standard: { query: { - term: { - text: term + bool: { + should: [ + { + constant_score: { + filter: { + term: { + text: term1 + } + }, + boost: 10.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term2 + } + }, + boost: 9.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term3 + } + }, + boost: 8.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term4 + } + }, + boost: 7.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term5 + } + }, + boost: 6.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term6 + } + }, + boost: 5.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term7 + } + }, + boost: 4.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term8 + } + }, + boost: 3.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term9 + } + }, + boost: 2.0 + } + } + ] } }, collapse: { field: keyword, inner_hits: { name: sub_hits, size: 1 } } @@ -303,8 +613,9 @@ setup: size: 3 - match: { hits.hits.0._id: "7" } - - match: { hits.hits.1._id: "1" } - - match: { hits.hits.2._id: "6" } + - close_to: { hits.hits.0._score: { value: 0.1623, error: 0.001 } } + - match: { hits.hits.1._id: "3" } + - close_to: { hits.hits.1._score: { value: 0.1602, error: 0.001 } } --- "rrf retriever highlighting results": @@ -331,7 +642,7 @@ setup: standard: { query: { term: { - keyword: technology + text: term5 } } } @@ -349,7 +660,7 @@ setup: } } - - match: { hits.total : 5 } + - match: { hits.total : 2 } - match: { hits.hits.0._id: "5" } - match: { hits.hits.0.highlight.text_to_highlight.0: "You know, for Search!" } @@ -357,9 +668,6 @@ setup: - match: { hits.hits.1._id: "2" } - match: { hits.hits.1.highlight.text_to_highlight.0: "search for the truth" } - - match: { hits.hits.2._id: "3" } - - not_exists: hits.hits.2.highlight - --- "rrf retriever with custom nested sort": @@ -374,12 +682,103 @@ setup: retrievers: [ { # this one retrievers docs 1, 2, 3, .., 9 - # but due to sorting, it will revert the order to 6, 5, .., 9 which due to + # but due to sorting, it will revert the order to 6, 5, 9, ... which due to # rank_window_size: 2 will only return 6 and 5 standard: { query: { - term: { - text: term + bool: { + should: [ + { + constant_score: { + filter: { + term: { + text: term1 + } + }, + boost: 10.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term2 + } + }, + boost: 9.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term3 + } + }, + boost: 8.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term4 + } + }, + boost: 7.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term5 + } + }, + boost: 6.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term6 + } + }, + boost: 5.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term7 + } + }, + boost: 4.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term8 + } + }, + boost: 3.0 + } + }, + { + constant_score: { + filter: { + term: { + text: term9 + } + }, + boost: 2.0 + } + } + ] } }, sort: [ @@ -410,7 +809,6 @@ setup: - length: {hits.hits: 2 } - match: { hits.hits.0._id: "6" } - - match: { hits.hits.1._id: "2" } --- "rrf retriever with nested query": @@ -427,7 +825,7 @@ setup: { knn: { field: vector, - query_vector: [ 7.0 ], + query_vector: [ 77.0 ], k: 1, num_candidates: 3 } From b76905939820dbf9340cdc4bd66d117645c83ffb Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Mon, 7 Oct 2024 14:31:47 -0300 Subject: [PATCH 142/194] IPinfo ASN and Country (#114192) Adding the building blocks to support IPinfo ASN and Country data --- .../elasticsearch/ingest/geoip/Database.java | 19 +- .../ingest/geoip/IpDataLookupFactories.java | 3 +- .../ingest/geoip/IpinfoIpDataLookups.java | 235 ++++++++++++++++++ .../ingest/geoip/GeoIpProcessorTests.java | 13 +- .../geoip/IpinfoIpDataLookupsTests.java | 223 +++++++++++++++++ .../ingest/geoip/MaxMindSupportTests.java | 6 + .../src/test/resources/ipinfo/asn_sample.mmdb | Bin 0 -> 25210 bytes .../test/resources/ipinfo/ip_asn_sample.mmdb | Bin 0 -> 23456 bytes .../resources/ipinfo/ip_country_sample.mmdb | Bin 0 -> 32292 bytes 9 files changed, 495 insertions(+), 4 deletions(-) create mode 100644 modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java create mode 100644 modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java create mode 100644 modules/ingest-geoip/src/test/resources/ipinfo/asn_sample.mmdb create mode 100644 modules/ingest-geoip/src/test/resources/ipinfo/ip_asn_sample.mmdb create mode 100644 modules/ingest-geoip/src/test/resources/ipinfo/ip_country_sample.mmdb diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java index 52ca5eea52c1a..31d7a43e3869a 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java @@ -22,6 +22,10 @@ *

    * A database has a set of properties that are valid to use with it (see {@link Database#properties()}), * as well as a list of default properties to use if no properties are specified (see {@link Database#defaultProperties()}). + *

    + * Some database providers have similar concepts but might have slightly different properties associated with those types. + * This can be accommodated, for example, by having a Foo value and a separate FooV2 value where the 'V' should be read as + * 'variant' or 'variation'. A V-less Database type is inherently the first variant/variation (i.e. V1). */ enum Database { @@ -137,6 +141,18 @@ enum Database { Property.MOBILE_COUNTRY_CODE, Property.MOBILE_NETWORK_CODE ) + ), + AsnV2( + Set.of( + Property.IP, + Property.ASN, + Property.ORGANIZATION_NAME, + Property.NETWORK, + Property.DOMAIN, + Property.COUNTRY_ISO_CODE, + Property.TYPE + ), + Set.of(Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK) ); private final Set properties; @@ -211,7 +227,8 @@ enum Property { MOBILE_COUNTRY_CODE, MOBILE_NETWORK_CODE, CONNECTION_TYPE, - USER_TYPE; + USER_TYPE, + TYPE; /** * Parses a string representation of a property into an actual Property instance. Not all properties that exist are diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java index 990788978a0ca..3379fdff0633a 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java @@ -76,6 +76,7 @@ static Database getDatabase(final String databaseType) { return database; } + @Nullable static Function, IpDataLookup> getMaxmindLookup(final Database database) { return switch (database) { case City -> MaxmindIpDataLookups.City::new; @@ -86,6 +87,7 @@ static Function, IpDataLookup> getMaxmindLookup(final Dat case Domain -> MaxmindIpDataLookups.Domain::new; case Enterprise -> MaxmindIpDataLookups.Enterprise::new; case Isp -> MaxmindIpDataLookups.Isp::new; + default -> null; }; } @@ -97,7 +99,6 @@ static IpDataLookupFactory get(final String databaseType, final String databaseF final Function, IpDataLookup> factoryMethod = getMaxmindLookup(database); - // note: this can't presently be null, but keep this check -- it will be useful in the near future if (factoryMethod == null) { throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]"); } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java new file mode 100644 index 0000000000000..ac7f56468f37e --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java @@ -0,0 +1,235 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import com.maxmind.db.DatabaseRecord; +import com.maxmind.db.MaxMindDbConstructor; +import com.maxmind.db.MaxMindDbParameter; +import com.maxmind.db.Reader; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.core.Nullable; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * A collection of {@link IpDataLookup} implementations for IPinfo databases + */ +final class IpinfoIpDataLookups { + + private IpinfoIpDataLookups() { + // utility class + } + + private static final Logger logger = LogManager.getLogger(IpinfoIpDataLookups.class); + + /** + * Lax-ly parses a string that (ideally) looks like 'AS123' into a Long like 123L (or null, if such parsing isn't possible). + * @param asn a potentially empty (or null) ASN string that is expected to contain 'AS' and then a parsable long + * @return the parsed asn + */ + static Long parseAsn(final String asn) { + if (asn == null || Strings.hasText(asn) == false) { + return null; + } else { + String stripped = asn.toUpperCase(Locale.ROOT).replaceAll("AS", "").trim(); + try { + return Long.parseLong(stripped); + } catch (NumberFormatException e) { + logger.trace("Unable to parse non-compliant ASN string [{}]", asn); + return null; + } + } + } + + public record AsnResult( + Long asn, + @Nullable String country, // not present in the free asn database + String domain, + String name, + @Nullable String type // not present in the free asn database + ) { + @SuppressWarnings("checkstyle:RedundantModifier") + @MaxMindDbConstructor + public AsnResult( + @MaxMindDbParameter(name = "asn") String asn, + @Nullable @MaxMindDbParameter(name = "country") String country, + @MaxMindDbParameter(name = "domain") String domain, + @MaxMindDbParameter(name = "name") String name, + @Nullable @MaxMindDbParameter(name = "type") String type + ) { + this(parseAsn(asn), country, domain, name, type); + } + } + + public record CountryResult( + @MaxMindDbParameter(name = "continent") String continent, + @MaxMindDbParameter(name = "continent_name") String continentName, + @MaxMindDbParameter(name = "country") String country, + @MaxMindDbParameter(name = "country_name") String countryName + ) { + @MaxMindDbConstructor + public CountryResult {} + } + + static class Asn extends AbstractBase { + Asn(Set properties) { + super(properties, AsnResult.class); + } + + @Override + protected Map transform(final Result result) { + AsnResult response = result.result; + Long asn = response.asn; + String organizationName = response.name; + String network = result.network; + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", result.ip); + case ASN -> { + if (asn != null) { + data.put("asn", asn); + } + } + case ORGANIZATION_NAME -> { + if (organizationName != null) { + data.put("organization_name", organizationName); + } + } + case NETWORK -> { + if (network != null) { + data.put("network", network); + } + } + case COUNTRY_ISO_CODE -> { + if (response.country != null) { + data.put("country_iso_code", response.country); + } + } + case DOMAIN -> { + if (response.domain != null) { + data.put("domain", response.domain); + } + } + case TYPE -> { + if (response.type != null) { + data.put("type", response.type); + } + } + } + } + return data; + } + } + + static class Country extends AbstractBase { + Country(Set properties) { + super(properties, CountryResult.class); + } + + @Override + protected Map transform(final Result result) { + CountryResult response = result.result; + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", result.ip); + case COUNTRY_ISO_CODE -> { + String countryIsoCode = response.country; + if (countryIsoCode != null) { + data.put("country_iso_code", countryIsoCode); + } + } + case COUNTRY_NAME -> { + String countryName = response.countryName; + if (countryName != null) { + data.put("country_name", countryName); + } + } + case CONTINENT_CODE -> { + String continentCode = response.continent; + if (continentCode != null) { + data.put("continent_code", continentCode); + } + } + case CONTINENT_NAME -> { + String continentName = response.continentName; + if (continentName != null) { + data.put("continent_name", continentName); + } + } + } + } + return data; + } + } + + /** + * Just a little record holder -- there's the data that we receive via the binding to our record objects from the Reader via the + * getRecord call, but then we also need to capture the passed-in ip address that came from the caller as well as the network for + * the returned DatabaseRecord from the Reader. + */ + public record Result(T result, String ip, String network) {} + + /** + * The {@link IpinfoIpDataLookups.AbstractBase} is an abstract base implementation of {@link IpDataLookup} that + * provides common functionality for getting a {@link IpinfoIpDataLookups.Result} that wraps a record from a {@link IpDatabase}. + * + * @param the record type that will be wrapped and returned + */ + private abstract static class AbstractBase implements IpDataLookup { + + protected final Set properties; + protected final Class clazz; + + AbstractBase(final Set properties, final Class clazz) { + this.properties = Set.copyOf(properties); + this.clazz = clazz; + } + + @Override + public Set getProperties() { + return this.properties; + } + + @Override + public final Map getData(final IpDatabase ipDatabase, final String ipAddress) { + final Result response = ipDatabase.getResponse(ipAddress, this::lookup); + return (response == null || response.result == null) ? Map.of() : transform(response); + } + + @Nullable + private Result lookup(final Reader reader, final String ipAddress) throws IOException { + final InetAddress ip = InetAddresses.forString(ipAddress); + final DatabaseRecord record = reader.getRecord(ip, clazz); + final RESPONSE data = record.getData(); + return (data == null) ? null : new Result<>(data, NetworkAddress.format(ip), record.getNetwork().toString()); + } + + /** + * Extract the configured properties from the retrieved response + * @param response the non-null response that was retrieved + * @return a mapping of properties for the ip from the response + */ + protected abstract Map transform(Result response); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index 793754ec316b2..46024cb6ad215 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; @@ -64,8 +65,16 @@ public void testDatabasePropertyInvariants() { assertThat(Sets.difference(Database.Asn.properties(), Database.Isp.properties()), is(empty())); assertThat(Sets.difference(Database.Asn.defaultProperties(), Database.Isp.defaultProperties()), is(empty())); - // the enterprise database is like everything joined together - for (Database type : Database.values()) { + // the enterprise database is like these other databases joined together + for (Database type : Set.of( + Database.City, + Database.Country, + Database.Asn, + Database.AnonymousIp, + Database.ConnectionType, + Database.Domain, + Database.Isp + )) { assertThat(Sets.difference(type.properties(), Database.Enterprise.properties()), is(empty())); } // but in terms of the default fields, it's like a drop-in replacement for the city database diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java new file mode 100644 index 0000000000000..905eb027626a1 --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java @@ -0,0 +1,223 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import com.maxmind.db.DatabaseRecord; +import com.maxmind.db.Networks; +import com.maxmind.db.Reader; + +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.junit.After; +import org.junit.Before; + +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.nio.file.Path; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; + +import static java.util.Map.entry; +import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase; +import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseAsn; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.startsWith; + +public class IpinfoIpDataLookupsTests extends ESTestCase { + + private ThreadPool threadPool; + private ResourceWatcherService resourceWatcherService; + + @Before + public void setup() { + threadPool = new TestThreadPool(ConfigDatabases.class.getSimpleName()); + Settings settings = Settings.builder().put("resource.reload.interval.high", TimeValue.timeValueMillis(100)).build(); + resourceWatcherService = new ResourceWatcherService(settings, threadPool); + } + + @After + public void cleanup() { + resourceWatcherService.close(); + threadPool.shutdownNow(); + } + + public void testDatabasePropertyInvariants() { + // the second ASN variant database is like a specialization of the ASN database + assertThat(Sets.difference(Database.Asn.properties(), Database.AsnV2.properties()), is(empty())); + assertThat(Database.Asn.defaultProperties(), equalTo(Database.AsnV2.defaultProperties())); + } + + public void testParseAsn() { + // expected case: "AS123" is 123 + assertThat(parseAsn("AS123"), equalTo(123L)); + // defensive cases: null and empty becomes null, this is not expected fwiw + assertThat(parseAsn(null), nullValue()); + assertThat(parseAsn(""), nullValue()); + // defensive cases: we strip whitespace and ignore case + assertThat(parseAsn(" as 456 "), equalTo(456L)); + // defensive cases: we ignore the absence of the 'AS' prefix + assertThat(parseAsn("123"), equalTo(123L)); + // bottom case: a non-parsable string is null + assertThat(parseAsn("anythingelse"), nullValue()); + } + + public void testAsn() throws IOException { + Path configDir = createTempDir(); + copyDatabase("ipinfo/ip_asn_sample.mmdb", configDir.resolve("ip_asn_sample.mmdb")); + copyDatabase("ipinfo/asn_sample.mmdb", configDir.resolve("asn_sample.mmdb")); + + GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload + ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache); + configDatabases.initialize(resourceWatcherService); + + // this is the 'free' ASN database (sample) + { + DatabaseReaderLazyLoader loader = configDatabases.getDatabase("ip_asn_sample.mmdb"); + IpDataLookup lookup = new IpinfoIpDataLookups.Asn(Set.of(Database.Property.values())); + Map data = lookup.getData(loader, "5.182.109.0"); + assertThat( + data, + equalTo( + Map.ofEntries( + entry("ip", "5.182.109.0"), + entry("organization_name", "M247 Europe SRL"), + entry("asn", 9009L), + entry("network", "5.182.109.0/24"), + entry("domain", "m247.com") + ) + ) + ); + } + + // this is the non-free or 'standard' ASN database (sample) + { + DatabaseReaderLazyLoader loader = configDatabases.getDatabase("asn_sample.mmdb"); + IpDataLookup lookup = new IpinfoIpDataLookups.Asn(Set.of(Database.Property.values())); + Map data = lookup.getData(loader, "23.53.116.0"); + assertThat( + data, + equalTo( + Map.ofEntries( + entry("ip", "23.53.116.0"), + entry("organization_name", "Akamai Technologies, Inc."), + entry("asn", 32787L), + entry("network", "23.53.116.0/24"), + entry("domain", "akamai.com"), + entry("type", "hosting"), + entry("country_iso_code", "US") + ) + ) + ); + } + } + + public void testAsnInvariants() { + Path configDir = createTempDir(); + copyDatabase("ipinfo/ip_asn_sample.mmdb", configDir.resolve("ip_asn_sample.mmdb")); + copyDatabase("ipinfo/asn_sample.mmdb", configDir.resolve("asn_sample.mmdb")); + + { + final Set expectedColumns = Set.of("network", "asn", "name", "domain"); + + Path databasePath = configDir.resolve("ip_asn_sample.mmdb"); + assertDatabaseInvariants(databasePath, (ip, row) -> { + assertThat(row.keySet(), equalTo(expectedColumns)); + String asn = (String) row.get("asn"); + assertThat(asn, startsWith("AS")); + assertThat(asn, equalTo(asn.trim())); + Long parsed = parseAsn(asn); + assertThat(parsed, notNullValue()); + assertThat(asn, equalTo("AS" + parsed)); // reverse it + }); + } + + { + final Set expectedColumns = Set.of("network", "asn", "name", "domain", "country", "type"); + + Path databasePath = configDir.resolve("asn_sample.mmdb"); + assertDatabaseInvariants(databasePath, (ip, row) -> { + assertThat(row.keySet(), equalTo(expectedColumns)); + String asn = (String) row.get("asn"); + assertThat(asn, startsWith("AS")); + assertThat(asn, equalTo(asn.trim())); + Long parsed = parseAsn(asn); + assertThat(parsed, notNullValue()); + assertThat(asn, equalTo("AS" + parsed)); // reverse it + }); + } + } + + public void testCountry() throws IOException { + Path configDir = createTempDir(); + copyDatabase("ipinfo/ip_country_sample.mmdb", configDir.resolve("ip_country_sample.mmdb")); + + GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload + ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache); + configDatabases.initialize(resourceWatcherService); + + // this is the 'free' Country database (sample) + { + DatabaseReaderLazyLoader loader = configDatabases.getDatabase("ip_country_sample.mmdb"); + IpDataLookup lookup = new IpinfoIpDataLookups.Country(Set.of(Database.Property.values())); + Map data = lookup.getData(loader, "4.221.143.168"); + assertThat( + data, + equalTo( + Map.ofEntries( + entry("ip", "4.221.143.168"), + entry("country_name", "South Africa"), + entry("country_iso_code", "ZA"), + entry("continent_name", "Africa"), + entry("continent_code", "AF") + ) + ) + ); + } + } + + private static void assertDatabaseInvariants(final Path databasePath, final BiConsumer> rowConsumer) { + try (Reader reader = new Reader(pathToFile(databasePath))) { + Networks networks = reader.networks(Map.class); + while (networks.hasNext()) { + DatabaseRecord dbr = networks.next(); + InetAddress address = dbr.getNetwork().getNetworkAddress(); + @SuppressWarnings("unchecked") + Map result = reader.get(address, Map.class); + try { + rowConsumer.accept(address, result); + } catch (AssertionError e) { + fail(e, "Assert failed for address [%s]", NetworkAddress.format(address)); + } catch (Exception e) { + fail(e, "Exception handling address [%s]", NetworkAddress.format(address)); + } + } + } catch (Exception e) { + fail(e); + } + } + + @SuppressForbidden(reason = "Maxmind API requires java.io.File") + private static File pathToFile(Path databasePath) { + return databasePath.toFile(); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java index 84ea5fd584352..3b12003637783 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java @@ -361,8 +361,14 @@ public class MaxMindSupportTests extends ESTestCase { private static final Set> KNOWN_UNSUPPORTED_RESPONSE_CLASSES = Set.of(IpRiskResponse.class); + private static final Set KNOWN_UNSUPPORTED_DATABASE_VARIANTS = Set.of(Database.AsnV2); + public void testMaxMindSupport() { for (Database databaseType : Database.values()) { + if (KNOWN_UNSUPPORTED_DATABASE_VARIANTS.contains(databaseType)) { + continue; + } + Class maxMindClass = TYPE_TO_MAX_MIND_CLASS.get(databaseType); Set supportedFields = TYPE_TO_SUPPORTED_FIELDS_MAP.get(databaseType); Set unsupportedFields = TYPE_TO_UNSUPPORTED_FIELDS_MAP.get(databaseType); diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/asn_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/asn_sample.mmdb new file mode 100644 index 0000000000000000000000000000000000000000..916a8252a5df1d5d2ea15dfb14061e55360d6cd0 GIT binary patch literal 25210 zcmbW71$Y}rw1s5~wGFi4w9Ph66UT{FF}P@s46+m1!Mdp|t!>4!5JLAF>d1s2Q5jVu;RAz1`-OZdg~OUM!ANOBZ8 znjAxJKyFB)Zdw0WavV9HoIq|&ZbD8ZCy|rMDdbdg8abVuL2gQJMs7~dBn@&4au$he zm*+d1+=`q-&LvC9d1M(mpIktelU8zT(ni`z2k9hTq?`1RUeZVU$qI5Ca$9mca(i+I za!0a~tRkz)8nTwGBkRe9WPofS8_6cJnGBLGWGlIdTug?@HnN>uLM|njk;};yYYw1Ke2PJ~CjeQ*T7#(KRd4np`F0|^{{tCIe`A|u=mi8-;?@YAzV{vpb?97-NW9!?%X9!VZW9*ugAQMPRFSn9`-$20ad@=m075_vLX z8zFWI%Bh-v8vN5W{|p{;CiJs3{cMY6RCYd{i+L~^@$*ogLpdMiZj=j9uH*3+k{3a{ znD!;)rHb-8E<^ls&A);{{-CBmMEzmtkI;Tpi$4baaq3TyPm)h* zc~2wn8S2mG_0jXFJx{)Xycbb^M!|VoK1O+&u~+iPSzd+yn&!U_{|(K56aHJ8|2F)0 zH2+=r?`i(~%C>v}{X>+Gls_8lOcdE?pJ@4?Qu~bj9LIb?`%CgGMR}~4*WYOVx6Jtt z`uDVd(BeNr|0(Z}Mju#y!Mq*8++X4UM*Da25Asj)FGbOQ0n#O56clQD5&UAtO7i-c za_A$ekIL%>qoI${{0*41At~A#OM9G_H=f!A#5bnB2|1CRq$tjFOh59bpe#h0n%4`a z!Jm$@fXB>$y{Q)8jQZx#XVNyb_!iJ-QQuP2XH(k>@j0~TYVlI&oT~+8u;;7T7_1jj z#5v3U75!$_{H@{JP_{#{GsmIDozPvH?`Dn%x>wVE@co)!!MtstZ>#JLvVF3>mbU}* zc0^t!?JBZb%d0`WR`ctaQ?Izeh4cf+YtZr<;WweQp)@lmNVaHst<)Bgi^-71vSGIG z+NmuemnzDBT}Ev=+F7CLD{*`eV;zXSg0eHpF(|80dKudVc9?djc6ok85{(8;d zjmPXx?t%E8%HC+QmbW+beVDVarte2>fARqGK=L5+U>tu4?L*1K6y^0CPVETtNb)H1 zXp3c|>^L2Zy6#0e4&^44<54a^If1z+k|&WTlcy-k_D)6qX_|jJbIyQ%rly}o{cQ3a z@?7MdNBew5QSU}~QOnNvi;#D5K3;GM{7W_eGUi`SUO{4B6kMg{T}}NO@>=pb@_OrmpOK#F#a|94f!qk9r-%&$SbR@3XK*F#@OJD|lIs5K(qr0fZ& zQwwT&E!0}cMdV^KgyY+2w=2qiSOR@1^<|pA9Qq2)UkQIF=5>%eYw=Zx@1pr(_}Tdr zVSbb}$u7kSTt|gH)MMmo)UlR!FByl{M?0Y?uOmsVAMssjr?fcco6O6Q1ISyGk8ivl z{<^$hILJJq??!ufExre}JrUna)Ay#n5A=O$@0X8n{0Z~}s2@lkL>`>aD?9{whoYRo z*kP~_r+ox@BzY8hG3#WD8^&(Qoc znRAxKvdKf5eh%`^W$ZlieDVUs?n1ec{zb5_L%CS_^4u-_+u7QG1(whkO_L?_Dft=lmm>djup0aVoCSW#_*dlDit^ZR5dW6?cgUYa`+Mj=z_)-uYWY7g{xkGn zH2qijzcKbZ`G*$&6Y;2Ihg>k)O0r9O@vPfj2=R+RahAb+CrCm+W6WO9m@H?vV`8&d|)ch)CPrZvV6UO>|u zs5g>Lh&OBTAp91_TFFJ^VlqUwk?rIX#VM!JUxvEwLs^b;4z(4qSE9t}?*zMp_Ri!g zau-F}FJa_$YJP+{QPM=bi*`5JqbQG$A-KSqX@iny9lIs*@{vhJ(sqd!gyIU+%FQmRFat@`v7wo;YynW#BtNHseZ-3|qX!?QF z4}yNMrXQm0X=2`By%wE_a<~>h0{)Sjf0VL|jwX*m{8%l19QEU&pODw5;T(%j%KJqp zGw&4Wr)qk3j-1Z;8Hk@r`z-Qo#c40nKUY!q=Xuo5$8i^+T#u6N$BVSQiy$rzE9Tsw#c!l`6Y_4ReG7Rjc^i4VqOA80YIl-% z;rP3?_&xCN&HK}Ftwr}U_kp}#^dPl|5Pz8VBjlsxW8~wCvi&EhJxM-AK8<>w(ej?9 z{v7!{;xA}%jMa>6zr4(xSIAe%*T~n&H^?{f9yf{hTd4PKly6bqq4qBM9{E1`0r?^M z5&1Fs3Hd4cnWDVzTZcD~f~ui$^p*f)872IgJScR0^t@cX=8^aK1KHUB3b`!n=k zH2qijzcKbZ`3L!@miL#k(Tt_IfOcU%UR-3cY&t{pOPDi)97&EMN28uGv^P)`*I&FL z3MlQdnm!Ktc+H;xe`C$xgvU(8d2kIEPex7?$`q6el&L85m^TgfbnSSad-0}>Z-)5h zv}bC02J|hc&my-ZXOo<7#dGM-Rg~9MYO!plw_Aq#=4a){=E(J-Lt!kPT#` zqNs0k%%kFFlrUpKvISaeKCgHY{Kbrg$TqT_T%st?b1CwdQD083AXkz*ksajD%BcUIa_h({kijT?r#mDlPI~n>ZTHdMD zPa{uPocSXCGm(FmvgP@l4gVa*&einusGX1a1+*_DFCs5il=WPKyi2KHMqW-{L0(B- zrD)8je+_vpc^!E@>U;*}2Ib3kZiIgm<2S>8fc7o0Z>4=3c{_QBqCEaiYIh<3Zrb;d z_iB0fA%4H|4fdft{z2wGL_UnXN3`P}h5s01kCRW3Pm)h5%KDzRShm=d`m^M7sN)@! z=e0bn1DT&4$Cv28OumBrSGD}tsJ{;V4cc##Z)tgND|-u!P4T;ozem0g?JJZIwEPd@ zf5iC5YtNekY6g!nnC|-@*72Yec$5v?==5==KMhZi1<&mf7bGTf&MG? z-xOzYofZE<{ZH~Q#S#lyfPJRdVks#^!LcPpD43Ha#VBnkNGzFyGJ^Rd$x+az&>l^W zAvYj5M9xOE$CBg7@rvTQOD0g;81^QbK9Tw)aEvU~TwM93Sm6*z=Vw+gkv?oG~l8wHCJ_Zl~_h zbSE_z=_Wm-S8+?swGuzI3fT21+tA;Z+>YFy+(A*E-;UHOaa^NVGx?e=Oj`8bZKY(%r{TmhKxZXtVX7U!~-Aem5@^(d;e+S}sQooD5 zTZ`XA?Ox>FNBe%oxgTnI4>I-;@*dXoN2osv{W02)Yw;(bKS}*5@@euJE$>-s&msSL zO@BezrC4hvFQI(Gyq96WLi<(nHS%>udHru7?@j7&k#CdlknfW3k?$+Y<3GUhdOv&w z|6>&|{f7CUlAmeEeU7{@H2+KHdan@hbE`aDf9gFm0K1!OsCCATJRq+L;-rvt}3HQ%M|GOX_r9_D*V zAL&<=$5&9>hTN9i4##h=e<3BX6Y^--+=Ka%aR>(cXm& zlbwq4JR-=8Qa8yiE#8fI5A~R)uU7W_PY{nY-Us_0lmz`G?0snW!`@ZdvfUK?wB~2v z4={HPxmJs>qc(`V^|W^*cPIByl=bgPZ7<}buY@Mb0`$pQ`&}H5N{D(FF5%`a4{$o7maqAAK?6dF5L% z7p3+h`4agu`3m_e`I@3U=5=askZ+=1%q~By$P`*6oL--$Q z{>RJ}>-|$r|BU+Q(7&Mlr567R`qz1X>v8bEW!`t>_xbpU9}xdh^M8W>GxL5Sf7Rl@ zA^yAO|Do)y_l5o!bqiUL&l_0?yNG@9z#ggjqhODwKSt9xfW4vew|+qp&1wIA zSRGC$>MKJ|hu7z=k0b{Ynbcrad#Em&>TmJwL$@>~5lEs-y+ z+9jaGqpYR=A4*`U$7^*stgD1dtg${?;>r@qD z^wx=vGs>1$C)XL($-cgUL@W}{#FB}$93vyJ%;Uq*s8gi*&l+7GpWDG2V|@`-V{4$f zy1cpvwRI;K7)?#pxLB{t>acsQeq3$Nl!LIab0Cd=O{djJxLgiDDy*un!bl_r!ik8f zD($cY+be_BwMKPIb8~wzP+i#;XbFalra*I`t+oau?RQu)!gj9{ZDehyQZf$7_1Z!K2P`}k!ZzQcrD@N5X<|c~GF6zuXs!}nk z=vBAV=k};R2qadQ$JVIWal0|KPE6<2 zfKj%xr9Xo{j2qSA&bVm=)sjhz35RoA>UUdR9;@FgYR#H|KQEuhAtrUG9@n}i8I73< zc4hGr>0y{NnAw<3p=5kOjeKjyR5QB6BCgtQvsoST>hm5Jl-{SfHjhs%%;x3B)K*NQ zHD)Ruj*F(OeS=s+o3v*&OtVD7jHs6MUBM`Ao@a!j39C`nBQ9>G-R87<{1{BpX`1to zmYpu24J}uq+gfAdV!BhYsBAfm4(zbF>kVuNOIsE<8ZE80i#cUl+m;(mZJ08$8N1DA z#dJ~4NQc{loHSzJ@!G|dX=pX3)+aI<_F+M=t?i%7raGDGPp0G=!noV97@cbM(8SsQ zW8B?7zaMok!M;~>k*Sn{%Ob9R0N(X~#eP0r9- zj30LMSQwM0Kb#m;ecak6B+u@m%SU+-1-DXa|Ui6%W*1H6lr=Nt@$z6xA6 zo7;h*6q|KJEYXz;$Hi3`WlaI)a3e%*=n}CliET=A^0lE~d^Rr!w7D-9NhQR6idbK#ob3#OB;1+XDRL|F27u=ZHsYBNe}d!sieHCsT~V9yS!us;$|k6 zj`bM}W9$3OOx#Sxj7n=ru1+szq0?=}rHICqgSPX7fNSu(-C_t@8;z-r{h5fmr}p<^ zVivZBd&REP6zhv+%&5_fbqintCymy&>L&EL1MA<87Tn^7p*eqR0sD?v{Q>Od%@H#m z7duWQX^mk!?-*IF1fwpt4s|z)TQ??B{w8NMSzC?33Wp6hO(!OkY)9Fi?AVIwge!5| zY~0^l#3mDu?K)s4I3A0{9!g~aE+K;cS?<7=;==`q?#bJK8)28viQW;<13~#LBW}KY zjOi$l&oPF$$+a{axREBpc8}kv3P*Z7F?&V#IB;jT^Q>v-2b-tu!i`Baf!-0@yx14y zgHp#>S*cN*UEfAH5j7T@X)~OP^knZeMm4U|Oo{6eI~i_);&POkYiz5p)J@Rg@`)*1 z6%>P!NUlky&D0v+x`RRZA>kWU)+Okmj)@MN&0gX6L@O%LK{TCRU0oF@|N0N_;bJsg z9&xJ-V9%&CV<~IQ#O6~QOQHL6Gb-CcXQwmUf-r|_A-jFJ>^_$m@wdgbn3*oz(vzuf z`9QcbDE(|7rHz*Uh|#(RlUY0lc1)936YX+Txa}@mw(Xq%e_S00&4JBabYly)5lk#I zkxW?=*ha)&C>$e*Gck?I5E`2y8$)+^E3p0KT9bnJKl)L2FglaX`E5?onRVC?7A4F9 zYzGN*9UcL(j*|V@yn>pRZ?V(q`Fo2(7^+3_2!`Rqtq@!O?2|C@aee7bN^ZqgaldZC zH3~r+#>w&$eQrz<&J<}6XCO<9D^)wC#X+O2w0KqS{CyseyTYgSm=rpP*JDnnLu@)V zn0|7Ada-Jwn1Z1ib0CwB^q9Dd#7(I3CMajRv%+rgQs=MCVV!uOBzU*uy#b|D!suOD+jnov{DE)$xk`QBQ78 z$vTBFoJl1fxBNc8OYB(0J$nWrcZ96IeR!g zDqjEGKCy?2^Qm7QOY#&fi^a36c-n5j2Ln6Ck2xh2hLJ@r32G z@dm#T?`b~I-|p?r$<7$w8vpjplQ!y8$$@@|Ry>Z%*GM_z{S`iU#FbnB;>4sdto1+o zaP^Q;I{FmM_|fNc6`_i7r#`1B{MQKoZKL|{IwPH(f9o%w5QfuVrTvf2c$3tA8KN8FE^GC;M^>o#0W>U$xInak!uj(EX5Axz&UVlZDTSK@a z;`Pc0A(xHzP$$5z8Fsw=+Hf-xQ?DVKz=xq&M18d?5O414f1_Ny%wQ@StplC$Sj4Eo zM-#Qtg)7X6J*sslg`v8WAI%(2zg@JB%P%ds8PD9j?x49YW7K0wX5WOxQzRCsQC+(j zZ{rTO9;t}fO;ek66o!~|@`1Bfe9Gb5Rv-i)YK!D9JSPls z;qs9U)6VJl+C6+bY>&peW0`PV?0xC}aKucj?I%>*AB(j28w->1D88S{4|=lOMPpqN zfBx|&Ck)kkcF?ie<2zWg=>g3It_P;6`2?}1oc zqJgL|MA=@>+AkaC*9Qd^H;_9lr=S zs_~ujYGfUq?#|r9Pfi%ljS9o(@VI>rF?{lRqU!6XSP@onJ@`D(h2KW-nJk41S!`_* zmonMm!!mS5(1&;+%-KUtOs)cl$LW$Ei>uIqt$pH4O7_vm+KDEYH#cU#dA8Ly)rwCD z_}E!#G~?G0tU@2wgxz8HA>#7aBl7?E^S5&Iu&W^R) z;m+S{hsAxP=1YAr)>(Y^-V2Rl}*(x!C-B5TlQ;27z@;H+BzG}nzb5Vb~>}S)q<)N?k4G;>=ep>!Al#h;-@7nC((|>=8-RlIeR#Li%WES z?D!Bj>k;ho_?VjLHs#MjJGDsLXidh`z1V~BQxQ~A>BIShP zOe8eybl{sF-kI7kq}&F@tvvf_x;?uS2Be7Hqd&V5ik9$AH2;d46Na@BPHJGYrIVIn zBb*e^U5~?O&p)xtN`)|->%pDGg`Zi)J{ZCcvmxCtf2OpiamVbi)DOhF#9gd^K<X%97d_%vj9tB>5``&nJOD`oA%2HY_&l*BJB_#v$hlPD67rA?z| zz`&buEGY)Mzejw;EjL0q3Z11UlcU1tvgdDY;zXn{tP!0u7X2>Hp>)b-7||(nv`W2r zsp0)nRSIF4y^7yi8|1F8Qe| zEbgwt9?mI<$1lIjC4K=DXRr+OHkw&$rh3ito^W5E8SfmxS7-5YaRu&U3>gb^q3kJ$ z&B%p!IiJ3GDGaCc(G_^AcX{wsHfJfuI+?KJRW)wKye(^MNf>s#(!}EiejLKDn;H2# zoa}jgF?8A7;)WK+W2m%;dD(7!!j#WUm|D&Ksc^TMDvxo;54B1ky92&ONAM=8`dM70 z7jI6H{HN!fFw`R9YW3juE56-|-zXOKBvY~VNxYh>2haq5G-_L@wCt@W`)h@0I$Yt6 zM6f1vONqizONmY6_kEZ6ja)QcA9F_4D^p|4StBj{EW#xl7ES2mNMWdP6z!BwuR{}+ z=+n~a?brvVlepZ+Jd9|#V^+Cc9>3y+m^U- zm{|XQ?b?(-)TN0V2iBik{c2lPiS^gol`^v*68T3qt#y#(^COwGgte5H`PjXPNsj1{8Z)B=@v69^Y*uMpYaB%qcE^{N+#L ztMM%nzyD_Mo1qvssZ5M#g9jlKFWI^2Wv4Kl-oO}m@ROJLlb=vEHUs=lD_>{$M>Gp# zviIe0YuM~jGk$24zv<=dVQxA2%Ht8O$Q6jcC)m|0mVBTUmbT8(fU655n_K5uVK@_4 zJOugFyP23zfmnYWzfaX zRSJKodB%rCxB5FCydX_&!nQAdxWiuytj7~-5YO7;7dt$S@vHb`b^Z>AU)&Ks&Bn|vU=BjkKuRm^D`}(4t zjZrflNyYG&W68ujRc7Kp(i&p@*_O^35s2fxX8i;29bph$KWOKi|7v~hmb?b zVdQ4yaB>7$OpYW+k)z2mx=9b|C4Hoy43I%GM25*SavO46ayxQ+atCrp zGD2bw$azpsR*;n>=7h|zBCE+7vX-nPqhvkVKsJ(1wo zYsr5-iD=sIpT;TJ%`kebS097w2`qLusfA4`=X0_0{S{lUr)Uo zdXJ_jsi&a#(%x0ePgBbvKdb3|)cc`t$m>JVXO01+!;m&HW;b$oau0G(McMAX5WhF| zeKdVvYWtDM3Hr&@Pa#hwPt)Q~hy5+}Gd2Azls}vLIpn!o{&~nhpE~YE-V^T2aUr#f zU|+21m%zVN^Dl#cxx+C`l(|At)FJ(=pxw)Qt|qS`uO+WT%=NTyAa5jZQj~piGvaQc zek*yKmVZ0)@1TAsc^7#%d5_{SdwlMren0tuqP*UN)E*)qCLh6d9;N*l`8fH6qAdR; z;+~@ZH2UKM+Rs3LHt!4F@jTK;NH1vRUu67C&|jwg3i&Genxeek>xg?p^WS95ThQOu z^mnMgOTLHv_i3|@vc3-;j?MhkKSs>&NS`QQ=6?$RGtK{;v0sp1BL6GeUz6V`%JTMH z{SI;8GyezjM=kCrYCj|H7o^{iens23#szI#*82zY|J2I=B_I2ag3(GVD1bjGpI=Z& zy$JeXO&>yiC^?MW3~|G0k06T`#q|qDQX8eX8OE<*47IW3IC8wAEH{DLM8r>`Jz2}& z9QqXMQ^{#sehKoY=l$VZz@LeP{wOez&P1Ap)P^)0Nwjqil8<%GCFhaz$pz#>#G+2o zR@vSy;crEI5m`z)$*mP-ybJMe>K;w^Ivm53)H&Z|e1KYz4556OcA27#+lJb<6-h+C%l%URC~=qok775*ybttNNU^4Cz?8F6bheHZv<-XDQJDTpH-fYgrE zhh*jR3p(I;YJL~|gyye) zn5G{N{Rkv+Z%1nSQSgsu{4wOQ7n}_J6r?klcPe=rdAedT zVnmW{KMV1X(LNjYIY_tDKNt3SNLSE5ANB>xmSr!bei8JGXuMZ%lji;jJX5)oq4_BF6wua_mKA@ z?mpW0E6REwfc~K7KLr0_#yz6xk2)M9F($IiQQxEQqp#HbC5)>=Ts7?) zEx#6eo#sa!jxm@^vL73ew*jdUDNd~k_EOr-TDchX7V68$X!AlL&w_5`5o{(HNT5-3FzxIeLehc=Jk+CGNr}! zB5qgeq77Nv8O1R#LhGZ}pVwsDHc}rz%ptTlk-L$*lY1a$Pi4!pdr{vT`aZPx)$;eF zwm*3Q;tr&JkQR5a!!dRk^+PrNFlvXBN03L7N0CRP-eYJVt0>!d9JS+7zkN?9Ivj;> zBK{<#zmQI5%qg(HL^>7eW~9@opH7|u?J}e@>7PZOtthW|4z+X1^ALYN?F+~Y6=nQI z)GkKcB}kVle=M#ql8BLUmm}{A+E;4&RnV`devPJI3;jCHzn(ESK);dpO^RdxpnnVU z-axt)=>=-H!M>gL9k3s!eJAX@lr7tLH~f2;cQ1J#dA}C-0JR4Z|B$lB)ob~WF#l2V zG4gT5KcU4vN&PAEY4RDwJxlvJ#c^VspLaL}bNwmS!%Ld~GRwR|zKZ^AYrqX@5d~N`9s&`}cEdUpO4&xqhVn74@&l zZ&2U2w7(<2Cx0M+B!41*CUI}F&R-R!{Tu50UGwd^@~6_q^ZpARSW^W^Gm!?Vyh4bg zj>00Ou}FjS^%M?a-caaV_l28j`NNq%0{O+7J`(;Y%^wYajPfVAHGLfP@yaipKu#nl zk(0^I$tjAm%~Mf+8ub!#x|Tmf*%SH^XCNJgG)v2$4Sx>OzDRSCwn3VQk zZIMUCmWF8 zn9ra1IQ*q~zpxp8O!HeAx*Lk z@o{C#_P0~F$PUFxh%fA-mVmtxX&q88we_&Ol`ZS(fuCevDz8tvj@qtd8gZF?eqok+ zA9T!FQJ1W5gTpbIYfbtC$lt`g-L$yfsqI1Ti8zjxEVnoOeN^6L_Iu%ejNP9+fIN^q zh&)(P)^iBzIh6Wg?@vA%{?SMeARU8r6U!V6`#7XC=pPUJgnYTe6XBnv z`KU);{}l36@-*^v#mPUR+?hz1Qa=m!*~*sX&w+og=AQ@ue9gZA{)L)<5&Vla{}N?y zZm*fklwWu`c?EeTc@=pzc@65jmbN`7uBU&4qHO1l4#(#9J>RV9w=nir@;35z@(%J& z@-FgjT=yQ@_mcOK_bX1BK>tBS*^Y;(J&f{?X!@hnA0x&3euDOsuwSJ86!|o?XOu1L zeU|!j(4VLMLOy>A)>Pq3)L$lF$>$foO6@hoy{_qRI2=>#__rAIHh$v%5OME-TZ8X{ zxYxq>kiKWk`>;Qtjabn}+0Kuke@y!m@>B9N@^eKQ{{^)#$*;(-QO`G8+_%)fb2z3B zq5T8eHJJ8~h{s%(b^fdpTT zc5~=cXiwGhr$H~#{OOFDLCz!%#Ld#;W>cR-&L!ue4>)#33$*x!D8B_`w$${kpf93c zs_9OLV_Fq;7h;Y#3ktNiIP=+b{F0;ny&)maHSA$XiOgo@^i+k=KO2;2IY!YGz(c zQEDwJrf69{R$g}n;#aCT>9@jP#qz5)eJ5&b$ej_lmi8{N+i9C*8yQzD!F(yQs9_Dr zYjx)Hq@O^{{z&VXzaI7g?QXJ%Od>C(Y+1IK`mSV}%pfkS#r47O*Zd8P+o)KAdn($b zGwpPu2X>;GeGfXDEBRSX*Zy z-KPAavtgfubQRLMTKswN&)5757=Iyo5%Moax{Qo2cE4xLau7n$MqZxAS)9-GR6}^Z7-0!M|Jc?}2}>=HCbZ zewKYe(;uYv5cx3qh~o5@Q0_6L7pXrE`w3;sc0Wn|De`IZ8N@wH`#D7!|2*^;^8Soj z@No^1C-Yy1{|fC_$=9^}*QvchzKQs^Xuqw+y#xJS>hF>7I~+5xR*ODR6xWmfNBIA7 zKIk8VCEzEF{}lG`v_FIWxw2(Dzo7ml`4#fN*7Coh{w?_(`91jq;(tW?mHtn#e@1)6 zJpV;ewomxKsTh$bul)!4C)zs#`F|;YutQPA3@)HHh%AIwq~#-c@DSz=C5LJGn<0O= z@@Me+vfg6GjwDARZnRcz4E3?(IC4BW0r3-&rqG`Rd$PkZa~t}bD+*tv!Bdf@5QF0&Lj;*8H0HxIUD8YAZ*K=MN513zKD}=!

    Z>K2RvpuyP$Q=T2SY!E`k}B7Q?_vn?gL5Ibp&~&qSTI}b~Jemc`SJx>N=kG3FL|7Ns6+b zlM#1{=AX)#)5z1wGZ1&C7Izl)v!S1(>E}{E4?5?Ur+r8O}~-aO~}8QHv3uDdn@$YsNYWBLEfo2 z3v*J$%J{nxe-G_@$@|Fr$p^>>$%n{?6=gk-pq@uH|1tQFGwuoUN%AT3Y4REJS@JpZ zdGZDFMa5Yk(SMnIg?yEK4f_)2_~6&+zd^nU-3R?G&;hkSnME*?vLjI~ayB1}C zL;9WiALO6pU->w3tsw=xYX9`yN|z5xD0=50Z4Np3|hQk3V?!C%U_W=)SlZ_)f^@Ru`gg{H5B-b#HHxtiRGT%$Obl=rxndApD% z*+#}uZ#!*EQI_j~-l_Rr@DrN9j%C(E@21^DCbhT}wO(>pGL3Q>+F7!X>?b#n8_5B3 zlcH?rZm4H>>U(JVp30v4DDwAa{yyZss@s?OMcLNBep${|0I| zk~bmlX4u%`xP`{VFkG!9JfP7GKo;`OTrv3fe#ylRuC@l0T6@lfRI^lE0C^lYfwZl7A@YeF*%YP1&*dYh8(LnpX+Goc=$f%O#mm-o08j=lH7`1M3$0Ha%<8>x4#6 zqo67h^LxWSuTfmzYYud$;)(2lsL?1{)@1uec`DtTN}Jh4DrvY^gq-fM)8liaww$Rd ztB%CHVZYDgF^cO_=K5}PLsHe%+8C*eMpiT!wTdTqA`rO_?*cCL28@u|<4OSxFo$6~>b=Frei>#=uL0yf@Dq@CvwcF=(dxK7QK-{P8 z1pUt3je5{?Zlk!l$4qAv$xMHuyW4X1o6f$BQL?feH*TaljCw6|k+D2txLZA5RP1pE ze5f~XYt_2le%x`q3q`H8YTfd3<)QC-dis)yxa_?Qn(6VO^IT51FC?m?6LzzivI|2l zHwL6J*}0KJm{FBR8>&_s4QfCS+~pOQf;j-?@A<1qs8jB z#Eg`cxV=879AH#JJ6DM}9CEt@MsZbgAnxo;^&2G>ODY;#MM2C0AEt-ssJxvYLAM_( zr?{abkhWUCT(U+wZhNkYzIVBC zJ5^?u{T_?7%x^JbR=PhCw=#A;tpSXn7uATVMcZ%Jf~nvR2f_iYrsiZO(Ut0MPb51t z-B~VxW9xQ^|uD;GLi}PxAwRG&M zWyA(DS*r)Fl(S)V5DPRYW`t;^?Rat1_C)gs{4S$7+L2Brvk7|ztQoO{{y!#}SY?qO zD~&Z__ruDd&xsW#)&;Hr+v^m)Ywu7nf?^Mlt3RH?Y{TEa^+ri+x!4cNQ{^dKBbl_~ z*?cc74+fng`dHd3IyZ_cYZ^(!BzkVRuo*w8{b2nC}Jfp%)V!v5$M0&ADwTAr85Ehc` zN!q#bMzefApSaOgnJkKwCX(%anQS_NValYs`^5H(ooY=%tR>Q1hxS!0i?uY@RK|>0 zV{}6kc|CsIN>wM8nq0Ljs#;q9Ipw%) z!)Oe)aXbh_UA7}OI`J%&doMZ$TU&K+cf3!H z{sp{F{6%d!J71Y6ga;COd4tuK*|pD<`{X9mC|SM2YBOR&$xYjZP1yk~;VhQ#YPSnZ z0e6juHXdYoCs(oC>kW%}(iD$p+R`brz0FK=Q)sElbeT!9k}+kBrhYVB)VaptcA-Xh z04rHO9yKpF8Xi|zj7F`w-o)ub-F<6BI@tH{XiLh6nAvSCb}q;Gib}Dmx-b@4yrQYL zXU`Kk)jWO=mRe(6Z00?wwgeu!VmmKcGoji_L{rJm+Ju?xH0ov6-v>^3y=HPCzp7is z(i7W+xSlYvd*)V^Rl`R6FTl(pT>1Y)uqywX*Afa$ME%A=?owEEon2E!SN=` z^UZ2E&JJR`!+s&IK}VY;ZXcdw*xhG;V3f@MRty*(EbfrYiD$4Vp&T?aKVUwWCyZ99 zPQZc_4>V^yiN=j9udZo`v{Xhb%Ny$r*%g^Sqc@%EPbA|w$MtRMO7$83Dg$#{oaSV= zV_o7Ij0G+)nDewpv=>tk^SCD7(}v!b{ZL+BzgU&V^zvg*56eLlCkS{R@f^x+xbBcY zj2(TGmFzYL>}45UMOQopu|v3Vd2|m7*nsMOb2 zwpKP9_Kl+xQG*9_DXb2HbnN=XsUzSEqq@4zRJ+;U&QoerRjR`F?B|s@tcl~5JYm_6 z-`+9ABngN80rbwwOv381l9|#j%k0i#ep;9oT=g-PWmIcfMx;u1pr{{Pl6qXp3rJ6N zq}WHWCwb(dwJw7yF%R3F>v}QNfo3aXndx|!(U|TulZj1g#^G^Tlk7;Pdz6c%oPPl4 z=4?$elTBp%vX*#y=ECFRK#76I>3-4wqAD@v)=Vgug4k8$k^H`k?z_Rt7`1)LUHhy=+B7Vq zsxN`Z^?m0-#zD(eR}@b_G1b)-ZBJV-0UTO!!N#J+9FHr`0DoJFEIS|x@=wuw%DO3Sl#Q)hRo&BRd4)0Q)t zGD@1NqK%6q(elQIhRX65Lk#k2*%)*b9yqK`IND+kxa2lnyCETZ%yQzXTGFsWIY!Lc z?2L*7gs298J$Nrv)kw#lFJh0uZYvHlP4TYIQgNni$FoFrePelbRcU=?b4^7}q`_Xf zDDQS*s(9r?Q1kM0+U0ZmaGYygpX$M)%zE7Y)_OS;S7O=-#aOu#Z=F~teR#pa)75UM z+wE1SB{`zPvF}$j)a7#pP;E`z+9c;BPMh`Rm8%*mTa1h|Eze7Mo{ARXxg#DX>V}o4 z?Ok{*`9oN9HQ4nsR@|AfjFOs&w2g?`-(@A3uer{bk0f}1yGbu*#`2`KF{?&2!&ej4 zd94kVD_aaTutu!1d3jAa7I4(rl)G8%47q9N7LGRW+#auZnX9+@GV8Ii;(UoEi1Dqo zeT;BK9UwwzXArli)|Tz*Pjrtf=nY~uEbS0)V5RuDfiDrMv@?l)EGFN|j7Dl)7MM18 zdSC!TYWC5wcL;3HVYfVzq|!LYi3bzL+L;z->lqbk9J%{b299aWz)rE=j3?Gv*tNy6 zkFwF8N*Rl-RA<_3FI{C}Ly>!hxH4MrmM;sk9rW^jA9MxX*eqi5ZAR24_9v0V9@Q-0 zXmanoIM3q?gqRrEi19`)_8~Db=;<#>7;Z7aqVaAsEjMDPDfYZEEmph>S23dX_)3B) zEO)**?gU$$Jk^=;Y@*-H=uC)%&#;P<4ce{)2|lqMt*U$6m@Ej zYK(Z@^82uJ)O7dZ$ z(|x_EE~{U@a>zz5EXF|#O~E&wK+q|l^f^cCaDT|}K?Ms7?l(%7G?>XOZb=mLIK|tq zJilvB?z%X%h_i8hW}v4nk#eg24==DW>D$L5@lX_{v74&hDd*TNKxYK~V#Jqbdpq@o zEZ(GJElpMVWo&<%L5&z`S3sRjZ6`Nkm_%-$$BTQaOC<5s&f>F#*g30elxMhi65oo% zK};T#ZAWYk>h>zpCh>h|VNnF@vq;QakIV1GXHa!uqT}(S7C8D$jZZI@*<&p36Q}r0##k2P%oc4y zJ85G_}p{A{NBL_7#EqrBo>RaPTB>~y)o4Z5*ftF7)t#&mWh(27c7 z8jS<%@L@|FtS}SgI4e_sqTm!Q4&%}M%bPfmudI!g8?!5$OIy^V3+G7rVG^hF)v`Wu z(i2Y(%~7opy&m+V#=1l=UTQP!>#9U=weaK(iBk%?QS2L{270+c77GM-geenkH?4N7 zyN~Z{%`IIPzO^aIs7j~$dQsm*Y&-5Ud|NIHn`LEUJIi?;_*3>T|ES+Yyk1WT7nvQx z8~zehG+7q)`@>~;jMz0P*Zj9yq^7FEr$!IXYYm-g%Sv{1rxN^9(^%D9S=nGLi8j_$ zs1L1crpT)Vg7Gp>M<8xrCg=Z4YwhcxYr`QI&Ro@a6G7j}PYa{e_w#@Iu##aHmE-$7 z&#-GIVViT8;dRy}uAH;~>#Fvpak#~Y_n-@Bj~KpN#@A=s6L@>W7Z$#v*W!C+d0kBW z+=HGRkD3GMMP5Xi-hUqp`x3aj03HOm0BTKaN+sL4Ce^ZNTCUXmQgyGH<_Wl5W$28K zG7~>2h&$2*+~R-TmKro)z~{vd*V2YV`cDdH6AxgTIz*qRLOTQ`vHZkOyO19 z&9e5mb{Xma>-4s}R4gQX?Zd=u%%<>DjoR$g?ZsO1icw>KqnBOkE9>x~H$B*$RF~QU zZs~t?sV5v354MF9R$_;qfZa#0(D%;@egC^cd|MSSLNR<(X-lPXtdify?EN4HHH}lP z8~b*|-nb{Bzx-uBQ_a4d{a@o@cNXTKH{iwuuf~qUmzmlMn1Ab=U|EN!&0S{U$2-+G zy6|6@z;s(M$tYeh((TxHryB8+i364RDY6;O#OKW1tA+fn zq`smuii39UgB!X(~5Gx*xme#ZBSMzu4K9TAO;~cntR9hY))qHZ=FOSxNq8 zYi+`H#-naO)_usMu0#8Ou6K7x`~tY57heE-#rJ;v#rJOc@ksVwQ>>*j5^bqA@WQ^L zu^HQ$x~I0XaL~RdX=Bjw=Rg0!c#C_&9xWaUQSsJYn#CKM{K60wzYmJ{HT63VM#yxv zWB>4Z%dj5g6YTH8e_cv#V*U_D7*AU>BQ5p$XLY@a^BTSa;zLQwh~OtHd>W5s`{MXM z&wE1a+tqw9v31*Y{po?V>XYW$9&=qPy;gp3PbJS8 z^v~QxGM{^PU0YwGyM3+In~HZ84eA{9@aXz>Gi$b)8Eb8Jpx0_m^x_>dWh8pn;y**I z&6qvC_%)%Yr@gJN-O9w%iC*y|**lA^$GwOecB ZDf}pQ@6g3{X}p@J+t+3io2+w;{{xjsLht|p literal 0 HcmV?d00001 diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/ip_country_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/ip_country_sample.mmdb new file mode 100644 index 0000000000000000000000000000000000000000..88428315ee8d6d164a89a40a9e8fff42338e3e2f GIT binary patch literal 32292 zcmbW62Yi%8_qLY+fe?D{EWIb$&896NB-GH0NEcHU2qnP~s@TN>_J(2u3#eePSWvNm z3id7*R1{Feh6USqKQq^62MqfC-}m~Yy;yZ*F0d=?2D`%^uqRA{$uI>5U=XIl5KM#VFau`7EEtBpU~kw59tZov zelQ#6!2WOm90&)&!Egx7g+t*mI2?|EBjG4G8jgWu;W(%>Y0l+%cmkXNC&Ec^GMoaZ zLY-w(=R`Oio&@uB^xxEZ&sW>4R-ks7S|Ra>>{GF4$exKk%hJ>?LYuAl97msvJ`WZv zUgE^(qn|9j6ng|$cCOS?2?o4H&r?FsU(fLBVpiaebM z-_>fX)z%ZgM)tK%d=2__(yzz9!HM4}{U*m>OU^p!H)G!dZ*}r+vv#fK#5Ta&;T`Z! zxDjqr-Mg%9)_6DiJ@8(5pYrZ^@*Y5cQ2IlT{;>Q87JMqVSzFN8`zQ--im`}>r zej9s>?5D7whR-uBHcJe+!{}k?qpDFKi>@VOR_$Bozc1|zbke(C*B>c2kfc5B((s3GEA|oJ=5p&TBV>`YD}vX zlCQO!xu(Z-<7Zmm*jdU8JANp@z;{GPWsJ`evABDiQVSt>!ojS{M*U7L;9WA8{sA=?=Hpf zM!yH%3-5#XTYB@XqqA7)LH#a$SbRu~&eQWBk>65$RPC_ZX0?54k5T7w+3%}8f&V1j zB5eouQ?j4Ne#X-5&sOQrqCW?pcjCHJm9{zlc5+^n{*t4=Z0$PQ>qyG~hId8(Z6n`809r&)LneThrE4Tj#%Ky-b@1oB~(m!_ePvn2<_`BtQM(*eE3%Cb< z3BOYAUTfE_>+^Zdn~Ryxe#O4V{sta^-@=3NJ4@5=d$b?a|3~bf;2|gPXT^U({}uig zi`VUk_Pg@_!2T2drE^mo`)}DkYkT9zRkC)R#JI|iUPXRYV)2e%4Xrv%fHjn#=;YOu zU(5P+OC7zAboMQ-o}<^7-oW~C4PhhL*vV_6yr$^Q9KE^p7S^x3o_H&16VzI(C9AbT z_rtc9W*^!q-rn&$$nWU*o#@jUc2T^m6YqxJU3w2k?X(HbwlwSLCB3&=Ut)dWahCP;{NnndWjk7q&*#NVjR9~V9As%?gVBbl zPA>LPCq4{qI2@t8k!oYqMrjOhexohDdd441?l?=MMPuWYe?m+*bC@WtSZxyhCc`O~ zCTFVR(;WXq`O_W$B>8!cpKtBB0_la09+5vot%&$cILor0=Ha!t*=loQTHIXu^J2c4 zQ;D=Q=s#cf$=L4wSRlR3>ZZ>^au&hGiZ8)l3YS@${8JQP?)ayYa~eF|vff(dor!i9 zJX_jjYUkjufahA8`sbmY4=+&uO6*neLU@s->31>OCGb-1+gsR|J9$^A-j$AjmHey8 zTkYuApk1r@8b`kl{d(y)VBhG(Z?bm%%8IWeezOz51?^VFZ^K^i#5YL4-SO`r=T5j0 zZi06y|88rWeYgkxUU;A4_hUccJ{HT1dmQZv_@wf;IPs^@ zpO*fNqi>b}tm8jN-t+JU#kXN^hcCjH;LFN+6?+HVDeV=XufFbsw|J&yk+IbXrOitlsc`_aFa{tfm4_^q@bun)rT;P;mGb>6%d_oLcR#16$`-Wtt1 ze}TWk-;{sY+9v+H{6C2O>F9q+|J(XH`leRpN-!>#S6RWzRnV(Cdc5>%=+$8Ytf5#< z>_p4@dInyL&bxAL<!w3*;A)7jg6%(r2R2a`YnUvmJj9Ide7lo3V?Pvktoi&X;yFcB$-@*b878)cVal z7s17F30w-7!BgOJcq%*%p02UZu(qjtCi+?MY^rwX|9>yS!c<^6J8R%Byc}GhPGqhER9c z=#60$*c3KX|K`>QyaQsa9S!%t=4_h``DXn+R_Qo*d90&Wt zelQ#6!2aq#z}lwYK>34+4R-V)Xt|0HwRWS;Xv3Yn5sHsQ9|cFlF;3oCv~kL3&Sw1a z=+WmfLGg)bli*}alRriAsWHEi?pBo($(!!;Ro8j1a+1$i^>Q%}Js%dpLKuNF;7m9R z7Fn8cXRF>E^tr09Ggf6DezEMI)JmNE`SMRz^PX2Jc?+C)8QMa)NO_B~m%ycP89c?( z%xAgsPj&p$NXUIR3erGxQ+0xHJU*YKIqMfJs`Pdi4;*Hy)ty26#?2BUYDi=$? z#PKhce;Ijt7G};@z$@WZ@M^dkUIVX%Yv6V8dUyl85#9vX!gcUwcniE0-ezg$wqEny zfPOo?1KtTY!cFilcsINU-V5)8_rnL^gYY5vFnk0)3ZwV!G5p8j6Yxp61wI9zhR?vQ z@LBjAd>+04x54f3MfehY8Sa2P;VbY}_!@j2z5(BaZ^5_WJMdlj9(*5u06&Dg;79Od z_zCzmgaoy*ZKGw{ToL=ApKj%KPdk@$Nygb4{ASJt?^u+ z&ud;yQ}>W!KRf;}@_%*w-{c=A_jmXQ{1g5K|F-n%R`sb>a{M^?l^wr|d?lIOcvuZq zhY7F-OoTOIEm+&q)T)D4SM}>**LUI#q&IZ@M&vYxO<+^l3^rH47T7H<&3LV(w|4wC z)^4o3SG6tq?VNaf=^fN&sdZEvMNTK!Sy~tDu1>#hXx$a>f!z}(!DN^M1270vVF;$d zbeLgj)|IJovK&85PA}=bvHQT|U|-nJ($vXD%YpsXe*pGCI0z1gLtri(sy@T4-9%@h z>Tt&%;qw`7q@_2e8F#eu#$b=NH2OH{U*!0U z>9Yhbh0Bz83ifhKv#wL6pN4)qJOiEy&w^)Lnttb~-wMY+mz?w9`S1d`Qu(W}FSImu zE|Pw+<6k2GQpdl{+D$e|zryFMrZZmkN~=}93SJFY!)xHRK3}}Gle!Zm` z{|4zdqTdA9N_!4_o$Q;jZ-KYM+u(Y*0p1SpfOo=;P-`(`-v#fsG}=9A_rm+&{qOeW@kzJ^J_VnK&%mwlSxc{u*Wc889=@P@+pxDg z3rA6{#o^Zv9{T-U*-Qs?69N%F8vSmKjB~SZ_9X|ZG0vCI9M50fmNY8cw@#_gVkYz zrHR!*ON2FHE%mF7T?f|H`O;mAuV-nVhvzpS-VipjH1!)R-UMBT(`e0LbJ#-lv`6tR zVJp}gwt;@w7Pf=!VF%a|c7mN@7uXecvov$+&RlwE9zBUC#o}guDT?V##|Lb#=@*or zs&*3nL$cGb(_se8gjp~Qds&)#z0vxp&T-g%op?XAY{hf1`@;cnARJ_A>JCO5qJFuK zK2-iNV#DDGI1-M6qv06!nSec3_Bfxfnd-;utQn5CG~=FN^E}mYV_5z*I-{G z`&!L;pY=UP?fB?AZXowYOH=PA^;?U+4&Lm-x<@J>kh_m ziuv(($-kT2d*HqBJ}2*fv?V{0Sa{Kf_-vP5och|2Oo*@OLNv z2il*?`^(Y)wswnj?MWqTSBry{V|mr8pjCzO%GV}(V|%Syb(kQ#26iH>DXkrLEm#}Y zfpuX$<<`e;02{(aurX|6X~u7g)(kdRofg2zR zyHKq+ejj+8rI|-xw0_FV#?FELoxA~x4@4gX2g4ywUasbIh3~=l;RoiW{|3&_gI6aEGN)_()_#qwI}8C8$7e)Y<-b+)Tl!LJJAVKt}% z-khr^SQ@*A{6w|W)oK#21#3&2q*g~Qq*fQbp3}cRS_9Y+HiC`SuL*WjOEYdW>CMqw zz?M$D6Fv=wIC@9vozOeOF0iYU*G+ld(R;w2FbO8Z6c~U(m}=RI zc~wudzFB9w>SvIb>F8N#Va0nndT;rCh#d#}!hSFt=D_}N02~Mh!NHbhK0{PL7k#Ls z50gF|eFPi{N5Ro>3><4|>W))?oh@UJhbK7k31|~7Tg_JfWV9)8s->whP4N>Qe>yoQ z!919+yaMb(OVd9heFpkWN1r9V2wih9xpUxLI1d)X5;z~8Y-#$IqAgI}GDlx1f05%a zmcNADrEnQM1uloD!qY5UJ%)bO|0;5>mcAPM8h9;S1FwVE`+TjdD*py}BfJT&h3nwW@D_M0 zybZ308!WwbnfiAy-koqG+yw6uLz>6kK3|Rd#e2~2_4yLMlz*Sv`)Z1ry?OvX2p@tE z!$;twa5IcP*T=1IYCWO4PZHk(pMp=rXW&-&tflGyocceH{sP;w2A+yy_fG&vumeWGzc#oi4+ zgP&WP{4bQZ2VKv~XkR(;y^8PCbJSg~zF(|K&e!l8cmRH@yo1=^!SCS@@JIL)JOqD+ zzrbHD&76KiI}Cr&tF{-4Hf{jbRhm6gGp+VGGz2wz4#H zY_0lj9N#a$EqU!6y}k4f=pA7vY2DR2Yi)W430*8r-LCSx#e6Sra(YPbY5N;LNq%z7 z_xhUt0qH^PR2YJ3Fdb&VOqd13mZn}WwBE1}JWk{E#qI~Qr47Z-f&HZo@cG&d#vcd= zNtfp72}59REYIk}$Q=$xSen>K^&5pg+R?{QYiz7mLUeD&+dQ+b6X-huPE@@~*puND zI2BHVC&KCQB$#Ju>gTIo0eYdMN6=;{KGV@>p%+P??dWsl&m}g`(TmYaEZeMB-pOj) z&`aS0Sfi9Js=XpR4$J zj(@)V3mku?{8f&Bq5O-ipK!6GUt;YxUlF@Zd6#2f0k4Et!K9hQFG#e|J9+Z)r2dl$T0@q4iE zh4;bx;REnN_z-*;J^~+wn=Q>akD)ydpU`+uVsC*@!KW=vzh{)U)$yMt=Q;Si;xEMF zem&!a?ZjS$FTs~%c?mnvb}IiB>{sDy@OAhGd=tI}-?lXKd`ETOb^Q0_zfax=j{YIq zF2z4`^pDX$fuF+N@H6#ApL9nZ{Pv=Ej$Rn zgWp@4_5Gm!KRW(T@(+>sGyDbq3V(x#;qUMd_$T}e{%u*q2P?riSQ%DM+DKG$oFcpSi8cc^7FcW6MFzf|;!#?mh*cbML*)Rw8 zhXde1OY>|8>A4R^9|CjXP&f<@ha=!fI0}x2W8hdg4(gnlF~`FbEIqAlbPp$LK9h(~ zhErmBHKr;)4gEwo9i9a9U_LB>g)jnVsMa~yGvO?0Mb|_`xHylf4THi9se}>r#t=`^f?or1<$r@8{L}~=;x~c zdDb@d&qu#N`bz9o@IrVIyck{rFNK#`ntGS3&J~V-B{^3~zZ!cryarxtY5K2G-gW5L zJNgZ1H!6M;_F5;tPG{;N^jqMq@HV&}Zm=}t-Hvt#yi@fzVsC-pLF~!zl8m=>^*8boc=rIze4;~M}JNF>*#O5H{o0G zZTJp+7rqDIw>0zo0PRE7-R0;X$^Y2#Kau~b5XVO2%{=%}|DEu!i%{X77?N$Ch zN8d01YhvHP1MpjT5Pk>0hd;m{;ZN`o{2BfNe}%un!eoa^l)`oRpU04s+hYc*f`FVBBJR8Br zkb9BXRO2^8YYtn$mar9U4ckCJYzy1L_OJugT{Ls(1Uthnuq*5ayTcx^r{)+`OY-^J zA15Zm6d17d#_-}MCpD%errA1X4(aNjp{8e_n5lN9T9(>kwXoU=YP}e@H|zsd+tlc5 zY3zQ=&vyJA`TZS#0DT6+L5dH?9s+aWP&mxe)E}<=5$GcwU3V&RwBwHBKSw8RGK&%WdbYhEQxn|x=$XTj>%dBnmQ_z>gQ{idKJ01HB zcqTjxo(<1|E8w~CJWDgq`DhoY?n-Q)Q{shA-bKVNR^BDpmpbvwq+gDHg=Gib6EntD z@M^ePx%XpV1Fw~~2KzdAy``yfgW@+j{!Q}NlDE#$ZG+l}@a{9N_Fz}^GD zgkQnEa39*>R3v8M}&D)zRaztBKXI6JQOP2y4Px@OZJdS{=u)i(OBw zkKMqDH*S?k(Um={0!_&F-vWrS{SVt z^zwRR_kqWWeX;w&Z0P0ZsP)&lW5of`n}_Fn@j+^Xu|0nXcCI)Sdl(!Jy}S|FBgIkJ zqv06nPWGUY<<9itym^s9*)vLu3i1XQlq@VR zD_xp1IeR#!2@fAvvaoEHpQPeQ+2WGYIa5MO!C)XM5K2u7q-XYwdBL=_J^o5LBeVUr zM~o~f^*YXrc;)>aa>kGICyy9kTvQe*^p7jcD~l}f=S~TzLP}avAT1>+rDwU5nna<2 zH;n{k8oI{T=)({K@{6{@h7v zLCrLj6bNXjs2!cBf3uReXqwBqyb(C51AR0vX=kMD6k+kDA{!UwTGTKpPPVrFgC95pJlW6=i6C zDM@L1j+vpJfZ>QU~bnhlHg7@Xrz9?5ea zR#ZH*uw1v9;TDbbz-2Kti+4~~@1%PZDsQ!{lgQZ>8OP?DZdxhFN4 z;!M%MO;2ct&O(;*0-9P9B<)!sGuZP8H`VKLNXOx@zeC&0{mH|}fV02Ho~N67lMIcQl{;I7c|5D<0zO8I;26J*XR(qp$dE0 zS?P&-mr*Cc4pQzPy(n*1Q_LV4SxKp>N#z}^XBJWMq_R?z^#4G*F1xphQP+D&?yh+Q z*sThtWNI1Os(_tTxu0eZowL)XPbIIcw@-l-Zx}O5xtmteI_%>N2K6G*t3WrwRIqL^ z_1L==)LBuwntdeYepW>jGrJYkZPu|#V_JB}-Ysw4ncfSMrAJ+N8975o>w*r=o1dpw zj<=RTlHR33?^cJriN##AVeKpVYqyWrn<{&}-iX-?7nGIe&GlY3nxO8Z_Bq2lGwN9G z1-+Mtj&kj`tJN=OtUq~BPHEn&GJ=|6 zdb(aECO_(#eTcpavh}JToIN++Tv+weOU>K8fZje@mE*Y=CTHAWEoz+JIeGI-N+Y_} z0q;WVr4z`~7Gm4aL-YmLw#c8{R$DhIpszt7l%zj1lpAx77~uE`TK(~PMT@=No)SpQ z(5zCD^g{CvLCmv9Ai9>}+UlXh7cPm+%P(12I@6qN?{%BzeRK4}3Iuuvj`Y3TK=0_= zZE_$(Pb8#+9n>q-ae_ze+aT}p3@XhlF3|FIgS-RkeYNzF3zV3-bm_ZJhQRR29}Sc*?C_aZMb(iva(Dc z$9MX4-tSNDe9(KsS!qe>8pB(bPKDzgvFl^?;T}A;MBgp@O4mnbCS$H(DV5zt-A+h(7n@ z78jK*vp*a))h4!Zz7`ysQC<_bg2|a7j$V3~xM}694iy%rg!5(u^+;v}%U6ArT0vp2 zCif_3$S_^OAtl8#{lmOJa;LS?+a)EOnW;ZgBPqeKK26?ST`{$y#h1^P%ANM=oE=cX z7N&1jK}6FG=rK_&CR8xP@?u9noYpFq7S7kB4(nT^KOz)7Qp~7mhTgLOrCv~vCmb** zyL=4&jibU-T|U3)Hs+4>ezF!9nuG04GAkTPNlyW&1wy)>&X7gNKj!DHW-qu+7f z4rYZjv(iH0^n&!%Xswu_S`|*ydC<|1p)O2$vAj?~H_zKkW@jo|HUEFASP;s{*TRpg zXw;1VWsZ5;Jk2pH?}&=1n&N-0I;X$)T>8(lUzhTwt3X~LFI;fsM5Ai@vCCF;d?Ml0 z)Pk`7#-t~mAD!nB;<4);{iMB<6VXdH-Mf1k;neh4JxBb{dS=;O;#cYR#VV%HsGy=1 z|79biKPYm&zbECEM(kf*rnL;F>t!2A$Wk{3p9&Gwiv%`y7p zvyp%y1->8S++pg#V9qMqd&CDAd|?4C+jW^|ZY&s=0q&HAnpa zuh-1f3#COu;mnlOwCK!Y!ZAzz<89bVrPB2+E(`~CH=?ipnD!qp!{}oel&iBcsHm8; z(md#0Uu||UB{dk1*ySD}9eak9EdN2(TD1bP%q!i zC`(@#rygQz=zpl^Ep$d8-+Z5<^UF74F!*0TA1auizMpy#=7%Eg{G#H~mn?etqQByD zykAZBr$clHQ^OhF2QQSCWT>KyAn3=9~6DpixKJ?K!#>9WG82y{kG@p)KFsxs4dc69Ldd$(aw8oniZ#o(=x)LeBN_MN)@m7A5X(OC3FnT@P5+=Q@Kz_s-d)F-O;0e)H@YZ z^9yw8GxcGNJ>ax(Os#m1(Z7H=ze~&K7)lRj=?7Ja*9dcrsmG{z^i%fUbkV+9`5EE# zqZS>?%FmRTf2=hhU9Xi>&->d z>C)*lO7yS&Wz*;7%`PdOz9>?Ag2nm;}IU$PeD&6__rk~D8#Vg9hf$by2>qWNVS zX?ISf_&?Hzs!DWT@wxgHHFII!%*cXDRn&E8aY&ki EKQa5Jod5s; literal 0 HcmV?d00001 From 7987c01f3643e0af0a85e050dfd906277f4ba4ff Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 8 Oct 2024 04:37:03 +1100 Subject: [PATCH 143/194] Mute org.elasticsearch.logsdb.datageneration.DataGeneratorTests testDataGeneratorProducesValidMappingAndDocument #114188 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 4e1d6bbf44614..0a9736c474798 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -368,6 +368,9 @@ tests: - class: org.elasticsearch.xpack.inference.services.openai.OpenAiServiceTests method: testInfer_StreamRequest issue: https://github.com/elastic/elasticsearch/issues/114232 +- class: org.elasticsearch.logsdb.datageneration.DataGeneratorTests + method: testDataGeneratorProducesValidMappingAndDocument + issue: https://github.com/elastic/elasticsearch/issues/114188 # Examples: # From 8f24f43aeda2fb7a1b65ca9711d0615e3a7544a8 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 7 Oct 2024 11:38:48 -0600 Subject: [PATCH 144/194] Fix issue with releasing resources in bulk tests (#114186) A recent commit incidentally changed a release resources call from doBefore to doAfter. Several tests depending on resources being released synchronously which requires doBefore. Closes #114181 Closes #114182 --- muted-tests.yml | 3 --- .../org/elasticsearch/action/bulk/IncrementalBulkService.java | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 0a9736c474798..f205c9ce44a08 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -345,9 +345,6 @@ tests: - class: org.elasticsearch.xpack.inference.InferenceCrudIT method: testGet issue: https://github.com/elastic/elasticsearch/issues/114135 -- class: org.elasticsearch.action.bulk.IncrementalBulkIT - method: testIncrementalBulkHighWatermarkBackOff - issue: https://github.com/elastic/elasticsearch/issues/114073 - class: org.elasticsearch.xpack.esql.expression.function.aggregate.AvgTests method: "testFold {TestCase= #7}" issue: https://github.com/elastic/elasticsearch/issues/114175 diff --git a/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java b/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java index d5ad3aa2d29a1..58ffe25e08e49 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/IncrementalBulkService.java @@ -194,7 +194,7 @@ public void lastItems(List> items, Releasable releasable, Act releasables.clear(); // We do not need to set this back to false as this will be the last request. bulkInProgress = true; - client.bulk(bulkRequest, ActionListener.runAfter(new ActionListener<>() { + client.bulk(bulkRequest, ActionListener.runBefore(new ActionListener<>() { private final boolean isFirstRequest = incrementalRequestSubmitted == false; From 49902761479e85da18d0521c3f9a4fba428b51ad Mon Sep 17 00:00:00 2001 From: Iraklis Psaroudakis Date: Mon, 7 Oct 2024 21:15:04 +0300 Subject: [PATCH 145/194] Fast refresh indices should use search shards (#113478) Fast refresh indices should now behave like non fast refresh indices in how they execute (m)gets and searches. I.e., they should use the search shards. For BWC, we define a new transport version. We expect search shards to be upgraded first, before promotable shards. Until the cluster is fully upgraded, the promotable shards (whether upgraded or not) will still receive and execute gets/searches locally. Relates ES-9573 Relates ES-9579 --- .../org/elasticsearch/TransportVersions.java | 1 + .../refresh/TransportShardRefreshAction.java | 32 +++++++------------ ...ansportUnpromotableShardRefreshAction.java | 15 +++++++++ .../action/get/TransportGetAction.java | 3 +- .../get/TransportShardMultiGetAction.java | 3 +- .../support/replication/PostWriteRefresh.java | 9 ++---- .../cluster/routing/OperationRouting.java | 9 +++++- .../index/cache/bitset/BitsetFilterCache.java | 2 +- .../routing/IndexRoutingTableTests.java | 24 ++++++++------ .../cache/bitset/BitSetFilterCacheTests.java | 2 +- 10 files changed, 56 insertions(+), 44 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index f6e4649aa4807..1911013cbe8e9 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -235,6 +235,7 @@ static TransportVersion def(int id) { public static final TransportVersion SEARCH_FAILURE_STATS = def(8_759_00_0); public static final TransportVersion INGEST_GEO_DATABASE_PROVIDERS = def(8_760_00_0); public static final TransportVersion DATE_TIME_DOC_VALUES_LOCALES = def(8_761_00_0); + public static final TransportVersion FAST_REFRESH_RCO = def(8_762_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java index 7857e9a22e9b9..cb667400240f0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java @@ -23,7 +23,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.injection.guice.Inject; @@ -120,27 +119,18 @@ public void onPrimaryOperationComplete( ActionListener listener ) { assert replicaRequest.primaryRefreshResult.refreshed() : "primary has not refreshed"; - boolean fastRefresh = IndexSettings.INDEX_FAST_REFRESH_SETTING.get( - clusterService.state().metadata().index(indexShardRoutingTable.shardId().getIndex()).getSettings() + UnpromotableShardRefreshRequest unpromotableReplicaRequest = new UnpromotableShardRefreshRequest( + indexShardRoutingTable, + replicaRequest.primaryRefreshResult.primaryTerm(), + replicaRequest.primaryRefreshResult.generation(), + false + ); + transportService.sendRequest( + transportService.getLocalNode(), + TransportUnpromotableShardRefreshAction.NAME, + unpromotableReplicaRequest, + new ActionListenerResponseHandler<>(listener.safeMap(r -> null), in -> ActionResponse.Empty.INSTANCE, refreshExecutor) ); - - // Indices marked with fast refresh do not rely on refreshing the unpromotables - if (fastRefresh) { - listener.onResponse(null); - } else { - UnpromotableShardRefreshRequest unpromotableReplicaRequest = new UnpromotableShardRefreshRequest( - indexShardRoutingTable, - replicaRequest.primaryRefreshResult.primaryTerm(), - replicaRequest.primaryRefreshResult.generation(), - false - ); - transportService.sendRequest( - transportService.getLocalNode(), - TransportUnpromotableShardRefreshAction.NAME, - unpromotableReplicaRequest, - new ActionListenerResponseHandler<>(listener.safeMap(r -> null), in -> ActionResponse.Empty.INSTANCE, refreshExecutor) - ); - } } } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java index 6c24ec2d17604..f91a983d47885 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportUnpromotableShardRefreshAction.java @@ -24,6 +24,9 @@ import java.util.List; +import static org.elasticsearch.TransportVersions.FAST_REFRESH_RCO; +import static org.elasticsearch.index.IndexSettings.INDEX_FAST_REFRESH_SETTING; + public class TransportUnpromotableShardRefreshAction extends TransportBroadcastUnpromotableAction< UnpromotableShardRefreshRequest, ActionResponse.Empty> { @@ -73,6 +76,18 @@ protected void unpromotableShardOperation( return; } + // During an upgrade to FAST_REFRESH_RCO, we expect search shards to be first upgraded before the primary is upgraded. Thus, + // when the primary is upgraded, and starts to deliver unpromotable refreshes, we expect the search shards to be upgraded already. + // Note that the fast refresh setting is final. + // TODO: remove assertion (ES-9563) + assert INDEX_FAST_REFRESH_SETTING.get(shard.indexSettings().getSettings()) == false + || transportService.getLocalNodeConnection().getTransportVersion().onOrAfter(FAST_REFRESH_RCO) + : "attempted to refresh a fast refresh search shard " + + shard + + " on transport version " + + transportService.getLocalNodeConnection().getTransportVersion() + + " (before FAST_REFRESH_RCO)"; + ActionListener.run(responseListener, listener -> { shard.waitForPrimaryTermAndGeneration( request.getPrimaryTerm(), diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java index 189aa1c95d865..99eac250641ae 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportGetAction.java @@ -125,11 +125,10 @@ protected void asyncShardOperation(GetRequest request, ShardId shardId, ActionLi IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); IndexShard indexShard = indexService.getShard(shardId.id()); if (indexShard.routingEntry().isPromotableToPrimary() == false) { - assert indexShard.indexSettings().isFastRefresh() == false - : "a search shard should not receive a TransportGetAction for an index with fast refresh"; handleGetOnUnpromotableShard(request, indexShard, listener); return; } + // TODO: adapt assertion to assert only that it is not stateless (ES-9563) assert DiscoveryNode.isStateless(clusterService.getSettings()) == false || indexShard.indexSettings().isFastRefresh() : "in Stateless a promotable to primary shard can receive a TransportGetAction only if an index has the fast refresh setting"; if (request.realtime()) { // we are not tied to a refresh cycle here anyway diff --git a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java index 8d5760307c3fe..633e7ef6793ab 100644 --- a/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java +++ b/server/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java @@ -124,11 +124,10 @@ protected void asyncShardOperation(MultiGetShardRequest request, ShardId shardId IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); IndexShard indexShard = indexService.getShard(shardId.id()); if (indexShard.routingEntry().isPromotableToPrimary() == false) { - assert indexShard.indexSettings().isFastRefresh() == false - : "a search shard should not receive a TransportShardMultiGetAction for an index with fast refresh"; handleMultiGetOnUnpromotableShard(request, indexShard, listener); return; } + // TODO: adapt assertion to assert only that it is not stateless (ES-9563) assert DiscoveryNode.isStateless(clusterService.getSettings()) == false || indexShard.indexSettings().isFastRefresh() : "in Stateless a promotable to primary shard can receive a TransportShardMultiGetAction only if an index has " + "the fast refresh setting"; diff --git a/server/src/main/java/org/elasticsearch/action/support/replication/PostWriteRefresh.java b/server/src/main/java/org/elasticsearch/action/support/replication/PostWriteRefresh.java index 683c3589c893d..7414aeeb2c405 100644 --- a/server/src/main/java/org/elasticsearch/action/support/replication/PostWriteRefresh.java +++ b/server/src/main/java/org/elasticsearch/action/support/replication/PostWriteRefresh.java @@ -19,7 +19,6 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.translog.Translog; @@ -53,9 +52,7 @@ public void refreshShard( case WAIT_UNTIL -> waitUntil(indexShard, location, new ActionListener<>() { @Override public void onResponse(Boolean forced) { - // Fast refresh indices do not depend on the unpromotables being refreshed - boolean fastRefresh = IndexSettings.INDEX_FAST_REFRESH_SETTING.get(indexShard.indexSettings().getSettings()); - if (location != null && (indexShard.routingEntry().isSearchable() == false && fastRefresh == false)) { + if (location != null && indexShard.routingEntry().isSearchable() == false) { refreshUnpromotables(indexShard, location, listener, forced, postWriteRefreshTimeout); } else { listener.onResponse(forced); @@ -68,9 +65,7 @@ public void onFailure(Exception e) { } }); case IMMEDIATE -> immediate(indexShard, listener.delegateFailureAndWrap((l, r) -> { - // Fast refresh indices do not depend on the unpromotables being refreshed - boolean fastRefresh = IndexSettings.INDEX_FAST_REFRESH_SETTING.get(indexShard.indexSettings().getSettings()); - if (indexShard.getReplicationGroup().getRoutingTable().unpromotableShards().size() > 0 && fastRefresh == false) { + if (indexShard.getReplicationGroup().getRoutingTable().unpromotableShards().size() > 0) { sendUnpromotableRequests(indexShard, r.generation(), true, l, postWriteRefreshTimeout); } else { l.onResponse(true); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java index f7812d284f2af..9120e25b443d7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java @@ -32,6 +32,7 @@ import java.util.Set; import java.util.stream.Collectors; +import static org.elasticsearch.TransportVersions.FAST_REFRESH_RCO; import static org.elasticsearch.index.IndexSettings.INDEX_FAST_REFRESH_SETTING; public class OperationRouting { @@ -305,8 +306,14 @@ public ShardId shardId(ClusterState clusterState, String index, String id, @Null } public static boolean canSearchShard(ShardRouting shardRouting, ClusterState clusterState) { + // TODO: remove if and always return isSearchable (ES-9563) if (INDEX_FAST_REFRESH_SETTING.get(clusterState.metadata().index(shardRouting.index()).getSettings())) { - return shardRouting.isPromotableToPrimary(); + // Until all the cluster is upgraded, we send searches/gets to the primary (even if it has been upgraded) to execute locally. + if (clusterState.getMinTransportVersion().onOrAfter(FAST_REFRESH_RCO)) { + return shardRouting.isSearchable(); + } else { + return shardRouting.isPromotableToPrimary(); + } } else { return shardRouting.isSearchable(); } diff --git a/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java b/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java index c19e3ca353569..3b37afc3b297b 100644 --- a/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java +++ b/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java @@ -105,7 +105,7 @@ static boolean shouldLoadRandomAccessFiltersEagerly(IndexSettings settings) { boolean loadFiltersEagerlySetting = settings.getValue(INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING); boolean isStateless = DiscoveryNode.isStateless(settings.getNodeSettings()); if (isStateless) { - return DiscoveryNode.hasRole(settings.getNodeSettings(), DiscoveryNodeRole.INDEX_ROLE) + return DiscoveryNode.hasRole(settings.getNodeSettings(), DiscoveryNodeRole.SEARCH_ROLE) && loadFiltersEagerlySetting && INDEX_FAST_REFRESH_SETTING.get(settings.getSettings()); } else { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTableTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTableTests.java index 21b30557cafea..6a7f4bb27a324 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTableTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTableTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.cluster.routing; +import org.elasticsearch.TransportVersion; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.Settings; @@ -19,6 +20,7 @@ import java.util.List; +import static org.elasticsearch.TransportVersions.FAST_REFRESH_RCO; import static org.elasticsearch.index.IndexSettings.INDEX_FAST_REFRESH_SETTING; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; @@ -27,16 +29,22 @@ public class IndexRoutingTableTests extends ESTestCase { public void testReadyForSearch() { - innerReadyForSearch(false); - innerReadyForSearch(true); + innerReadyForSearch(false, false); + innerReadyForSearch(false, true); + innerReadyForSearch(true, false); + innerReadyForSearch(true, true); } - private void innerReadyForSearch(boolean fastRefresh) { + // TODO: remove if (fastRefresh && beforeFastRefreshRCO) branches (ES-9563) + private void innerReadyForSearch(boolean fastRefresh, boolean beforeFastRefreshRCO) { Index index = new Index(randomIdentifier(), UUIDs.randomBase64UUID()); ClusterState clusterState = mock(ClusterState.class, Mockito.RETURNS_DEEP_STUBS); when(clusterState.metadata().index(any(Index.class)).getSettings()).thenReturn( Settings.builder().put(INDEX_FAST_REFRESH_SETTING.getKey(), fastRefresh).build() ); + when(clusterState.getMinTransportVersion()).thenReturn( + beforeFastRefreshRCO ? TransportVersion.fromId(FAST_REFRESH_RCO.id() - 1_00_0) : TransportVersion.current() + ); // 2 primaries that are search and index ShardId p1 = new ShardId(index, 0); IndexShardRoutingTable shardTable1 = new IndexShardRoutingTable( @@ -55,7 +63,7 @@ private void innerReadyForSearch(boolean fastRefresh) { shardTable1 = new IndexShardRoutingTable(p1, List.of(getShard(p1, true, ShardRoutingState.STARTED, ShardRouting.Role.INDEX_ONLY))); shardTable2 = new IndexShardRoutingTable(p2, List.of(getShard(p2, true, ShardRoutingState.STARTED, ShardRouting.Role.INDEX_ONLY))); indexRoutingTable = new IndexRoutingTable(index, new IndexShardRoutingTable[] { shardTable1, shardTable2 }); - if (fastRefresh) { + if (fastRefresh && beforeFastRefreshRCO) { assertTrue(indexRoutingTable.readyForSearch(clusterState)); } else { assertFalse(indexRoutingTable.readyForSearch(clusterState)); @@ -91,7 +99,7 @@ private void innerReadyForSearch(boolean fastRefresh) { ) ); indexRoutingTable = new IndexRoutingTable(index, new IndexShardRoutingTable[] { shardTable1, shardTable2 }); - if (fastRefresh) { + if (fastRefresh && beforeFastRefreshRCO) { assertTrue(indexRoutingTable.readyForSearch(clusterState)); } else { assertFalse(indexRoutingTable.readyForSearch(clusterState)); @@ -118,8 +126,6 @@ private void innerReadyForSearch(boolean fastRefresh) { assertTrue(indexRoutingTable.readyForSearch(clusterState)); // 2 unassigned primaries that are index only with some replicas that are all available - // Fast refresh indices do not support replicas so this can not practically happen. If we add support we will want to ensure - // that readyForSearch allows for searching replicas when the index shard is not available. shardTable1 = new IndexShardRoutingTable( p1, List.of( @@ -137,8 +143,8 @@ private void innerReadyForSearch(boolean fastRefresh) { ) ); indexRoutingTable = new IndexRoutingTable(index, new IndexShardRoutingTable[] { shardTable1, shardTable2 }); - if (fastRefresh) { - assertFalse(indexRoutingTable.readyForSearch(clusterState)); // if we support replicas for fast refreshes this needs to change + if (fastRefresh && beforeFastRefreshRCO) { + assertFalse(indexRoutingTable.readyForSearch(clusterState)); } else { assertTrue(indexRoutingTable.readyForSearch(clusterState)); } diff --git a/server/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java b/server/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java index 77635fd0312f8..4cb3ce418f761 100644 --- a/server/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java +++ b/server/src/test/java/org/elasticsearch/index/cache/bitset/BitSetFilterCacheTests.java @@ -276,7 +276,7 @@ public void testShouldLoadRandomAccessFiltersEagerly() { for (var isStateless : values) { if (isStateless) { assertEquals( - loadFiltersEagerly && indexFastRefresh && hasIndexRole, + loadFiltersEagerly && indexFastRefresh && hasIndexRole == false, BitsetFilterCache.shouldLoadRandomAccessFiltersEagerly( bitsetFilterCacheSettings(isStateless, hasIndexRole, loadFiltersEagerly, indexFastRefresh) ) From ce73a908cce972a62d4e5c4f3abb9aaf283266b9 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Mon, 7 Oct 2024 20:06:13 +0100 Subject: [PATCH 146/194] [ML] Use the same chunking configurations for models in the Elasticsearch service (#111336) --- docs/changelog/111336.yaml | 5 + .../chunking/EmbeddingRequestChunker.java | 54 +++- .../BaseElasticsearchInternalService.java | 5 +- .../ElasticsearchInternalModel.java | 5 + .../ElasticsearchInternalService.java | 137 +++++---- .../EmbeddingRequestChunkerTests.java | 80 ++++++ .../ElasticsearchInternalServiceTests.java | 262 ++++++++++-------- 7 files changed, 386 insertions(+), 162 deletions(-) create mode 100644 docs/changelog/111336.yaml diff --git a/docs/changelog/111336.yaml b/docs/changelog/111336.yaml new file mode 100644 index 0000000000000..d5bf602cb7a88 --- /dev/null +++ b/docs/changelog/111336.yaml @@ -0,0 +1,5 @@ +pr: 111336 +summary: Use the same chunking configurations for models in the Elasticsearch service +area: Machine Learning +type: enhancement +issues: [] diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java index 81ebebdb47e4f..3ae8dc0550391 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunker.java @@ -16,10 +16,13 @@ import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; +import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingByteResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingByteResults; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults; import java.util.ArrayList; import java.util.List; @@ -42,7 +45,8 @@ public class EmbeddingRequestChunker { public enum EmbeddingType { FLOAT, - BYTE; + BYTE, + SPARSE; public static EmbeddingType fromDenseVectorElementType(DenseVectorFieldMapper.ElementType elementType) { return switch (elementType) { @@ -67,6 +71,7 @@ public static EmbeddingType fromDenseVectorElementType(DenseVectorFieldMapper.El private List> chunkedInputs; private List>> floatResults; private List>> byteResults; + private List>> sparseResults; private AtomicArray errors; private ActionListener> finalListener; @@ -117,6 +122,7 @@ private void splitIntoBatchedRequests(List inputs) { switch (embeddingType) { case FLOAT -> floatResults = new ArrayList<>(inputs.size()); case BYTE -> byteResults = new ArrayList<>(inputs.size()); + case SPARSE -> sparseResults = new ArrayList<>(inputs.size()); } errors = new AtomicArray<>(inputs.size()); @@ -127,6 +133,7 @@ private void splitIntoBatchedRequests(List inputs) { switch (embeddingType) { case FLOAT -> floatResults.add(new AtomicArray<>(numberOfSubBatches)); case BYTE -> byteResults.add(new AtomicArray<>(numberOfSubBatches)); + case SPARSE -> sparseResults.add(new AtomicArray<>(numberOfSubBatches)); } chunkedInputs.add(chunks); } @@ -217,6 +224,7 @@ public void onResponse(InferenceServiceResults inferenceServiceResults) { switch (embeddingType) { case FLOAT -> handleFloatResults(inferenceServiceResults); case BYTE -> handleByteResults(inferenceServiceResults); + case SPARSE -> handleSparseResults(inferenceServiceResults); } } @@ -266,6 +274,29 @@ private void handleByteResults(InferenceServiceResults inferenceServiceResults) } } + private void handleSparseResults(InferenceServiceResults inferenceServiceResults) { + if (inferenceServiceResults instanceof SparseEmbeddingResults sparseEmbeddings) { + if (failIfNumRequestsDoNotMatch(sparseEmbeddings.embeddings().size())) { + return; + } + + int start = 0; + for (var pos : positions) { + sparseResults.get(pos.inputIndex()) + .setOnce(pos.chunkIndex(), sparseEmbeddings.embeddings().subList(start, start + pos.embeddingCount())); + start += pos.embeddingCount(); + } + + if (resultCount.incrementAndGet() == totalNumberOfRequests) { + sendResponse(); + } + } else { + onFailure( + unexpectedResultTypeException(inferenceServiceResults.getWriteableName(), InferenceTextEmbeddingByteResults.NAME) + ); + } + } + private boolean failIfNumRequestsDoNotMatch(int numberOfResults) { int numberOfRequests = positions.stream().mapToInt(SubBatchPositionsAndCount::embeddingCount).sum(); if (numberOfRequests != numberOfResults) { @@ -319,6 +350,7 @@ private ChunkedInferenceServiceResults mergeResultsWithInputs(int resultIndex) { return switch (embeddingType) { case FLOAT -> mergeFloatResultsWithInputs(chunkedInputs.get(resultIndex), floatResults.get(resultIndex)); case BYTE -> mergeByteResultsWithInputs(chunkedInputs.get(resultIndex), byteResults.get(resultIndex)); + case SPARSE -> mergeSparseResultsWithInputs(chunkedInputs.get(resultIndex), sparseResults.get(resultIndex)); }; } @@ -366,6 +398,26 @@ private InferenceChunkedTextEmbeddingByteResults mergeByteResultsWithInputs( return new InferenceChunkedTextEmbeddingByteResults(embeddingChunks, false); } + private InferenceChunkedSparseEmbeddingResults mergeSparseResultsWithInputs( + List chunks, + AtomicArray> debatchedResults + ) { + var all = new ArrayList(); + for (int i = 0; i < debatchedResults.length(); i++) { + var subBatch = debatchedResults.get(i); + all.addAll(subBatch); + } + + assert chunks.size() == all.size(); + + var embeddingChunks = new ArrayList(); + for (int i = 0; i < chunks.size(); i++) { + embeddingChunks.add(new MlChunkedTextExpansionResults.ChunkedResult(chunks.get(i), all.get(i).tokens())); + } + + return new InferenceChunkedSparseEmbeddingResults(embeddingChunks); + } + public record BatchRequest(List subBatches) { public int size() { return subBatches.stream().mapToInt(SubBatch::size).sum(); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java index 0dd41db2f016c..881e2e82b766a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/BaseElasticsearchInternalService.java @@ -248,15 +248,14 @@ public static InferModelAction.Request buildInferenceRequest( InferenceConfigUpdate update, List inputs, InputType inputType, - TimeValue timeout, - boolean chunk + TimeValue timeout ) { var request = InferModelAction.Request.forTextInput(id, update, inputs, true, timeout); request.setPrefixType( InputType.SEARCH == inputType ? TrainedModelPrefixStrings.PrefixType.SEARCH : TrainedModelPrefixStrings.PrefixType.INGEST ); request.setHighPriority(InputType.SEARCH == inputType); - request.setChunked(chunk); + request.setChunked(false); return request; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java index 07d0cc14b2ac8..a593e1dfb6d9d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalModel.java @@ -58,6 +58,11 @@ public abstract ActionListener getC ActionListener listener ); + @Override + public ElasticsearchInternalServiceSettings getServiceSettings() { + return (ElasticsearchInternalServiceSettings) super.getServiceSettings(); + } + @Override public String toString() { return Strings.toString(this.getConfigurations()); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java index 9b4c0e50bdebe..739f514bee1c9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java @@ -28,21 +28,19 @@ import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; -import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; -import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.RankedDocsResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsAction; import org.elasticsearch.xpack.core.ml.action.InferModelAction; import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; -import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextEmbeddingFloatResults; -import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults; +import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; +import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; +import org.elasticsearch.xpack.core.ml.inference.trainedmodel.EmptyConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextExpansionConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextSimilarityConfigUpdate; -import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate; +import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.ServiceUtils; @@ -74,6 +72,7 @@ public class ElasticsearchInternalService extends BaseElasticsearchInternalServi MULTILINGUAL_E5_SMALL_MODEL_ID_LINUX_X86 ); + public static final int EMBEDDING_MAX_BATCH_SIZE = 10; public static final String DEFAULT_ELSER_ID = ".elser-2"; private static final Logger logger = LogManager.getLogger(ElasticsearchInternalService.class); @@ -501,8 +500,7 @@ public void inferTextEmbedding( TextEmbeddingConfigUpdate.EMPTY_INSTANCE, inputs, inputType, - timeout, - false + timeout ); ActionListener mlResultsListener = listener.delegateFailureAndWrap( @@ -528,8 +526,7 @@ public void inferSparseEmbedding( TextExpansionConfigUpdate.EMPTY_UPDATE, inputs, inputType, - timeout, - false + timeout ); ActionListener mlResultsListener = listener.delegateFailureAndWrap( @@ -557,8 +554,7 @@ public void inferRerank( new TextSimilarityConfigUpdate(query), inputs, inputType, - timeout, - false + timeout ); var modelSettings = (CustomElandRerankTaskSettings) model.getTaskSettings(); @@ -610,52 +606,80 @@ public void chunkedInfer( if (model instanceof ElasticsearchInternalModel esModel) { - var configUpdate = chunkingOptions != null - ? new TokenizationConfigUpdate(chunkingOptions.windowSize(), chunkingOptions.span()) - : new TokenizationConfigUpdate(null, null); - - var request = buildInferenceRequest( - model.getConfigurations().getInferenceEntityId(), - configUpdate, + var batchedRequests = new EmbeddingRequestChunker( input, - inputType, - timeout, - true - ); + EMBEDDING_MAX_BATCH_SIZE, + embeddingTypeFromTaskTypeAndSettings(model.getTaskType(), esModel.internalServiceSettings) + ).batchRequestsWithListeners(listener); + + for (var batch : batchedRequests) { + var inferenceRequest = buildInferenceRequest( + model.getConfigurations().getInferenceEntityId(), + EmptyConfigUpdate.INSTANCE, + batch.batch().inputs(), + inputType, + timeout + ); - ActionListener mlResultsListener = listener.delegateFailureAndWrap( - (l, inferenceResult) -> l.onResponse(translateToChunkedResults(inferenceResult.getInferenceResults())) - ); + ActionListener mlResultsListener = batch.listener() + .delegateFailureAndWrap( + (l, inferenceResult) -> translateToChunkedResult(model.getTaskType(), inferenceResult.getInferenceResults(), l) + ); - var maybeDeployListener = mlResultsListener.delegateResponse( - (l, exception) -> maybeStartDeployment(esModel, exception, request, mlResultsListener) - ); + var maybeDeployListener = mlResultsListener.delegateResponse( + (l, exception) -> maybeStartDeployment(esModel, exception, inferenceRequest, mlResultsListener) + ); - client.execute(InferModelAction.INSTANCE, request, maybeDeployListener); + client.execute(InferModelAction.INSTANCE, inferenceRequest, maybeDeployListener); + } } else { listener.onFailure(notElasticsearchModelException(model)); } } - private static List translateToChunkedResults(List inferenceResults) { - var translated = new ArrayList(); - - for (var inferenceResult : inferenceResults) { - translated.add(translateToChunkedResult(inferenceResult)); - } - - return translated; - } + private static void translateToChunkedResult( + TaskType taskType, + List inferenceResults, + ActionListener chunkPartListener + ) { + if (taskType == TaskType.TEXT_EMBEDDING) { + var translated = new ArrayList(); - private static ChunkedInferenceServiceResults translateToChunkedResult(InferenceResults inferenceResult) { - if (inferenceResult instanceof MlChunkedTextEmbeddingFloatResults mlChunkedResult) { - return InferenceChunkedTextEmbeddingFloatResults.ofMlResults(mlChunkedResult); - } else if (inferenceResult instanceof MlChunkedTextExpansionResults mlChunkedResult) { - return InferenceChunkedSparseEmbeddingResults.ofMlResult(mlChunkedResult); - } else if (inferenceResult instanceof ErrorInferenceResults error) { - return new ErrorChunkedInferenceResults(error.getException()); - } else { - throw createInvalidChunkedResultException(MlChunkedTextEmbeddingFloatResults.NAME, inferenceResult.getWriteableName()); + for (var inferenceResult : inferenceResults) { + if (inferenceResult instanceof MlTextEmbeddingResults mlTextEmbeddingResult) { + translated.add( + new InferenceTextEmbeddingFloatResults.InferenceFloatEmbedding(mlTextEmbeddingResult.getInferenceAsFloat()) + ); + } else if (inferenceResult instanceof ErrorInferenceResults error) { + chunkPartListener.onFailure(error.getException()); + return; + } else { + chunkPartListener.onFailure( + createInvalidChunkedResultException(MlTextEmbeddingResults.NAME, inferenceResult.getWriteableName()) + ); + return; + } + } + chunkPartListener.onResponse(new InferenceTextEmbeddingFloatResults(translated)); + } else { // sparse + var translated = new ArrayList(); + + for (var inferenceResult : inferenceResults) { + if (inferenceResult instanceof TextExpansionResults textExpansionResult) { + translated.add( + new SparseEmbeddingResults.Embedding(textExpansionResult.getWeightedTokens(), textExpansionResult.isTruncated()) + ); + } else if (inferenceResult instanceof ErrorInferenceResults error) { + chunkPartListener.onFailure(error.getException()); + return; + } else { + chunkPartListener.onFailure( + createInvalidChunkedResultException(TextExpansionResults.NAME, inferenceResult.getWriteableName()) + ); + return; + } + } + chunkPartListener.onResponse(new SparseEmbeddingResults(translated)); } } @@ -738,4 +762,21 @@ public List defaultConfigs() { protected boolean isDefaultId(String inferenceId) { return DEFAULT_ELSER_ID.equals(inferenceId); } + + static EmbeddingRequestChunker.EmbeddingType embeddingTypeFromTaskTypeAndSettings( + TaskType taskType, + ElasticsearchInternalServiceSettings serviceSettings + ) { + return switch (taskType) { + case SPARSE_EMBEDDING -> EmbeddingRequestChunker.EmbeddingType.SPARSE; + case TEXT_EMBEDDING -> serviceSettings.elementType() == null + ? EmbeddingRequestChunker.EmbeddingType.FLOAT + : EmbeddingRequestChunker.EmbeddingType.fromDenseVectorElementType(serviceSettings.elementType()); + default -> throw new ElasticsearchStatusException( + "Chunking is not supported for task type [{}]", + RestStatus.BAD_REQUEST, + taskType + ); + }; + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java index cf862ee6fb4b8..c1be537a6b0a7 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/chunking/EmbeddingRequestChunkerTests.java @@ -11,10 +11,13 @@ import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; +import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingByteResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingByteResults; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.ml.search.WeightedToken; import java.util.ArrayList; import java.util.List; @@ -357,6 +360,83 @@ public void testMergingListener_Byte() { } } + public void testMergingListener_Sparse() { + int batchSize = 4; + int chunkSize = 10; + int overlap = 0; + // passage will be chunked into 2.1 batches + // and spread over 3 batch requests + int numberOfWordsInPassage = (chunkSize * batchSize * 2) + 5; + + var passageBuilder = new StringBuilder(); + for (int i = 0; i < numberOfWordsInPassage; i++) { + passageBuilder.append("passage_input").append(i).append(" "); // chunk on whitespace + } + List inputs = List.of("1st small", "2nd small", "3rd small", passageBuilder.toString()); + + var finalListener = testListener(); + var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap, EmbeddingRequestChunker.EmbeddingType.SPARSE) + .batchRequestsWithListeners(finalListener); + assertThat(batches, hasSize(3)); + + // 4 inputs in 3 batches + { + var embeddings = new ArrayList(); + for (int i = 0; i < batchSize; i++) { + embeddings.add(new SparseEmbeddingResults.Embedding(List.of(new WeightedToken(randomAlphaOfLength(4), 1.0f)), false)); + } + batches.get(0).listener().onResponse(new SparseEmbeddingResults(embeddings)); + } + { + var embeddings = new ArrayList(); + for (int i = 0; i < batchSize; i++) { + embeddings.add(new SparseEmbeddingResults.Embedding(List.of(new WeightedToken(randomAlphaOfLength(4), 1.0f)), false)); + } + batches.get(1).listener().onResponse(new SparseEmbeddingResults(embeddings)); + } + { + var embeddings = new ArrayList(); + for (int i = 0; i < 4; i++) { // 4 chunks in the final batch + embeddings.add(new SparseEmbeddingResults.Embedding(List.of(new WeightedToken(randomAlphaOfLength(4), 1.0f)), false)); + } + batches.get(2).listener().onResponse(new SparseEmbeddingResults(embeddings)); + } + + assertNotNull(finalListener.results); + assertThat(finalListener.results, hasSize(4)); + { + var chunkedResult = finalListener.results.get(0); + assertThat(chunkedResult, instanceOf(InferenceChunkedSparseEmbeddingResults.class)); + var chunkedSparseResult = (InferenceChunkedSparseEmbeddingResults) chunkedResult; + assertThat(chunkedSparseResult.getChunkedResults(), hasSize(1)); + assertEquals("1st small", chunkedSparseResult.getChunkedResults().get(0).matchedText()); + } + { + var chunkedResult = finalListener.results.get(1); + assertThat(chunkedResult, instanceOf(InferenceChunkedSparseEmbeddingResults.class)); + var chunkedSparseResult = (InferenceChunkedSparseEmbeddingResults) chunkedResult; + assertThat(chunkedSparseResult.getChunkedResults(), hasSize(1)); + assertEquals("2nd small", chunkedSparseResult.getChunkedResults().get(0).matchedText()); + } + { + var chunkedResult = finalListener.results.get(2); + assertThat(chunkedResult, instanceOf(InferenceChunkedSparseEmbeddingResults.class)); + var chunkedSparseResult = (InferenceChunkedSparseEmbeddingResults) chunkedResult; + assertThat(chunkedSparseResult.getChunkedResults(), hasSize(1)); + assertEquals("3rd small", chunkedSparseResult.getChunkedResults().get(0).matchedText()); + } + { + // this is the large input split in multiple chunks + var chunkedResult = finalListener.results.get(3); + assertThat(chunkedResult, instanceOf(InferenceChunkedSparseEmbeddingResults.class)); + var chunkedSparseResult = (InferenceChunkedSparseEmbeddingResults) chunkedResult; + assertThat(chunkedSparseResult.getChunkedResults(), hasSize(9)); // passage is split into 9 chunks, 10 words each + assertThat(chunkedSparseResult.getChunkedResults().get(0).matchedText(), startsWith("passage_input0 ")); + assertThat(chunkedSparseResult.getChunkedResults().get(1).matchedText(), startsWith(" passage_input10 ")); + assertThat(chunkedSparseResult.getChunkedResults().get(8).matchedText(), startsWith(" passage_input80 ")); + } + } + public void testListenerErrorsWithWrongNumberOfResponses() { List inputs = List.of("1st small", "2nd small", "3rd small"); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java index cd6da4c0ad8d8..db7189dc1af17 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.core.action.util.QueryPage; @@ -44,15 +43,14 @@ import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig; import org.elasticsearch.xpack.core.ml.inference.TrainedModelPrefixStrings; import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; -import org.elasticsearch.xpack.core.ml.inference.results.InferenceChunkedTextExpansionResultsTests; -import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextEmbeddingFloatResults; -import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextEmbeddingFloatResultsTests; -import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; +import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResultsTests; +import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; +import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResultsTests; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate; -import org.elasticsearch.xpack.core.utils.FloatConversionUtils; import org.elasticsearch.xpack.inference.InferencePlugin; +import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; import org.elasticsearch.xpack.inference.services.ServiceFields; import org.junit.After; import org.junit.Before; @@ -663,14 +661,12 @@ public void testParsePersistedConfig() { @SuppressWarnings("unchecked") public void testChunkInfer_e5() { var mlTrainedModelResults = new ArrayList(); - mlTrainedModelResults.add(MlChunkedTextEmbeddingFloatResultsTests.createRandomResults()); - mlTrainedModelResults.add(MlChunkedTextEmbeddingFloatResultsTests.createRandomResults()); - mlTrainedModelResults.add(new ErrorInferenceResults(new RuntimeException("boom"))); + mlTrainedModelResults.add(MlTextEmbeddingResultsTests.createRandomResults()); + mlTrainedModelResults.add(MlTextEmbeddingResultsTests.createRandomResults()); var response = new InferModelAction.Response(mlTrainedModelResults, "foo", true); - ThreadPool threadpool = new TestThreadPool("test"); Client client = mock(Client.class); - when(client.threadPool()).thenReturn(threadpool); + when(client.threadPool()).thenReturn(threadPool); doAnswer(invocationOnMock -> { var listener = (ActionListener) invocationOnMock.getArguments()[2]; listener.onResponse(response); @@ -687,47 +683,26 @@ public void testChunkInfer_e5() { var gotResults = new AtomicBoolean(); var resultsListener = ActionListener.>wrap(chunkedResponse -> { - assertThat(chunkedResponse, hasSize(3)); + assertThat(chunkedResponse, hasSize(2)); assertThat(chunkedResponse.get(0), instanceOf(InferenceChunkedTextEmbeddingFloatResults.class)); var result1 = (InferenceChunkedTextEmbeddingFloatResults) chunkedResponse.get(0); - assertEquals( - ((MlChunkedTextEmbeddingFloatResults) mlTrainedModelResults.get(0)).getChunks().size(), - result1.getChunks().size() - ); - assertEquals( - ((MlChunkedTextEmbeddingFloatResults) mlTrainedModelResults.get(0)).getChunks().get(0).matchedText(), - result1.getChunks().get(0).matchedText() - ); + assertThat(result1.chunks(), hasSize(1)); assertArrayEquals( - (FloatConversionUtils.floatArrayOf( - ((MlChunkedTextEmbeddingFloatResults) mlTrainedModelResults.get(0)).getChunks().get(0).embedding() - )), + ((MlTextEmbeddingResults) mlTrainedModelResults.get(0)).getInferenceAsFloat(), result1.getChunks().get(0).embedding(), 0.0001f ); + assertEquals("foo", result1.getChunks().get(0).matchedText()); assertThat(chunkedResponse.get(1), instanceOf(InferenceChunkedTextEmbeddingFloatResults.class)); var result2 = (InferenceChunkedTextEmbeddingFloatResults) chunkedResponse.get(1); - // assertEquals(((MlChunkedTextEmbeddingFloatResults) mlTrainedModelResults.get(1)).getChunks(), result2.getChunks()); - - assertEquals( - ((MlChunkedTextEmbeddingFloatResults) mlTrainedModelResults.get(1)).getChunks().size(), - result2.getChunks().size() - ); - assertEquals( - ((MlChunkedTextEmbeddingFloatResults) mlTrainedModelResults.get(1)).getChunks().get(0).matchedText(), - result2.getChunks().get(0).matchedText() - ); + assertThat(result2.chunks(), hasSize(1)); assertArrayEquals( - (FloatConversionUtils.floatArrayOf( - ((MlChunkedTextEmbeddingFloatResults) mlTrainedModelResults.get(1)).getChunks().get(0).embedding() - )), + ((MlTextEmbeddingResults) mlTrainedModelResults.get(1)).getInferenceAsFloat(), result2.getChunks().get(0).embedding(), 0.0001f ); + assertEquals("bar", result2.getChunks().get(0).matchedText()); - var result3 = (ErrorChunkedInferenceResults) chunkedResponse.get(2); - assertThat(result3.getException(), instanceOf(RuntimeException.class)); - assertThat(result3.getException().getMessage(), containsString("boom")); gotResults.set(true); }, ESTestCase::fail); @@ -739,26 +714,21 @@ public void testChunkInfer_e5() { InputType.SEARCH, new ChunkingOptions(null, null), InferenceAction.Request.DEFAULT_TIMEOUT, - ActionListener.runAfter(resultsListener, () -> terminate(threadpool)) + ActionListener.runAfter(resultsListener, () -> terminate(threadPool)) ); - if (gotResults.get() == false) { - terminate(threadpool); - } assertTrue("Listener not called", gotResults.get()); } @SuppressWarnings("unchecked") public void testChunkInfer_Sparse() { var mlTrainedModelResults = new ArrayList(); - mlTrainedModelResults.add(InferenceChunkedTextExpansionResultsTests.createRandomResults()); - mlTrainedModelResults.add(InferenceChunkedTextExpansionResultsTests.createRandomResults()); - mlTrainedModelResults.add(new ErrorInferenceResults(new RuntimeException("boom"))); + mlTrainedModelResults.add(TextExpansionResultsTests.createRandomResults()); + mlTrainedModelResults.add(TextExpansionResultsTests.createRandomResults()); var response = new InferModelAction.Response(mlTrainedModelResults, "foo", true); - ThreadPool threadpool = new TestThreadPool("test"); Client client = mock(Client.class); - when(client.threadPool()).thenReturn(threadpool); + when(client.threadPool()).thenReturn(threadPool); doAnswer(invocationOnMock -> { var listener = (ActionListener) invocationOnMock.getArguments()[2]; listener.onResponse(response); @@ -775,16 +745,21 @@ public void testChunkInfer_Sparse() { var gotResults = new AtomicBoolean(); var resultsListener = ActionListener.>wrap(chunkedResponse -> { - assertThat(chunkedResponse, hasSize(3)); + assertThat(chunkedResponse, hasSize(2)); assertThat(chunkedResponse.get(0), instanceOf(InferenceChunkedSparseEmbeddingResults.class)); var result1 = (InferenceChunkedSparseEmbeddingResults) chunkedResponse.get(0); - assertEquals(((MlChunkedTextExpansionResults) mlTrainedModelResults.get(0)).getChunks(), result1.getChunkedResults()); + assertEquals( + ((TextExpansionResults) mlTrainedModelResults.get(0)).getWeightedTokens(), + result1.getChunkedResults().get(0).weightedTokens() + ); + assertEquals("foo", result1.getChunkedResults().get(0).matchedText()); assertThat(chunkedResponse.get(1), instanceOf(InferenceChunkedSparseEmbeddingResults.class)); var result2 = (InferenceChunkedSparseEmbeddingResults) chunkedResponse.get(1); - assertEquals(((MlChunkedTextExpansionResults) mlTrainedModelResults.get(1)).getChunks(), result2.getChunkedResults()); - var result3 = (ErrorChunkedInferenceResults) chunkedResponse.get(2); - assertThat(result3.getException(), instanceOf(RuntimeException.class)); - assertThat(result3.getException().getMessage(), containsString("boom")); + assertEquals( + ((TextExpansionResults) mlTrainedModelResults.get(1)).getWeightedTokens(), + result2.getChunkedResults().get(0).weightedTokens() + ); + assertEquals("bar", result2.getChunkedResults().get(0).matchedText()); gotResults.set(true); }, ESTestCase::fail); @@ -796,12 +771,9 @@ public void testChunkInfer_Sparse() { InputType.SEARCH, new ChunkingOptions(null, null), InferenceAction.Request.DEFAULT_TIMEOUT, - ActionListener.runAfter(resultsListener, () -> terminate(threadpool)) + ActionListener.runAfter(resultsListener, () -> terminate(threadPool)) ); - if (gotResults.get() == false) { - terminate(threadpool); - } assertTrue("Listener not called", gotResults.get()); } @@ -811,57 +783,103 @@ public void testChunkInferSetsTokenization() { var expectedWindowSize = new AtomicReference(); Client client = mock(Client.class); - ThreadPool threadpool = new TestThreadPool("test"); - try { - when(client.threadPool()).thenReturn(threadpool); - doAnswer(invocationOnMock -> { - var request = (InferTrainedModelDeploymentAction.Request) invocationOnMock.getArguments()[1]; - assertThat(request.getUpdate(), instanceOf(TokenizationConfigUpdate.class)); - var update = (TokenizationConfigUpdate) request.getUpdate(); - assertEquals(update.getSpanSettings().span(), expectedSpan.get()); - assertEquals(update.getSpanSettings().maxSequenceLength(), expectedWindowSize.get()); - return null; - }).when(client) - .execute( - same(InferTrainedModelDeploymentAction.INSTANCE), - any(InferTrainedModelDeploymentAction.Request.class), - any(ActionListener.class) - ); - - var model = new MultilingualE5SmallModel( - "foo", - TaskType.TEXT_EMBEDDING, - "e5", - new MultilingualE5SmallInternalServiceSettings(1, 1, "cross-platform", null) + when(client.threadPool()).thenReturn(threadPool); + doAnswer(invocationOnMock -> { + var request = (InferTrainedModelDeploymentAction.Request) invocationOnMock.getArguments()[1]; + assertThat(request.getUpdate(), instanceOf(TokenizationConfigUpdate.class)); + var update = (TokenizationConfigUpdate) request.getUpdate(); + assertEquals(update.getSpanSettings().span(), expectedSpan.get()); + assertEquals(update.getSpanSettings().maxSequenceLength(), expectedWindowSize.get()); + return null; + }).when(client) + .execute( + same(InferTrainedModelDeploymentAction.INSTANCE), + any(InferTrainedModelDeploymentAction.Request.class), + any(ActionListener.class) ); - var service = createService(client); - expectedSpan.set(-1); - expectedWindowSize.set(null); - service.chunkedInfer( - model, - List.of("foo", "bar"), - Map.of(), - InputType.SEARCH, - null, - InferenceAction.Request.DEFAULT_TIMEOUT, - ActionListener.wrap(r -> fail("unexpected result"), e -> fail(e.getMessage())) - ); + var model = new MultilingualE5SmallModel( + "foo", + TaskType.TEXT_EMBEDDING, + "e5", + new MultilingualE5SmallInternalServiceSettings(1, 1, "cross-platform", null) + ); + var service = createService(client); + + expectedSpan.set(-1); + expectedWindowSize.set(null); + service.chunkedInfer( + model, + List.of("foo", "bar"), + Map.of(), + InputType.SEARCH, + null, + InferenceAction.Request.DEFAULT_TIMEOUT, + ActionListener.wrap(r -> fail("unexpected result"), e -> fail(e.getMessage())) + ); + + expectedSpan.set(-1); + expectedWindowSize.set(256); + service.chunkedInfer( + model, + List.of("foo", "bar"), + Map.of(), + InputType.SEARCH, + new ChunkingOptions(256, null), + InferenceAction.Request.DEFAULT_TIMEOUT, + ActionListener.wrap(r -> fail("unexpected result"), e -> fail(e.getMessage())) + ); - expectedSpan.set(-1); - expectedWindowSize.set(256); - service.chunkedInfer( - model, - List.of("foo", "bar"), - Map.of(), - InputType.SEARCH, - new ChunkingOptions(256, null), - InferenceAction.Request.DEFAULT_TIMEOUT, - ActionListener.wrap(r -> fail("unexpected result"), e -> fail(e.getMessage())) - ); - } finally { - terminate(threadpool); - } + } + + @SuppressWarnings("unchecked") + public void testChunkInfer_FailsBatch() { + var mlTrainedModelResults = new ArrayList(); + mlTrainedModelResults.add(MlTextEmbeddingResultsTests.createRandomResults()); + mlTrainedModelResults.add(MlTextEmbeddingResultsTests.createRandomResults()); + mlTrainedModelResults.add(new ErrorInferenceResults(new RuntimeException("boom"))); + var response = new InferModelAction.Response(mlTrainedModelResults, "foo", true); + + Client client = mock(Client.class); + when(client.threadPool()).thenReturn(threadPool); + doAnswer(invocationOnMock -> { + var listener = (ActionListener) invocationOnMock.getArguments()[2]; + listener.onResponse(response); + return null; + }).when(client).execute(same(InferModelAction.INSTANCE), any(InferModelAction.Request.class), any(ActionListener.class)); + + var model = new MultilingualE5SmallModel( + "foo", + TaskType.TEXT_EMBEDDING, + "e5", + new MultilingualE5SmallInternalServiceSettings(1, 1, "cross-platform", null) + ); + var service = createService(client); + + var gotResults = new AtomicBoolean(); + var resultsListener = ActionListener.>wrap(chunkedResponse -> { + assertThat(chunkedResponse, hasSize(3)); + // a single failure fails the batch + for (var er : chunkedResponse) { + assertThat(er, instanceOf(ErrorChunkedInferenceResults.class)); + assertEquals("boom", ((ErrorChunkedInferenceResults) er).getException().getMessage()); + } + + gotResults.set(true); + }, ESTestCase::fail); + + service.chunkedInfer( + model, + null, + List.of("foo", "bar", "baz"), + Map.of(), + InputType.SEARCH, + new ChunkingOptions(null, null), + InferenceAction.Request.DEFAULT_TIMEOUT, + ActionListener.runAfter(resultsListener, () -> terminate(threadPool)) + ); + + assertTrue("Listener not called", gotResults.get()); } public void testParsePersistedConfig_Rerank() { @@ -992,14 +1010,12 @@ public void testBuildInferenceRequest() { var inputs = randomList(1, 3, () -> randomAlphaOfLength(4)); var inputType = randomFrom(InputType.SEARCH, InputType.INGEST); var timeout = randomTimeValue(); - var chunk = randomBoolean(); var request = ElasticsearchInternalService.buildInferenceRequest( id, TextEmbeddingConfigUpdate.EMPTY_INSTANCE, inputs, inputType, - timeout, - chunk + timeout ); assertEquals(id, request.getId()); @@ -1009,7 +1025,7 @@ public void testBuildInferenceRequest() { request.getPrefixType() ); assertEquals(timeout, request.getInferenceTimeout()); - assertEquals(chunk, request.isChunked()); + assertEquals(false, request.isChunked()); } @SuppressWarnings("unchecked") @@ -1132,6 +1148,32 @@ public void testModelVariantDoesNotMatchArchitecturesAndIsNotPlatformAgnostic() } } + public void testEmbeddingTypeFromTaskTypeAndSettings() { + assertEquals( + EmbeddingRequestChunker.EmbeddingType.SPARSE, + ElasticsearchInternalService.embeddingTypeFromTaskTypeAndSettings( + TaskType.SPARSE_EMBEDDING, + new ElasticsearchInternalServiceSettings(1, 1, "foo", null) + ) + ); + assertEquals( + EmbeddingRequestChunker.EmbeddingType.FLOAT, + ElasticsearchInternalService.embeddingTypeFromTaskTypeAndSettings( + TaskType.TEXT_EMBEDDING, + new MultilingualE5SmallInternalServiceSettings(1, 1, "foo", null) + ) + ); + + var e = expectThrows( + ElasticsearchStatusException.class, + () -> ElasticsearchInternalService.embeddingTypeFromTaskTypeAndSettings( + TaskType.COMPLETION, + new ElasticsearchInternalServiceSettings(1, 1, "foo", null) + ) + ); + assertThat(e.getMessage(), containsString("Chunking is not supported for task type [completion]")); + } + private ElasticsearchInternalService createService(Client client) { var context = new InferenceServiceExtension.InferenceServiceFactoryContext(client, threadPool); return new ElasticsearchInternalService(context); From e065a3789be6939d8411811909f5e9370b418052 Mon Sep 17 00:00:00 2001 From: Pat Whelan Date: Mon, 7 Oct 2024 15:07:59 -0400 Subject: [PATCH 147/194] [ML] Stream Cohere Completion (#114080) Implement and enable streaming for Cohere chat completions (v1). Includes processor for ND JSON streaming responses. Co-authored-by: Elastic Machine --- docs/changelog/114080.yaml | 5 + .../inference/common/DelegatingProcessor.java | 4 +- .../cohere/CohereResponseHandler.java | 23 ++- .../cohere/CohereStreamingProcessor.java | 101 ++++++++++ .../CohereCompletionRequestManager.java | 9 +- .../CohereEmbeddingsRequestManager.java | 2 +- .../sender/CohereRerankRequestManager.java | 2 +- .../completion/CohereCompletionRequest.java | 15 +- .../CohereCompletionRequestEntity.java | 8 +- .../NewlineDelimitedByteProcessor.java | 67 +++++++ .../services/cohere/CohereService.java | 6 + .../cohere/CohereResponseHandlerTests.java | 2 +- .../cohere/CohereStreamingProcessorTests.java | 189 ++++++++++++++++++ .../CohereCompletionRequestEntityTests.java | 8 +- .../cohere/CohereCompletionRequestTests.java | 8 +- .../NewlineDelimitedByteProcessorTests.java | 112 +++++++++++ .../services/cohere/CohereServiceTests.java | 50 +++++ 17 files changed, 585 insertions(+), 26 deletions(-) create mode 100644 docs/changelog/114080.yaml create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereStreamingProcessor.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/streaming/NewlineDelimitedByteProcessor.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/cohere/CohereStreamingProcessorTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/streaming/NewlineDelimitedByteProcessorTests.java diff --git a/docs/changelog/114080.yaml b/docs/changelog/114080.yaml new file mode 100644 index 0000000000000..395768c46369a --- /dev/null +++ b/docs/changelog/114080.yaml @@ -0,0 +1,5 @@ +pr: 114080 +summary: Stream Cohere Completion +area: Machine Learning +type: enhancement +issues: [] diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/DelegatingProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/DelegatingProcessor.java index 9af5668ecf75b..fc2d890dd89e6 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/DelegatingProcessor.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/DelegatingProcessor.java @@ -21,7 +21,7 @@ public abstract class DelegatingProcessor implements Flow.Processor { private static final Logger log = LogManager.getLogger(DelegatingProcessor.class); private final AtomicLong pendingRequests = new AtomicLong(); - private final AtomicBoolean isClosed = new AtomicBoolean(false); + protected final AtomicBoolean isClosed = new AtomicBoolean(false); private Flow.Subscriber downstream; private Flow.Subscription upstream; @@ -49,7 +49,7 @@ private Flow.Subscription forwardingSubscription() { @Override public void request(long n) { if (isClosed.get()) { - downstream.onComplete(); // shouldn't happen, but reinforce that we're no longer listening + downstream.onComplete(); } else if (upstream != null) { upstream.request(n); } else { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandler.java index b5af0b474834f..3579cd4100bbb 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandler.java @@ -8,14 +8,19 @@ package org.elasticsearch.xpack.inference.external.cohere; import org.apache.logging.log4j.Logger; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler; import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser; import org.elasticsearch.xpack.inference.external.http.retry.RetryException; import org.elasticsearch.xpack.inference.external.request.Request; import org.elasticsearch.xpack.inference.external.response.cohere.CohereErrorResponseEntity; +import org.elasticsearch.xpack.inference.external.response.streaming.NewlineDelimitedByteProcessor; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import java.util.concurrent.Flow; + import static org.elasticsearch.xpack.inference.external.http.HttpUtils.checkForEmptyBody; /** @@ -33,9 +38,11 @@ public class CohereResponseHandler extends BaseResponseHandler { static final String TEXTS_ARRAY_TOO_LARGE_MESSAGE_MATCHER = "invalid request: total number of texts must be at most"; static final String TEXTS_ARRAY_ERROR_MESSAGE = "Received a texts array too large response"; + private final boolean canHandleStreamingResponse; - public CohereResponseHandler(String requestType, ResponseParser parseFunction) { + public CohereResponseHandler(String requestType, ResponseParser parseFunction, boolean canHandleStreamingResponse) { super(requestType, parseFunction, CohereErrorResponseEntity::fromResponse); + this.canHandleStreamingResponse = canHandleStreamingResponse; } @Override @@ -45,6 +52,20 @@ public void validateResponse(ThrottlerManager throttlerManager, Logger logger, R checkForEmptyBody(throttlerManager, logger, request, result); } + @Override + public boolean canHandleStreamingResponses() { + return canHandleStreamingResponse; + } + + @Override + public InferenceServiceResults parseResult(Request request, Flow.Publisher flow) { + var ndProcessor = new NewlineDelimitedByteProcessor(); + var cohereProcessor = new CohereStreamingProcessor(); + flow.subscribe(ndProcessor); + ndProcessor.subscribe(cohereProcessor); + return new StreamingChatCompletionResults(cohereProcessor); + } + /** * Validates the status code throws an RetryException if not in the range [200, 300). * diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereStreamingProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereStreamingProcessor.java new file mode 100644 index 0000000000000..2516a647a91fd --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereStreamingProcessor.java @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.cohere; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; +import org.elasticsearch.xpack.inference.common.DelegatingProcessor; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Map; +import java.util.Optional; + +class CohereStreamingProcessor extends DelegatingProcessor, StreamingChatCompletionResults.Results> { + private static final Logger log = LogManager.getLogger(CohereStreamingProcessor.class); + + @Override + protected void next(Deque item) throws Exception { + if (item.isEmpty()) { + // discard empty result and go to the next + upstream().request(1); + return; + } + + var results = new ArrayDeque(item.size()); + for (String json : item) { + try (var jsonParser = jsonParser(json)) { + var responseMap = jsonParser.map(); + var eventType = (String) responseMap.get("event_type"); + switch (eventType) { + case "text-generation" -> parseText(responseMap).ifPresent(results::offer); + case "stream-end" -> validateResponse(responseMap); + case "stream-start", "search-queries-generation", "search-results", "citation-generation", "tool-calls-generation", + "tool-calls-chunk" -> { + log.debug("Skipping event type [{}] for line [{}].", eventType, item); + } + default -> throw new IOException("Unknown eventType found: " + eventType); + } + } catch (ElasticsearchStatusException e) { + throw e; + } catch (Exception e) { + log.warn("Failed to parse json from cohere: {}", json); + throw e; + } + } + + if (results.isEmpty()) { + upstream().request(1); + } else { + downstream().onNext(new StreamingChatCompletionResults.Results(results)); + } + } + + private static XContentParser jsonParser(String line) throws IOException { + return XContentFactory.xContent(XContentType.JSON).createParser(XContentParserConfiguration.EMPTY, line); + } + + private Optional parseText(Map responseMap) throws IOException { + var text = (String) responseMap.get("text"); + if (text != null) { + return Optional.of(new StreamingChatCompletionResults.Result(text)); + } else { + throw new IOException("Null text found in text-generation cohere event"); + } + } + + private void validateResponse(Map responseMap) { + var finishReason = (String) responseMap.get("finish_reason"); + switch (finishReason) { + case "ERROR", "ERROR_TOXIC" -> throw new ElasticsearchStatusException( + "Cohere stopped the stream due to an error: {}", + RestStatus.INTERNAL_SERVER_ERROR, + parseErrorMessage(responseMap) + ); + case "ERROR_LIMIT" -> throw new ElasticsearchStatusException( + "Cohere stopped the stream due to an error: {}", + RestStatus.TOO_MANY_REQUESTS, + parseErrorMessage(responseMap) + ); + } + } + + @SuppressWarnings("unchecked") + private String parseErrorMessage(Map responseMap) { + var innerResponseMap = (Map) responseMap.get("response"); + return (String) innerResponseMap.get("text"); + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereCompletionRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereCompletionRequestManager.java index 423093a14a9f0..ae46fbe0fef87 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereCompletionRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereCompletionRequestManager.java @@ -19,7 +19,6 @@ import org.elasticsearch.xpack.inference.external.response.cohere.CohereCompletionResponseEntity; import org.elasticsearch.xpack.inference.services.cohere.completion.CohereCompletionModel; -import java.util.List; import java.util.Objects; import java.util.function.Supplier; @@ -30,7 +29,7 @@ public class CohereCompletionRequestManager extends CohereRequestManager { private static final ResponseHandler HANDLER = createCompletionHandler(); private static ResponseHandler createCompletionHandler() { - return new CohereResponseHandler("cohere completion", CohereCompletionResponseEntity::fromResponse); + return new CohereResponseHandler("cohere completion", CohereCompletionResponseEntity::fromResponse, true); } public static CohereCompletionRequestManager of(CohereCompletionModel model, ThreadPool threadPool) { @@ -51,8 +50,10 @@ public void execute( Supplier hasRequestCompletedFunction, ActionListener listener ) { - List docsInput = DocumentsOnlyInput.of(inferenceInputs).getInputs(); - CohereCompletionRequest request = new CohereCompletionRequest(docsInput, model); + var docsOnly = DocumentsOnlyInput.of(inferenceInputs); + var docsInput = docsOnly.getInputs(); + var stream = docsOnly.stream(); + CohereCompletionRequest request = new CohereCompletionRequest(docsInput, model, stream); execute(new ExecutableInferenceRequest(requestSender, logger, request, HANDLER, hasRequestCompletedFunction, listener)); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereEmbeddingsRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereEmbeddingsRequestManager.java index 402f91a0838dc..80617ea56e63c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereEmbeddingsRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereEmbeddingsRequestManager.java @@ -28,7 +28,7 @@ public class CohereEmbeddingsRequestManager extends CohereRequestManager { private static final ResponseHandler HANDLER = createEmbeddingsHandler(); private static ResponseHandler createEmbeddingsHandler() { - return new CohereResponseHandler("cohere text embedding", CohereEmbeddingsResponseEntity::fromResponse); + return new CohereResponseHandler("cohere text embedding", CohereEmbeddingsResponseEntity::fromResponse, false); } public static CohereEmbeddingsRequestManager of(CohereEmbeddingsModel model, ThreadPool threadPool) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRerankRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRerankRequestManager.java index 9d565e7124b03..d27812b17399b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRerankRequestManager.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRerankRequestManager.java @@ -27,7 +27,7 @@ public class CohereRerankRequestManager extends CohereRequestManager { private static final ResponseHandler HANDLER = createCohereResponseHandler(); private static ResponseHandler createCohereResponseHandler() { - return new CohereResponseHandler("cohere rerank", (request, response) -> CohereRankedResponseEntity.fromResponse(response)); + return new CohereResponseHandler("cohere rerank", (request, response) -> CohereRankedResponseEntity.fromResponse(response), false); } public static CohereRerankRequestManager of(CohereRerankModel model, ThreadPool threadPool) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/completion/CohereCompletionRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/completion/CohereCompletionRequest.java index f68f919a7d85b..2172dcd4d791f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/completion/CohereCompletionRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/completion/CohereCompletionRequest.java @@ -25,22 +25,20 @@ import java.util.Objects; public class CohereCompletionRequest extends CohereRequest { - private final CohereAccount account; - private final List input; - private final String modelId; - private final String inferenceEntityId; + private final boolean stream; - public CohereCompletionRequest(List input, CohereCompletionModel model) { + public CohereCompletionRequest(List input, CohereCompletionModel model, boolean stream) { Objects.requireNonNull(model); this.account = CohereAccount.of(model, CohereCompletionRequest::buildDefaultUri); this.input = Objects.requireNonNull(input); this.modelId = model.getServiceSettings().modelId(); this.inferenceEntityId = model.getInferenceEntityId(); + this.stream = stream; } @Override @@ -48,7 +46,7 @@ public HttpRequest createHttpRequest() { HttpPost httpPost = new HttpPost(account.uri()); ByteArrayEntity byteEntity = new ByteArrayEntity( - Strings.toString(new CohereCompletionRequestEntity(input, modelId)).getBytes(StandardCharsets.UTF_8) + Strings.toString(new CohereCompletionRequestEntity(input, modelId, isStreaming())).getBytes(StandardCharsets.UTF_8) ); httpPost.setEntity(byteEntity); @@ -62,6 +60,11 @@ public String getInferenceEntityId() { return inferenceEntityId; } + @Override + public boolean isStreaming() { + return stream; + } + @Override public URI getURI() { return account.uri(); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/completion/CohereCompletionRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/completion/CohereCompletionRequestEntity.java index 8cb3dc6e3c8e8..b834e4335d73c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/completion/CohereCompletionRequestEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/completion/CohereCompletionRequestEntity.java @@ -15,11 +15,11 @@ import java.util.List; import java.util.Objects; -public record CohereCompletionRequestEntity(List input, @Nullable String model) implements ToXContentObject { +public record CohereCompletionRequestEntity(List input, @Nullable String model, boolean stream) implements ToXContentObject { private static final String MESSAGE_FIELD = "message"; - private static final String MODEL = "model"; + private static final String STREAM = "stream"; public CohereCompletionRequestEntity { Objects.requireNonNull(input); @@ -36,6 +36,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(MODEL, model); } + if (stream) { + builder.field(STREAM, true); + } + builder.endObject(); return builder; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/streaming/NewlineDelimitedByteProcessor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/streaming/NewlineDelimitedByteProcessor.java new file mode 100644 index 0000000000000..7c44b202a816b --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/streaming/NewlineDelimitedByteProcessor.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.response.streaming; + +import org.elasticsearch.xpack.inference.common.DelegatingProcessor; +import org.elasticsearch.xpack.inference.external.http.HttpResult; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.regex.Pattern; + +/** + * Processes HttpResult bytes into lines separated by newlines, delimited by either line-feed or carriage-return line-feed. + * Downstream is responsible for validating the structure of the lines after they have been separated. + * Because Upstream (Apache) can send us a single line split between two HttpResults, this processor will aggregate bytes from the last + * HttpResult and append them to the front of the next HttpResult. + * When onComplete is called, the last batch is always flushed to the downstream onNext. + */ +public class NewlineDelimitedByteProcessor extends DelegatingProcessor> { + private static final Pattern END_OF_LINE_REGEX = Pattern.compile("\\n|\\r\\n"); + private volatile String previousTokens = ""; + + @Override + protected void next(HttpResult item) { + // discard empty result and go to the next + if (item.isBodyEmpty()) { + upstream().request(1); + return; + } + + var body = previousTokens + new String(item.body(), StandardCharsets.UTF_8); + var lines = END_OF_LINE_REGEX.split(body, -1); // -1 because we actually want trailing empty strings + + var results = new ArrayDeque(lines.length); + for (var i = 0; i < lines.length - 1; i++) { + var line = lines[i].trim(); + if (line.isBlank() == false) { + results.offer(line); + } + } + + previousTokens = lines[lines.length - 1].trim(); + + if (results.isEmpty()) { + upstream().request(1); + } else { + downstream().onNext(results); + } + } + + @Override + public void onComplete() { + if (previousTokens.isBlank()) { + super.onComplete(); + } else if (isClosed.compareAndSet(false, true)) { + var results = new ArrayDeque(1); + results.offer(previousTokens); + downstream().onNext(results); + } + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java index 728a4ac137dff..3ba93dd8d1b66 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java @@ -39,6 +39,7 @@ import java.util.List; import java.util.Map; +import java.util.Set; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; @@ -288,4 +289,9 @@ static SimilarityMeasure defaultSimilarity() { public TransportVersion getMinimalSupportedVersion() { return TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED; } + + @Override + public Set supportedStreamingTasks() { + return COMPLETION_ONLY; + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandlerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandlerTests.java index d64ac495c8c99..444415dfc8e48 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandlerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandlerTests.java @@ -132,7 +132,7 @@ private static void callCheckForFailureStatusCode(int statusCode, @Nullable Stri var mockRequest = mock(Request.class); when(mockRequest.getInferenceEntityId()).thenReturn(modelId); var httpResult = new HttpResult(httpResponse, errorMessage == null ? new byte[] {} : responseJson.getBytes(StandardCharsets.UTF_8)); - var handler = new CohereResponseHandler("", (request, result) -> null); + var handler = new CohereResponseHandler("", (request, result) -> null, false); handler.checkForFailureStatusCode(mockRequest, httpResult); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/cohere/CohereStreamingProcessorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/cohere/CohereStreamingProcessorTests.java new file mode 100644 index 0000000000000..87d6d63bb8c51 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/cohere/CohereStreamingProcessorTests.java @@ -0,0 +1,189 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.cohere; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.common.xcontent.ChunkedToXContent; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentParseException; +import org.elasticsearch.xpack.core.inference.results.StreamingChatCompletionResults; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.concurrent.Flow; +import java.util.function.Consumer; + +import static org.elasticsearch.xpack.inference.common.DelegatingProcessorTests.onError; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.isA; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.assertArg; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class CohereStreamingProcessorTests extends ESTestCase { + + public void testParseErrorCallsOnError() { + var item = new ArrayDeque(); + item.offer("this is not json"); + + var exception = onError(new CohereStreamingProcessor(), item); + assertThat(exception, instanceOf(XContentParseException.class)); + } + + public void testUnrecognizedEventCallsOnError() { + var item = new ArrayDeque(); + item.offer("{\"event_type\":\"test\"}"); + + var exception = onError(new CohereStreamingProcessor(), item); + assertThat(exception, instanceOf(IOException.class)); + assertThat(exception.getMessage(), equalTo("Unknown eventType found: test")); + } + + public void testMissingTextCallsOnError() { + var item = new ArrayDeque(); + item.offer("{\"event_type\":\"text-generation\"}"); + + var exception = onError(new CohereStreamingProcessor(), item); + assertThat(exception, instanceOf(IOException.class)); + assertThat(exception.getMessage(), equalTo("Null text found in text-generation cohere event")); + } + + public void testEmptyResultsRequestsMoreData() throws Exception { + var emptyDeque = new ArrayDeque(); + + var processor = new CohereStreamingProcessor(); + + Flow.Subscriber downstream = mock(); + processor.subscribe(downstream); + + Flow.Subscription upstream = mock(); + processor.onSubscribe(upstream); + + processor.next(emptyDeque); + + verify(upstream, times(1)).request(1); + verify(downstream, times(0)).onNext(any()); + } + + public void testNonDataEventsAreSkipped() throws Exception { + var item = new ArrayDeque(); + item.offer("{\"event_type\":\"stream-start\"}"); + item.offer("{\"event_type\":\"search-queries-generation\"}"); + item.offer("{\"event_type\":\"search-results\"}"); + item.offer("{\"event_type\":\"citation-generation\"}"); + item.offer("{\"event_type\":\"tool-calls-generation\"}"); + item.offer("{\"event_type\":\"tool-calls-chunk\"}"); + + var processor = new CohereStreamingProcessor(); + + Flow.Subscriber downstream = mock(); + processor.subscribe(downstream); + + Flow.Subscription upstream = mock(); + processor.onSubscribe(upstream); + + processor.next(item); + + verify(upstream, times(1)).request(1); + verify(downstream, times(0)).onNext(any()); + } + + public void testParseError() { + var json = "{\"event_type\":\"stream-end\", \"finish_reason\":\"ERROR\", \"response\":{ \"text\": \"a wild error appears\" }}"; + testError(json, e -> { + assertThat(e.status().getStatus(), equalTo(500)); + assertThat(e.getMessage(), containsString("a wild error appears")); + }); + } + + private void testError(String json, Consumer test) { + var item = new ArrayDeque(); + item.offer(json); + + var processor = new CohereStreamingProcessor(); + + Flow.Subscriber downstream = mock(); + processor.subscribe(downstream); + + Flow.Subscription upstream = mock(); + processor.onSubscribe(upstream); + + try { + processor.next(item); + fail("Expected an exception to be thrown"); + } catch (ElasticsearchStatusException e) { + test.accept(e); + } catch (Exception e) { + fail(e, "Expected an exception of type ElasticsearchStatusException to be thrown"); + } + } + + public void testParseToxic() { + var json = "{\"event_type\":\"stream-end\", \"finish_reason\":\"ERROR_TOXIC\", \"response\":{ \"text\": \"by britney spears\" }}"; + testError(json, e -> { + assertThat(e.status().getStatus(), equalTo(500)); + assertThat(e.getMessage(), containsString("by britney spears")); + }); + } + + public void testParseLimit() { + var json = "{\"event_type\":\"stream-end\", \"finish_reason\":\"ERROR_LIMIT\", \"response\":{ \"text\": \"over the limit\" }}"; + testError(json, e -> { + assertThat(e.status().getStatus(), equalTo(429)); + assertThat(e.getMessage(), containsString("over the limit")); + }); + } + + public void testNonErrorFinishesAreSkipped() throws Exception { + var item = new ArrayDeque(); + item.offer("{\"event_type\":\"stream-end\", \"finish_reason\":\"COMPLETE\"}"); + item.offer("{\"event_type\":\"stream-end\", \"finish_reason\":\"STOP_SEQUENCE\"}"); + item.offer("{\"event_type\":\"stream-end\", \"finish_reason\":\"USER_CANCEL\"}"); + item.offer("{\"event_type\":\"stream-end\", \"finish_reason\":\"MAX_TOKENS\"}"); + + var processor = new CohereStreamingProcessor(); + + Flow.Subscriber downstream = mock(); + processor.subscribe(downstream); + + Flow.Subscription upstream = mock(); + processor.onSubscribe(upstream); + + processor.next(item); + + verify(upstream, times(1)).request(1); + verify(downstream, times(0)).onNext(any()); + } + + public void testParseCohereData() throws Exception { + var item = new ArrayDeque(); + item.offer("{\"event_type\":\"text-generation\", \"text\":\"hello there\"}"); + + var processor = new CohereStreamingProcessor(); + + Flow.Subscriber downstream = mock(); + processor.subscribe(downstream); + + Flow.Subscription upstream = mock(); + processor.onSubscribe(upstream); + + processor.next(item); + + verify(upstream, times(0)).request(1); + verify(downstream, times(1)).onNext(assertArg(chunks -> { + assertThat(chunks, isA(StreamingChatCompletionResults.Results.class)); + var results = (StreamingChatCompletionResults.Results) chunks; + assertThat(results.results().size(), equalTo(1)); + assertThat(results.results().getFirst().delta(), equalTo("hello there")); + })); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereCompletionRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereCompletionRequestEntityTests.java index dbe6a9438d884..c3b534f42e7ee 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereCompletionRequestEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereCompletionRequestEntityTests.java @@ -22,7 +22,7 @@ public class CohereCompletionRequestEntityTests extends ESTestCase { public void testXContent_WritesAllFields() throws IOException { - var entity = new CohereCompletionRequestEntity(List.of("some input"), "model"); + var entity = new CohereCompletionRequestEntity(List.of("some input"), "model", false); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); entity.toXContent(builder, null); @@ -33,7 +33,7 @@ public void testXContent_WritesAllFields() throws IOException { } public void testXContent_DoesNotWriteModelIfNotSpecified() throws IOException { - var entity = new CohereCompletionRequestEntity(List.of("some input"), null); + var entity = new CohereCompletionRequestEntity(List.of("some input"), null, false); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); entity.toXContent(builder, null); @@ -44,10 +44,10 @@ public void testXContent_DoesNotWriteModelIfNotSpecified() throws IOException { } public void testXContent_ThrowsIfInputIsNull() { - expectThrows(NullPointerException.class, () -> new CohereCompletionRequestEntity(null, null)); + expectThrows(NullPointerException.class, () -> new CohereCompletionRequestEntity(null, null, false)); } public void testXContent_ThrowsIfMessageInInputIsNull() { - expectThrows(NullPointerException.class, () -> new CohereCompletionRequestEntity(List.of((String) null), null)); + expectThrows(NullPointerException.class, () -> new CohereCompletionRequestEntity(List.of((String) null), null, false)); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereCompletionRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereCompletionRequestTests.java index d6d0d5c00eaf4..f2e6d4305f9e6 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereCompletionRequestTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereCompletionRequestTests.java @@ -26,7 +26,7 @@ public class CohereCompletionRequestTests extends ESTestCase { public void testCreateRequest_UrlDefined() throws IOException { - var request = new CohereCompletionRequest(List.of("abc"), CohereCompletionModelTests.createModel("url", "secret", null)); + var request = new CohereCompletionRequest(List.of("abc"), CohereCompletionModelTests.createModel("url", "secret", null), false); var httpRequest = request.createHttpRequest(); assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); @@ -43,7 +43,7 @@ public void testCreateRequest_UrlDefined() throws IOException { } public void testCreateRequest_ModelDefined() throws IOException { - var request = new CohereCompletionRequest(List.of("abc"), CohereCompletionModelTests.createModel("url", "secret", "model")); + var request = new CohereCompletionRequest(List.of("abc"), CohereCompletionModelTests.createModel("url", "secret", "model"), false); var httpRequest = request.createHttpRequest(); assertThat(httpRequest.httpRequestBase(), instanceOf(HttpPost.class)); @@ -60,14 +60,14 @@ public void testCreateRequest_ModelDefined() throws IOException { } public void testTruncate_ReturnsSameInstance() { - var request = new CohereCompletionRequest(List.of("abc"), CohereCompletionModelTests.createModel("url", "secret", "model")); + var request = new CohereCompletionRequest(List.of("abc"), CohereCompletionModelTests.createModel("url", "secret", "model"), false); var truncatedRequest = request.truncate(); assertThat(truncatedRequest, sameInstance(request)); } public void testTruncationInfo_ReturnsNull() { - var request = new CohereCompletionRequest(List.of("abc"), CohereCompletionModelTests.createModel("url", "secret", "model")); + var request = new CohereCompletionRequest(List.of("abc"), CohereCompletionModelTests.createModel("url", "secret", "model"), false); assertNull(request.getTruncationInfo()); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/streaming/NewlineDelimitedByteProcessorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/streaming/NewlineDelimitedByteProcessorTests.java new file mode 100644 index 0000000000000..488cbccd0e7c3 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/response/streaming/NewlineDelimitedByteProcessorTests.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.response.streaming; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.external.http.HttpResult; +import org.junit.Before; +import org.mockito.ArgumentCaptor; + +import java.nio.charset.StandardCharsets; +import java.util.Deque; +import java.util.concurrent.Flow; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.assertArg; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class NewlineDelimitedByteProcessorTests extends ESTestCase { + private Flow.Subscription upstream; + private Flow.Subscriber> downstream; + private NewlineDelimitedByteProcessor processor; + + @Before + public void setUp() throws Exception { + super.setUp(); + upstream = mock(); + downstream = mock(); + processor = new NewlineDelimitedByteProcessor(); + processor.onSubscribe(upstream); + processor.subscribe(downstream); + } + + public void testEmptyBody() { + processor.next(result(null)); + processor.onComplete(); + verify(upstream, times(1)).request(1); + verify(downstream, times(0)).onNext(any()); + } + + private HttpResult result(String response) { + return new HttpResult(mock(), response == null ? new byte[0] : response.getBytes(StandardCharsets.UTF_8)); + } + + public void testEmptyParseResponse() { + processor.next(result("")); + verify(upstream, times(1)).request(1); + verify(downstream, times(0)).onNext(any()); + } + + public void testValidResponse() { + processor.next(result("{\"hello\":\"there\"}\n")); + verify(downstream, times(1)).onNext(assertArg(deque -> { + assertThat(deque, notNullValue()); + assertThat(deque.size(), is(1)); + assertThat(deque.getFirst(), is("{\"hello\":\"there\"}")); + })); + } + + public void testMultipleValidResponse() { + processor.next(result(""" + {"value": 1} + {"value": 2} + {"value": 3} + """)); + verify(upstream, times(0)).request(1); + verify(downstream, times(1)).onNext(assertArg(deque -> { + assertThat(deque, notNullValue()); + assertThat(deque.size(), is(3)); + var items = deque.iterator(); + IntStream.range(1, 4).forEach(i -> { + assertThat(items.hasNext(), is(true)); + assertThat(items.next(), containsString(String.valueOf(i))); + }); + })); + } + + public void testOnCompleteFlushesResponse() { + processor.next(result(""" + {"value": 1}""")); + + // onNext should not be called with only one value + verify(downstream, times(0)).onNext(any()); + verify(downstream, times(0)).onComplete(); + + // onComplete should flush the value pending, and onNext should be called + processor.onComplete(); + verify(downstream, times(1)).onNext(assertArg(deque -> { + assertThat(deque, notNullValue()); + assertThat(deque.size(), is(1)); + var item = deque.getFirst(); + assertThat(item, containsString(String.valueOf(1))); + })); + verify(downstream, times(0)).onComplete(); + + // next time the downstream requests data, onComplete is called + var downstreamSubscription = ArgumentCaptor.forClass(Flow.Subscription.class); + verify(downstream).onSubscribe(downstreamSubscription.capture()); + downstreamSubscription.getValue().request(1); + verify(downstream, times(1)).onComplete(); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java index 22503108b5262..420a635963a29 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java @@ -38,6 +38,8 @@ import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import org.elasticsearch.xpack.inference.services.InferenceEventsAssertion; +import org.elasticsearch.xpack.inference.services.cohere.completion.CohereCompletionModelTests; import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingType; import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsModel; import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsModelTests; @@ -1349,6 +1351,54 @@ public void testDefaultSimilarity() { assertEquals(SimilarityMeasure.DOT_PRODUCT, CohereService.defaultSimilarity()); } + public void testInfer_StreamRequest() throws Exception { + String responseJson = """ + {"event_type":"text-generation", "text":"hello"} + {"event_type":"text-generation", "text":"there"} + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var result = streamChatCompletion(); + + InferenceEventsAssertion.assertThat(result).hasFinishedStream().hasNoErrors().hasEvent(""" + {"completion":[{"delta":"hello"},{"delta":"there"}]}"""); + } + + private InferenceServiceResults streamChatCompletion() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + try (var service = new CohereService(senderFactory, createWithEmptySettings(threadPool))) { + var model = CohereCompletionModelTests.createModel(getUrl(webServer), "secret", "model"); + PlainActionFuture listener = new PlainActionFuture<>(); + service.infer( + model, + null, + List.of("abc"), + true, + new HashMap<>(), + InputType.INGEST, + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + return listener.actionGet(TIMEOUT); + } + } + + public void testInfer_StreamRequest_ErrorResponse() throws Exception { + String responseJson = """ + { "event_type":"stream-end", "finish_reason":"ERROR", "response":{ "text": "how dare you" } } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var result = streamChatCompletion(); + + InferenceEventsAssertion.assertThat(result) + .hasFinishedStream() + .hasNoEvents() + .hasErrorWithStatusCode(500) + .hasErrorContaining("how dare you"); + } + private Map getRequestConfigMap( Map serviceSettings, Map taskSettings, From 7753c5216a5195890c3d4a72dcb7c6f9056c8eaa Mon Sep 17 00:00:00 2001 From: Pat Whelan Date: Mon, 7 Oct 2024 15:10:36 -0400 Subject: [PATCH 148/194] [ML] Add Streaming Inference spec (#113812) API for `/_inference/{task_type}/{inference_id}/_stream` and `/_inference/{inference_id}/_stream` Request is `application/json` Response is `text/event-stream` --- docs/changelog/113812.yaml | 5 ++ .../api/inference.stream_inference.json | 49 +++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 docs/changelog/113812.yaml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_inference.json diff --git a/docs/changelog/113812.yaml b/docs/changelog/113812.yaml new file mode 100644 index 0000000000000..04498b4ae5f7e --- /dev/null +++ b/docs/changelog/113812.yaml @@ -0,0 +1,5 @@ +pr: 113812 +summary: Add Streaming Inference spec +area: Machine Learning +type: enhancement +issues: [] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_inference.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_inference.json new file mode 100644 index 0000000000000..32b4b2f311837 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.stream_inference.json @@ -0,0 +1,49 @@ +{ + "inference.stream_inference":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/post-stream-inference-api.html", + "description":"Perform streaming inference" + }, + "stability":"experimental", + "visibility":"public", + "headers":{ + "accept": [ "text/event-stream"], + "content_type": ["application/json"] + }, + "url":{ + "paths":[ + { + "path":"/_inference/{inference_id}/_stream", + "methods":[ + "POST" + ], + "parts":{ + "inference_id":{ + "type":"string", + "description":"The inference Id" + } + } + }, + { + "path":"/_inference/{task_type}/{inference_id}/_stream", + "methods":[ + "POST" + ], + "parts":{ + "task_type":{ + "type":"string", + "description":"The task type" + }, + "inference_id":{ + "type":"string", + "description":"The inference Id" + } + } + } + ] + }, + "body":{ + "description":"The inference payload" + } + } +} From e7cd5c9dea0e200af9f2bfb791e41ad555150846 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Mon, 7 Oct 2024 16:31:05 -0300 Subject: [PATCH 149/194] Add postal_code support to the City and Enterprise databases (#114193) --- .../org/elasticsearch/ingest/geoip/Database.java | 9 ++++++--- .../ingest/geoip/MaxmindIpDataLookups.java | 13 +++++++++++++ .../ingest/geoip/GeoIpProcessorFactoryTests.java | 3 ++- .../ingest/geoip/GeoIpProcessorTests.java | 6 ++++-- .../ingest/geoip/MaxMindSupportTests.java | 6 +++--- 5 files changed, 28 insertions(+), 9 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java index 31d7a43e3869a..10817c920e1e5 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java @@ -40,7 +40,8 @@ enum Database { Property.REGION_NAME, Property.CITY_NAME, Property.TIMEZONE, - Property.LOCATION + Property.LOCATION, + Property.POSTAL_CODE ), Set.of( Property.COUNTRY_ISO_CODE, @@ -108,7 +109,8 @@ enum Database { Property.MOBILE_COUNTRY_CODE, Property.MOBILE_NETWORK_CODE, Property.USER_TYPE, - Property.CONNECTION_TYPE + Property.CONNECTION_TYPE, + Property.POSTAL_CODE ), Set.of( Property.COUNTRY_ISO_CODE, @@ -228,7 +230,8 @@ enum Property { MOBILE_NETWORK_CODE, CONNECTION_TYPE, USER_TYPE, - TYPE; + TYPE, + POSTAL_CODE; /** * Parses a string representation of a property into an actual Property instance. Not all properties that exist are diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java index 5b22b3f4005a9..2e0d4a031a072 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java @@ -23,6 +23,7 @@ import com.maxmind.geoip2.model.IspResponse; import com.maxmind.geoip2.record.Continent; import com.maxmind.geoip2.record.Location; +import com.maxmind.geoip2.record.Postal; import com.maxmind.geoip2.record.Subdivision; import org.elasticsearch.common.network.InetAddresses; @@ -139,6 +140,7 @@ protected Map transform(final CityResponse response) { Location location = response.getLocation(); Continent continent = response.getContinent(); Subdivision subdivision = response.getMostSpecificSubdivision(); + Postal postal = response.getPostal(); Map data = new HashMap<>(); for (Database.Property property : this.properties) { @@ -206,6 +208,11 @@ protected Map transform(final CityResponse response) { data.put("location", locationObject); } } + case POSTAL_CODE -> { + if (postal != null && postal.getCode() != null) { + data.put("postal_code", postal.getCode()); + } + } } } return data; @@ -324,6 +331,7 @@ protected Map transform(final EnterpriseResponse response) { Location location = response.getLocation(); Continent continent = response.getContinent(); Subdivision subdivision = response.getMostSpecificSubdivision(); + Postal postal = response.getPostal(); Long asn = response.getTraits().getAutonomousSystemNumber(); String organizationName = response.getTraits().getAutonomousSystemOrganization(); @@ -413,6 +421,11 @@ protected Map transform(final EnterpriseResponse response) { data.put("location", locationObject); } } + case POSTAL_CODE -> { + if (postal != null && postal.getCode() != null) { + data.put("postal_code", postal.getCode()); + } + } case ASN -> { if (asn != null) { data.put("asn", asn); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java index 9972db26b3642..d4017268b53db 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java @@ -274,7 +274,8 @@ public void testBuildIllegalFieldOption() { e.getMessage(), equalTo( "[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_ISO_CODE, " - + "COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, CITY_NAME, TIMEZONE, LOCATION]" + + "COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, CITY_NAME, TIMEZONE, " + + "LOCATION, POSTAL_CODE]" ) ); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index 46024cb6ad215..3fb082f33b3f6 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -222,7 +222,7 @@ public void testCity_withIpV6() throws Exception { @SuppressWarnings("unchecked") Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(10)); + assertThat(geoData.size(), equalTo(11)); assertThat(geoData.get("ip"), equalTo(ip)); assertThat(geoData.get("country_iso_code"), equalTo("US")); assertThat(geoData.get("country_name"), equalTo("United States")); @@ -233,6 +233,7 @@ public void testCity_withIpV6() throws Exception { assertThat(geoData.get("city_name"), equalTo("Homestead")); assertThat(geoData.get("timezone"), equalTo("America/New_York")); assertThat(geoData.get("location"), equalTo(Map.of("lat", 25.4573d, "lon", -80.4572d))); + assertThat(geoData.get("postal_code"), equalTo("33035")); } public void testCityWithMissingLocation() throws Exception { @@ -470,7 +471,7 @@ public void testEnterprise() throws Exception { @SuppressWarnings("unchecked") Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(24)); + assertThat(geoData.size(), equalTo(25)); assertThat(geoData.get("ip"), equalTo(ip)); assertThat(geoData.get("country_iso_code"), equalTo("US")); assertThat(geoData.get("country_name"), equalTo("United States")); @@ -481,6 +482,7 @@ public void testEnterprise() throws Exception { assertThat(geoData.get("city_name"), equalTo("Chatham")); assertThat(geoData.get("timezone"), equalTo("America/New_York")); assertThat(geoData.get("location"), equalTo(Map.of("lat", 42.3478, "lon", -73.5549))); + assertThat(geoData.get("postal_code"), equalTo("12037")); assertThat(geoData.get("asn"), equalTo(14671L)); assertThat(geoData.get("organization_name"), equalTo("FairPoint Communications")); assertThat(geoData.get("network"), equalTo("74.209.16.0/20")); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java index 3b12003637783..7a3de6ca199aa 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java @@ -84,7 +84,8 @@ public class MaxMindSupportTests extends ESTestCase { "location.longitude", "location.timeZone", "mostSpecificSubdivision.isoCode", - "mostSpecificSubdivision.name" + "mostSpecificSubdivision.name", + "postal.code" ); private static final Set CITY_UNSUPPORTED_FIELDS = Set.of( "city.confidence", @@ -109,7 +110,6 @@ public class MaxMindSupportTests extends ESTestCase { "mostSpecificSubdivision.confidence", "mostSpecificSubdivision.geoNameId", "mostSpecificSubdivision.names", - "postal.code", "postal.confidence", "registeredCountry.confidence", "registeredCountry.geoNameId", @@ -223,6 +223,7 @@ public class MaxMindSupportTests extends ESTestCase { "location.timeZone", "mostSpecificSubdivision.isoCode", "mostSpecificSubdivision.name", + "postal.code", "traits.anonymous", "traits.anonymousVpn", "traits.autonomousSystemNumber", @@ -263,7 +264,6 @@ public class MaxMindSupportTests extends ESTestCase { "mostSpecificSubdivision.confidence", "mostSpecificSubdivision.geoNameId", "mostSpecificSubdivision.names", - "postal.code", "postal.confidence", "registeredCountry.confidence", "registeredCountry.geoNameId", From 3595956d7ee7bfd5272def08ce419bd998e3b2f5 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 8 Oct 2024 08:06:42 +1100 Subject: [PATCH 150/194] Mute org.elasticsearch.ingest.geoip.IpinfoIpDataLookupsTests org.elasticsearch.ingest.geoip.IpinfoIpDataLookupsTests #114266 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index f205c9ce44a08..93893d7103afb 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -368,6 +368,8 @@ tests: - class: org.elasticsearch.logsdb.datageneration.DataGeneratorTests method: testDataGeneratorProducesValidMappingAndDocument issue: https://github.com/elastic/elasticsearch/issues/114188 +- class: org.elasticsearch.ingest.geoip.IpinfoIpDataLookupsTests + issue: https://github.com/elastic/elasticsearch/issues/114266 # Examples: # From 7bbebbd37d7510e32a8b50ffd501bd0ba09e6d56 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Mon, 7 Oct 2024 20:24:12 -0500 Subject: [PATCH 151/194] Supporting more maxmind fields in the geoip processor (#114268) --- .../elasticsearch/ingest/geoip/Database.java | 27 +++++++++-- .../ingest/geoip/MaxmindIpDataLookups.java | 48 +++++++++++++++++++ .../geoip/GeoIpProcessorFactoryTests.java | 6 +-- .../ingest/geoip/GeoIpProcessorTests.java | 17 +++++-- .../ingest/geoip/MaxMindSupportTests.java | 16 +++---- 5 files changed, 95 insertions(+), 19 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java index 10817c920e1e5..128c16e163764 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java @@ -32,6 +32,7 @@ enum Database { City( Set.of( Property.IP, + Property.COUNTRY_IN_EUROPEAN_UNION, Property.COUNTRY_ISO_CODE, Property.CONTINENT_CODE, Property.COUNTRY_NAME, @@ -41,7 +42,8 @@ enum Database { Property.CITY_NAME, Property.TIMEZONE, Property.LOCATION, - Property.POSTAL_CODE + Property.POSTAL_CODE, + Property.ACCURACY_RADIUS ), Set.of( Property.COUNTRY_ISO_CODE, @@ -54,7 +56,14 @@ enum Database { ) ), Country( - Set.of(Property.IP, Property.CONTINENT_CODE, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE), + Set.of( + Property.IP, + Property.CONTINENT_CODE, + Property.CONTINENT_NAME, + Property.COUNTRY_NAME, + Property.COUNTRY_IN_EUROPEAN_UNION, + Property.COUNTRY_ISO_CODE + ), Set.of(Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE) ), Asn( @@ -85,12 +94,15 @@ enum Database { Enterprise( Set.of( Property.IP, + Property.COUNTRY_CONFIDENCE, + Property.COUNTRY_IN_EUROPEAN_UNION, Property.COUNTRY_ISO_CODE, Property.COUNTRY_NAME, Property.CONTINENT_CODE, Property.CONTINENT_NAME, Property.REGION_ISO_CODE, Property.REGION_NAME, + Property.CITY_CONFIDENCE, Property.CITY_NAME, Property.TIMEZONE, Property.LOCATION, @@ -110,7 +122,9 @@ enum Database { Property.MOBILE_NETWORK_CODE, Property.USER_TYPE, Property.CONNECTION_TYPE, - Property.POSTAL_CODE + Property.POSTAL_CODE, + Property.POSTAL_CONFIDENCE, + Property.ACCURACY_RADIUS ), Set.of( Property.COUNTRY_ISO_CODE, @@ -205,12 +219,15 @@ public Set parseProperties(@Nullable final List propertyNames) enum Property { IP, + COUNTRY_CONFIDENCE, + COUNTRY_IN_EUROPEAN_UNION, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, + CITY_CONFIDENCE, CITY_NAME, TIMEZONE, LOCATION, @@ -231,7 +248,9 @@ enum Property { CONNECTION_TYPE, USER_TYPE, TYPE, - POSTAL_CODE; + POSTAL_CODE, + POSTAL_CONFIDENCE, + ACCURACY_RADIUS; /** * Parses a string representation of a property into an actual Property instance. Not all properties that exist are diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java index 2e0d4a031a072..e7c3481938033 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java @@ -146,6 +146,12 @@ protected Map transform(final CityResponse response) { for (Database.Property property : this.properties) { switch (property) { case IP -> data.put("ip", response.getTraits().getIpAddress()); + case COUNTRY_IN_EUROPEAN_UNION -> { + if (country.getIsoCode() != null) { + // isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country + data.put("country_in_european_union", country.isInEuropeanUnion()); + } + } case COUNTRY_ISO_CODE -> { String countryIsoCode = country.getIsoCode(); if (countryIsoCode != null) { @@ -208,6 +214,12 @@ protected Map transform(final CityResponse response) { data.put("location", locationObject); } } + case ACCURACY_RADIUS -> { + Integer accuracyRadius = location.getAccuracyRadius(); + if (accuracyRadius != null) { + data.put("accuracy_radius", accuracyRadius); + } + } case POSTAL_CODE -> { if (postal != null && postal.getCode() != null) { data.put("postal_code", postal.getCode()); @@ -261,6 +273,12 @@ protected Map transform(final CountryResponse response) { for (Database.Property property : this.properties) { switch (property) { case IP -> data.put("ip", response.getTraits().getIpAddress()); + case COUNTRY_IN_EUROPEAN_UNION -> { + if (country.getIsoCode() != null) { + // isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country + data.put("country_in_european_union", country.isInEuropeanUnion()); + } + } case COUNTRY_ISO_CODE -> { String countryIsoCode = country.getIsoCode(); if (countryIsoCode != null) { @@ -359,6 +377,18 @@ protected Map transform(final EnterpriseResponse response) { for (Database.Property property : this.properties) { switch (property) { case IP -> data.put("ip", response.getTraits().getIpAddress()); + case COUNTRY_CONFIDENCE -> { + Integer countryConfidence = country.getConfidence(); + if (countryConfidence != null) { + data.put("country_confidence", countryConfidence); + } + } + case COUNTRY_IN_EUROPEAN_UNION -> { + if (country.getIsoCode() != null) { + // isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country + data.put("country_in_european_union", country.isInEuropeanUnion()); + } + } case COUNTRY_ISO_CODE -> { String countryIsoCode = country.getIsoCode(); if (countryIsoCode != null) { @@ -399,6 +429,12 @@ protected Map transform(final EnterpriseResponse response) { data.put("region_name", subdivisionName); } } + case CITY_CONFIDENCE -> { + Integer cityConfidence = city.getConfidence(); + if (cityConfidence != null) { + data.put("city_confidence", cityConfidence); + } + } case CITY_NAME -> { String cityName = city.getName(); if (cityName != null) { @@ -421,11 +457,23 @@ protected Map transform(final EnterpriseResponse response) { data.put("location", locationObject); } } + case ACCURACY_RADIUS -> { + Integer accuracyRadius = location.getAccuracyRadius(); + if (accuracyRadius != null) { + data.put("accuracy_radius", accuracyRadius); + } + } case POSTAL_CODE -> { if (postal != null && postal.getCode() != null) { data.put("postal_code", postal.getCode()); } } + case POSTAL_CONFIDENCE -> { + Integer postalConfidence = postal.getConfidence(); + if (postalConfidence != null) { + data.put("postal_confidence", postalConfidence); + } + } case ASN -> { if (asn != null) { data.put("asn", asn); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java index d4017268b53db..cfea54d2520bd 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java @@ -195,7 +195,7 @@ public void testBuildWithCountryDbAndAsnFields() { equalTo( "[properties] illegal property value [" + asnProperty - + "]. valid values are [IP, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME]" + + "]. valid values are [IP, COUNTRY_IN_EUROPEAN_UNION, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME]" ) ); } @@ -273,9 +273,9 @@ public void testBuildIllegalFieldOption() { assertThat( e.getMessage(), equalTo( - "[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_ISO_CODE, " + "[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_IN_EUROPEAN_UNION, COUNTRY_ISO_CODE, " + "COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, CITY_NAME, TIMEZONE, " - + "LOCATION, POSTAL_CODE]" + + "LOCATION, POSTAL_CODE, ACCURACY_RADIUS]" ) ); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index 3fb082f33b3f6..ffc40324bd886 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -106,8 +106,9 @@ public void testCity() throws Exception { @SuppressWarnings("unchecked") Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(7)); + assertThat(geoData.size(), equalTo(9)); assertThat(geoData.get("ip"), equalTo(ip)); + assertThat(geoData.get("country_in_european_union"), equalTo(false)); assertThat(geoData.get("country_iso_code"), equalTo("US")); assertThat(geoData.get("country_name"), equalTo("United States")); assertThat(geoData.get("continent_code"), equalTo("NA")); @@ -222,8 +223,9 @@ public void testCity_withIpV6() throws Exception { @SuppressWarnings("unchecked") Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(11)); + assertThat(geoData.size(), equalTo(13)); assertThat(geoData.get("ip"), equalTo(ip)); + assertThat(geoData.get("country_in_european_union"), equalTo(false)); assertThat(geoData.get("country_iso_code"), equalTo("US")); assertThat(geoData.get("country_name"), equalTo("United States")); assertThat(geoData.get("continent_code"), equalTo("NA")); @@ -233,6 +235,7 @@ public void testCity_withIpV6() throws Exception { assertThat(geoData.get("city_name"), equalTo("Homestead")); assertThat(geoData.get("timezone"), equalTo("America/New_York")); assertThat(geoData.get("location"), equalTo(Map.of("lat", 25.4573d, "lon", -80.4572d))); + assertThat(geoData.get("accuracy_radius"), equalTo(50)); assertThat(geoData.get("postal_code"), equalTo("33035")); } @@ -288,8 +291,9 @@ public void testCountry() throws Exception { @SuppressWarnings("unchecked") Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(5)); + assertThat(geoData.size(), equalTo(6)); assertThat(geoData.get("ip"), equalTo(ip)); + assertThat(geoData.get("country_in_european_union"), equalTo(true)); assertThat(geoData.get("country_iso_code"), equalTo("NL")); assertThat(geoData.get("country_name"), equalTo("Netherlands")); assertThat(geoData.get("continent_code"), equalTo("EU")); @@ -471,18 +475,23 @@ public void testEnterprise() throws Exception { @SuppressWarnings("unchecked") Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("target_field"); assertThat(geoData, notNullValue()); - assertThat(geoData.size(), equalTo(25)); + assertThat(geoData.size(), equalTo(30)); assertThat(geoData.get("ip"), equalTo(ip)); + assertThat(geoData.get("country_confidence"), equalTo(99)); + assertThat(geoData.get("country_in_european_union"), equalTo(false)); assertThat(geoData.get("country_iso_code"), equalTo("US")); assertThat(geoData.get("country_name"), equalTo("United States")); assertThat(geoData.get("continent_code"), equalTo("NA")); assertThat(geoData.get("continent_name"), equalTo("North America")); assertThat(geoData.get("region_iso_code"), equalTo("US-NY")); assertThat(geoData.get("region_name"), equalTo("New York")); + assertThat(geoData.get("city_confidence"), equalTo(11)); assertThat(geoData.get("city_name"), equalTo("Chatham")); assertThat(geoData.get("timezone"), equalTo("America/New_York")); assertThat(geoData.get("location"), equalTo(Map.of("lat", 42.3478, "lon", -73.5549))); + assertThat(geoData.get("accuracy_radius"), equalTo(27)); assertThat(geoData.get("postal_code"), equalTo("12037")); + assertThat(geoData.get("city_confidence"), equalTo(11)); assertThat(geoData.get("asn"), equalTo(14671L)); assertThat(geoData.get("organization_name"), equalTo("FairPoint Communications")); assertThat(geoData.get("network"), equalTo("74.209.16.0/20")); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java index 7a3de6ca199aa..1e05cf2b3ba33 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java @@ -78,8 +78,10 @@ public class MaxMindSupportTests extends ESTestCase { "city.name", "continent.code", "continent.name", + "country.inEuropeanUnion", "country.isoCode", "country.name", + "location.accuracyRadius", "location.latitude", "location.longitude", "location.timeZone", @@ -95,14 +97,12 @@ public class MaxMindSupportTests extends ESTestCase { "continent.names", "country.confidence", "country.geoNameId", - "country.inEuropeanUnion", "country.names", "leastSpecificSubdivision.confidence", "leastSpecificSubdivision.geoNameId", "leastSpecificSubdivision.isoCode", "leastSpecificSubdivision.name", "leastSpecificSubdivision.names", - "location.accuracyRadius", "location.averageIncome", "location.metroCode", "location.populationDensity", @@ -159,6 +159,7 @@ public class MaxMindSupportTests extends ESTestCase { private static final Set COUNTRY_SUPPORTED_FIELDS = Set.of( "continent.name", + "country.inEuropeanUnion", "country.isoCode", "continent.code", "country.name" @@ -168,7 +169,6 @@ public class MaxMindSupportTests extends ESTestCase { "continent.names", "country.confidence", "country.geoNameId", - "country.inEuropeanUnion", "country.names", "maxMind", "registeredCountry.confidence", @@ -213,17 +213,22 @@ public class MaxMindSupportTests extends ESTestCase { private static final Set DOMAIN_UNSUPPORTED_FIELDS = Set.of("ipAddress", "network"); private static final Set ENTERPRISE_SUPPORTED_FIELDS = Set.of( + "city.confidence", "city.name", "continent.code", "continent.name", + "country.confidence", + "country.inEuropeanUnion", "country.isoCode", "country.name", + "location.accuracyRadius", "location.latitude", "location.longitude", "location.timeZone", "mostSpecificSubdivision.isoCode", "mostSpecificSubdivision.name", "postal.code", + "postal.confidence", "traits.anonymous", "traits.anonymousVpn", "traits.autonomousSystemNumber", @@ -242,21 +247,17 @@ public class MaxMindSupportTests extends ESTestCase { "traits.userType" ); private static final Set ENTERPRISE_UNSUPPORTED_FIELDS = Set.of( - "city.confidence", "city.geoNameId", "city.names", "continent.geoNameId", "continent.names", - "country.confidence", "country.geoNameId", - "country.inEuropeanUnion", "country.names", "leastSpecificSubdivision.confidence", "leastSpecificSubdivision.geoNameId", "leastSpecificSubdivision.isoCode", "leastSpecificSubdivision.name", "leastSpecificSubdivision.names", - "location.accuracyRadius", "location.averageIncome", "location.metroCode", "location.populationDensity", @@ -264,7 +265,6 @@ public class MaxMindSupportTests extends ESTestCase { "mostSpecificSubdivision.confidence", "mostSpecificSubdivision.geoNameId", "mostSpecificSubdivision.names", - "postal.confidence", "registeredCountry.confidence", "registeredCountry.geoNameId", "registeredCountry.inEuropeanUnion", From 44f379189925e9738873196f7c86788085ab0f98 Mon Sep 17 00:00:00 2001 From: Panagiotis Bailis Date: Tue, 8 Oct 2024 08:27:06 +0300 Subject: [PATCH 152/194] Updating toXContent implementation for retrievers (#114017) --- .../search/retriever/RetrieverBuilder.java | 8 +++ .../builder/SearchSourceBuilderTests.java | 71 +++++++++++++++++++ .../KnnRetrieverBuilderParsingTests.java | 2 +- .../StandardRetrieverBuilderParsingTests.java | 2 +- .../random/RandomRankRetrieverBuilder.java | 5 +- .../TextSimilarityRankRetrieverBuilder.java | 8 +-- .../RandomRankRetrieverBuilderTests.java | 10 ++- ...xtSimilarityRankRetrieverBuilderTests.java | 46 +++++++++++- .../xpack/rank/rrf/RRFRetrieverBuilder.java | 3 - .../rrf/RRFRetrieverBuilderParsingTests.java | 57 ++++++++++++++- 10 files changed, 187 insertions(+), 25 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java index 1328106896bcb..1c6f8c4a7ce44 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java @@ -251,11 +251,19 @@ public ActionRequestValidationException validate( @Override public final XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { builder.startObject(); + builder.startObject(getName()); if (preFilterQueryBuilders.isEmpty() == false) { builder.field(PRE_FILTER_FIELD.getPreferredName(), preFilterQueryBuilders); } + if (minScore != null) { + builder.field(MIN_SCORE_FIELD.getPreferredName(), minScore); + } + if (retrieverName != null) { + builder.field(NAME_FIELD.getPreferredName(), retrieverName); + } doToXContent(builder, params); builder.endObject(); + builder.endObject(); return builder; } diff --git a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index 3f33bbfe6f6cb..240a677f4cbfd 100644 --- a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -41,6 +41,8 @@ import org.elasticsearch.search.collapse.CollapseBuilderTests; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.rescore.QueryRescorerBuilder; +import org.elasticsearch.search.retriever.KnnRetrieverBuilder; +import org.elasticsearch.search.retriever.StandardRetrieverBuilder; import org.elasticsearch.search.slice.SliceBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; @@ -600,6 +602,75 @@ public void testNegativeTrackTotalHits() throws IOException { } } + public void testStandardRetrieverParsing() throws IOException { + String restContent = "{" + + " \"retriever\": {" + + " \"standard\": {" + + " \"query\": {" + + " \"match_all\": {}" + + " }," + + " \"min_score\": 10," + + " \"_name\": \"foo_standard\"" + + " }" + + " }" + + "}"; + SearchUsageHolder searchUsageHolder = new UsageService().getSearchUsageHolder(); + try (XContentParser jsonParser = createParser(JsonXContent.jsonXContent, restContent)) { + SearchSourceBuilder source = new SearchSourceBuilder().parseXContent(jsonParser, true, searchUsageHolder, nf -> true); + assertThat(source.retriever(), instanceOf(StandardRetrieverBuilder.class)); + StandardRetrieverBuilder parsed = (StandardRetrieverBuilder) source.retriever(); + assertThat(parsed.minScore(), equalTo(10f)); + assertThat(parsed.retrieverName(), equalTo("foo_standard")); + try (XContentParser parseSerialized = createParser(JsonXContent.jsonXContent, Strings.toString(source))) { + SearchSourceBuilder deserializedSource = new SearchSourceBuilder().parseXContent( + parseSerialized, + true, + searchUsageHolder, + nf -> true + ); + assertThat(deserializedSource.retriever(), instanceOf(StandardRetrieverBuilder.class)); + StandardRetrieverBuilder deserialized = (StandardRetrieverBuilder) source.retriever(); + assertThat(parsed, equalTo(deserialized)); + } + } + } + + public void testKnnRetrieverParsing() throws IOException { + String restContent = "{" + + " \"retriever\": {" + + " \"knn\": {" + + " \"query_vector\": [" + + " 3" + + " ]," + + " \"field\": \"vector\"," + + " \"k\": 10," + + " \"num_candidates\": 15," + + " \"min_score\": 10," + + " \"_name\": \"foo_knn\"" + + " }" + + " }" + + "}"; + SearchUsageHolder searchUsageHolder = new UsageService().getSearchUsageHolder(); + try (XContentParser jsonParser = createParser(JsonXContent.jsonXContent, restContent)) { + SearchSourceBuilder source = new SearchSourceBuilder().parseXContent(jsonParser, true, searchUsageHolder, nf -> true); + assertThat(source.retriever(), instanceOf(KnnRetrieverBuilder.class)); + KnnRetrieverBuilder parsed = (KnnRetrieverBuilder) source.retriever(); + assertThat(parsed.minScore(), equalTo(10f)); + assertThat(parsed.retrieverName(), equalTo("foo_knn")); + try (XContentParser parseSerialized = createParser(JsonXContent.jsonXContent, Strings.toString(source))) { + SearchSourceBuilder deserializedSource = new SearchSourceBuilder().parseXContent( + parseSerialized, + true, + searchUsageHolder, + nf -> true + ); + assertThat(deserializedSource.retriever(), instanceOf(KnnRetrieverBuilder.class)); + KnnRetrieverBuilder deserialized = (KnnRetrieverBuilder) source.retriever(); + assertThat(parsed, equalTo(deserialized)); + } + } + } + public void testStoredFieldsUsage() throws IOException { Set storedFieldRestVariations = Set.of( "{\"stored_fields\" : [\"_none_\"]}", diff --git a/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java b/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java index f3dd86e0b1fa2..b0bf7e6636498 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java +++ b/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java @@ -74,7 +74,7 @@ protected KnnRetrieverBuilder createTestInstance() { @Override protected KnnRetrieverBuilder doParseInstance(XContentParser parser) throws IOException { - return KnnRetrieverBuilder.fromXContent( + return (KnnRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( parser, new RetrieverParserContext( new SearchUsage(), diff --git a/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java b/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java index d2a1cac43c154..eacd949077bc4 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java +++ b/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java @@ -98,7 +98,7 @@ protected StandardRetrieverBuilder createTestInstance() { @Override protected StandardRetrieverBuilder doParseInstance(XContentParser parser) throws IOException { - return StandardRetrieverBuilder.fromXContent( + return (StandardRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( parser, new RetrieverParserContext( new SearchUsage(), diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java index eb36c445506a7..134f8af0e083d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java @@ -103,10 +103,7 @@ public int rankWindowSize() { @Override protected void doToXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(RETRIEVER_FIELD.getPreferredName()); - builder.startObject(); - builder.field(retrieverBuilder.getName(), retrieverBuilder); - builder.endObject(); + builder.field(RETRIEVER_FIELD.getPreferredName(), retrieverBuilder); builder.field(FIELD_FIELD.getPreferredName(), field); builder.field(RANK_WINDOW_SIZE_FIELD.getPreferredName(), rankWindowSize); if (seed != null) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java index ab013e0275a69..50d762e7b90aa 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java @@ -179,17 +179,11 @@ public int rankWindowSize() { @Override protected void doToXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(RETRIEVER_FIELD.getPreferredName()); - builder.startObject(); - builder.field(retrieverBuilder.getName(), retrieverBuilder); - builder.endObject(); + builder.field(RETRIEVER_FIELD.getPreferredName(), retrieverBuilder); builder.field(INFERENCE_ID_FIELD.getPreferredName(), inferenceId); builder.field(INFERENCE_TEXT_FIELD.getPreferredName(), inferenceText); builder.field(FIELD_FIELD.getPreferredName(), field); builder.field(RANK_WINDOW_SIZE_FIELD.getPreferredName(), rankWindowSize); - if (minScore != null) { - builder.field(MIN_SCORE_FIELD.getPreferredName(), minScore); - } } @Override diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java index c33f30d461350..c0ef4e45f101f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java @@ -17,8 +17,6 @@ import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.json.JsonXContent; -import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankBuilder; -import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankRetrieverBuilder; import java.io.IOException; import java.util.ArrayList; @@ -48,8 +46,8 @@ protected RandomRankRetrieverBuilder createTestInstance() { } @Override - protected RandomRankRetrieverBuilder doParseInstance(XContentParser parser) { - return RandomRankRetrieverBuilder.PARSER.apply( + protected RandomRankRetrieverBuilder doParseInstance(XContentParser parser) throws IOException { + return (RandomRankRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( parser, new RetrieverParserContext( new SearchUsage(), @@ -77,8 +75,8 @@ protected NamedXContentRegistry xContentRegistry() { entries.add( new NamedXContentRegistry.Entry( RetrieverBuilder.class, - new ParseField(TextSimilarityRankBuilder.NAME), - (p, c) -> TextSimilarityRankRetrieverBuilder.PARSER.apply(p, (RetrieverParserContext) c) + new ParseField(RandomRankBuilder.NAME), + (p, c) -> RandomRankRetrieverBuilder.PARSER.apply(p, (RetrieverParserContext) c) ) ); return new NamedXContentRegistry(entries); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java index 1a72cb0da2899..140b181a42a0a 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.inference.rank.textsimilarity; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.common.Strings; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.MatchNoneQueryBuilder; @@ -25,6 +26,8 @@ import org.elasticsearch.test.AbstractXContentTestCase; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.usage.SearchUsage; +import org.elasticsearch.usage.SearchUsageHolder; +import org.elasticsearch.usage.UsageService; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentParser; @@ -72,8 +75,8 @@ protected TextSimilarityRankRetrieverBuilder createTestInstance() { } @Override - protected TextSimilarityRankRetrieverBuilder doParseInstance(XContentParser parser) { - return TextSimilarityRankRetrieverBuilder.PARSER.apply( + protected TextSimilarityRankRetrieverBuilder doParseInstance(XContentParser parser) throws IOException { + return (TextSimilarityRankRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( parser, new RetrieverParserContext( new SearchUsage(), @@ -208,6 +211,45 @@ public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder } } + public void testTextSimilarityRetrieverParsing() throws IOException { + String restContent = "{" + + " \"retriever\": {" + + " \"text_similarity_reranker\": {" + + " \"retriever\": {" + + " \"test\": {" + + " \"value\": \"my-test-retriever\"" + + " }" + + " }," + + " \"field\": \"my-field\"," + + " \"inference_id\": \"my-inference-id\"," + + " \"inference_text\": \"my-inference-text\"," + + " \"rank_window_size\": 100," + + " \"min_score\": 20.0," + + " \"_name\": \"foo_reranker\"" + + " }" + + " }" + + "}"; + SearchUsageHolder searchUsageHolder = new UsageService().getSearchUsageHolder(); + try (XContentParser jsonParser = createParser(JsonXContent.jsonXContent, restContent)) { + SearchSourceBuilder source = new SearchSourceBuilder().parseXContent(jsonParser, true, searchUsageHolder, nf -> true); + assertThat(source.retriever(), instanceOf(TextSimilarityRankRetrieverBuilder.class)); + TextSimilarityRankRetrieverBuilder parsed = (TextSimilarityRankRetrieverBuilder) source.retriever(); + assertThat(parsed.minScore(), equalTo(20f)); + assertThat(parsed.retrieverName(), equalTo("foo_reranker")); + try (XContentParser parseSerialized = createParser(JsonXContent.jsonXContent, Strings.toString(source))) { + SearchSourceBuilder deserializedSource = new SearchSourceBuilder().parseXContent( + parseSerialized, + true, + searchUsageHolder, + nf -> true + ); + assertThat(deserializedSource.retriever(), instanceOf(TextSimilarityRankRetrieverBuilder.class)); + TextSimilarityRankRetrieverBuilder deserialized = (TextSimilarityRankRetrieverBuilder) source.retriever(); + assertThat(parsed, equalTo(deserialized)); + } + } + } + public void testIsCompound() { RetrieverBuilder compoundInnerRetriever = new TestRetrieverBuilder(ESTestCase.randomAlphaOfLengthBetween(5, 10)) { @Override diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java index 496af99574431..5f19e361d857d 100644 --- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java +++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilder.java @@ -180,10 +180,7 @@ public void doToXContent(XContentBuilder builder, Params params) throws IOExcept builder.startArray(RETRIEVERS_FIELD.getPreferredName()); for (var entry : innerRetrievers) { - builder.startObject(); - builder.field(entry.retriever().getName()); entry.retriever().toXContent(builder, params); - builder.endObject(); } builder.endArray(); } diff --git a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java index e360237371a82..d324effe41c22 100644 --- a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java +++ b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRetrieverBuilderParsingTests.java @@ -8,19 +8,27 @@ package org.elasticsearch.xpack.rank.rrf; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.retriever.RetrieverBuilder; import org.elasticsearch.search.retriever.RetrieverParserContext; import org.elasticsearch.search.retriever.TestRetrieverBuilder; import org.elasticsearch.test.AbstractXContentTestCase; import org.elasticsearch.usage.SearchUsage; +import org.elasticsearch.usage.SearchUsageHolder; +import org.elasticsearch.usage.UsageService; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + public class RRFRetrieverBuilderParsingTests extends AbstractXContentTestCase { /** @@ -53,7 +61,10 @@ protected RRFRetrieverBuilder createTestInstance() { @Override protected RRFRetrieverBuilder doParseInstance(XContentParser parser) throws IOException { - return RRFRetrieverBuilder.PARSER.apply(parser, new RetrieverParserContext(new SearchUsage(), nf -> true)); + return (RRFRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( + parser, + new RetrieverParserContext(new SearchUsage(), nf -> true) + ); } @Override @@ -81,4 +92,48 @@ protected NamedXContentRegistry xContentRegistry() { ); return new NamedXContentRegistry(entries); } + + public void testRRFRetrieverParsing() throws IOException { + String restContent = "{" + + " \"retriever\": {" + + " \"rrf\": {" + + " \"retrievers\": [" + + " {" + + " \"test\": {" + + " \"value\": \"foo\"" + + " }" + + " }," + + " {" + + " \"test\": {" + + " \"value\": \"bar\"" + + " }" + + " }" + + " ]," + + " \"rank_window_size\": 100," + + " \"rank_constant\": 10," + + " \"min_score\": 20.0," + + " \"_name\": \"foo_rrf\"" + + " }" + + " }" + + "}"; + SearchUsageHolder searchUsageHolder = new UsageService().getSearchUsageHolder(); + try (XContentParser jsonParser = createParser(JsonXContent.jsonXContent, restContent)) { + SearchSourceBuilder source = new SearchSourceBuilder().parseXContent(jsonParser, true, searchUsageHolder, nf -> true); + assertThat(source.retriever(), instanceOf(RRFRetrieverBuilder.class)); + RRFRetrieverBuilder parsed = (RRFRetrieverBuilder) source.retriever(); + assertThat(parsed.minScore(), equalTo(20f)); + assertThat(parsed.retrieverName(), equalTo("foo_rrf")); + try (XContentParser parseSerialized = createParser(JsonXContent.jsonXContent, Strings.toString(source))) { + SearchSourceBuilder deserializedSource = new SearchSourceBuilder().parseXContent( + parseSerialized, + true, + searchUsageHolder, + nf -> true + ); + assertThat(deserializedSource.retriever(), instanceOf(RRFRetrieverBuilder.class)); + RRFRetrieverBuilder deserialized = (RRFRetrieverBuilder) source.retriever(); + assertThat(parsed, equalTo(deserialized)); + } + } + } } From 740cb2e0c78baf0a746563864674cbb2f0ff75af Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 8 Oct 2024 06:59:30 +0100 Subject: [PATCH 153/194] Document that `?wait_for_active_shards=0` is permitted (#114091) Today the docs for the `?wait_for_active_shards` parameter say that it must be a positive integer, proscribing `0`, yet `0` is a legitimate value for this parameter. This commit fixes this point and rewords the docs slightly for clarity. --- docs/reference/rest-api/common-parms.asciidoc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index fabd495cdc525..993bb8cb894f9 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -1298,10 +1298,11 @@ tag::wait_for_active_shards[] `wait_for_active_shards`:: + -- -(Optional, string) The number of shard copies that must be active before -proceeding with the operation. Set to `all` or any positive integer up -to the total number of shards in the index (`number_of_replicas+1`). -Default: 1, the primary shard. +(Optional, string) The number of copies of each shard that must be active +before proceeding with the operation. Set to `all` or any non-negative integer +up to the total number of copies of each shard in the index +(`number_of_replicas+1`). Defaults to `1`, meaning to wait just for each +primary shard to be active. See <>. -- From 1c40954037596995911dc42701f9e2f9fb230dd2 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Tue, 8 Oct 2024 17:01:42 +1100 Subject: [PATCH 154/194] Publish APM metrics from the Azure BlobStore (#113913) Closes ES-9550 --- .../AzureBlobStoreRepositoryMetricsTests.java | 468 ++++++++++++++++++ .../azure/AzureBlobStoreRepositoryTests.java | 119 ++++- .../repositories/azure/AzureBlobStore.java | 149 +++--- .../azure/AzureClientProvider.java | 133 ++++- .../repositories/azure/AzureRepository.java | 8 +- .../azure/AzureRepositoryPlugin.java | 10 +- .../azure/AzureStorageService.java | 4 +- .../azure/AbstractAzureServerTestCase.java | 6 +- .../azure/AzureClientProviderTests.java | 6 +- .../azure/AzureRepositorySettingsTests.java | 4 +- .../repositories/s3/S3BlobStore.java | 12 +- .../repositories/RepositoriesMetrics.java | 78 +++ .../org/elasticsearch/rest/RestStatus.java | 12 + 13 files changed, 912 insertions(+), 97 deletions(-) create mode 100644 modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryMetricsTests.java diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryMetricsTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryMetricsTests.java new file mode 100644 index 0000000000000..a9bf0afa37e18 --- /dev/null +++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryMetricsTests.java @@ -0,0 +1,468 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.repositories.azure; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; + +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.blobstore.OperationPurpose; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.repositories.RepositoriesMetrics; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.repositories.blobstore.RequestedRangeNotSatisfiedException; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.telemetry.Measurement; +import org.elasticsearch.telemetry.TestTelemetryPlugin; +import org.junit.After; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.elasticsearch.repositories.azure.AbstractAzureServerTestCase.randomBlobContent; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +@SuppressForbidden(reason = "we use a HttpServer to emulate Azure") +public class AzureBlobStoreRepositoryMetricsTests extends AzureBlobStoreRepositoryTests { + + private static final Predicate GET_BLOB_REQUEST_PREDICATE = request -> GET_BLOB_PATTERN.test( + request.getRequestMethod() + " " + request.getRequestURI() + ); + private static final int MAX_RETRIES = 3; + + private final Queue requestHandlers = new ConcurrentLinkedQueue<>(); + + @Override + protected Map createHttpHandlers() { + Map httpHandlers = super.createHttpHandlers(); + assert httpHandlers.size() == 1 : "This assumes there's a single handler"; + return httpHandlers.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> new ResponseInjectingAzureHttpHandler(requestHandlers, e.getValue()))); + } + + /** + * We want to control the errors in this test + */ + @Override + protected HttpHandler createErroneousHttpHandler(HttpHandler delegate) { + return delegate; + } + + @After + public void checkRequestHandlerQueue() { + if (requestHandlers.isEmpty() == false) { + fail("There were unused request handlers left in the queue, this is probably a broken test"); + } + } + + private static BlobContainer getBlobContainer(String dataNodeName, String repository) { + final var blobStoreRepository = (BlobStoreRepository) internalCluster().getInstance(RepositoriesService.class, dataNodeName) + .repository(repository); + return blobStoreRepository.blobStore().blobContainer(BlobPath.EMPTY.add(randomIdentifier())); + } + + public void testThrottleResponsesAreCountedInMetrics() throws IOException { + final String repository = createRepository(randomRepositoryName()); + final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData); + final BlobContainer blobContainer = getBlobContainer(dataNodeName, repository); + + // Create a blob + final String blobName = "index-" + randomIdentifier(); + final OperationPurpose purpose = randomFrom(OperationPurpose.values()); + blobContainer.writeBlob(purpose, blobName, BytesReference.fromByteBuffer(ByteBuffer.wrap(randomBlobContent())), false); + clearMetrics(dataNodeName); + + // Queue up some throttle responses + final int numThrottles = randomIntBetween(1, MAX_RETRIES); + IntStream.range(0, numThrottles).forEach(i -> requestHandlers.offer(new FixedRequestHandler(RestStatus.TOO_MANY_REQUESTS))); + + // Check that the blob exists + blobContainer.blobExists(purpose, blobName); + + // Correct metrics are recorded + metricsAsserter(dataNodeName, purpose, AzureBlobStore.Operation.GET_BLOB_PROPERTIES, repository).expectMetrics() + .withRequests(numThrottles + 1) + .withThrottles(numThrottles) + .withExceptions(numThrottles) + .forResult(MetricsAsserter.Result.Success); + } + + public void testRangeNotSatisfiedAreCountedInMetrics() throws IOException { + final String repository = createRepository(randomRepositoryName()); + final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData); + final BlobContainer blobContainer = getBlobContainer(dataNodeName, repository); + + // Create a blob + final String blobName = "index-" + randomIdentifier(); + final OperationPurpose purpose = randomFrom(OperationPurpose.values()); + blobContainer.writeBlob(purpose, blobName, BytesReference.fromByteBuffer(ByteBuffer.wrap(randomBlobContent())), false); + clearMetrics(dataNodeName); + + // Queue up a range-not-satisfied error + requestHandlers.offer(new FixedRequestHandler(RestStatus.REQUESTED_RANGE_NOT_SATISFIED, null, GET_BLOB_REQUEST_PREDICATE)); + + // Attempt to read the blob + assertThrows(RequestedRangeNotSatisfiedException.class, () -> blobContainer.readBlob(purpose, blobName)); + + // Correct metrics are recorded + metricsAsserter(dataNodeName, purpose, AzureBlobStore.Operation.GET_BLOB, repository).expectMetrics() + .withRequests(1) + .withThrottles(0) + .withExceptions(1) + .forResult(MetricsAsserter.Result.RangeNotSatisfied); + } + + public void testErrorResponsesAreCountedInMetrics() throws IOException { + final String repository = createRepository(randomRepositoryName()); + final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData); + final BlobContainer blobContainer = getBlobContainer(dataNodeName, repository); + + // Create a blob + final String blobName = "index-" + randomIdentifier(); + final OperationPurpose purpose = randomFrom(OperationPurpose.values()); + blobContainer.writeBlob(purpose, blobName, BytesReference.fromByteBuffer(ByteBuffer.wrap(randomBlobContent())), false); + clearMetrics(dataNodeName); + + // Queue some retry-able error responses + final int numErrors = randomIntBetween(1, MAX_RETRIES); + final AtomicInteger throttles = new AtomicInteger(); + IntStream.range(0, numErrors).forEach(i -> { + RestStatus status = randomFrom(RestStatus.INTERNAL_SERVER_ERROR, RestStatus.TOO_MANY_REQUESTS, RestStatus.SERVICE_UNAVAILABLE); + if (status == RestStatus.TOO_MANY_REQUESTS) { + throttles.incrementAndGet(); + } + requestHandlers.offer(new FixedRequestHandler(status)); + }); + + // Check that the blob exists + blobContainer.blobExists(purpose, blobName); + + // Correct metrics are recorded + metricsAsserter(dataNodeName, purpose, AzureBlobStore.Operation.GET_BLOB_PROPERTIES, repository).expectMetrics() + .withRequests(numErrors + 1) + .withThrottles(throttles.get()) + .withExceptions(numErrors) + .forResult(MetricsAsserter.Result.Success); + } + + public void testRequestFailuresAreCountedInMetrics() { + final String repository = createRepository(randomRepositoryName()); + final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData); + final BlobContainer blobContainer = getBlobContainer(dataNodeName, repository); + clearMetrics(dataNodeName); + + // Repeatedly cause a connection error to exhaust retries + IntStream.range(0, MAX_RETRIES + 1).forEach(i -> requestHandlers.offer((exchange, delegate) -> exchange.close())); + + // Hit the API + OperationPurpose purpose = randomFrom(OperationPurpose.values()); + assertThrows(IOException.class, () -> blobContainer.listBlobs(purpose)); + + // Correct metrics are recorded + metricsAsserter(dataNodeName, purpose, AzureBlobStore.Operation.LIST_BLOBS, repository).expectMetrics() + .withRequests(4) + .withThrottles(0) + .withExceptions(4) + .forResult(MetricsAsserter.Result.Exception); + } + + public void testRequestTimeIsAccurate() throws IOException { + final String repository = createRepository(randomRepositoryName()); + final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData); + final BlobContainer blobContainer = getBlobContainer(dataNodeName, repository); + clearMetrics(dataNodeName); + + AtomicLong totalDelayMillis = new AtomicLong(0); + // Add some artificial delays + IntStream.range(0, randomIntBetween(1, MAX_RETRIES)).forEach(i -> { + long thisDelay = randomLongBetween(10, 100); + totalDelayMillis.addAndGet(thisDelay); + requestHandlers.offer((exchange, delegate) -> { + safeSleep(thisDelay); + // return a retry-able error + exchange.sendResponseHeaders(RestStatus.INTERNAL_SERVER_ERROR.getStatus(), -1); + }); + }); + + // Hit the API + final long startTimeMillis = System.currentTimeMillis(); + blobContainer.listBlobs(randomFrom(OperationPurpose.values())); + final long elapsedTimeMillis = System.currentTimeMillis() - startTimeMillis; + + List longHistogramMeasurement = getTelemetryPlugin(dataNodeName).getLongHistogramMeasurement( + RepositoriesMetrics.HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM + ); + long recordedRequestTime = longHistogramMeasurement.get(0).getLong(); + // Request time should be >= the delays we simulated + assertThat(recordedRequestTime, greaterThanOrEqualTo(totalDelayMillis.get())); + // And <= the elapsed time for the request + assertThat(recordedRequestTime, lessThanOrEqualTo(elapsedTimeMillis)); + } + + private void clearMetrics(String discoveryNode) { + internalCluster().getInstance(PluginsService.class, discoveryNode) + .filterPlugins(TestTelemetryPlugin.class) + .forEach(TestTelemetryPlugin::resetMeter); + } + + private MetricsAsserter metricsAsserter( + String dataNodeName, + OperationPurpose operationPurpose, + AzureBlobStore.Operation operation, + String repository + ) { + return new MetricsAsserter(dataNodeName, operationPurpose, operation, repository); + } + + private class MetricsAsserter { + private final String dataNodeName; + private final OperationPurpose purpose; + private final AzureBlobStore.Operation operation; + private final String repository; + + enum Result { + Success, + Failure, + RangeNotSatisfied, + Exception + } + + enum MetricType { + LongHistogram { + @Override + List getMeasurements(TestTelemetryPlugin testTelemetryPlugin, String name) { + return testTelemetryPlugin.getLongHistogramMeasurement(name); + } + }, + LongCounter { + @Override + List getMeasurements(TestTelemetryPlugin testTelemetryPlugin, String name) { + return testTelemetryPlugin.getLongCounterMeasurement(name); + } + }; + + abstract List getMeasurements(TestTelemetryPlugin testTelemetryPlugin, String name); + } + + private MetricsAsserter(String dataNodeName, OperationPurpose purpose, AzureBlobStore.Operation operation, String repository) { + this.dataNodeName = dataNodeName; + this.purpose = purpose; + this.operation = operation; + this.repository = repository; + } + + private class Expectations { + private int expectedRequests; + private int expectedThrottles; + private int expectedExceptions; + + public Expectations withRequests(int expectedRequests) { + this.expectedRequests = expectedRequests; + return this; + } + + public Expectations withThrottles(int expectedThrottles) { + this.expectedThrottles = expectedThrottles; + return this; + } + + public Expectations withExceptions(int expectedExceptions) { + this.expectedExceptions = expectedExceptions; + return this; + } + + public void forResult(Result result) { + assertMetricsRecorded(expectedRequests, expectedThrottles, expectedExceptions, result); + } + } + + Expectations expectMetrics() { + return new Expectations(); + } + + private void assertMetricsRecorded(int expectedRequests, int expectedThrottles, int expectedExceptions, Result result) { + assertIntMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_OPERATIONS_TOTAL, 1); + assertIntMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_REQUESTS_TOTAL, expectedRequests); + + if (expectedThrottles > 0) { + assertIntMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_THROTTLES_TOTAL, expectedThrottles); + assertIntMetricRecorded(MetricType.LongHistogram, RepositoriesMetrics.METRIC_THROTTLES_HISTOGRAM, expectedThrottles); + } else { + assertNoMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_THROTTLES_TOTAL); + assertNoMetricRecorded(MetricType.LongHistogram, RepositoriesMetrics.METRIC_THROTTLES_HISTOGRAM); + } + + if (expectedExceptions > 0) { + assertIntMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_EXCEPTIONS_TOTAL, expectedExceptions); + assertIntMetricRecorded(MetricType.LongHistogram, RepositoriesMetrics.METRIC_EXCEPTIONS_HISTOGRAM, expectedExceptions); + } else { + assertNoMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_EXCEPTIONS_TOTAL); + assertNoMetricRecorded(MetricType.LongHistogram, RepositoriesMetrics.METRIC_EXCEPTIONS_HISTOGRAM); + } + + if (result == Result.RangeNotSatisfied || result == Result.Failure || result == Result.Exception) { + assertIntMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_UNSUCCESSFUL_OPERATIONS_TOTAL, 1); + } else { + assertNoMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_UNSUCCESSFUL_OPERATIONS_TOTAL); + } + + if (result == Result.RangeNotSatisfied) { + assertIntMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_EXCEPTIONS_REQUEST_RANGE_NOT_SATISFIED_TOTAL, 1); + } else { + assertNoMetricRecorded(MetricType.LongCounter, RepositoriesMetrics.METRIC_EXCEPTIONS_REQUEST_RANGE_NOT_SATISFIED_TOTAL); + } + + assertMatchingMetricRecorded( + MetricType.LongHistogram, + RepositoriesMetrics.HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM, + m -> assertThat("No request time metric found", m.getLong(), greaterThanOrEqualTo(0L)) + ); + } + + private void assertIntMetricRecorded(MetricType metricType, String metricName, int expectedValue) { + assertMatchingMetricRecorded( + metricType, + metricName, + measurement -> assertEquals("Unexpected value for " + metricType + " " + metricName, expectedValue, measurement.getLong()) + ); + } + + private void assertNoMetricRecorded(MetricType metricType, String metricName) { + assertThat( + "Expected no values for " + metricType + " " + metricName, + metricType.getMeasurements(getTelemetryPlugin(dataNodeName), metricName), + hasSize(0) + ); + } + + private void assertMatchingMetricRecorded(MetricType metricType, String metricName, Consumer assertion) { + List measurements = metricType.getMeasurements(getTelemetryPlugin(dataNodeName), metricName); + Measurement measurement = measurements.stream() + .filter( + m -> m.attributes().get("operation").equals(operation.getKey()) + && m.attributes().get("purpose").equals(purpose.getKey()) + && m.attributes().get("repo_name").equals(repository) + && m.attributes().get("repo_type").equals("azure") + ) + .findFirst() + .orElseThrow( + () -> new IllegalStateException( + "No metric found with name=" + + metricName + + " and operation=" + + operation.getKey() + + " and purpose=" + + purpose.getKey() + + " and repo_name=" + + repository + + " in " + + measurements + ) + ); + + assertion.accept(measurement); + } + } + + @SuppressForbidden(reason = "we use a HttpServer to emulate Azure") + private static class ResponseInjectingAzureHttpHandler implements DelegatingHttpHandler { + + private final HttpHandler delegate; + private final Queue requestHandlerQueue; + + ResponseInjectingAzureHttpHandler(Queue requestHandlerQueue, HttpHandler delegate) { + this.delegate = delegate; + this.requestHandlerQueue = requestHandlerQueue; + } + + @Override + public void handle(HttpExchange exchange) throws IOException { + RequestHandler nextHandler = requestHandlerQueue.peek(); + if (nextHandler != null && nextHandler.matchesRequest(exchange)) { + requestHandlerQueue.poll().writeResponse(exchange, delegate); + } else { + delegate.handle(exchange); + } + } + + @Override + public HttpHandler getDelegate() { + return delegate; + } + } + + @SuppressForbidden(reason = "we use a HttpServer to emulate Azure") + @FunctionalInterface + private interface RequestHandler { + void writeResponse(HttpExchange exchange, HttpHandler delegate) throws IOException; + + default boolean matchesRequest(HttpExchange exchange) { + return true; + } + } + + @SuppressForbidden(reason = "we use a HttpServer to emulate Azure") + private static class FixedRequestHandler implements RequestHandler { + + private final RestStatus status; + private final String responseBody; + private final Predicate requestMatcher; + + FixedRequestHandler(RestStatus status) { + this(status, null, req -> true); + } + + /** + * Create a handler that only gets executed for requests that match the supplied predicate. Note + * that because the errors are stored in a queue this will prevent any subsequently queued errors from + * being returned until after it returns. + */ + FixedRequestHandler(RestStatus status, String responseBody, Predicate requestMatcher) { + this.status = status; + this.responseBody = responseBody; + this.requestMatcher = requestMatcher; + } + + @Override + public boolean matchesRequest(HttpExchange exchange) { + return requestMatcher.test(exchange); + } + + @Override + public void writeResponse(HttpExchange exchange, HttpHandler delegateHandler) throws IOException { + if (responseBody != null) { + byte[] responseBytes = responseBody.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(status.getStatus(), responseBytes.length); + exchange.getResponseBody().write(responseBytes); + } else { + exchange.sendResponseHeaders(status.getStatus(), -1); + } + } + } +} diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java index 1b7628cc0ad8e..473d91da6e34c 100644 --- a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java +++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java @@ -16,11 +16,13 @@ import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; +import org.elasticsearch.action.support.broadcast.BroadcastResponse; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.MockSecureSettings; @@ -30,8 +32,15 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.repositories.RepositoryMissingException; +import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.blobstore.ESMockAPIBasedRepositoryIntegTestCase; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.telemetry.Measurement; +import org.elasticsearch.telemetry.TestTelemetryPlugin; +import org.elasticsearch.test.BackgroundIndexer; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -41,22 +50,33 @@ import java.util.Base64; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.LongAdder; import java.util.function.Predicate; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_OPERATIONS_TOTAL; import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.is; @SuppressForbidden(reason = "this test uses a HttpServer to emulate an Azure endpoint") public class AzureBlobStoreRepositoryTests extends ESMockAPIBasedRepositoryIntegTestCase { - private static final String DEFAULT_ACCOUNT_NAME = "account"; + protected static final String DEFAULT_ACCOUNT_NAME = "account"; + protected static final Predicate LIST_PATTERN = Pattern.compile("GET /[a-zA-Z0-9]+/[a-zA-Z0-9]+\\?.+").asMatchPredicate(); + protected static final Predicate GET_BLOB_PATTERN = Pattern.compile("GET /[a-zA-Z0-9]+/[a-zA-Z0-9]+/.+").asMatchPredicate(); @Override protected String repositoryType() { @@ -78,7 +98,7 @@ protected Settings repositorySettings(String repoName) { @Override protected Collection> nodePlugins() { - return Collections.singletonList(TestAzureRepositoryPlugin.class); + return List.of(TestAzureRepositoryPlugin.class, TestTelemetryPlugin.class); } @Override @@ -91,7 +111,7 @@ protected Map createHttpHandlers() { @Override protected HttpHandler createErroneousHttpHandler(final HttpHandler delegate) { - return new AzureErroneousHttpHandler(delegate, AzureStorageSettings.DEFAULT_MAX_RETRIES); + return new AzureHTTPStatsCollectorHandler(new AzureErroneousHttpHandler(delegate, AzureStorageSettings.DEFAULT_MAX_RETRIES)); } @Override @@ -119,6 +139,13 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .build(); } + protected TestTelemetryPlugin getTelemetryPlugin(String dataNodeName) { + return internalCluster().getInstance(PluginsService.class, dataNodeName) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + } + /** * AzureRepositoryPlugin that allows to set low values for the Azure's client retry policy * and for BlobRequestOptions#getSingleBlobPutThresholdInBytes(). @@ -195,9 +222,6 @@ protected String requestUniqueId(final HttpExchange exchange) { */ @SuppressForbidden(reason = "this test uses a HttpServer to emulate an Azure endpoint") private static class AzureHTTPStatsCollectorHandler extends HttpStatsCollectorHandler { - private static final Predicate LIST_PATTERN = Pattern.compile("GET /[a-zA-Z0-9]+/[a-zA-Z0-9]+\\?.+").asMatchPredicate(); - private static final Predicate GET_BLOB_PATTERN = Pattern.compile("GET /[a-zA-Z0-9]+/[a-zA-Z0-9]+/.+").asMatchPredicate(); - private final Set seenRequestIds = ConcurrentCollections.newConcurrentSet(); private AzureHTTPStatsCollectorHandler(HttpHandler delegate) { @@ -303,4 +327,87 @@ public void testReadByteByByte() throws Exception { container.delete(randomPurpose()); } } + + public void testMetrics() throws Exception { + // Reset all the metrics so there's none lingering from previous tests + internalCluster().getInstances(PluginsService.class) + .forEach(ps -> ps.filterPlugins(TestTelemetryPlugin.class).forEach(TestTelemetryPlugin::resetMeter)); + + // Create the repository and perform some activities + final String repository = createRepository(randomRepositoryName(), false); + final String index = "index-no-merges"; + createIndex(index, 1, 0); + + final long nbDocs = randomLongBetween(10_000L, 20_000L); + try (BackgroundIndexer indexer = new BackgroundIndexer(index, client(), (int) nbDocs)) { + waitForDocs(nbDocs, indexer); + } + flushAndRefresh(index); + BroadcastResponse forceMerge = client().admin().indices().prepareForceMerge(index).setFlush(true).setMaxNumSegments(1).get(); + assertThat(forceMerge.getSuccessfulShards(), equalTo(1)); + assertHitCount(prepareSearch(index).setSize(0).setTrackTotalHits(true), nbDocs); + + final String snapshot = "snapshot"; + assertSuccessfulSnapshot( + clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repository, snapshot).setWaitForCompletion(true).setIndices(index) + ); + assertAcked(client().admin().indices().prepareDelete(index)); + assertSuccessfulRestore( + clusterAdmin().prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, repository, snapshot).setWaitForCompletion(true) + ); + ensureGreen(index); + assertHitCount(prepareSearch(index).setSize(0).setTrackTotalHits(true), nbDocs); + assertAcked(clusterAdmin().prepareDeleteSnapshot(TEST_REQUEST_TIMEOUT, repository, snapshot).get()); + + final Map aggregatedMetrics = new HashMap<>(); + // Compare collected stats and metrics for each node and they should be the same + for (var nodeName : internalCluster().getNodeNames()) { + final BlobStoreRepository blobStoreRepository; + try { + blobStoreRepository = (BlobStoreRepository) internalCluster().getInstance(RepositoriesService.class, nodeName) + .repository(repository); + } catch (RepositoryMissingException e) { + continue; + } + + final AzureBlobStore blobStore = (AzureBlobStore) blobStoreRepository.blobStore(); + final Map statsCollectors = blobStore.getMetricsRecorder().opsCounters; + + final List metrics = Measurement.combine( + getTelemetryPlugin(nodeName).getLongCounterMeasurement(METRIC_OPERATIONS_TOTAL) + ); + + assertThat( + statsCollectors.keySet().stream().map(AzureBlobStore.StatsKey::operation).collect(Collectors.toSet()), + equalTo( + metrics.stream() + .map(m -> AzureBlobStore.Operation.fromKey((String) m.attributes().get("operation"))) + .collect(Collectors.toSet()) + ) + ); + metrics.forEach(metric -> { + assertThat( + metric.attributes(), + allOf(hasEntry("repo_type", AzureRepository.TYPE), hasKey("repo_name"), hasKey("operation"), hasKey("purpose")) + ); + final AzureBlobStore.Operation operation = AzureBlobStore.Operation.fromKey((String) metric.attributes().get("operation")); + final AzureBlobStore.StatsKey statsKey = new AzureBlobStore.StatsKey( + operation, + OperationPurpose.parse((String) metric.attributes().get("purpose")) + ); + assertThat(nodeName + "/" + statsKey + " exists", statsCollectors, hasKey(statsKey)); + assertThat(nodeName + "/" + statsKey + " has correct sum", metric.getLong(), equalTo(statsCollectors.get(statsKey).sum())); + aggregatedMetrics.compute(statsKey.operation(), (k, v) -> v == null ? metric.getLong() : v + metric.getLong()); + }); + } + + // Metrics number should be consistent with server side request count as well. + assertThat(aggregatedMetrics, equalTo(getServerMetrics())); + } + + private Map getServerMetrics() { + return getMockRequestCounts().entrySet() + .stream() + .collect(Collectors.toMap(e -> AzureBlobStore.Operation.fromKey(e.getKey()), Map.Entry::getValue)); + } } diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java index 5466989082129..d520d30f2bac6 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java @@ -60,6 +60,7 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; +import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.azure.AzureRepository.Repository; import org.elasticsearch.repositories.blobstore.ChunkedBlobOutputStream; import org.elasticsearch.rest.RestStatus; @@ -86,11 +87,11 @@ import java.util.Spliterator; import java.util.Spliterators; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.LongAdder; import java.util.function.BiPredicate; -import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -102,59 +103,54 @@ public class AzureBlobStore implements BlobStore { private static final int DEFAULT_UPLOAD_BUFFERS_SIZE = (int) new ByteSizeValue(64, ByteSizeUnit.KB).getBytes(); private final AzureStorageService service; - private final BigArrays bigArrays; + private final RepositoryMetadata repositoryMetadata; private final String clientName; private final String container; private final LocationMode locationMode; private final ByteSizeValue maxSinglePartUploadSize; - private final StatsCollectors statsCollectors = new StatsCollectors(); - private final AzureClientProvider.SuccessfulRequestHandler statsConsumer; + private final RequestMetricsRecorder requestMetricsRecorder; + private final AzureClientProvider.RequestMetricsHandler requestMetricsHandler; - public AzureBlobStore(RepositoryMetadata metadata, AzureStorageService service, BigArrays bigArrays) { + public AzureBlobStore( + RepositoryMetadata metadata, + AzureStorageService service, + BigArrays bigArrays, + RepositoriesMetrics repositoriesMetrics + ) { this.container = Repository.CONTAINER_SETTING.get(metadata.settings()); this.clientName = Repository.CLIENT_NAME.get(metadata.settings()); this.service = service; this.bigArrays = bigArrays; + this.requestMetricsRecorder = new RequestMetricsRecorder(repositoriesMetrics); + this.repositoryMetadata = metadata; // locationMode is set per repository, not per client this.locationMode = Repository.LOCATION_MODE_SETTING.get(metadata.settings()); this.maxSinglePartUploadSize = Repository.MAX_SINGLE_PART_UPLOAD_SIZE_SETTING.get(metadata.settings()); - List requestStatsCollectors = List.of( - RequestStatsCollector.create( - (httpMethod, url) -> httpMethod == HttpMethod.HEAD, - purpose -> statsCollectors.onSuccessfulRequest(Operation.GET_BLOB_PROPERTIES, purpose) - ), - RequestStatsCollector.create( + List requestMatchers = List.of( + new RequestMatcher((httpMethod, url) -> httpMethod == HttpMethod.HEAD, Operation.GET_BLOB_PROPERTIES), + new RequestMatcher( (httpMethod, url) -> httpMethod == HttpMethod.GET && isListRequest(httpMethod, url) == false, - purpose -> statsCollectors.onSuccessfulRequest(Operation.GET_BLOB, purpose) - ), - RequestStatsCollector.create( - AzureBlobStore::isListRequest, - purpose -> statsCollectors.onSuccessfulRequest(Operation.LIST_BLOBS, purpose) - ), - RequestStatsCollector.create( - AzureBlobStore::isPutBlockRequest, - purpose -> statsCollectors.onSuccessfulRequest(Operation.PUT_BLOCK, purpose) + Operation.GET_BLOB ), - RequestStatsCollector.create( - AzureBlobStore::isPutBlockListRequest, - purpose -> statsCollectors.onSuccessfulRequest(Operation.PUT_BLOCK_LIST, purpose) - ), - RequestStatsCollector.create( + new RequestMatcher(AzureBlobStore::isListRequest, Operation.LIST_BLOBS), + new RequestMatcher(AzureBlobStore::isPutBlockRequest, Operation.PUT_BLOCK), + new RequestMatcher(AzureBlobStore::isPutBlockListRequest, Operation.PUT_BLOCK_LIST), + new RequestMatcher( // https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob#uri-parameters // The only URI parameter allowed for put-blob operation is "timeout", but if a sas token is used, // it's possible that the URI parameters contain additional parameters unrelated to the upload type. (httpMethod, url) -> httpMethod == HttpMethod.PUT && isPutBlockRequest(httpMethod, url) == false && isPutBlockListRequest(httpMethod, url) == false, - purpose -> statsCollectors.onSuccessfulRequest(Operation.PUT_BLOB, purpose) + Operation.PUT_BLOB ) ); - this.statsConsumer = (purpose, httpMethod, url) -> { + this.requestMetricsHandler = (purpose, method, url, metrics) -> { try { URI uri = url.toURI(); String path = uri.getPath() == null ? "" : uri.getPath(); @@ -167,9 +163,9 @@ && isPutBlockListRequest(httpMethod, url) == false, return; } - for (RequestStatsCollector requestStatsCollector : requestStatsCollectors) { - if (requestStatsCollector.shouldConsumeRequestInfo(httpMethod, url)) { - requestStatsCollector.consumeHttpRequestInfo(purpose); + for (RequestMatcher requestMatcher : requestMatchers) { + if (requestMatcher.matches(method, url)) { + requestMetricsRecorder.onRequestComplete(requestMatcher.operation, purpose, metrics); return; } } @@ -665,12 +661,12 @@ private BlobServiceAsyncClient asyncClient(OperationPurpose purpose) { } private AzureBlobServiceClient getAzureBlobServiceClientClient(OperationPurpose purpose) { - return service.client(clientName, locationMode, purpose, statsConsumer); + return service.client(clientName, locationMode, purpose, requestMetricsHandler); } @Override public Map stats() { - return statsCollectors.statsMap(service.isStateless()); + return requestMetricsRecorder.statsMap(service.isStateless()); } // visible for testing @@ -691,26 +687,43 @@ public String getKey() { Operation(String key) { this.key = key; } + + public static Operation fromKey(String key) { + for (Operation operation : Operation.values()) { + if (operation.key.equals(key)) { + return operation; + } + } + throw new IllegalArgumentException("No matching key: " + key); + } } - private record StatsKey(Operation operation, OperationPurpose purpose) { + // visible for testing + record StatsKey(Operation operation, OperationPurpose purpose) { @Override public String toString() { return purpose.getKey() + "_" + operation.getKey(); } } - private static class StatsCollectors { - final Map collectors = new ConcurrentHashMap<>(); + // visible for testing + class RequestMetricsRecorder { + private final RepositoriesMetrics repositoriesMetrics; + final Map opsCounters = new ConcurrentHashMap<>(); + final Map> opsAttributes = new ConcurrentHashMap<>(); + + RequestMetricsRecorder(RepositoriesMetrics repositoriesMetrics) { + this.repositoriesMetrics = repositoriesMetrics; + } Map statsMap(boolean stateless) { if (stateless) { - return collectors.entrySet() + return opsCounters.entrySet() .stream() .collect(Collectors.toUnmodifiableMap(e -> e.getKey().toString(), e -> e.getValue().sum())); } else { Map normalisedStats = Arrays.stream(Operation.values()).collect(Collectors.toMap(Operation::getKey, o -> 0L)); - collectors.forEach( + opsCounters.forEach( (key, value) -> normalisedStats.compute( key.operation.getKey(), (k, current) -> Objects.requireNonNull(current) + value.sum() @@ -720,11 +733,50 @@ Map statsMap(boolean stateless) { } } - public void onSuccessfulRequest(Operation operation, OperationPurpose purpose) { - collectors.computeIfAbsent(new StatsKey(operation, purpose), k -> new LongAdder()).increment(); + public void onRequestComplete(Operation operation, OperationPurpose purpose, AzureClientProvider.RequestMetrics requestMetrics) { + final StatsKey statsKey = new StatsKey(operation, purpose); + final LongAdder counter = opsCounters.computeIfAbsent(statsKey, k -> new LongAdder()); + final Map attributes = opsAttributes.computeIfAbsent( + statsKey, + k -> RepositoriesMetrics.createAttributesMap(repositoryMetadata, purpose, operation.getKey()) + ); + + counter.add(1); + + // range not satisfied is not retried, so we count them by checking the final response + if (requestMetrics.getStatusCode() == RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus()) { + repositoriesMetrics.requestRangeNotSatisfiedExceptionCounter().incrementBy(1, attributes); + } + + repositoriesMetrics.operationCounter().incrementBy(1, attributes); + if (RestStatus.isSuccessful(requestMetrics.getStatusCode()) == false) { + repositoriesMetrics.unsuccessfulOperationCounter().incrementBy(1, attributes); + } + + repositoriesMetrics.requestCounter().incrementBy(requestMetrics.getRequestCount(), attributes); + if (requestMetrics.getErrorCount() > 0) { + repositoriesMetrics.exceptionCounter().incrementBy(requestMetrics.getErrorCount(), attributes); + repositoriesMetrics.exceptionHistogram().record(requestMetrics.getErrorCount(), attributes); + } + + if (requestMetrics.getThrottleCount() > 0) { + repositoriesMetrics.throttleCounter().incrementBy(requestMetrics.getThrottleCount(), attributes); + repositoriesMetrics.throttleHistogram().record(requestMetrics.getThrottleCount(), attributes); + } + + // We use nanosecond precision, so a zero value indicates that no requests were executed + if (requestMetrics.getTotalRequestTimeNanos() > 0) { + repositoriesMetrics.httpRequestTimeInMillisHistogram() + .record(TimeUnit.NANOSECONDS.toMillis(requestMetrics.getTotalRequestTimeNanos()), attributes); + } } } + // visible for testing + RequestMetricsRecorder getMetricsRecorder() { + return requestMetricsRecorder; + } + private static class AzureInputStream extends InputStream { private final CancellableRateLimitedFluxIterator cancellableRateLimitedFluxIterator; private ByteBuf byteBuf; @@ -846,26 +898,11 @@ private ByteBuf getNextByteBuf() throws IOException { } } - private static class RequestStatsCollector { - private final BiPredicate filter; - private final Consumer onHttpRequest; - - private RequestStatsCollector(BiPredicate filter, Consumer onHttpRequest) { - this.filter = filter; - this.onHttpRequest = onHttpRequest; - } - - static RequestStatsCollector create(BiPredicate filter, Consumer consumer) { - return new RequestStatsCollector(filter, consumer); - } + private record RequestMatcher(BiPredicate filter, Operation operation) { - private boolean shouldConsumeRequestInfo(HttpMethod httpMethod, URL url) { + private boolean matches(HttpMethod httpMethod, URL url) { return filter.test(httpMethod, url); } - - private void consumeHttpRequestInfo(OperationPurpose operationPurpose) { - onHttpRequest.accept(operationPurpose); - } } OptionalBytesReference getRegister(OperationPurpose purpose, String blobPath, String containerPath, String blobKey) { diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java index ae497ff159576..654742c980268 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java @@ -24,6 +24,7 @@ import com.azure.core.http.HttpMethod; import com.azure.core.http.HttpPipelineCallContext; import com.azure.core.http.HttpPipelineNextPolicy; +import com.azure.core.http.HttpPipelinePosition; import com.azure.core.http.HttpRequest; import com.azure.core.http.HttpResponse; import com.azure.core.http.ProxyOptions; @@ -44,11 +45,13 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.repositories.azure.executors.PrivilegedExecutor; import org.elasticsearch.repositories.azure.executors.ReactorScheduledExecutorService; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.netty4.NettyAllocator; import java.net.URL; import java.time.Duration; +import java.util.Optional; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadFactory; @@ -57,6 +60,8 @@ import static org.elasticsearch.repositories.azure.AzureRepositoryPlugin.REPOSITORY_THREAD_POOL_NAME; class AzureClientProvider extends AbstractLifecycleComponent { + private static final Logger logger = LogManager.getLogger(AzureClientProvider.class); + private static final TimeValue DEFAULT_CONNECTION_TIMEOUT = TimeValue.timeValueSeconds(30); private static final TimeValue DEFAULT_MAX_CONNECTION_IDLE_TIME = TimeValue.timeValueSeconds(60); private static final int DEFAULT_MAX_CONNECTIONS = 50; @@ -160,7 +165,7 @@ AzureBlobServiceClient createClient( LocationMode locationMode, RequestRetryOptions retryOptions, ProxyOptions proxyOptions, - SuccessfulRequestHandler successfulRequestHandler, + RequestMetricsHandler requestMetricsHandler, OperationPurpose purpose ) { if (closed) { @@ -189,8 +194,9 @@ AzureBlobServiceClient createClient( builder.credential(credentialBuilder.build()); } - if (successfulRequestHandler != null) { - builder.addPolicy(new SuccessfulRequestTracker(purpose, successfulRequestHandler)); + if (requestMetricsHandler != null) { + builder.addPolicy(new RequestMetricsTracker(purpose, requestMetricsHandler)); + builder.addPolicy(RetryMetricsTracker.INSTANCE); } if (locationMode.isSecondary()) { @@ -259,38 +265,135 @@ protected void doStop() { @Override protected void doClose() {} - private static final class SuccessfulRequestTracker implements HttpPipelinePolicy { - private static final Logger logger = LogManager.getLogger(SuccessfulRequestTracker.class); + static class RequestMetrics { + private volatile long totalRequestTimeNanos = 0; + private volatile int requestCount; + private volatile int errorCount; + private volatile int throttleCount; + private volatile int statusCode; + + int getRequestCount() { + return requestCount; + } + + int getErrorCount() { + return errorCount; + } + + int getStatusCode() { + return statusCode; + } + + int getThrottleCount() { + return throttleCount; + } + + /** + * Total time spent executing requests to complete operation in nanoseconds + */ + long getTotalRequestTimeNanos() { + return totalRequestTimeNanos; + } + + @Override + public String toString() { + return "RequestMetrics{" + + "totalRequestTimeNanos=" + + totalRequestTimeNanos + + ", requestCount=" + + requestCount + + ", errorCount=" + + errorCount + + ", throttleCount=" + + throttleCount + + ", statusCode=" + + statusCode + + '}'; + } + } + + private enum RetryMetricsTracker implements HttpPipelinePolicy { + INSTANCE; + + @Override + public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { + Optional metricsData = context.getData(RequestMetricsTracker.ES_REQUEST_METRICS_CONTEXT_KEY); + if (metricsData.isPresent() == false) { + assert false : "No metrics object associated with request " + context.getHttpRequest(); + return next.process(); + } + RequestMetrics metrics = (RequestMetrics) metricsData.get(); + metrics.requestCount++; + long requestStartTimeNanos = System.nanoTime(); + return next.process().doOnError(throwable -> { + metrics.totalRequestTimeNanos += System.nanoTime() - requestStartTimeNanos; + logger.debug("Detected error in RetryMetricsTracker", throwable); + metrics.errorCount++; + }).doOnSuccess(response -> { + metrics.totalRequestTimeNanos += System.nanoTime() - requestStartTimeNanos; + if (RestStatus.isSuccessful(response.getStatusCode()) == false) { + metrics.errorCount++; + // Azure always throttles with a 429 response, see + // https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/request-limits-and-throttling#error-code + if (response.getStatusCode() == RestStatus.TOO_MANY_REQUESTS.getStatus()) { + metrics.throttleCount++; + } + } + }); + } + + @Override + public HttpPipelinePosition getPipelinePosition() { + return HttpPipelinePosition.PER_RETRY; + } + } + + private static final class RequestMetricsTracker implements HttpPipelinePolicy { + private static final String ES_REQUEST_METRICS_CONTEXT_KEY = "_es_azure_repo_request_stats"; + private static final Logger logger = LogManager.getLogger(RequestMetricsTracker.class); private final OperationPurpose purpose; - private final SuccessfulRequestHandler onSuccessfulRequest; + private final RequestMetricsHandler requestMetricsHandler; - private SuccessfulRequestTracker(OperationPurpose purpose, SuccessfulRequestHandler onSuccessfulRequest) { + private RequestMetricsTracker(OperationPurpose purpose, RequestMetricsHandler requestMetricsHandler) { this.purpose = purpose; - this.onSuccessfulRequest = onSuccessfulRequest; + this.requestMetricsHandler = requestMetricsHandler; } @Override public Mono process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) { - return next.process().doOnSuccess(httpResponse -> trackSuccessfulRequest(context.getHttpRequest(), httpResponse)); + final RequestMetrics requestMetrics = new RequestMetrics(); + context.setData(ES_REQUEST_METRICS_CONTEXT_KEY, requestMetrics); + return next.process().doOnSuccess((httpResponse) -> { + requestMetrics.statusCode = httpResponse.getStatusCode(); + trackCompletedRequest(context.getHttpRequest(), requestMetrics); + }).doOnError(throwable -> { + logger.debug("Detected error in RequestMetricsTracker", throwable); + trackCompletedRequest(context.getHttpRequest(), requestMetrics); + }); } - private void trackSuccessfulRequest(HttpRequest httpRequest, HttpResponse httpResponse) { + private void trackCompletedRequest(HttpRequest httpRequest, RequestMetrics requestMetrics) { HttpMethod method = httpRequest.getHttpMethod(); - if (httpResponse != null && method != null && httpResponse.getStatusCode() > 199 && httpResponse.getStatusCode() <= 299) { + if (method != null) { try { - onSuccessfulRequest.onSuccessfulRequest(purpose, method, httpRequest.getUrl()); + requestMetricsHandler.requestCompleted(purpose, method, httpRequest.getUrl(), requestMetrics); } catch (Exception e) { logger.warn("Unable to notify a successful request", e); } } } + + @Override + public HttpPipelinePosition getPipelinePosition() { + return HttpPipelinePosition.PER_CALL; + } } /** - * The {@link SuccessfulRequestTracker} calls this when a request completes successfully + * The {@link RequestMetricsTracker} calls this when a request completes */ - interface SuccessfulRequestHandler { + interface RequestMetricsHandler { - void onSuccessfulRequest(OperationPurpose purpose, HttpMethod method, URL url); + void requestCompleted(OperationPurpose purpose, HttpMethod method, URL url, RequestMetrics metrics); } } diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index aec148adf9aa8..80e662343baee 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.indices.recovery.RecoverySettings; +import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -91,6 +92,7 @@ public static final class Repository { private final ByteSizeValue chunkSize; private final AzureStorageService storageService; private final boolean readonly; + private final RepositoriesMetrics repositoriesMetrics; public AzureRepository( final RepositoryMetadata metadata, @@ -98,7 +100,8 @@ public AzureRepository( final AzureStorageService storageService, final ClusterService clusterService, final BigArrays bigArrays, - final RecoverySettings recoverySettings + final RecoverySettings recoverySettings, + final RepositoriesMetrics repositoriesMetrics ) { super( metadata, @@ -111,6 +114,7 @@ public AzureRepository( ); this.chunkSize = Repository.CHUNK_SIZE_SETTING.get(metadata.settings()); this.storageService = storageService; + this.repositoriesMetrics = repositoriesMetrics; // If the user explicitly did not define a readonly value, we set it by ourselves depending on the location mode setting. // For secondary_only setting, the repository should be read only @@ -152,7 +156,7 @@ protected BlobStore getBlobStore() { @Override protected AzureBlobStore createBlobStore() { - final AzureBlobStore blobStore = new AzureBlobStore(metadata, storageService, bigArrays); + final AzureBlobStore blobStore = new AzureBlobStore(metadata, storageService, bigArrays, repositoriesMetrics); logger.debug( () -> format( diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java index c3cd5e78c5dbe..4556e63378fea 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java @@ -71,7 +71,15 @@ public Map getRepositories( return Collections.singletonMap(AzureRepository.TYPE, metadata -> { AzureStorageService storageService = azureStoreService.get(); assert storageService != null; - return new AzureRepository(metadata, namedXContentRegistry, storageService, clusterService, bigArrays, recoverySettings); + return new AzureRepository( + metadata, + namedXContentRegistry, + storageService, + clusterService, + bigArrays, + recoverySettings, + repositoriesMetrics + ); }); } diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java index c6e85e44d24dd..7373ed9485784 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java @@ -91,7 +91,7 @@ public AzureBlobServiceClient client( String clientName, LocationMode locationMode, OperationPurpose purpose, - AzureClientProvider.SuccessfulRequestHandler successfulRequestHandler + AzureClientProvider.RequestMetricsHandler requestMetricsHandler ) { final AzureStorageSettings azureStorageSettings = getClientSettings(clientName); @@ -102,7 +102,7 @@ public AzureBlobServiceClient client( locationMode, retryOptions, proxyOptions, - successfulRequestHandler, + requestMetricsHandler, purpose ); } diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java index 1962bddd8fdb3..cb9facc061a28 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AbstractAzureServerTestCase.java @@ -29,6 +29,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.mocksocket.MockHttpServer; +import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -168,7 +169,10 @@ int getMaxReadRetries(String clientName) { .build() ); - return new AzureBlobContainer(BlobPath.EMPTY, new AzureBlobStore(repositoryMetadata, service, BigArrays.NON_RECYCLING_INSTANCE)); + return new AzureBlobContainer( + BlobPath.EMPTY, + new AzureBlobStore(repositoryMetadata, service, BigArrays.NON_RECYCLING_INSTANCE, RepositoriesMetrics.NOOP) + ); } protected static byte[] randomBlobContent() { diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java index 7d82f2d5029f6..2699438de8ac6 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureClientProviderTests.java @@ -26,7 +26,7 @@ import java.util.concurrent.TimeUnit; public class AzureClientProviderTests extends ESTestCase { - private static final AzureClientProvider.SuccessfulRequestHandler EMPTY_CONSUMER = (purpose, method, url) -> {}; + private static final AzureClientProvider.RequestMetricsHandler NOOP_HANDLER = (purpose, method, url, metrics) -> {}; private ThreadPool threadPool; private AzureClientProvider azureClientProvider; @@ -76,7 +76,7 @@ public void testCanCreateAClientWithSecondaryLocation() { locationMode, requestRetryOptions, null, - EMPTY_CONSUMER, + NOOP_HANDLER, randomFrom(OperationPurpose.values()) ); } @@ -106,7 +106,7 @@ public void testCanNotCreateAClientWithSecondaryLocationWithoutAProperEndpoint() locationMode, requestRetryOptions, null, - EMPTY_CONSUMER, + NOOP_HANDLER, randomFrom(OperationPurpose.values()) ) ); diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java index 7037dd4eaf111..3afacb5b7426e 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.util.MockBigArrays; import org.elasticsearch.env.Environment; import org.elasticsearch.indices.recovery.RecoverySettings; +import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -40,7 +41,8 @@ private AzureRepository azureRepository(Settings settings) { mock(AzureStorageService.class), BlobStoreTestUtil.mockClusterService(), MockBigArrays.NON_RECYCLING_INSTANCE, - new RecoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)) + new RecoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), + RepositoriesMetrics.NOOP ); assertThat(azureRepository.getBlobStore(), is(nullValue())); return azureRepository; diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index bd5723b4dbcc4..3e6b7c356cb11 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.repositories.RepositoriesMetrics; import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; @@ -144,16 +145,7 @@ class IgnoreNoResponseMetricsCollector extends RequestMetricCollector { private IgnoreNoResponseMetricsCollector(Operation operation, OperationPurpose purpose) { this.operation = operation; - this.attributes = Map.of( - "repo_type", - S3Repository.TYPE, - "repo_name", - repositoryMetadata.name(), - "operation", - operation.getKey(), - "purpose", - purpose.getKey() - ); + this.attributes = RepositoriesMetrics.createAttributesMap(repositoryMetadata, purpose, operation.getKey()); } @Override diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java index cce3c764fe7a4..2cd6e2b11ef7a 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java @@ -9,10 +9,17 @@ package org.elasticsearch.repositories; +import org.elasticsearch.cluster.metadata.RepositoryMetadata; +import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.telemetry.metric.LongCounter; import org.elasticsearch.telemetry.metric.LongHistogram; import org.elasticsearch.telemetry.metric.MeterRegistry; +import java.util.Map; + +/** + * The common set of metrics that we publish for {@link org.elasticsearch.repositories.blobstore.BlobStoreRepository} implementations. + */ public record RepositoriesMetrics( MeterRegistry meterRegistry, LongCounter requestCounter, @@ -28,15 +35,65 @@ public record RepositoriesMetrics( public static RepositoriesMetrics NOOP = new RepositoriesMetrics(MeterRegistry.NOOP); + /** + * Is incremented for each request sent to the blob store (including retries) + * + * Exposed as {@link #requestCounter()} + */ public static final String METRIC_REQUESTS_TOTAL = "es.repositories.requests.total"; + /** + * Is incremented for each request which returns a non 2xx response OR fails to return a response + * (includes throttling and retryable errors) + * + * Exposed as {@link #exceptionCounter()} + */ public static final String METRIC_EXCEPTIONS_TOTAL = "es.repositories.exceptions.total"; + /** + * Is incremented each time an operation ends with a 416 response + * + * Exposed as {@link #requestRangeNotSatisfiedExceptionCounter()} + */ public static final String METRIC_EXCEPTIONS_REQUEST_RANGE_NOT_SATISFIED_TOTAL = "es.repositories.exceptions.request_range_not_satisfied.total"; + /** + * Is incremented each time we are throttled by the blob store, e.g. upon receiving an HTTP 429 response + * + * Exposed as {@link #throttleCounter()} + */ public static final String METRIC_THROTTLES_TOTAL = "es.repositories.throttles.total"; + /** + * Is incremented for each operation we attempt, whether it succeeds or fails, this doesn't include retries + * + * Exposed via {@link #operationCounter()} + */ public static final String METRIC_OPERATIONS_TOTAL = "es.repositories.operations.total"; + /** + * Is incremented for each operation that ends with a non 2xx response or throws an exception + * + * Exposed via {@link #unsuccessfulOperationCounter()} + */ public static final String METRIC_UNSUCCESSFUL_OPERATIONS_TOTAL = "es.repositories.operations.unsuccessful.total"; + /** + * Each time an operation has one or more failed requests (from non 2xx response or exception), the + * count of those is sampled + * + * Exposed via {@link #exceptionHistogram()} + */ public static final String METRIC_EXCEPTIONS_HISTOGRAM = "es.repositories.exceptions.histogram"; + /** + * Each time an operation has one or more throttled requests, the count of those is sampled + * + * Exposed via {@link #throttleHistogram()} + */ public static final String METRIC_THROTTLES_HISTOGRAM = "es.repositories.throttles.histogram"; + /** + * Every operation that is attempted will record a time. The value recorded here is the sum of the duration of + * each of the requests executed to try and complete the operation. The duration of each request is the time + * between sending the request and either a response being received, or the request failing. Does not include + * the consumption of the body of the response or any time spent pausing between retries. + * + * Exposed via {@link #httpRequestTimeInMillisHistogram()} + */ public static final String HTTP_REQUEST_TIME_IN_MILLIS_HISTOGRAM = "es.repositories.requests.http_request_time.histogram"; public RepositoriesMetrics(MeterRegistry meterRegistry) { @@ -61,4 +118,25 @@ public RepositoriesMetrics(MeterRegistry meterRegistry) { ) ); } + + /** + * Create the map of attributes we expect to see on repository metrics + */ + public static Map createAttributesMap( + RepositoryMetadata repositoryMetadata, + OperationPurpose purpose, + String operation + ) { + return Map.of( + "repo_type", + repositoryMetadata.type(), + "repo_name", + repositoryMetadata.name(), + "operation", + operation, + "purpose", + purpose.getKey() + ); + } + } diff --git a/server/src/main/java/org/elasticsearch/rest/RestStatus.java b/server/src/main/java/org/elasticsearch/rest/RestStatus.java index 72227b2d26ec0..569b63edda00b 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestStatus.java +++ b/server/src/main/java/org/elasticsearch/rest/RestStatus.java @@ -571,4 +571,16 @@ public static RestStatus status(int successfulShards, int totalShards, ShardOper public static RestStatus fromCode(int code) { return CODE_TO_STATUS.get(code); } + + /** + * Utility method to determine if an HTTP status code is "Successful" + * + * as defined by RFC 9110 + * + * @param code An HTTP status code + * @return true if it is a 2xx code, false otherwise + */ + public static boolean isSuccessful(int code) { + return code >= 200 && code < 300; + } } From 40bddafd92ea3ed1d3e28e694883b11dfd6516ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Istv=C3=A1n=20Zolt=C3=A1n=20Szab=C3=B3?= Date: Tue, 8 Oct 2024 08:41:11 +0200 Subject: [PATCH 155/194] [DOCS] Adds DeBERTa v2 tokenization params to infer trained model API docs (#114242) * [DOCS] Adds DeBERTa v2 tokenization params to infer trained model API docs. * [DOCS] Mode edits. --- .../apis/infer-trained-model.asciidoc | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc b/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc index 99c3ecad03a9d..7acbc0bd23859 100644 --- a/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc +++ b/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc @@ -225,6 +225,17 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, string) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -301,6 +312,17 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, string) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -397,6 +419,21 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, string) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`span`:::: +(Optional, integer) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] + +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -517,6 +554,21 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, string) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`span`:::: +(Optional, integer) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] + +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -608,6 +660,17 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, string) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] @@ -687,6 +750,21 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, integer) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +`with_special_tokens`:::: +(Optional, boolean) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] +======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`span`:::: +(Optional, integer) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] + `with_special_tokens`:::: (Optional, boolean) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] @@ -790,6 +868,17 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizatio (Optional, string) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= +`deberta_v2`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-deberta-v2] ++ +.Properties of deberta_v2 +[%collapsible%open] +======= +`truncate`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate-deberta-v2] +======= `roberta`:::: (Optional, object) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] From 07c3acf1c0aecc53234602aab55258530cfd8efc Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 8 Oct 2024 07:59:57 +0100 Subject: [PATCH 156/194] Remove cluster state from `/_cluster/reroute` response (#114231) Including the cluster state in responses to the `POST _cluster/state` API was deprecated in #90399 (v8.6.0) requiring callers to pass `?metric=none` to avoid the deprecation warning. This commit adjusts the behaviour as promised in v9 so that this API never returns the cluster state, and deprecates the `?metric` parameter itself. Closes #88978 --- docs/changelog/114231.yaml | 17 ++++++++ docs/reference/cluster/reroute.asciidoc | 4 +- docs/reference/commands/shard-tool.asciidoc | 2 +- .../red-yellow-cluster-status.asciidoc | 22 +++++----- .../upgrades/SnapshotBasedRecoveryIT.java | 2 +- .../test/cluster.reroute/10_basic.yml | 7 +++- .../test/cluster.reroute/11_explain.yml | 9 +++- .../20_deprecated_response_filtering.yml | 42 +++++++++++++++---- server/src/main/java/module-info.java | 1 + .../reroute/ClusterRerouteResponse.java | 2 - .../decider/MaxRetryAllocationDecider.java | 2 +- .../admin/cluster/ClusterRerouteFeatures.java | 24 +++++++++++ .../cluster/RestClusterRerouteAction.java | 34 ++++++++++++--- ...lasticsearch.features.FeatureSpecification | 1 + .../MaxRetryAllocationDeciderTests.java | 4 +- .../actions/SearchableSnapshotActionIT.java | 2 +- 16 files changed, 136 insertions(+), 39 deletions(-) create mode 100644 docs/changelog/114231.yaml create mode 100644 server/src/main/java/org/elasticsearch/rest/action/admin/cluster/ClusterRerouteFeatures.java diff --git a/docs/changelog/114231.yaml b/docs/changelog/114231.yaml new file mode 100644 index 0000000000000..61c447688edcf --- /dev/null +++ b/docs/changelog/114231.yaml @@ -0,0 +1,17 @@ +pr: 114231 +summary: Remove cluster state from `/_cluster/reroute` response +area: Allocation +type: breaking +issues: + - 88978 +breaking: + title: Remove cluster state from `/_cluster/reroute` response + area: REST API + details: >- + The `POST /_cluster/reroute` API no longer returns the cluster state in its + response. The `?metric` query parameter to this API now has no effect and + its use will be forbidden in a future version. + impact: >- + Cease usage of the `?metric` query parameter when calling the + `POST /_cluster/reroute` API. + notable: false diff --git a/docs/reference/cluster/reroute.asciidoc b/docs/reference/cluster/reroute.asciidoc index b4e4809ae73b4..429070f80b9bf 100644 --- a/docs/reference/cluster/reroute.asciidoc +++ b/docs/reference/cluster/reroute.asciidoc @@ -10,7 +10,7 @@ Changes the allocation of shards in a cluster. [[cluster-reroute-api-request]] ==== {api-request-title} -`POST /_cluster/reroute?metric=none` +`POST /_cluster/reroute` [[cluster-reroute-api-prereqs]] ==== {api-prereq-title} @@ -193,7 +193,7 @@ This is a short example of a simple reroute API call: [source,console] -------------------------------------------------- -POST /_cluster/reroute?metric=none +POST /_cluster/reroute { "commands": [ { diff --git a/docs/reference/commands/shard-tool.asciidoc b/docs/reference/commands/shard-tool.asciidoc index a2d9d557adf5e..b1e63740cede0 100644 --- a/docs/reference/commands/shard-tool.asciidoc +++ b/docs/reference/commands/shard-tool.asciidoc @@ -95,7 +95,7 @@ Changing allocation id V8QXk-QXSZinZMT-NvEq4w to tjm9Ve6uTBewVFAlfUMWjA You should run the following command to allocate this shard: -POST /_cluster/reroute?metric=none +POST /_cluster/reroute { "commands" : [ { diff --git a/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc b/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc index cae4eb99dd54a..eb56a37562c31 100644 --- a/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc +++ b/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc @@ -2,12 +2,12 @@ === Red or yellow cluster health status A red or yellow cluster health status indicates one or more shards are not assigned to -a node. +a node. * **Red health status**: The cluster has some unassigned primary shards, which -means that some operations such as searches and indexing may fail. -* **Yellow health status**: The cluster has no unassigned primary shards but some -unassigned replica shards. This increases your risk of data loss and can degrade +means that some operations such as searches and indexing may fail. +* **Yellow health status**: The cluster has no unassigned primary shards but some +unassigned replica shards. This increases your risk of data loss and can degrade cluster performance. When your cluster has a red or yellow health status, it will continue to process @@ -16,8 +16,8 @@ cleanup activities until the cluster returns to green health status. For instanc some <> actions require the index on which they operate to have a green health status. -In many cases, your cluster will recover to green health status automatically. -If the cluster doesn't automatically recover, then you must <> +In many cases, your cluster will recover to green health status automatically. +If the cluster doesn't automatically recover, then you must <> the remaining problems so management and cleanup activities can proceed. [discrete] @@ -107,7 +107,7 @@ asynchronously in the background. [source,console] ---- -POST _cluster/reroute?metric=none +POST _cluster/reroute ---- [discrete] @@ -231,10 +231,10 @@ unassigned. See <>. If a node containing a primary shard is lost, {es} can typically replace it using a replica on another node. If you can't recover the node and replicas -don't exist or are irrecoverable, <> will report `no_valid_shard_copy` and you'll need to do one of the following: +don't exist or are irrecoverable, <> will report `no_valid_shard_copy` and you'll need to do one of the following: -* restore the missing data from <> +* restore the missing data from <> * index the missing data from its original data source * accept data loss on the index-level by running <> * accept data loss on the shard-level by executing <> allocate_stale_primary or allocate_empty_primary command with `accept_data_loss: true` @@ -246,7 +246,7 @@ resulting in data loss. + [source,console] ---- -POST _cluster/reroute?metric=none +POST _cluster/reroute { "commands": [ { diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java index 6f4c37f9e56a7..3343a683bbd11 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SnapshotBasedRecoveryIT.java @@ -203,7 +203,7 @@ private void cancelShard(String indexName, int shard, String nodeName) throws IO } builder.endObject(); - Request request = new Request(HttpPost.METHOD_NAME, "/_cluster/reroute?pretty&metric=none"); + Request request = new Request(HttpPost.METHOD_NAME, "/_cluster/reroute?pretty"); request.setJsonEntity(Strings.toString(builder)); Response response = client().performRequest(request); logger.info("--> Relocated primary to an older version {}", EntityUtils.toString(response.getEntity())); diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.reroute/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.reroute/10_basic.yml index f7378cc01dc0a..d73efed1f7571 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.reroute/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.reroute/10_basic.yml @@ -1,5 +1,8 @@ --- "Basic sanity check": + - requires: + cluster_features: ["cluster.reroute.ignores_metric_param"] + reason: requires this feature + - do: - cluster.reroute: - metric: none + cluster.reroute: {} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.reroute/11_explain.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.reroute/11_explain.yml index 7543c96b232dc..3584ce9666705 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.reroute/11_explain.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.reroute/11_explain.yml @@ -13,12 +13,14 @@ setup: --- "Explain API with empty command list": + - requires: + cluster_features: ["cluster.reroute.ignores_metric_param"] + reason: requires this feature - do: cluster.reroute: explain: true dry_run: true - metric: none body: commands: [] @@ -26,6 +28,10 @@ setup: --- "Explain API for non-existent node & shard": + - requires: + cluster_features: ["cluster.reroute.ignores_metric_param"] + reason: requires this feature + - skip: features: [arbitrary_key] @@ -39,7 +45,6 @@ setup: cluster.reroute: explain: true dry_run: true - metric: none body: commands: - cancel: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.reroute/20_deprecated_response_filtering.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.reroute/20_deprecated_response_filtering.yml index 3bc27f53ad679..9775fbd1bea83 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.reroute/20_deprecated_response_filtering.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.reroute/20_deprecated_response_filtering.yml @@ -1,21 +1,45 @@ --- -"Do not return metadata by default and produce deprecation warning": +"Do not return metadata by default and emit no warning": + - requires: + cluster_features: ["cluster.reroute.ignores_metric_param"] + reason: requires this feature + + - do: + cluster.reroute: {} + - is_false: state + +--- +"Do not return metadata with ?metric=none and produce deprecation warning": + - requires: + cluster_features: ["cluster.reroute.ignores_metric_param"] + reason: requires this feature + - skip: features: [ "allowed_warnings" ] + - do: - cluster.reroute: {} + cluster.reroute: + metric: none allowed_warnings: - - "The [state] field in the response to the reroute API is deprecated and will be removed in a future version. Specify ?metric=none to adopt the future behaviour." - - is_false: state.metadata + - >- + the [?metric] query parameter to the [POST /_cluster/reroute] API has no effect; + its use will be forbidden in a future version + - is_false: state + --- -"If requested return metadata and produce deprecation warning": +"Do not return metadata with ?metric=metadata and produce deprecation warning": + - requires: + cluster_features: ["cluster.reroute.ignores_metric_param"] + reason: requires this feature + - skip: features: [ "allowed_warnings" ] + - do: cluster.reroute: metric: metadata allowed_warnings: - - "The [state] field in the response to the reroute API is deprecated and will be removed in a future version. Specify ?metric=none to adopt the future behaviour." - - is_true: state.metadata - - is_false: state.nodes - + - >- + the [?metric] query parameter to the [POST /_cluster/reroute] API has no effect; + its use will be forbidden in a future version + - is_false: state diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 56672957dd571..11965abf1dcd2 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -429,6 +429,7 @@ org.elasticsearch.indices.IndicesFeatures, org.elasticsearch.repositories.RepositoriesFeatures, org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures, + org.elasticsearch.rest.action.admin.cluster.ClusterRerouteFeatures, org.elasticsearch.index.mapper.MapperFeatures, org.elasticsearch.ingest.IngestGeoIpFeatures, org.elasticsearch.search.SearchFeatures, diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java index 4aa6ed60afe43..b0ec0968f8d1d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.UpdateForV10; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.xcontent.ToXContent; @@ -45,7 +44,6 @@ public class ClusterRerouteResponse extends ActionResponse implements IsAcknowle /** * To be removed when REST compatibility with {@link org.elasticsearch.Version#V_8_6_0} / {@link RestApiVersion#V_8} no longer needed */ - @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) // to remove from the v9 API only @UpdateForV10(owner = UpdateForV10.Owner.DISTRIBUTED_COORDINATION) // to remove entirely private final ClusterState state; private final RoutingExplanations explanations; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java index b20cd3ecaf992..a55522ff14c83 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java @@ -35,7 +35,7 @@ public class MaxRetryAllocationDecider extends AllocationDecider { Setting.Property.NotCopyableOnResize ); - private static final String RETRY_FAILED_API = "POST /_cluster/reroute?retry_failed&metric=none"; + private static final String RETRY_FAILED_API = "POST /_cluster/reroute?retry_failed"; public static final String NAME = "max_retry"; diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/ClusterRerouteFeatures.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/ClusterRerouteFeatures.java new file mode 100644 index 0000000000000..c6582cab4a2da --- /dev/null +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/ClusterRerouteFeatures.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.rest.action.admin.cluster; + +import org.elasticsearch.features.FeatureSpecification; +import org.elasticsearch.features.NodeFeature; + +import java.util.Set; + +public class ClusterRerouteFeatures implements FeatureSpecification { + public static final NodeFeature CLUSTER_REROUTE_IGNORES_METRIC_PARAM = new NodeFeature("cluster.reroute.ignores_metric_param"); + + @Override + public Set getFeatures() { + return Set.of(CLUSTER_REROUTE_IGNORES_METRIC_PARAM); + } +} diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java index 66d6aee30d00a..fada07d60b74e 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterRerouteAction.java @@ -15,8 +15,11 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -39,6 +42,8 @@ @ServerlessScope(Scope.INTERNAL) public class RestClusterRerouteAction extends BaseRestHandler { + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestClusterRerouteAction.class); + private static final Set RESPONSE_PARAMS = addToCopy(Settings.FORMAT_PARAMS, "metric"); private static final ObjectParser PARSER = new ObjectParser<>("cluster_reroute"); @@ -51,7 +56,8 @@ public class RestClusterRerouteAction extends BaseRestHandler { PARSER.declareBoolean(ClusterRerouteRequest::dryRun, new ParseField("dry_run")); } - private static final String DEFAULT_METRICS = Strings.arrayToCommaDelimitedString( + @UpdateForV10(owner = UpdateForV10.Owner.DISTRIBUTED_COORDINATION) // no longer used, so can be removed + private static final String V8_DEFAULT_METRICS = Strings.arrayToCommaDelimitedString( EnumSet.complementOf(EnumSet.of(ClusterState.Metric.METADATA)).toArray() ); @@ -76,6 +82,11 @@ public boolean allowSystemIndexAccessByDefault() { return true; } + @UpdateForV10(owner = UpdateForV10.Owner.DISTRIBUTED_COORDINATION) + // actually UpdateForV11 because V10 still supports the V9 API including this deprecation message + private static final String METRIC_DEPRECATION_MESSAGE = """ + the [?metric] query parameter to the [POST /_cluster/reroute] API has no effect; its use will be forbidden in a future version"""; + @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { ClusterRerouteRequest clusterRerouteRequest = createRequest(request); @@ -83,11 +94,24 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC if (clusterRerouteRequest.explain()) { request.params().put("explain", Boolean.TRUE.toString()); } - // by default, return everything but metadata - final String metric = request.param("metric"); - if (metric == null) { - request.params().put("metric", DEFAULT_METRICS); + + switch (request.getRestApiVersion()) { + case V_9 -> { + // always avoid returning the cluster state by forcing `?metric=none`; emit a warning if `?metric` is even present + if (request.hasParam("metric")) { + deprecationLogger.critical(DeprecationCategory.API, "cluster-reroute-metric-param", METRIC_DEPRECATION_MESSAGE); + } + request.params().put("metric", "none"); + } + case V_8, V_7 -> { + // by default, return everything but metadata + final String metric = request.param("metric"); + if (metric == null) { + request.params().put("metric", V8_DEFAULT_METRICS); + } + } } + return channel -> client.execute( TransportClusterRerouteAction.TYPE, clusterRerouteRequest, diff --git a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification index 143c0293c5ab7..5cd8935f72403 100644 --- a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification +++ b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification @@ -16,6 +16,7 @@ org.elasticsearch.rest.RestFeatures org.elasticsearch.indices.IndicesFeatures org.elasticsearch.repositories.RepositoriesFeatures org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures +org.elasticsearch.rest.action.admin.cluster.ClusterRerouteFeatures org.elasticsearch.index.mapper.MapperFeatures org.elasticsearch.ingest.IngestGeoIpFeatures org.elasticsearch.search.SearchFeatures diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java index 9d889f24acb6c..c20d84fcf4b10 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java @@ -181,7 +181,7 @@ public void testFailedAllocation() { decision.getExplanation(), allOf( containsString("shard has exceeded the maximum number of retries"), - containsString("POST /_cluster/reroute?retry_failed&metric=none") + containsString("POST /_cluster/reroute?retry_failed") ) ); } @@ -280,7 +280,7 @@ public void testFailedRelocation() { decision.getExplanation(), allOf( containsString("shard has exceeded the maximum number of retries"), - containsString("POST /_cluster/reroute?retry_failed&metric=none") + containsString("POST /_cluster/reroute?retry_failed") ) ); }); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java index fefeaa95319ed..f00b5b566c156 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java @@ -982,7 +982,7 @@ public void testSearchableSnapshotTotalShardsPerNode() throws Exception { * notification that partial-index is now GREEN. */ private void triggerStateChange() throws IOException { - Request rerouteRequest = new Request("POST", "/_cluster/reroute?metric=none"); + Request rerouteRequest = new Request("POST", "/_cluster/reroute"); client().performRequest(rerouteRequest); } } From 4ecc5bd53e473174523fdb9c38655e326408e313 Mon Sep 17 00:00:00 2001 From: Panagiotis Bailis Date: Tue, 8 Oct 2024 10:15:10 +0300 Subject: [PATCH 157/194] Text_similarity_reranker retriever rework to be evaluated during rewrite phase (#114085) --- .../org/elasticsearch/TransportVersions.java | 1 + .../action/search/SearchPhaseController.java | 9 + .../elasticsearch/search/rank/RankDoc.java | 11 +- .../retriever/CompoundRetrieverBuilder.java | 4 +- .../search/sort/ShardDocSortField.java | 6 +- ...bstractRankDocWireSerializingTestCase.java | 57 +++ .../search/rank/RankDocTests.java | 16 +- .../xpack/inference/InferenceFeatures.java | 3 +- .../xpack/inference/InferencePlugin.java | 3 + .../textsimilarity/TextSimilarityRankDoc.java | 103 ++++++ .../TextSimilarityRankRetrieverBuilder.java | 127 ++++--- .../TextSimilarityRankDocTests.java | 88 +++++ ...xtSimilarityRankRetrieverBuilderTests.java | 121 +------ .../70_text_similarity_rank_retriever.yml | 40 ++- x-pack/plugin/rank-rrf/build.gradle | 4 + .../xpack/rank/rrf/RRFRankDoc.java | 6 + .../xpack/rank/rrf/RRFRankDocTests.java | 21 +- .../rrf/RRFRankClientYamlTestSuiteIT.java | 2 + ...ith_text_similarity_reranker_retriever.yml | 334 ++++++++++++++++++ 19 files changed, 760 insertions(+), 196 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/search/rank/AbstractRankDocWireSerializingTestCase.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDoc.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDocTests.java create mode 100644 x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 1911013cbe8e9..2095ba47ee377 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -236,6 +236,7 @@ static TransportVersion def(int id) { public static final TransportVersion INGEST_GEO_DATABASE_PROVIDERS = def(8_760_00_0); public static final TransportVersion DATE_TIME_DOC_VALUES_LOCALES = def(8_761_00_0); public static final TransportVersion FAST_REFRESH_RCO = def(8_762_00_0); + public static final TransportVersion TEXT_SIMILARITY_RERANKER_QUERY_REWRITE = def(8_763_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index a6acb3ee2a52e..1c4eb1c191370 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -36,6 +36,7 @@ import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchService; +import org.elasticsearch.search.SearchSortValues; import org.elasticsearch.search.aggregations.AggregationReduceContext; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.InternalAggregations; @@ -51,6 +52,7 @@ import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rank.RankDoc; import org.elasticsearch.search.rank.context.QueryPhaseRankCoordinatorContext; +import org.elasticsearch.search.sort.ShardDocSortField; import org.elasticsearch.search.suggest.Suggest; import org.elasticsearch.search.suggest.Suggest.Suggestion; import org.elasticsearch.search.suggest.completion.CompletionSuggestion; @@ -464,6 +466,13 @@ private static SearchHits getHits( assert shardDoc instanceof RankDoc; searchHit.setRank(((RankDoc) shardDoc).rank); searchHit.score(shardDoc.score); + long shardAndDoc = ShardDocSortField.encodeShardAndDoc(shardDoc.shardIndex, shardDoc.doc); + searchHit.sortValues( + new SearchSortValues( + new Object[] { shardDoc.score, shardAndDoc }, + new DocValueFormat[] { DocValueFormat.RAW, DocValueFormat.RAW } + ) + ); } else if (sortedTopDocs.isSortedByField) { FieldDoc fieldDoc = (FieldDoc) shardDoc; searchHit.sortValues(fieldDoc.fields, reducedQueryPhase.sortValueFormats); diff --git a/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java b/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java index b16a234931115..9ab14aa9362b5 100644 --- a/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java +++ b/server/src/main/java/org/elasticsearch/search/rank/RankDoc.java @@ -11,9 +11,11 @@ import org.apache.lucene.search.Explanation; import org.apache.lucene.search.ScoreDoc; -import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.VersionedNamedWriteable; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; @@ -24,7 +26,7 @@ * {@code RankDoc} is the base class for all ranked results. * Subclasses should extend this with additional information required for their global ranking method. */ -public class RankDoc extends ScoreDoc implements NamedWriteable, ToXContentFragment, Comparable { +public class RankDoc extends ScoreDoc implements VersionedNamedWriteable, ToXContentFragment, Comparable { public static final String NAME = "rank_doc"; @@ -40,6 +42,11 @@ public String getWriteableName() { return NAME; } + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.RANK_DOCS_RETRIEVER; + } + @Override public final int compareTo(RankDoc other) { if (score != other.score) { diff --git a/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java index 1962145d7336d..22bef026523e9 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java @@ -160,7 +160,7 @@ public void onFailure(Exception e) { @Override public final QueryBuilder topDocsQuery() { - throw new IllegalStateException(getName() + " cannot be nested"); + throw new IllegalStateException("Should not be called, missing a rewrite?"); } @Override @@ -208,7 +208,7 @@ public int doHashCode() { return Objects.hash(innerRetrievers); } - private SearchSourceBuilder createSearchSourceBuilder(PointInTimeBuilder pit, RetrieverBuilder retrieverBuilder) { + protected SearchSourceBuilder createSearchSourceBuilder(PointInTimeBuilder pit, RetrieverBuilder retrieverBuilder) { var sourceBuilder = new SearchSourceBuilder().pointInTimeBuilder(pit) .trackTotalHits(false) .storedFields(new StoredFieldsContext(false)) diff --git a/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java b/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java index a38a24eb75fca..be1ce9c925037 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java +++ b/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java @@ -64,7 +64,7 @@ public void setTopValue(Long value) { @Override public Long value(int slot) { - return (((long) shardRequestIndex) << 32) | (delegate.value(slot) & 0xFFFFFFFFL); + return encodeShardAndDoc(shardRequestIndex, delegate.value(slot)); } @Override @@ -87,4 +87,8 @@ public static int decodeDoc(long value) { public static int decodeShardRequestIndex(long value) { return (int) (value >> 32); } + + public static long encodeShardAndDoc(int shardIndex, int doc) { + return (((long) shardIndex) << 32) | (doc & 0xFFFFFFFFL); + } } diff --git a/server/src/test/java/org/elasticsearch/search/rank/AbstractRankDocWireSerializingTestCase.java b/server/src/test/java/org/elasticsearch/search/rank/AbstractRankDocWireSerializingTestCase.java new file mode 100644 index 0000000000000..d0c85a33acf09 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/rank/AbstractRankDocWireSerializingTestCase.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.search.rank; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public abstract class AbstractRankDocWireSerializingTestCase extends AbstractWireSerializingTestCase { + + protected abstract T createTestRankDoc(); + + @Override + protected NamedWriteableRegistry writableRegistry() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); + List entries = searchModule.getNamedWriteables(); + entries.addAll(getAdditionalNamedWriteables()); + return new NamedWriteableRegistry(entries); + } + + protected abstract List getAdditionalNamedWriteables(); + + @Override + protected T createTestInstance() { + return createTestRankDoc(); + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + public void testRankDocSerialization() throws IOException { + int totalDocs = randomIntBetween(10, 100); + Set docs = new HashSet<>(); + for (int i = 0; i < totalDocs; i++) { + docs.add(createTestRankDoc()); + } + RankDocsQueryBuilder rankDocsQueryBuilder = new RankDocsQueryBuilder(docs.toArray((T[]) new RankDoc[0]), null, randomBoolean()); + RankDocsQueryBuilder copy = (RankDocsQueryBuilder) copyNamedWriteable(rankDocsQueryBuilder, writableRegistry(), QueryBuilder.class); + assertThat(rankDocsQueryBuilder, equalTo(copy)); + } +} diff --git a/server/src/test/java/org/elasticsearch/search/rank/RankDocTests.java b/server/src/test/java/org/elasticsearch/search/rank/RankDocTests.java index d190139309c31..21101b2bc7db1 100644 --- a/server/src/test/java/org/elasticsearch/search/rank/RankDocTests.java +++ b/server/src/test/java/org/elasticsearch/search/rank/RankDocTests.java @@ -9,27 +9,29 @@ package org.elasticsearch.search.rank; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.test.AbstractWireSerializingTestCase; import java.io.IOException; +import java.util.Collections; +import java.util.List; -public class RankDocTests extends AbstractWireSerializingTestCase { +public class RankDocTests extends AbstractRankDocWireSerializingTestCase { - static RankDoc createTestRankDoc() { + protected RankDoc createTestRankDoc() { RankDoc rankDoc = new RankDoc(randomNonNegativeInt(), randomFloat(), randomIntBetween(0, 1)); rankDoc.rank = randomNonNegativeInt(); return rankDoc; } @Override - protected Writeable.Reader instanceReader() { - return RankDoc::new; + protected List getAdditionalNamedWriteables() { + return Collections.emptyList(); } @Override - protected RankDoc createTestInstance() { - return createTestRankDoc(); + protected Writeable.Reader instanceReader() { + return RankDoc::new; } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java index 30ccb48d5c709..a3f2105054639 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java @@ -27,7 +27,8 @@ public Set getFeatures() { TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_RETRIEVER_SUPPORTED, RandomRankRetrieverBuilder.RANDOM_RERANKER_RETRIEVER_SUPPORTED, SemanticTextFieldMapper.SEMANTIC_TEXT_SEARCH_INFERENCE_ID, - SemanticQueryBuilder.SEMANTIC_TEXT_INNER_HITS + SemanticQueryBuilder.SEMANTIC_TEXT_INNER_HITS, + TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_COMPOSITION_SUPPORTED ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index dbb9130ab91e1..927fd94809886 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -36,6 +36,7 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.search.rank.RankBuilder; +import org.elasticsearch.search.rank.RankDoc; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.ScalingExecutorBuilder; import org.elasticsearch.xcontent.ParseField; @@ -66,6 +67,7 @@ import org.elasticsearch.xpack.inference.rank.random.RandomRankBuilder; import org.elasticsearch.xpack.inference.rank.random.RandomRankRetrieverBuilder; import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankBuilder; +import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankDoc; import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankRetrieverBuilder; import org.elasticsearch.xpack.inference.registry.ModelRegistry; import org.elasticsearch.xpack.inference.rest.RestDeleteInferenceEndpointAction; @@ -253,6 +255,7 @@ public List getNamedWriteables() { var entries = new ArrayList<>(InferenceNamedWriteablesProvider.getNamedWriteables()); entries.add(new NamedWriteableRegistry.Entry(RankBuilder.class, TextSimilarityRankBuilder.NAME, TextSimilarityRankBuilder::new)); entries.add(new NamedWriteableRegistry.Entry(RankBuilder.class, RandomRankBuilder.NAME, RandomRankBuilder::new)); + entries.add(new NamedWriteableRegistry.Entry(RankDoc.class, TextSimilarityRankDoc.NAME, TextSimilarityRankDoc::new)); return entries; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDoc.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDoc.java new file mode 100644 index 0000000000000..d208623e53324 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDoc.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.rank.textsimilarity; + +import org.apache.lucene.search.Explanation; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.search.rank.RankDoc; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +public class TextSimilarityRankDoc extends RankDoc { + + public static final String NAME = "text_similarity_rank_doc"; + + public final String inferenceId; + public final String field; + + public TextSimilarityRankDoc(int doc, float score, int shardIndex, String inferenceId, String field) { + super(doc, score, shardIndex); + this.inferenceId = inferenceId; + this.field = field; + } + + public TextSimilarityRankDoc(StreamInput in) throws IOException { + super(in); + inferenceId = in.readString(); + field = in.readString(); + } + + @Override + public Explanation explain(Explanation[] sources, String[] queryNames) { + final String queryAlias = queryNames[0] == null ? "" : "[" + queryNames[0] + "]"; + return Explanation.match( + score, + "text_similarity_reranker match using inference endpoint: [" + + inferenceId + + "] on document field: [" + + field + + "] matching on source query " + + queryAlias, + sources + ); + } + + @Override + public void doWriteTo(StreamOutput out) throws IOException { + out.writeString(inferenceId); + out.writeString(field); + } + + @Override + public boolean doEquals(RankDoc rd) { + TextSimilarityRankDoc tsrd = (TextSimilarityRankDoc) rd; + return Objects.equals(inferenceId, tsrd.inferenceId) && Objects.equals(field, tsrd.field); + } + + @Override + public int doHashCode() { + return Objects.hash(inferenceId, field); + } + + @Override + public String toString() { + return "TextSimilarityRankDoc{" + + "doc=" + + doc + + ", shardIndex=" + + shardIndex + + ", score=" + + score + + ", inferenceId=" + + inferenceId + + ", field=" + + field + + '}'; + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + protected void doToXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("inferenceId", inferenceId); + builder.field("field", field); + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.TEXT_SIMILARITY_RERANKER_QUERY_REWRITE; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java index 50d762e7b90aa..3ddaab12eca14 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java @@ -7,14 +7,20 @@ package org.elasticsearch.xpack.inference.rank.textsimilarity; +import org.apache.lucene.search.ScoreDoc; import org.elasticsearch.common.ParsingException; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.StoredFieldsContext; +import org.elasticsearch.search.rank.RankDoc; +import org.elasticsearch.search.retriever.CompoundRetrieverBuilder; import org.elasticsearch.search.retriever.RetrieverBuilder; import org.elasticsearch.search.retriever.RetrieverParserContext; +import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentBuilder; @@ -32,11 +38,14 @@ /** * A {@code RetrieverBuilder} for parsing and constructing a text similarity reranker retriever. */ -public class TextSimilarityRankRetrieverBuilder extends RetrieverBuilder { +public class TextSimilarityRankRetrieverBuilder extends CompoundRetrieverBuilder { public static final NodeFeature TEXT_SIMILARITY_RERANKER_RETRIEVER_SUPPORTED = new NodeFeature( "text_similarity_reranker_retriever_supported" ); + public static final NodeFeature TEXT_SIMILARITY_RERANKER_COMPOSITION_SUPPORTED = new NodeFeature( + "text_similarity_reranker_retriever_composition_supported" + ); public static final ParseField RETRIEVER_FIELD = new ParseField("retriever"); public static final ParseField INFERENCE_ID_FIELD = new ParseField("inference_id"); @@ -51,7 +60,6 @@ public class TextSimilarityRankRetrieverBuilder extends RetrieverBuilder { String inferenceText = (String) args[2]; String field = (String) args[3]; int rankWindowSize = args[4] == null ? DEFAULT_RANK_WINDOW_SIZE : (int) args[4]; - return new TextSimilarityRankRetrieverBuilder(retrieverBuilder, inferenceId, inferenceText, field, rankWindowSize); }); @@ -70,17 +78,20 @@ public static TextSimilarityRankRetrieverBuilder fromXContent(XContentParser par if (context.clusterSupportsFeature(TEXT_SIMILARITY_RERANKER_RETRIEVER_SUPPORTED) == false) { throw new ParsingException(parser.getTokenLocation(), "unknown retriever [" + TextSimilarityRankBuilder.NAME + "]"); } + if (context.clusterSupportsFeature(TEXT_SIMILARITY_RERANKER_COMPOSITION_SUPPORTED) == false) { + throw new UnsupportedOperationException( + "[text_similarity_reranker] retriever composition feature is not supported by all nodes in the cluster" + ); + } if (TextSimilarityRankBuilder.TEXT_SIMILARITY_RERANKER_FEATURE.check(XPackPlugin.getSharedLicenseState()) == false) { throw LicenseUtils.newComplianceException(TextSimilarityRankBuilder.NAME); } return PARSER.apply(parser, context); } - private final RetrieverBuilder retrieverBuilder; private final String inferenceId; private final String inferenceText; private final String field; - private final int rankWindowSize; public TextSimilarityRankRetrieverBuilder( RetrieverBuilder retrieverBuilder, @@ -89,15 +100,14 @@ public TextSimilarityRankRetrieverBuilder( String field, int rankWindowSize ) { - this.retrieverBuilder = retrieverBuilder; + super(List.of(new RetrieverSource(retrieverBuilder, null)), rankWindowSize); this.inferenceId = inferenceId; this.inferenceText = inferenceText; this.field = field; - this.rankWindowSize = rankWindowSize; } public TextSimilarityRankRetrieverBuilder( - RetrieverBuilder retrieverBuilder, + List retrieverSource, String inferenceId, String inferenceText, String field, @@ -106,66 +116,75 @@ public TextSimilarityRankRetrieverBuilder( String retrieverName, List preFilterQueryBuilders ) { - this.retrieverBuilder = retrieverBuilder; + super(retrieverSource, rankWindowSize); + if (retrieverSource.size() != 1) { + throw new IllegalArgumentException("[" + getName() + "] retriever should have exactly one inner retriever"); + } this.inferenceId = inferenceId; this.inferenceText = inferenceText; this.field = field; - this.rankWindowSize = rankWindowSize; this.minScore = minScore; this.retrieverName = retrieverName; this.preFilterQueryBuilders = preFilterQueryBuilders; } @Override - public QueryBuilder topDocsQuery() { - // the original matching set of the TextSimilarityRank retriever is specified by its nested retriever - return retrieverBuilder.topDocsQuery(); + protected TextSimilarityRankRetrieverBuilder clone(List newChildRetrievers) { + return new TextSimilarityRankRetrieverBuilder( + newChildRetrievers, + inferenceId, + inferenceText, + field, + rankWindowSize, + minScore, + retrieverName, + preFilterQueryBuilders + ); } @Override - public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException { - // rewrite prefilters - boolean hasChanged = false; - var newPreFilters = rewritePreFilters(ctx); - hasChanged |= newPreFilters != preFilterQueryBuilders; - - // rewrite nested retriever - RetrieverBuilder newRetriever = retrieverBuilder.rewrite(ctx); - hasChanged |= newRetriever != retrieverBuilder; - if (hasChanged) { - return new TextSimilarityRankRetrieverBuilder( - newRetriever, - field, - inferenceText, - inferenceId, - rankWindowSize, - minScore, - this.retrieverName, - newPreFilters - ); + protected RankDoc[] combineInnerRetrieverResults(List rankResults) { + assert rankResults.size() == 1; + ScoreDoc[] scoreDocs = rankResults.getFirst(); + TextSimilarityRankDoc[] textSimilarityRankDocs = new TextSimilarityRankDoc[scoreDocs.length]; + for (int i = 0; i < scoreDocs.length; i++) { + ScoreDoc scoreDoc = scoreDocs[i]; + textSimilarityRankDocs[i] = new TextSimilarityRankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex, inferenceId, field); } - return this; + return textSimilarityRankDocs; } @Override - public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) { - retrieverBuilder.getPreFilterQueryBuilders().addAll(preFilterQueryBuilders); - retrieverBuilder.extractToSearchSourceBuilder(searchSourceBuilder, compoundUsed); - // Combining with other rank builder (such as RRF) is not supported yet - if (searchSourceBuilder.rankBuilder() != null) { - throw new IllegalArgumentException("text similarity rank builder cannot be combined with other rank builders"); - } + public QueryBuilder explainQuery() { + // the original matching set of the TextSimilarityRank retriever is specified by its nested retriever + return new RankDocsQueryBuilder(rankDocs, new QueryBuilder[] { innerRetrievers.getFirst().retriever().explainQuery() }, true); + } - searchSourceBuilder.rankBuilder( + @Override + protected SearchSourceBuilder createSearchSourceBuilder(PointInTimeBuilder pit, RetrieverBuilder retrieverBuilder) { + var sourceBuilder = new SearchSourceBuilder().pointInTimeBuilder(pit) + .trackTotalHits(false) + .storedFields(new StoredFieldsContext(false)) + .size(rankWindowSize); + if (preFilterQueryBuilders.isEmpty() == false) { + retrieverBuilder.getPreFilterQueryBuilders().addAll(preFilterQueryBuilders); + } + retrieverBuilder.extractToSearchSourceBuilder(sourceBuilder, true); + + // apply the pre-filters + if (preFilterQueryBuilders.size() > 0) { + QueryBuilder query = sourceBuilder.query(); + BoolQueryBuilder newQuery = new BoolQueryBuilder(); + if (query != null) { + newQuery.must(query); + } + preFilterQueryBuilders.forEach(newQuery::filter); + sourceBuilder.query(newQuery); + } + sourceBuilder.rankBuilder( new TextSimilarityRankBuilder(this.field, this.inferenceId, this.inferenceText, this.rankWindowSize, this.minScore) ); - } - - /** - * Determines if this retriever contains sub-retrievers that need to be executed prior to search. - */ - public boolean isCompound() { - return retrieverBuilder.isCompound(); + return sourceBuilder; } @Override @@ -179,7 +198,7 @@ public int rankWindowSize() { @Override protected void doToXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(RETRIEVER_FIELD.getPreferredName(), retrieverBuilder); + builder.field(RETRIEVER_FIELD.getPreferredName(), innerRetrievers.getFirst().retriever()); builder.field(INFERENCE_ID_FIELD.getPreferredName(), inferenceId); builder.field(INFERENCE_TEXT_FIELD.getPreferredName(), inferenceText); builder.field(FIELD_FIELD.getPreferredName(), field); @@ -187,9 +206,9 @@ protected void doToXContent(XContentBuilder builder, Params params) throws IOExc } @Override - protected boolean doEquals(Object other) { + public boolean doEquals(Object other) { TextSimilarityRankRetrieverBuilder that = (TextSimilarityRankRetrieverBuilder) other; - return Objects.equals(retrieverBuilder, that.retrieverBuilder) + return super.doEquals(other) && Objects.equals(inferenceId, that.inferenceId) && Objects.equals(inferenceText, that.inferenceText) && Objects.equals(field, that.field) @@ -198,7 +217,7 @@ protected boolean doEquals(Object other) { } @Override - protected int doHashCode() { - return Objects.hash(retrieverBuilder, inferenceId, inferenceText, field, rankWindowSize, minScore); + public int doHashCode() { + return Objects.hash(inferenceId, inferenceText, field, rankWindowSize, minScore); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDocTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDocTests.java new file mode 100644 index 0000000000000..fed4565c54bd4 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDocTests.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.rank.textsimilarity; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.search.rank.AbstractRankDocWireSerializingTestCase; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.inference.InferencePlugin; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.search.rank.RankDoc.NO_RANK; + +public class TextSimilarityRankDocTests extends AbstractRankDocWireSerializingTestCase { + + static TextSimilarityRankDoc createTestTextSimilarityRankDoc() { + TextSimilarityRankDoc instance = new TextSimilarityRankDoc( + randomNonNegativeInt(), + randomFloat(), + randomBoolean() ? -1 : randomNonNegativeInt(), + randomAlphaOfLength(randomIntBetween(2, 5)), + randomAlphaOfLength(randomIntBetween(2, 5)) + ); + instance.rank = randomBoolean() ? NO_RANK : randomIntBetween(1, 10000); + return instance; + } + + @Override + protected List getAdditionalNamedWriteables() { + try (InferencePlugin plugin = new InferencePlugin(Settings.EMPTY)) { + return plugin.getNamedWriteables(); + } + } + + @Override + protected Writeable.Reader instanceReader() { + return TextSimilarityRankDoc::new; + } + + @Override + protected TextSimilarityRankDoc createTestRankDoc() { + return createTestTextSimilarityRankDoc(); + } + + @Override + protected TextSimilarityRankDoc mutateInstance(TextSimilarityRankDoc instance) throws IOException { + int doc = instance.doc; + int shardIndex = instance.shardIndex; + float score = instance.score; + int rank = instance.rank; + String inferenceId = instance.inferenceId; + String field = instance.field; + + switch (randomInt(5)) { + case 0: + doc = randomValueOtherThan(doc, ESTestCase::randomNonNegativeInt); + break; + case 1: + shardIndex = shardIndex == -1 ? randomNonNegativeInt() : -1; + break; + case 2: + score = randomValueOtherThan(score, ESTestCase::randomFloat); + break; + case 3: + rank = rank == NO_RANK ? randomIntBetween(1, 10000) : NO_RANK; + break; + case 4: + inferenceId = randomValueOtherThan(inferenceId, () -> randomAlphaOfLength(randomIntBetween(2, 5))); + break; + case 5: + field = randomValueOtherThan(field, () -> randomAlphaOfLength(randomIntBetween(2, 5))); + break; + default: + throw new AssertionError(); + } + TextSimilarityRankDoc mutated = new TextSimilarityRankDoc(doc, score, shardIndex, inferenceId, field); + mutated.rank = rank; + return mutated; + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java index 140b181a42a0a..32301bf9efea9 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java @@ -9,17 +9,9 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.common.Strings; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.MatchAllQueryBuilder; -import org.elasticsearch.index.query.MatchNoneQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryRewriteContext; -import org.elasticsearch.index.query.RandomQueryBuilder; -import org.elasticsearch.index.query.RangeQueryBuilder; -import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.builder.SubSearchSourceBuilder; import org.elasticsearch.search.retriever.RetrieverBuilder; import org.elasticsearch.search.retriever.RetrieverParserContext; import org.elasticsearch.search.retriever.TestRetrieverBuilder; @@ -38,10 +30,8 @@ import java.util.List; import static org.elasticsearch.search.rank.RankBuilder.DEFAULT_RANK_WINDOW_SIZE; -import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -import static org.mockito.Mockito.mock; public class TextSimilarityRankRetrieverBuilderTests extends AbstractXContentTestCase { @@ -82,6 +72,7 @@ protected TextSimilarityRankRetrieverBuilder doParseInstance(XContentParser pars new SearchUsage(), nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED || nf == TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_RETRIEVER_SUPPORTED + || nf == TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_COMPOSITION_SUPPORTED ) ); } @@ -131,86 +122,6 @@ public void testParserDefaults() throws IOException { } } - public void testRewriteInnerRetriever() throws IOException { - final boolean[] rewritten = { false }; - List preFilterQueryBuilders = new ArrayList<>(); - if (randomBoolean()) { - for (int i = 0; i < randomIntBetween(1, 5); i++) { - preFilterQueryBuilders.add(RandomQueryBuilder.createQuery(random())); - } - } - RetrieverBuilder innerRetriever = new TestRetrieverBuilder("top-level-retriever") { - @Override - public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException { - if (randomBoolean()) { - return this; - } - rewritten[0] = true; - return new TestRetrieverBuilder("nested-rewritten-retriever") { - @Override - public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) { - if (preFilterQueryBuilders.isEmpty() == false) { - BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - - for (QueryBuilder preFilterQueryBuilder : preFilterQueryBuilders) { - boolQueryBuilder.filter(preFilterQueryBuilder); - } - boolQueryBuilder.must(new RangeQueryBuilder("some_field")); - searchSourceBuilder.subSearches().add(new SubSearchSourceBuilder(boolQueryBuilder)); - } else { - searchSourceBuilder.subSearches().add(new SubSearchSourceBuilder(new RangeQueryBuilder("some_field"))); - } - } - }; - } - - @Override - public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) { - if (preFilterQueryBuilders.isEmpty() == false) { - BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - - for (QueryBuilder preFilterQueryBuilder : preFilterQueryBuilders) { - boolQueryBuilder.filter(preFilterQueryBuilder); - } - boolQueryBuilder.must(new TermQueryBuilder("field", "value")); - searchSourceBuilder.subSearches().add(new SubSearchSourceBuilder(boolQueryBuilder)); - } else { - searchSourceBuilder.subSearches().add(new SubSearchSourceBuilder(new TermQueryBuilder("field", "value"))); - } - } - }; - TextSimilarityRankRetrieverBuilder textSimilarityRankRetrieverBuilder = createRandomTextSimilarityRankRetrieverBuilder( - innerRetriever - ); - textSimilarityRankRetrieverBuilder.getPreFilterQueryBuilders().addAll(preFilterQueryBuilders); - SearchSourceBuilder source = new SearchSourceBuilder().retriever(textSimilarityRankRetrieverBuilder); - QueryRewriteContext queryRewriteContext = mock(QueryRewriteContext.class); - source = Rewriteable.rewrite(source, queryRewriteContext); - assertNull(source.retriever()); - if (false == preFilterQueryBuilders.isEmpty()) { - if (source.query() instanceof MatchAllQueryBuilder == false && source.query() instanceof MatchNoneQueryBuilder == false) { - assertThat(source.query(), instanceOf(BoolQueryBuilder.class)); - BoolQueryBuilder bq = (BoolQueryBuilder) source.query(); - assertFalse(bq.must().isEmpty()); - assertThat(bq.must().size(), equalTo(1)); - if (rewritten[0]) { - assertThat(bq.must().get(0), instanceOf(RangeQueryBuilder.class)); - } else { - assertThat(bq.must().get(0), instanceOf(TermQueryBuilder.class)); - } - for (int j = 0; j < bq.filter().size(); j++) { - assertEqualQueryOrMatchAllNone(bq.filter().get(j), preFilterQueryBuilders.get(j)); - } - } - } else { - if (rewritten[0]) { - assertThat(source.query(), instanceOf(RangeQueryBuilder.class)); - } else { - assertThat(source.query(), instanceOf(TermQueryBuilder.class)); - } - } - } - public void testTextSimilarityRetrieverParsing() throws IOException { String restContent = "{" + " \"retriever\": {" @@ -250,29 +161,6 @@ public void testTextSimilarityRetrieverParsing() throws IOException { } } - public void testIsCompound() { - RetrieverBuilder compoundInnerRetriever = new TestRetrieverBuilder(ESTestCase.randomAlphaOfLengthBetween(5, 10)) { - @Override - public boolean isCompound() { - return true; - } - }; - RetrieverBuilder nonCompoundInnerRetriever = new TestRetrieverBuilder(ESTestCase.randomAlphaOfLengthBetween(5, 10)) { - @Override - public boolean isCompound() { - return false; - } - }; - TextSimilarityRankRetrieverBuilder compoundTextSimilarityRankRetrieverBuilder = createRandomTextSimilarityRankRetrieverBuilder( - compoundInnerRetriever - ); - assertTrue(compoundTextSimilarityRankRetrieverBuilder.isCompound()); - TextSimilarityRankRetrieverBuilder nonCompoundTextSimilarityRankRetrieverBuilder = createRandomTextSimilarityRankRetrieverBuilder( - nonCompoundInnerRetriever - ); - assertFalse(nonCompoundTextSimilarityRankRetrieverBuilder.isCompound()); - } - public void testTopDocsQuery() { RetrieverBuilder innerRetriever = new TestRetrieverBuilder(ESTestCase.randomAlphaOfLengthBetween(5, 10)) { @Override @@ -281,11 +169,6 @@ public QueryBuilder topDocsQuery() { } }; TextSimilarityRankRetrieverBuilder retriever = createRandomTextSimilarityRankRetrieverBuilder(innerRetriever); - assertThat(retriever.topDocsQuery(), instanceOf(TermQueryBuilder.class)); - } - - private static void assertEqualQueryOrMatchAllNone(QueryBuilder actual, QueryBuilder expected) { - assertThat(actual, anyOf(instanceOf(MatchAllQueryBuilder.class), instanceOf(MatchNoneQueryBuilder.class), equalTo(expected))); + expectThrows(IllegalStateException.class, "Should not be called, missing a rewrite?", retriever::topDocsQuery); } - } diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml index e2c1417057578..9a4d7f4416164 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml @@ -87,11 +87,9 @@ setup: - length: { hits.hits: 2 } - match: { hits.hits.0._id: "doc_2" } - - match: { hits.hits.0._rank: 1 } - close_to: { hits.hits.0._score: { value: 0.4, error: 0.001 } } - match: { hits.hits.1._id: "doc_1" } - - match: { hits.hits.1._rank: 2 } - close_to: { hits.hits.1._score: { value: 0.2, error: 0.001 } } --- @@ -123,7 +121,6 @@ setup: - length: { hits.hits: 1 } - match: { hits.hits.0._id: "doc_1" } - - match: { hits.hits.0._rank: 1 } - close_to: { hits.hits.0._score: { value: 0.2, error: 0.001 } } @@ -178,3 +175,40 @@ setup: field: text size: 10 + +--- +"text similarity reranking with explain": + + - do: + search: + index: test-index + body: + track_total_hits: true + fields: [ "text", "topic" ] + retriever: { + text_similarity_reranker: { + retriever: + { + standard: { + query: { + term: { + topic: "science" + } + } + } + }, + rank_window_size: 10, + inference_id: my-rerank-model, + inference_text: "How often does the moon hide the sun?", + field: text + } + } + size: 10 + explain: true + + - match: { hits.hits.0._id: "doc_2" } + - match: { hits.hits.1._id: "doc_1" } + + - close_to: { hits.hits.0._explanation.value: { value: 0.4, error: 0.000001 } } + - match: {hits.hits.0._explanation.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[text\\].*/" } + - match: {hits.hits.0._explanation.details.0.description: "/weight.*science.*/" } diff --git a/x-pack/plugin/rank-rrf/build.gradle b/x-pack/plugin/rank-rrf/build.gradle index 2db33fa0f2c8d..2c3f217243aa4 100644 --- a/x-pack/plugin/rank-rrf/build.gradle +++ b/x-pack/plugin/rank-rrf/build.gradle @@ -20,7 +20,11 @@ dependencies { compileOnly project(path: xpackModule('core')) testImplementation(testArtifact(project(xpackModule('core')))) + testImplementation(testArtifact(project(':server'))) clusterModules project(xpackModule('rank-rrf')) + clusterModules project(xpackModule('inference')) clusterModules project(':modules:lang-painless') + + clusterPlugins project(':x-pack:plugin:inference:qa:test-service-plugin') } diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java index 500ed17395127..272df248e53e9 100644 --- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java +++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankDoc.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.rank.rrf; import org.apache.lucene.search.Explanation; +import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -169,4 +170,9 @@ protected void doToXContent(XContentBuilder builder, Params params) throws IOExc builder.field("scores", scores); builder.field("rankConstant", rankConstant); } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.RRF_QUERY_REWRITE; + } } diff --git a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankDocTests.java b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankDocTests.java index 4b64b6c173c92..5548392270a08 100644 --- a/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankDocTests.java +++ b/x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/rrf/RRFRankDocTests.java @@ -7,15 +7,17 @@ package org.elasticsearch.xpack.rank.rrf; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable.Reader; -import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.search.rank.AbstractRankDocWireSerializingTestCase; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.List; import static org.elasticsearch.xpack.rank.rrf.RRFRankDoc.NO_RANK; -public class RRFRankDocTests extends AbstractWireSerializingTestCase { +public class RRFRankDocTests extends AbstractRankDocWireSerializingTestCase { static RRFRankDoc createTestRRFRankDoc(int queryCount) { RRFRankDoc instance = new RRFRankDoc( @@ -35,9 +37,13 @@ static RRFRankDoc createTestRRFRankDoc(int queryCount) { return instance; } - static RRFRankDoc createTestRRFRankDoc() { - int queryCount = randomIntBetween(2, 20); - return createTestRRFRankDoc(queryCount); + @Override + protected List getAdditionalNamedWriteables() { + try (RRFRankPlugin rrfRankPlugin = new RRFRankPlugin()) { + return rrfRankPlugin.getNamedWriteables(); + } catch (IOException ex) { + throw new AssertionError("Failed to create RRFRankPlugin", ex); + } } @Override @@ -46,8 +52,9 @@ protected Reader instanceReader() { } @Override - protected RRFRankDoc createTestInstance() { - return createTestRRFRankDoc(); + protected RRFRankDoc createTestRankDoc() { + int queryCount = randomIntBetween(2, 20); + return createTestRRFRankDoc(queryCount); } @Override diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankClientYamlTestSuiteIT.java b/x-pack/plugin/rank-rrf/src/yamlRestTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankClientYamlTestSuiteIT.java index 3a577eb62faa3..32b5aedd5d99a 100644 --- a/x-pack/plugin/rank-rrf/src/yamlRestTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankClientYamlTestSuiteIT.java +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankClientYamlTestSuiteIT.java @@ -23,7 +23,9 @@ public class RRFRankClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { .nodes(2) .module("rank-rrf") .module("lang-painless") + .module("x-pack-inference") .setting("xpack.license.self_generated.type", "trial") + .plugin("inference-service-test") .build(); public RRFRankClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { diff --git a/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml new file mode 100644 index 0000000000000..3e758ae11f7e6 --- /dev/null +++ b/x-pack/plugin/rank-rrf/src/yamlRestTest/resources/rest-api-spec/test/rrf/800_rrf_with_text_similarity_reranker_retriever.yml @@ -0,0 +1,334 @@ +setup: + - requires: + cluster_features: ['rrf_retriever_composition_supported', 'text_similarity_reranker_retriever_supported'] + reason: need to have support for rrf and semantic reranking composition + test_runner_features: "close_to" + + - do: + inference.put: + task_type: rerank + inference_id: my-rerank-model + body: > + { + "service": "test_reranking_service", + "service_settings": { + "model_id": "my_model", + "api_key": "abc64" + }, + "task_settings": { + } + } + + + - do: + indices.create: + index: test-index + body: + settings: + number_of_shards: 1 + mappings: + properties: + text: + type: text + topic: + type: keyword + subtopic: + type: keyword + integer: + type: integer + + - do: + index: + index: test-index + id: doc_1 + body: + text: "Sun Moon Lake is a lake in Nantou County, Taiwan. It is the largest lake in Taiwan." + topic: [ "geography" ] + integer: 1 + + - do: + index: + index: test-index + id: doc_2 + body: + text: "The phases of the Moon come from the position of the Moon relative to the Earth and Sun." + topic: [ "science" ] + subtopic: [ "astronomy" ] + integer: 2 + + - do: + index: + index: test-index + id: doc_3 + body: + text: "As seen from Earth, a solar eclipse happens when the Moon is directly between the Earth and the Sun." + topic: [ "science" ] + subtopic: [ "technology" ] + integer: 3 + + - do: + indices.refresh: {} + +--- +"rrf retriever with a nested text similarity reranker": + + - do: + search: + index: test-index + body: + track_total_hits: true + fields: [ "text", "topic" ] + retriever: + rrf: { + retrievers: + [ + { + standard: { + query: { + bool: { + should: + [ + { + constant_score: { + filter: { + term: { + integer: 1 + } + }, + boost: 10 + } + }, + { + constant_score: + { + filter: + { + term: + { + integer: 2 + } + }, + boost: 1 + } + } + ] + } + } + } + }, + { + text_similarity_reranker: { + retriever: + { + standard: { + query: { + term: { + topic: "science" + } + } + } + }, + rank_window_size: 10, + inference_id: my-rerank-model, + inference_text: "How often does the moon hide the sun?", + field: text + } + } + ], + rank_window_size: 10, + rank_constant: 1 + } + size: 10 + from: 1 + aggs: + topics: + terms: + field: topic + size: 10 + + - match: { hits.total.value: 3 } + - length: { hits.hits: 2 } + + - match: { hits.hits.0._id: "doc_1" } + - match: { hits.hits.1._id: "doc_3" } + + - match: { aggregations.topics.buckets.0.key: "science" } + - match: { aggregations.topics.buckets.0.doc_count: 2 } + - match: { aggregations.topics.buckets.1.key: "geography" } + - match: { aggregations.topics.buckets.1.doc_count: 1 } + +--- +"Text similarity reranker on top of an RRF retriever": + + - do: + search: + index: test-index + body: + track_total_hits: true + fields: [ "text", "topic" ] + retriever: + { + text_similarity_reranker: { + retriever: + { + rrf: { + retrievers: + [ + { + standard: { + query: { + bool: { + should: + [ + { + constant_score: { + filter: { + term: { + integer: 1 + } + }, + boost: 10 + } + }, + { + constant_score: + { + filter: + { + term: + { + integer: 3 + } + }, + boost: 1 + } + } + ] + } + } + } + }, + { + standard: { + query: { + term: { + topic: "geography" + } + } + } + } + ], + rank_window_size: 10, + rank_constant: 1 + } + }, + rank_window_size: 10, + inference_id: my-rerank-model, + inference_text: "How often does the moon hide the sun?", + field: text + } + } + size: 10 + aggs: + topics: + terms: + field: topic + size: 10 + + - match: { hits.total.value: 2 } + - length: { hits.hits: 2 } + + - match: { hits.hits.0._id: "doc_3" } + - match: { hits.hits.1._id: "doc_1" } + + - match: { aggregations.topics.buckets.0.key: "geography" } + - match: { aggregations.topics.buckets.0.doc_count: 1 } + - match: { aggregations.topics.buckets.1.key: "science" } + - match: { aggregations.topics.buckets.1.doc_count: 1 } + + +--- +"explain using rrf retriever and text-similarity": + + - do: + search: + index: test-index + body: + track_total_hits: true + fields: [ "text", "topic" ] + retriever: + rrf: { + retrievers: + [ + { + standard: { + query: { + bool: { + should: + [ + { + constant_score: { + filter: { + term: { + integer: 1 + } + }, + boost: 10 + } + }, + { + constant_score: + { + filter: + { + term: + { + integer: 2 + } + }, + boost: 1 + } + } + ] + } + } + } + }, + { + text_similarity_reranker: { + retriever: + { + standard: { + query: { + term: { + topic: "science" + } + } + } + }, + rank_window_size: 10, + inference_id: my-rerank-model, + inference_text: "How often does the moon hide the sun?", + field: text + } + } + ], + rank_window_size: 10, + rank_constant: 1 + } + size: 10 + explain: true + + - match: { hits.hits.0._id: "doc_2" } + - match: { hits.hits.1._id: "doc_1" } + - match: { hits.hits.2._id: "doc_3" } + + - close_to: { hits.hits.0._explanation.value: { value: 0.6666667, error: 0.000001 } } + - match: {hits.hits.0._explanation.description: "/rrf.score:.\\[0.6666667\\].*/" } + - match: {hits.hits.0._explanation.details.0.value: 2} + - match: {hits.hits.0._explanation.details.0.description: "/rrf.score:.\\[0.33333334\\].*/" } + - match: {hits.hits.0._explanation.details.0.details.0.details.0.description: "/ConstantScore.*/" } + - match: {hits.hits.0._explanation.details.1.value: 2} + - match: {hits.hits.0._explanation.details.1.description: "/rrf.score:.\\[0.33333334\\].*/" } + - match: {hits.hits.0._explanation.details.1.details.0.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[text\\].*/" } + - match: {hits.hits.0._explanation.details.1.details.0.details.0.description: "/weight.*science.*/" } From bb9d612eb61d0d7d27be86bce356f4a64dfc7192 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 8 Oct 2024 08:47:28 +0100 Subject: [PATCH 158/194] Show only committed cluster UUID in `GET /` (#114275) Today we show `Metadata#clusterUUID` in the response to `GET /` regardless of whether this value is committed or not, which means that in theory users may see this value change even if nothing is going wrong. To avoid any doubt about the stability of this cluster UUID, this commit suppresses the cluster UUID in this API response until it is committed. --- .../rest/root/TransportMainAction.java | 3 +- .../rest/root/MainActionTests.java | 42 +++++++++++-------- .../cluster/metadata/Metadata.java | 5 +++ 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java index 15f23f7511445..2598f943ea27c 100644 --- a/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java +++ b/modules/rest-root/src/main/java/org/elasticsearch/rest/root/TransportMainAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; @@ -48,7 +49,7 @@ protected void doExecute(Task task, MainRequest request, ActionListener responseRef = new AtomicReference<>(); - action.doExecute(mock(Task.class), new MainRequest(), new ActionListener<>() { - @Override - public void onResponse(MainResponse mainResponse) { - responseRef.set(mainResponse); - } - - @Override - public void onFailure(Exception e) { - logger.error("unexpected error", e); - } - }); + final AtomicBoolean listenerCalled = new AtomicBoolean(); + new TransportMainAction(settings, transportService, mock(ActionFilters.class), clusterService).doExecute( + mock(Task.class), + new MainRequest(), + ActionTestUtils.assertNoFailureListener(mainResponse -> { + assertNotNull(mainResponse); + assertEquals( + state.metadata().clusterUUIDCommitted() ? state.metadata().clusterUUID() : Metadata.UNKNOWN_CLUSTER_UUID, + mainResponse.getClusterUuid() + ); + assertFalse(listenerCalled.getAndSet(true)); + }) + ); - assertNotNull(responseRef.get()); + assertTrue(listenerCalled.get()); verify(clusterService, times(1)).state(); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 566571d82c8ab..0756080c16d00 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -695,6 +695,11 @@ public long version() { return this.version; } + /** + * @return A UUID which identifies this cluster. Nodes record the UUID of the cluster they first join on disk, and will then refuse to + * join clusters with different UUIDs. Note that when the cluster is forming for the first time this value may not yet be committed, + * and therefore it may change. Check {@link #clusterUUIDCommitted()} to verify that the value is committed if needed. + */ public String clusterUUID() { return this.clusterUUID; } From 4af241b5d62cc36de7b617a2b7e5b6c068c49f34 Mon Sep 17 00:00:00 2001 From: kosabogi <105062005+kosabogi@users.noreply.github.com> Date: Tue, 8 Oct 2024 09:58:18 +0200 Subject: [PATCH 159/194] Adds note on reindexing existing data for semantic_text usage (#113590) * Adds note on reindexing existing data for semantic_text usage * Adds note about full crawl and full sync * Style guide related fix * Update docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> --------- Co-authored-by: Liam Thompson <32779855+leemthompo@users.noreply.github.com> --- .../semantic-search-semantic-text.asciidoc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc index f1bd238a64fbf..dbcfbb1b615f9 100644 --- a/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc @@ -89,6 +89,16 @@ PUT semantic-embeddings It will be used to generate the embeddings based on the input text. Every time you ingest data into the related `semantic_text` field, this endpoint will be used for creating the vector representation of the text. +[NOTE] +==== +If you're using web crawlers or connectors to generate indices, you have to +<> for these indices to +include the `semantic_text` field. Once the mapping is updated, you'll need to run +a full web crawl or a full connector sync. This ensures that all existing +documents are reprocessed and updated with the new semantic embeddings, +enabling semantic search on the updated data. +==== + [discrete] [[semantic-text-load-data]] @@ -118,6 +128,13 @@ Create the embeddings from the text by reindexing the data from the `test-data` The data in the `content` field will be reindexed into the `content` semantic text field of the destination index. The reindexed data will be processed by the {infer} endpoint associated with the `content` semantic text field. +[NOTE] +==== +This step uses the reindex API to simulate data ingestion. If you are working with data that has already been indexed, +rather than using the test-data set, reindexing is required to ensure that the data is processed by the {infer} endpoint +and the necessary embeddings are generated. +==== + [source,console] ------------------------------------------------------------ POST _reindex?wait_for_completion=false From c3fd8a7a9a2645a29570955f7cf7a7b6d446f07d Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 10:19:03 +0200 Subject: [PATCH 160/194] Configure Renovate to auto update chainguard image (#114172) * Adds renovate.json * Only auto update chainguard image --------- Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Co-authored-by: Rene Groeschke --- renovate.json | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 renovate.json diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000000000..7dde3a9440ed5 --- /dev/null +++ b/renovate.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "github>elastic/renovate-config:only-chainguard" + ], + "customManagers": [ + { + "description": "Extract Wolfi images from elasticsearch DockerBase configuration", + "customType": "regex", + "fileMatch": [ + "build\\-tools\\-internal\\/src\\/main\\/java\\/org\\/elasticsearch\\/gradle\\/internal\\/DockerBase\\.java$" + ], + "matchStrings": [ + "\\s*\"?(?[^\\s:@\"]+)(?::(?[-a-zA-Z0-9.]+))?(?:@(?sha256:[a-zA-Z0-9]+))?\"?" + ], + "currentValueTemplate": "{{#if currentValue}}{{{currentValue}}}{{else}}latest{{/if}}", + "autoReplaceStringTemplate": "\"{{{depName}}}{{#if newValue}}:{{{newValue}}}{{/if}}{{#if newDigest}}@{{{newDigest}}}{{/if}}\"", + "datasourceTemplate": "docker" + } + ] +} From 09d5e7f20f8e218861ec4e80456844cf47b80517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 8 Oct 2024 10:31:16 +0200 Subject: [PATCH 161/194] Remove dangling AwaitsFix (#113941) The issue referenced seems to be closed, maybe this just wasn't removed properly. --- .../SearchQueryThenFetchAsyncActionTests.java | 483 +++++++++--------- 1 file changed, 253 insertions(+), 230 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java index 3c4976d9bfa86..b63c88f623e21 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java @@ -258,11 +258,10 @@ public void run() { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/101932") public void testMinimumVersionSameAsNewVersion() throws Exception { var newVersion = VersionInformation.CURRENT; var oldVersion = new VersionInformation( - VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), VersionUtils.getPreviousVersion()), + VersionUtils.randomCompatibleVersion(random(), VersionUtils.getPreviousVersion()), IndexVersions.MINIMUM_COMPATIBLE, IndexVersionUtils.randomCompatibleVersion(random()) ); @@ -340,65 +339,69 @@ private void testMixedVersionsShardsSearch(VersionInformation oldVersion, Versio SearchTransportService searchTransportService = new SearchTransportService(null, null, null); SearchPhaseController controller = new SearchPhaseController((t, r) -> InternalAggregationTestCase.emptyReduceContextBuilder()); SearchTask task = new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap()); - QueryPhaseResultConsumer resultConsumer = new QueryPhaseResultConsumer( - searchRequest, - EsExecutors.DIRECT_EXECUTOR_SERVICE, - new NoopCircuitBreaker(CircuitBreaker.REQUEST), - controller, - task::isCancelled, - task.getProgressListener(), - shardsIter.size(), - exc -> {} - ); - final List responses = new ArrayList<>(); - SearchQueryThenFetchAsyncAction newSearchAsyncAction = new SearchQueryThenFetchAsyncAction( - logger, - null, - searchTransportService, - (clusterAlias, node) -> lookup.get(node), - Collections.singletonMap("_na_", AliasFilter.EMPTY), - Collections.emptyMap(), - EsExecutors.DIRECT_EXECUTOR_SERVICE, - resultConsumer, - searchRequest, - new ActionListener<>() { - @Override - public void onFailure(Exception e) { - responses.add(e); - } + try ( + QueryPhaseResultConsumer resultConsumer = new QueryPhaseResultConsumer( + searchRequest, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + new NoopCircuitBreaker(CircuitBreaker.REQUEST), + controller, + task::isCancelled, + task.getProgressListener(), + shardsIter.size(), + exc -> {} + ) + ) { + final List responses = new ArrayList<>(); + SearchQueryThenFetchAsyncAction newSearchAsyncAction = new SearchQueryThenFetchAsyncAction( + logger, + null, + searchTransportService, + (clusterAlias, node) -> lookup.get(node), + Collections.singletonMap("_na_", AliasFilter.EMPTY), + Collections.emptyMap(), + EsExecutors.DIRECT_EXECUTOR_SERVICE, + resultConsumer, + searchRequest, + new ActionListener<>() { + @Override + public void onFailure(Exception e) { + responses.add(e); + } - public void onResponse(SearchResponse response) { - responses.add(response); - }; - }, - shardsIter, - timeProvider, - new ClusterState.Builder(new ClusterName("test")).build(), - task, - SearchResponse.Clusters.EMPTY, - null - ); + public void onResponse(SearchResponse response) { + responses.add(response); + } - newSearchAsyncAction.start(); - assertThat(responses, hasSize(1)); - assertThat(responses.get(0), instanceOf(SearchPhaseExecutionException.class)); - SearchPhaseExecutionException e = (SearchPhaseExecutionException) responses.get(0); - assertThat(e.getCause(), instanceOf(VersionMismatchException.class)); - assertThat( - e.getCause().getMessage(), - equalTo("One of the shards is incompatible with the required minimum version [" + minVersion + "]") - ); + ; + }, + shardsIter, + timeProvider, + new ClusterState.Builder(new ClusterName("test")).build(), + task, + SearchResponse.Clusters.EMPTY, + null + ); + + newSearchAsyncAction.start(); + assertThat(responses, hasSize(1)); + assertThat(responses.get(0), instanceOf(SearchPhaseExecutionException.class)); + SearchPhaseExecutionException e = (SearchPhaseExecutionException) responses.get(0); + assertThat(e.getCause(), instanceOf(VersionMismatchException.class)); + assertThat( + e.getCause().getMessage(), + equalTo("One of the shards is incompatible with the required minimum version [" + minVersion + "]") + ); + } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/101932") public void testMinimumVersionSameAsOldVersion() throws Exception { - Version newVersion = Version.CURRENT; - Version oldVersion = VersionUtils.randomVersionBetween( - random(), - Version.CURRENT.minimumCompatibilityVersion(), - VersionUtils.getPreviousVersion(newVersion) + var newVersion = VersionInformation.CURRENT; + var oldVersion = new VersionInformation( + VersionUtils.randomCompatibleVersion(random(), VersionUtils.getPreviousVersion()), + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersionUtils.randomCompatibleVersion(random()) ); - Version minVersion = oldVersion; + Version minVersion = oldVersion.nodeVersion(); final TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider( 0, @@ -456,98 +459,106 @@ public void sendExecuteQuery( new SearchShardTarget("node1", new ShardId("idx", "na", shardId), null), null ); - SortField sortField = new SortField("timestamp", SortField.Type.LONG); - if (shardId == 0) { - queryResult.topDocs( - new TopDocsAndMaxScore( - new TopFieldDocs( - new TotalHits(1, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), - new FieldDoc[] { new FieldDoc(randomInt(1000), Float.NaN, new Object[] { shardId }) }, - new SortField[] { sortField } + try { + SortField sortField = new SortField("timestamp", SortField.Type.LONG); + if (shardId == 0) { + queryResult.topDocs( + new TopDocsAndMaxScore( + new TopFieldDocs( + new TotalHits(1, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), + new FieldDoc[] { new FieldDoc(randomInt(1000), Float.NaN, new Object[] { shardId }) }, + new SortField[] { sortField } + ), + Float.NaN ), - Float.NaN - ), - new DocValueFormat[] { DocValueFormat.RAW } - ); - } else if (shardId == 1) { - queryResult.topDocs( - new TopDocsAndMaxScore( - new TopFieldDocs( - new TotalHits(1, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), - new FieldDoc[] { new FieldDoc(randomInt(1000), Float.NaN, new Object[] { shardId }) }, - new SortField[] { sortField } + new DocValueFormat[] { DocValueFormat.RAW } + ); + } else if (shardId == 1) { + queryResult.topDocs( + new TopDocsAndMaxScore( + new TopFieldDocs( + new TotalHits(1, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), + new FieldDoc[] { new FieldDoc(randomInt(1000), Float.NaN, new Object[] { shardId }) }, + new SortField[] { sortField } + ), + Float.NaN ), - Float.NaN - ), - new DocValueFormat[] { DocValueFormat.RAW } - ); + new DocValueFormat[] { DocValueFormat.RAW } + ); + } + queryResult.from(0); + queryResult.size(1); + successfulOps.incrementAndGet(); + queryResult.incRef(); + new Thread(() -> ActionListener.respondAndRelease(listener, queryResult)).start(); + } finally { + queryResult.decRef(); } - queryResult.from(0); - queryResult.size(1); - successfulOps.incrementAndGet(); - new Thread(() -> listener.onResponse(queryResult)).start(); } }; SearchPhaseController controller = new SearchPhaseController((t, r) -> InternalAggregationTestCase.emptyReduceContextBuilder()); SearchTask task = new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap()); - QueryPhaseResultConsumer resultConsumer = new QueryPhaseResultConsumer( - searchRequest, - EsExecutors.DIRECT_EXECUTOR_SERVICE, - new NoopCircuitBreaker(CircuitBreaker.REQUEST), - controller, - task::isCancelled, - task.getProgressListener(), - shardsIter.size(), - exc -> {} - ); - CountDownLatch latch = new CountDownLatch(1); - SearchQueryThenFetchAsyncAction action = new SearchQueryThenFetchAsyncAction( - logger, - null, - searchTransportService, - (clusterAlias, node) -> lookup.get(node), - Collections.singletonMap("_na_", AliasFilter.EMPTY), - Collections.emptyMap(), - EsExecutors.DIRECT_EXECUTOR_SERVICE, - resultConsumer, - searchRequest, - null, - shardsIter, - timeProvider, - new ClusterState.Builder(new ClusterName("test")).build(), - task, - SearchResponse.Clusters.EMPTY, - null + try ( + QueryPhaseResultConsumer resultConsumer = new QueryPhaseResultConsumer( + searchRequest, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + new NoopCircuitBreaker(CircuitBreaker.REQUEST), + controller, + task::isCancelled, + task.getProgressListener(), + shardsIter.size(), + exc -> {} + ) ) { - @Override - protected SearchPhase getNextPhase(SearchPhaseResults results, SearchPhaseContext context) { - return new SearchPhase("test") { - @Override - public void run() { - latch.countDown(); - } - }; - } - }; + CountDownLatch latch = new CountDownLatch(1); + SearchQueryThenFetchAsyncAction action = new SearchQueryThenFetchAsyncAction( + logger, + null, + searchTransportService, + (clusterAlias, node) -> lookup.get(node), + Collections.singletonMap("_na_", AliasFilter.EMPTY), + Collections.emptyMap(), + EsExecutors.DIRECT_EXECUTOR_SERVICE, + resultConsumer, + searchRequest, + null, + shardsIter, + timeProvider, + new ClusterState.Builder(new ClusterName("test")).build(), + task, + SearchResponse.Clusters.EMPTY, + null + ) { + @Override + protected SearchPhase getNextPhase(SearchPhaseResults results, SearchPhaseContext context) { + return new SearchPhase("test") { + @Override + public void run() { + latch.countDown(); + } + }; + } + }; - action.start(); - latch.await(); - assertThat(successfulOps.get(), equalTo(2)); - SearchPhaseController.ReducedQueryPhase phase = action.results.reduce(); - assertThat(phase.numReducePhases(), greaterThanOrEqualTo(1)); - assertThat(phase.totalHits().value, equalTo(2L)); - assertThat(phase.totalHits().relation, equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO)); + action.start(); + latch.await(); + assertThat(successfulOps.get(), equalTo(2)); + SearchPhaseController.ReducedQueryPhase phase = action.results.reduce(); + assertThat(phase.numReducePhases(), greaterThanOrEqualTo(1)); + assertThat(phase.totalHits().value, equalTo(2L)); + assertThat(phase.totalHits().relation, equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO)); + } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/101932") public void testMinimumVersionShardDuringPhaseExecution() throws Exception { - Version newVersion = Version.CURRENT; - Version oldVersion = VersionUtils.randomVersionBetween( - random(), - Version.CURRENT.minimumCompatibilityVersion(), - VersionUtils.getPreviousVersion(newVersion) + var newVersion = VersionInformation.CURRENT; + var oldVersion = new VersionInformation( + VersionUtils.randomCompatibleVersion(random(), VersionUtils.getPreviousVersion()), + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersionUtils.randomCompatibleVersion(random()) ); - Version minVersion = newVersion; + + Version minVersion = newVersion.nodeVersion(); final TransportSearchAction.SearchTimeProvider timeProvider = new TransportSearchAction.SearchTimeProvider( 0, @@ -607,111 +618,123 @@ public void sendExecuteQuery( new SearchShardTarget("node1", new ShardId("idx", "na", shardId), null), null ); - SortField sortField = new SortField("timestamp", SortField.Type.LONG); - if (shardId == 0) { - queryResult.topDocs( - new TopDocsAndMaxScore( - new TopFieldDocs( - new TotalHits(1, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), - new FieldDoc[] { new FieldDoc(randomInt(1000), Float.NaN, new Object[] { shardId }) }, - new SortField[] { sortField } + try { + SortField sortField = new SortField("timestamp", SortField.Type.LONG); + if (shardId == 0) { + queryResult.topDocs( + new TopDocsAndMaxScore( + new TopFieldDocs( + new TotalHits(1, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), + new FieldDoc[] { new FieldDoc(randomInt(1000), Float.NaN, new Object[] { shardId }) }, + new SortField[] { sortField } + ), + Float.NaN ), - Float.NaN - ), - new DocValueFormat[] { DocValueFormat.RAW } - ); - } else if (shardId == 1) { - queryResult.topDocs( - new TopDocsAndMaxScore( - new TopFieldDocs( - new TotalHits(1, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), - new FieldDoc[] { new FieldDoc(randomInt(1000), Float.NaN, new Object[] { shardId }) }, - new SortField[] { sortField } + new DocValueFormat[] { DocValueFormat.RAW } + ); + } else if (shardId == 1) { + queryResult.topDocs( + new TopDocsAndMaxScore( + new TopFieldDocs( + new TotalHits(1, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), + new FieldDoc[] { new FieldDoc(randomInt(1000), Float.NaN, new Object[] { shardId }) }, + new SortField[] { sortField } + ), + Float.NaN ), - Float.NaN - ), - new DocValueFormat[] { DocValueFormat.RAW } - ); + new DocValueFormat[] { DocValueFormat.RAW } + ); + } + queryResult.from(0); + queryResult.size(1); + successfulOps.incrementAndGet(); + queryResult.incRef(); + new Thread(() -> ActionListener.respondAndRelease(listener, queryResult)).start(); + } finally { + queryResult.decRef(); } - queryResult.from(0); - queryResult.size(1); - successfulOps.incrementAndGet(); - new Thread(() -> listener.onResponse(queryResult)).start(); } }; SearchPhaseController controller = new SearchPhaseController((t, r) -> InternalAggregationTestCase.emptyReduceContextBuilder()); SearchTask task = new SearchTask(0, "n/a", "n/a", () -> "test", null, Collections.emptyMap()); - QueryPhaseResultConsumer resultConsumer = new QueryPhaseResultConsumer( - searchRequest, - EsExecutors.DIRECT_EXECUTOR_SERVICE, - new NoopCircuitBreaker(CircuitBreaker.REQUEST), - controller, - task::isCancelled, - task.getProgressListener(), - shardsIter.size(), - exc -> {} - ); + CountDownLatch latch = new CountDownLatch(1); - SearchQueryThenFetchAsyncAction action = new SearchQueryThenFetchAsyncAction( - logger, - null, - searchTransportService, - (clusterAlias, node) -> lookup.get(node), - Collections.singletonMap("_na_", AliasFilter.EMPTY), - Collections.emptyMap(), - EsExecutors.DIRECT_EXECUTOR_SERVICE, - resultConsumer, - searchRequest, - null, - shardsIter, - timeProvider, - new ClusterState.Builder(new ClusterName("test")).build(), - task, - SearchResponse.Clusters.EMPTY, - null + try ( + QueryPhaseResultConsumer resultConsumer = new QueryPhaseResultConsumer( + searchRequest, + EsExecutors.DIRECT_EXECUTOR_SERVICE, + new NoopCircuitBreaker(CircuitBreaker.REQUEST), + controller, + task::isCancelled, + task.getProgressListener(), + shardsIter.size(), + exc -> {} + ) ) { - @Override - protected SearchPhase getNextPhase(SearchPhaseResults results, SearchPhaseContext context) { - return new SearchPhase("test") { - @Override - public void run() { - latch.countDown(); - } - }; - } - }; - ShardRouting routingOldVersionShard = ShardRouting.newUnassigned( - new ShardId(new Index("idx", "_na_"), 2), - true, - RecoverySource.EmptyStoreRecoverySource.INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"), - ShardRouting.Role.DEFAULT - ); - SearchShardIterator shardIt = new SearchShardIterator( - null, - new ShardId(new Index("idx", "_na_"), 2), - singletonList(routingOldVersionShard), - idx - ); - routingOldVersionShard = routingOldVersionShard.initialize(oldVersionNode.getId(), "p2", 0); - routingOldVersionShard.started(); - action.start(); - latch.await(); - assertThat(successfulOps.get(), equalTo(2)); - SearchPhaseController.ReducedQueryPhase phase = action.results.reduce(); - assertThat(phase.numReducePhases(), greaterThanOrEqualTo(1)); - assertThat(phase.totalHits().value, equalTo(2L)); - assertThat(phase.totalHits().relation, equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO)); - - SearchShardTarget searchShardTarget = new SearchShardTarget("node3", shardIt.shardId(), null); - SearchActionListener listener = new SearchActionListener(searchShardTarget, 0) { - @Override - public void onFailure(Exception e) {} + SearchQueryThenFetchAsyncAction action = new SearchQueryThenFetchAsyncAction( + logger, + null, + searchTransportService, + (clusterAlias, node) -> lookup.get(node), + Collections.singletonMap("_na_", AliasFilter.EMPTY), + Collections.emptyMap(), + EsExecutors.DIRECT_EXECUTOR_SERVICE, + resultConsumer, + searchRequest, + null, + shardsIter, + timeProvider, + new ClusterState.Builder(new ClusterName("test")).build(), + task, + SearchResponse.Clusters.EMPTY, + null + ) { + @Override + protected SearchPhase getNextPhase(SearchPhaseResults results, SearchPhaseContext context) { + return new SearchPhase("test") { + @Override + public void run() { + latch.countDown(); + } + }; + } + }; + ShardRouting routingOldVersionShard = ShardRouting.newUnassigned( + new ShardId(new Index("idx", "_na_"), 2), + true, + RecoverySource.EmptyStoreRecoverySource.INSTANCE, + new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"), + ShardRouting.Role.DEFAULT + ); + SearchShardIterator shardIt = new SearchShardIterator( + null, + new ShardId(new Index("idx", "_na_"), 2), + singletonList(routingOldVersionShard), + idx + ); + routingOldVersionShard = routingOldVersionShard.initialize(oldVersionNode.getId(), "p2", 0); + routingOldVersionShard.started(); + action.start(); + latch.await(); + assertThat(successfulOps.get(), equalTo(2)); + SearchPhaseController.ReducedQueryPhase phase = action.results.reduce(); + assertThat(phase.numReducePhases(), greaterThanOrEqualTo(1)); + assertThat(phase.totalHits().value, equalTo(2L)); + assertThat(phase.totalHits().relation, equalTo(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO)); - @Override - protected void innerOnResponse(SearchPhaseResult response) {} - }; - Exception e = expectThrows(VersionMismatchException.class, () -> action.executePhaseOnShard(shardIt, searchShardTarget, listener)); - assertThat(e.getMessage(), equalTo("One of the shards is incompatible with the required minimum version [" + minVersion + "]")); + SearchShardTarget searchShardTarget = new SearchShardTarget("node3", shardIt.shardId(), null); + SearchActionListener listener = new SearchActionListener(searchShardTarget, 0) { + @Override + public void onFailure(Exception e) {} + + @Override + protected void innerOnResponse(SearchPhaseResult response) {} + }; + Exception e = expectThrows( + VersionMismatchException.class, + () -> action.executePhaseOnShard(shardIt, searchShardTarget, listener) + ); + assertThat(e.getMessage(), equalTo("One of the shards is incompatible with the required minimum version [" + minVersion + "]")); + } } } From 2a0d5ffc020df11bddcde9cd3b8ae666c4fc6457 Mon Sep 17 00:00:00 2001 From: ChrisHegarty Date: Tue, 8 Oct 2024 09:39:21 +0100 Subject: [PATCH 162/194] Fix simdvec gradle runtime java check --- libs/simdvec/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/simdvec/build.gradle b/libs/simdvec/build.gradle index dab5c25b34679..eee56be72d0bf 100644 --- a/libs/simdvec/build.gradle +++ b/libs/simdvec/build.gradle @@ -7,6 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ +import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.precommit.CheckForbiddenApisTask apply plugin: 'elasticsearch.publish' @@ -32,7 +33,7 @@ tasks.matching { it.name == "compileMain21Java" }.configureEach { } tasks.named('test').configure { - if (JavaVersion.current().majorVersion.toInteger() >= 21) { + if (BuildParams.getRuntimeJavaVersion().majorVersion.toInteger() >= 21) { jvmArgs '--add-modules=jdk.incubator.vector' } } From 6d6fc66e90da3eddcd9f37d93361b77a6596fed9 Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Tue, 8 Oct 2024 10:45:35 +0100 Subject: [PATCH 163/194] Delegate xorBitCount to Lucene (#114249) Now that we're on Lucene 9.12 we don't need our own optimized xorBitCount, can just delegate to Lucene's optimized one (which is identical). --- .../vectors/ES815BitFlatVectorsFormat.java | 4 +- .../field/vectors/ByteBinaryDenseVector.java | 2 +- .../field/vectors/ByteKnnDenseVector.java | 2 +- .../script/field/vectors/ESVectorUtil.java | 73 ------------------- 4 files changed, 4 insertions(+), 77 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/script/field/vectors/ESVectorUtil.java diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java index 5969c9d5db6d7..f0f25bd702749 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES815BitFlatVectorsFormat.java @@ -17,11 +17,11 @@ import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.util.VectorUtil; import org.apache.lucene.util.hnsw.RandomAccessVectorValues; import org.apache.lucene.util.hnsw.RandomVectorScorer; import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier; import org.apache.lucene.util.quantization.RandomAccessQuantizedByteVectorValues; -import org.elasticsearch.script.field.vectors.ESVectorUtil; import java.io.IOException; @@ -105,7 +105,7 @@ public RandomVectorScorer getRandomVectorScorer( } static float hammingScore(byte[] a, byte[] b) { - return ((a.length * Byte.SIZE) - ESVectorUtil.xorBitCount(a, b)) / (float) (a.length * Byte.SIZE); + return ((a.length * Byte.SIZE) - VectorUtil.xorBitCount(a, b)) / (float) (a.length * Byte.SIZE); } static class HammingVectorScorer extends RandomVectorScorer.AbstractRandomVectorScorer { diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteBinaryDenseVector.java b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteBinaryDenseVector.java index a01d1fcbdb4ed..8f13ada2fd604 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteBinaryDenseVector.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteBinaryDenseVector.java @@ -103,7 +103,7 @@ public double l1Norm(List queryVector) { @Override public int hamming(byte[] queryVector) { - return ESVectorUtil.xorBitCount(queryVector, vectorValue); + return VectorUtil.xorBitCount(queryVector, vectorValue); } @Override diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVector.java b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVector.java index a4219583824c3..42e5b5250199e 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVector.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVector.java @@ -104,7 +104,7 @@ public double l1Norm(List queryVector) { @Override public int hamming(byte[] queryVector) { - return ESVectorUtil.xorBitCount(queryVector, docVector); + return VectorUtil.xorBitCount(queryVector, docVector); } @Override diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/ESVectorUtil.java b/server/src/main/java/org/elasticsearch/script/field/vectors/ESVectorUtil.java deleted file mode 100644 index 045a0e5e75b04..0000000000000 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/ESVectorUtil.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.script.field.vectors; - -import org.apache.lucene.util.BitUtil; -import org.apache.lucene.util.Constants; - -/** - * This class consists of a single utility method that provides XOR bit count computed over signed bytes. - * Remove this class when Lucene version > 9.11 is released, and replace with Lucene's VectorUtil directly. - */ -public class ESVectorUtil { - - /** - * For xorBitCount we stride over the values as either 64-bits (long) or 32-bits (int) at a time. - * On ARM Long::bitCount is not vectorized, and therefore produces less than optimal code, when - * compared to Integer::bitCount. While Long::bitCount is optimal on x64. - */ - static final boolean XOR_BIT_COUNT_STRIDE_AS_INT = Constants.OS_ARCH.equals("aarch64"); - - /** - * XOR bit count computed over signed bytes. - * - * @param a bytes containing a vector - * @param b bytes containing another vector, of the same dimension - * @return the value of the XOR bit count of the two vectors - */ - public static int xorBitCount(byte[] a, byte[] b) { - if (a.length != b.length) { - throw new IllegalArgumentException("vector dimensions differ: " + a.length + "!=" + b.length); - } - if (XOR_BIT_COUNT_STRIDE_AS_INT) { - return xorBitCountInt(a, b); - } else { - return xorBitCountLong(a, b); - } - } - - /** XOR bit count striding over 4 bytes at a time. */ - static int xorBitCountInt(byte[] a, byte[] b) { - int distance = 0, i = 0; - for (final int upperBound = a.length & -Integer.BYTES; i < upperBound; i += Integer.BYTES) { - distance += Integer.bitCount((int) BitUtil.VH_NATIVE_INT.get(a, i) ^ (int) BitUtil.VH_NATIVE_INT.get(b, i)); - } - // tail: - for (; i < a.length; i++) { - distance += Integer.bitCount((a[i] ^ b[i]) & 0xFF); - } - return distance; - } - - /** XOR bit count striding over 8 bytes at a time. */ - static int xorBitCountLong(byte[] a, byte[] b) { - int distance = 0, i = 0; - for (final int upperBound = a.length & -Long.BYTES; i < upperBound; i += Long.BYTES) { - distance += Long.bitCount((long) BitUtil.VH_NATIVE_LONG.get(a, i) ^ (long) BitUtil.VH_NATIVE_LONG.get(b, i)); - } - // tail: - for (; i < a.length; i++) { - distance += Integer.bitCount((a[i] ^ b[i]) & 0xFF); - } - return distance; - } - - private ESVectorUtil() {} -} From cbde7f456d7ccd98556302fccf3238bb4557fc91 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Tue, 8 Oct 2024 11:42:22 +0100 Subject: [PATCH 164/194] Add a size limit to outputs from mustache (#114002) --- docs/changelog/114002.yaml | 5 ++ .../script/ScriptServiceBridge.java | 2 +- .../script/mustache/MustachePlugin.java | 2 +- .../script/mustache/MustacheScriptEngine.java | 27 +++++++- .../mustache/CustomMustacheFactoryTests.java | 7 +- .../mustache/MustacheScriptEngineTests.java | 28 +++++++- .../script/mustache/MustacheTests.java | 3 +- .../ingest/AbstractScriptTestCase.java | 2 +- .../common/text/SizeLimitingStringWriter.java | 69 +++++++++++++++++++ .../text/SizeLimitingStringWriterTests.java | 29 ++++++++ .../support/mapper/TemplateRoleNameTests.java | 14 ++-- .../SecurityQueryTemplateEvaluatorTests.java | 3 +- .../WildcardServiceProviderResolverTests.java | 2 +- .../ltr/LearningToRankServiceTests.java | 2 +- .../authc/ldap/ActiveDirectoryRealmTests.java | 2 +- .../security/authc/ldap/LdapRealmTests.java | 2 +- .../mapper/ClusterStateRoleMapperTests.java | 2 +- .../mapper/NativeRoleMappingStoreTests.java | 2 +- .../watcher/support/WatcherTemplateTests.java | 2 +- 19 files changed, 181 insertions(+), 24 deletions(-) create mode 100644 docs/changelog/114002.yaml create mode 100644 server/src/main/java/org/elasticsearch/common/text/SizeLimitingStringWriter.java create mode 100644 server/src/test/java/org/elasticsearch/common/text/SizeLimitingStringWriterTests.java diff --git a/docs/changelog/114002.yaml b/docs/changelog/114002.yaml new file mode 100644 index 0000000000000..b6bc7e25bcdea --- /dev/null +++ b/docs/changelog/114002.yaml @@ -0,0 +1,5 @@ +pr: 114002 +summary: Add a `mustache.max_output_size_bytes` setting to limit the length of results from mustache scripts +area: Infra/Scripting +type: enhancement +issues: [] diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/ScriptServiceBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/ScriptServiceBridge.java index caf6e87c1a530..1f7a19e333308 100644 --- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/ScriptServiceBridge.java +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/ScriptServiceBridge.java @@ -53,7 +53,7 @@ private static ScriptService getScriptService(final Settings settings, final Lon PainlessScriptEngine.NAME, new PainlessScriptEngine(settings, scriptContexts), MustacheScriptEngine.NAME, - new MustacheScriptEngine() + new MustacheScriptEngine(settings) ); return new ScriptService(settings, scriptEngines, ScriptModule.CORE_CONTEXTS, timeProvider); } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java index 64bc2799c24d0..b24d60cb8d887 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java @@ -44,7 +44,7 @@ public class MustachePlugin extends Plugin implements ScriptPlugin, ActionPlugin @Override public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { - return new MustacheScriptEngine(); + return new MustacheScriptEngine(settings); } @Override diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java index ca06a853b1ed6..e7b1727791510 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java @@ -14,6 +14,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.text.SizeLimitingStringWriter; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.MemorySizeValue; import org.elasticsearch.script.GeneralScriptException; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; @@ -47,6 +54,19 @@ public final class MustacheScriptEngine implements ScriptEngine { public static final String NAME = "mustache"; + public static final Setting MUSTACHE_RESULT_SIZE_LIMIT = new Setting<>( + "mustache.max_output_size_bytes", + s -> "1mb", + s -> MemorySizeValue.parseBytesSizeValueOrHeapRatio(s, "mustache.max_output_size_bytes"), + Setting.Property.NodeScope + ); + + private final int sizeLimit; + + public MustacheScriptEngine(Settings settings) { + sizeLimit = (int) MUSTACHE_RESULT_SIZE_LIMIT.get(settings).getBytes(); + } + /** * Compile a template string to (in this case) a Mustache object than can * later be re-used for execution to fill in missing parameter values. @@ -118,10 +138,15 @@ private class MustacheExecutableScript extends TemplateScript { @Override public String execute() { - final StringWriter writer = new StringWriter(); + StringWriter writer = new SizeLimitingStringWriter(sizeLimit); try { template.execute(writer, params); } catch (Exception e) { + // size limit exception can appear at several places in the causal list depending on script & context + if (ExceptionsHelper.unwrap(e, SizeLimitingStringWriter.SizeLimitExceededException.class) != null) { + // don't log, client problem + throw new ElasticsearchParseException("Mustache script result size limit exceeded", e); + } if (shouldLogException(e)) { logger.error(() -> format("Error running %s", template), e); } diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java index 014d6854121b6..eb9c1a6dc3031 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/CustomMustacheFactoryTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.script.mustache; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.script.TemplateScript; @@ -65,7 +66,7 @@ public void testCreateEncoder() { } public void testJsonEscapeEncoder() { - final ScriptEngine engine = new MustacheScriptEngine(); + final ScriptEngine engine = new MustacheScriptEngine(Settings.EMPTY); final Map params = randomBoolean() ? Map.of(Script.CONTENT_TYPE_OPTION, JSON_MEDIA_TYPE) : Map.of(); TemplateScript.Factory compiled = engine.compile(null, "{\"field\": \"{{value}}\"}", TemplateScript.CONTEXT, params); @@ -75,7 +76,7 @@ public void testJsonEscapeEncoder() { } public void testDefaultEncoder() { - final ScriptEngine engine = new MustacheScriptEngine(); + final ScriptEngine engine = new MustacheScriptEngine(Settings.EMPTY); final Map params = Map.of(Script.CONTENT_TYPE_OPTION, PLAIN_TEXT_MEDIA_TYPE); TemplateScript.Factory compiled = engine.compile(null, "{\"field\": \"{{value}}\"}", TemplateScript.CONTEXT, params); @@ -85,7 +86,7 @@ public void testDefaultEncoder() { } public void testUrlEncoder() { - final ScriptEngine engine = new MustacheScriptEngine(); + final ScriptEngine engine = new MustacheScriptEngine(Settings.EMPTY); final Map params = Map.of(Script.CONTENT_TYPE_OPTION, X_WWW_FORM_URLENCODED_MEDIA_TYPE); TemplateScript.Factory compiled = engine.compile(null, "{\"field\": \"{{value}}\"}", TemplateScript.CONTEXT, params); diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java index 089a154079a83..bc1cd30ad45bf 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java @@ -8,8 +8,13 @@ */ package org.elasticsearch.script.mustache; +import com.github.mustachejava.MustacheException; import com.github.mustachejava.MustacheFactory; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.text.SizeLimitingStringWriter; import org.elasticsearch.script.GeneralScriptException; import org.elasticsearch.script.Script; import org.elasticsearch.script.TemplateScript; @@ -24,6 +29,9 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.test.LambdaMatchers.transformedMatch; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.startsWith; @@ -37,7 +45,7 @@ public class MustacheScriptEngineTests extends ESTestCase { @Before public void setup() { - qe = new MustacheScriptEngine(); + qe = new MustacheScriptEngine(Settings.builder().put(MustacheScriptEngine.MUSTACHE_RESULT_SIZE_LIMIT.getKey(), "1kb").build()); factory = CustomMustacheFactory.builder().build(); } @@ -402,6 +410,24 @@ public void testEscapeJson() throws IOException { } } + public void testResultSizeLimit() throws IOException { + String vals = "\"" + "{{val}}".repeat(200) + "\""; + String params = "\"val\":\"aaaaaaaaaa\""; + XContentParser parser = createParser(JsonXContent.jsonXContent, Strings.format("{\"source\":%s,\"params\":{%s}}", vals, params)); + Script script = Script.parse(parser); + var compiled = qe.compile(null, script.getIdOrCode(), TemplateScript.CONTEXT, Map.of()); + TemplateScript templateScript = compiled.newInstance(script.getParams()); + var ex = expectThrows(ElasticsearchParseException.class, templateScript::execute); + assertThat(ex.getCause(), instanceOf(MustacheException.class)); + assertThat( + ex.getCause().getCause(), + allOf( + instanceOf(SizeLimitingStringWriter.SizeLimitExceededException.class), + transformedMatch(Throwable::getMessage, endsWith("has exceeded the size limit [1024]")) + ) + ); + } + private String getChars() { String string = randomRealisticUnicodeOfCodepointLengthBetween(0, 10); for (int i = 0; i < string.length(); i++) { diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java index 82c4637f600fe..335cfe91df87d 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.script.mustache; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Strings; import org.elasticsearch.script.ScriptEngine; @@ -39,7 +40,7 @@ public class MustacheTests extends ESTestCase { - private ScriptEngine engine = new MustacheScriptEngine(); + private ScriptEngine engine = new MustacheScriptEngine(Settings.EMPTY); public void testBasics() { String template = """ diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/AbstractScriptTestCase.java b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/AbstractScriptTestCase.java index ff6e75bed047f..8d877bd48c1e3 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/AbstractScriptTestCase.java +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/AbstractScriptTestCase.java @@ -31,7 +31,7 @@ public abstract class AbstractScriptTestCase extends ESTestCase { @Before public void init() throws Exception { - MustacheScriptEngine engine = new MustacheScriptEngine(); + MustacheScriptEngine engine = new MustacheScriptEngine(Settings.EMPTY); Map engines = Collections.singletonMap(engine.getType(), engine); scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS, () -> 1L); } diff --git a/server/src/main/java/org/elasticsearch/common/text/SizeLimitingStringWriter.java b/server/src/main/java/org/elasticsearch/common/text/SizeLimitingStringWriter.java new file mode 100644 index 0000000000000..2df7e6537c609 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/text/SizeLimitingStringWriter.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.common.text; + +import org.elasticsearch.common.Strings; + +import java.io.StringWriter; + +/** + * A {@link StringWriter} that throws an exception if the string exceeds a specified size. + */ +public class SizeLimitingStringWriter extends StringWriter { + + public static class SizeLimitExceededException extends IllegalStateException { + public SizeLimitExceededException(String message) { + super(message); + } + } + + private final int sizeLimit; + + public SizeLimitingStringWriter(int sizeLimit) { + this.sizeLimit = sizeLimit; + } + + private void checkSizeLimit(int additionalChars) { + int bufLen = getBuffer().length(); + if (bufLen + additionalChars > sizeLimit) { + throw new SizeLimitExceededException( + Strings.format("String [%s...] has exceeded the size limit [%s]", getBuffer().substring(0, Math.min(bufLen, 20)), sizeLimit) + ); + } + } + + @Override + public void write(int c) { + checkSizeLimit(1); + super.write(c); + } + + // write(char[]) delegates to write(char[], int, int) + + @Override + public void write(char[] cbuf, int off, int len) { + checkSizeLimit(len); + super.write(cbuf, off, len); + } + + @Override + public void write(String str) { + checkSizeLimit(str.length()); + super.write(str); + } + + @Override + public void write(String str, int off, int len) { + checkSizeLimit(len); + super.write(str, off, len); + } + + // append(...) delegates to write(...) methods +} diff --git a/server/src/test/java/org/elasticsearch/common/text/SizeLimitingStringWriterTests.java b/server/src/test/java/org/elasticsearch/common/text/SizeLimitingStringWriterTests.java new file mode 100644 index 0000000000000..32a8de20df9aa --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/text/SizeLimitingStringWriterTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.common.text; + +import org.elasticsearch.test.ESTestCase; + +public class SizeLimitingStringWriterTests extends ESTestCase { + public void testSizeIsLimited() { + SizeLimitingStringWriter writer = new SizeLimitingStringWriter(10); + + writer.write("a".repeat(10)); + + // test all the methods + expectThrows(SizeLimitingStringWriter.SizeLimitExceededException.class, () -> writer.write('a')); + expectThrows(SizeLimitingStringWriter.SizeLimitExceededException.class, () -> writer.write("a")); + expectThrows(SizeLimitingStringWriter.SizeLimitExceededException.class, () -> writer.write(new char[1])); + expectThrows(SizeLimitingStringWriter.SizeLimitExceededException.class, () -> writer.write(new char[1], 0, 1)); + expectThrows(SizeLimitingStringWriter.SizeLimitExceededException.class, () -> writer.append('a')); + expectThrows(SizeLimitingStringWriter.SizeLimitExceededException.class, () -> writer.append("a")); + expectThrows(SizeLimitingStringWriter.SizeLimitExceededException.class, () -> writer.append("a", 0, 1)); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java index 195e126662481..05044303561d8 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/support/mapper/TemplateRoleNameTests.java @@ -89,7 +89,7 @@ public void testEqualsAndHashCode() throws Exception { public void testEvaluateRoles() throws Exception { final ScriptService scriptService = new ScriptService( Settings.EMPTY, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine(Settings.EMPTY)), ScriptModule.CORE_CONTEXTS, () -> 1L ); @@ -145,7 +145,7 @@ public void tryEquals(TemplateRoleName original) { public void testValidate() { final ScriptService scriptService = new ScriptService( Settings.EMPTY, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine(Settings.EMPTY)), ScriptModule.CORE_CONTEXTS, () -> 1L ); @@ -173,7 +173,7 @@ public void testValidate() { public void testValidateWillPassWithEmptyContext() { final ScriptService scriptService = new ScriptService( Settings.EMPTY, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine(Settings.EMPTY)), ScriptModule.CORE_CONTEXTS, () -> 1L ); @@ -204,7 +204,7 @@ public void testValidateWillPassWithEmptyContext() { public void testValidateWillFailForSyntaxError() { final ScriptService scriptService = new ScriptService( Settings.EMPTY, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine(Settings.EMPTY)), ScriptModule.CORE_CONTEXTS, () -> 1L ); @@ -268,7 +268,7 @@ public void testValidationWillFailWhenInlineScriptIsNotEnabled() { final Settings settings = Settings.builder().put("script.allowed_types", ScriptService.ALLOW_NONE).build(); final ScriptService scriptService = new ScriptService( settings, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine(Settings.EMPTY)), ScriptModule.CORE_CONTEXTS, () -> 1L ); @@ -285,7 +285,7 @@ public void testValidateWillFailWhenStoredScriptIsNotEnabled() { final Settings settings = Settings.builder().put("script.allowed_types", ScriptService.ALLOW_NONE).build(); final ScriptService scriptService = new ScriptService( settings, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine(Settings.EMPTY)), ScriptModule.CORE_CONTEXTS, () -> 1L ); @@ -314,7 +314,7 @@ public void testValidateWillFailWhenStoredScriptIsNotEnabled() { public void testValidateWillFailWhenStoredScriptIsNotFound() { final ScriptService scriptService = new ScriptService( Settings.EMPTY, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine(Settings.EMPTY)), ScriptModule.CORE_CONTEXTS, () -> 1L ); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluatorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluatorTests.java index fed11c75715b5..58ae99df60bcd 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluatorTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/support/SecurityQueryTemplateEvaluatorTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.core.security.authz.support; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptService; @@ -94,7 +95,7 @@ public void testDocLevelSecurityTemplateWithOpenIdConnectStyleMetadata() throws true ); - final MustacheScriptEngine mustache = new MustacheScriptEngine(); + final MustacheScriptEngine mustache = new MustacheScriptEngine(Settings.EMPTY); when(scriptService.compile(any(Script.class), eq(TemplateScript.CONTEXT))).thenAnswer(inv -> { assertThat(inv.getArguments(), arrayWithSize(2)); diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/WildcardServiceProviderResolverTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/WildcardServiceProviderResolverTests.java index 70e5325878c0a..832a6e8163aca 100644 --- a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/WildcardServiceProviderResolverTests.java +++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/WildcardServiceProviderResolverTests.java @@ -95,7 +95,7 @@ public void setUpResolver() { final Settings settings = Settings.EMPTY; final ScriptService scriptService = new ScriptService( settings, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine(Settings.EMPTY)), ScriptModule.CORE_CONTEXTS, () -> 1L ); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankServiceTests.java index 6ca9ae4296789..46e54ff3f8c3d 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/ltr/LearningToRankServiceTests.java @@ -241,7 +241,7 @@ private LearningToRankService getTestLearningToRankService(TrainedModelProvider } private ScriptService getTestScriptService() { - ScriptEngine scriptEngine = new MustacheScriptEngine(); + ScriptEngine scriptEngine = new MustacheScriptEngine(Settings.EMPTY); return new ScriptService(Settings.EMPTY, Map.of(DEFAULT_TEMPLATE_LANG, scriptEngine), ScriptModule.CORE_CONTEXTS, () -> 1L); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java index b0821864aacc7..e72bbd77697c2 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java @@ -434,7 +434,7 @@ public void testRealmWithTemplatedRoleMapping() throws Exception { final ScriptService scriptService = new ScriptService( settings, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine(Settings.EMPTY)), ScriptModule.CORE_CONTEXTS, () -> 1L ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java index 7083d1301a3e6..83e2ff88f9dc3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealmTests.java @@ -533,7 +533,7 @@ public void testLdapRealmWithTemplatedRoleMapping() throws Exception { final ScriptService scriptService = new ScriptService( defaultGlobalSettings, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine(Settings.EMPTY)), ScriptModule.CORE_CONTEXTS, () -> 1L ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapperTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapperTests.java index 515b5ef741a00..063245e004476 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapperTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ClusterStateRoleMapperTests.java @@ -51,7 +51,7 @@ public class ClusterStateRoleMapperTests extends ESTestCase { public void setup() { scriptService = new ScriptService( Settings.EMPTY, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine(Settings.EMPTY)), ScriptModule.CORE_CONTEXTS, () -> 1L ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java index 2a084bacfaf76..38f01d4d18bc7 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java @@ -87,7 +87,7 @@ public class NativeRoleMappingStoreTests extends ESTestCase { public void setup() { scriptService = new ScriptService( Settings.EMPTY, - Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine()), + Collections.singletonMap(MustacheScriptEngine.NAME, new MustacheScriptEngine(Settings.EMPTY)), ScriptModule.CORE_CONTEXTS, () -> 1L ); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java index 5ddff34a0ac45..3018afbe97338 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherTemplateTests.java @@ -37,7 +37,7 @@ public class WatcherTemplateTests extends ESTestCase { @Before public void init() throws Exception { - MustacheScriptEngine engine = new MustacheScriptEngine(); + MustacheScriptEngine engine = new MustacheScriptEngine(Settings.EMPTY); Map engines = Collections.singletonMap(engine.getType(), engine); Map> contexts = Collections.singletonMap( Watcher.SCRIPT_TEMPLATE_CONTEXT.name, From 436d4cc877a33f945ae3f4200efef3e76802623e Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 8 Oct 2024 11:44:31 +0100 Subject: [PATCH 165/194] Remove `SearchableSnapshotIndexMetadataUpgrader` (#114290) This service only exists to fix up the metadata on 7.x searchable snapshot indices during/after an upgrade to 8.x, so we don't need it in 9.x. --- .../SearchableSnapshots.java | 2 - ...archableSnapshotIndexMetadataUpgrader.java | 140 ------------ ...bleSnapshotIndexMetadataUpgraderTests.java | 205 ------------------ 3 files changed, 347 deletions(-) delete mode 100644 x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/upgrade/SearchableSnapshotIndexMetadataUpgrader.java delete mode 100644 x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/upgrade/SearchableSnapshotIndexMetadataUpgraderTests.java diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index 4eea006b4c2f2..5ac8cdb43aa33 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -108,7 +108,6 @@ import org.elasticsearch.xpack.searchablesnapshots.rest.RestSearchableSnapshotsNodeCachesStatsAction; import org.elasticsearch.xpack.searchablesnapshots.rest.RestSearchableSnapshotsStatsAction; import org.elasticsearch.xpack.searchablesnapshots.store.SearchableSnapshotDirectory; -import org.elasticsearch.xpack.searchablesnapshots.upgrade.SearchableSnapshotIndexMetadataUpgrader; import java.io.IOException; import java.io.UncheckedIOException; @@ -359,7 +358,6 @@ public Collection createComponents(PluginServices services) { components.add(new FrozenCacheServiceSupplier(frozenCacheService.get())); components.add(new CacheServiceSupplier(cacheService.get())); if (DiscoveryNode.isMasterNode(settings)) { - new SearchableSnapshotIndexMetadataUpgrader(clusterService, threadPool).initialize(); clusterService.addListener(new RepositoryUuidWatcher(services.rerouteService())); } return Collections.unmodifiableList(components); diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/upgrade/SearchableSnapshotIndexMetadataUpgrader.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/upgrade/SearchableSnapshotIndexMetadataUpgrader.java deleted file mode 100644 index ccdad61adee52..0000000000000 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/upgrade/SearchableSnapshotIndexMetadataUpgrader.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.searchablesnapshots.upgrade; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateListener; -import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.index.IndexVersions; -import org.elasticsearch.indices.ShardLimitValidator; -import org.elasticsearch.threadpool.ThreadPool; - -import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * This class upgrades frozen indices to apply the index.shard_limit.group=frozen setting after all nodes have been upgraded to 7.13+ - */ -public class SearchableSnapshotIndexMetadataUpgrader { - private static final Logger logger = LogManager.getLogger(SearchableSnapshotIndexMetadataUpgrader.class); - - private final ClusterService clusterService; - private final ThreadPool threadPool; - private final AtomicBoolean upgraded = new AtomicBoolean(); - private final ClusterStateListener listener = this::clusterChanged; - - public SearchableSnapshotIndexMetadataUpgrader(ClusterService clusterService, ThreadPool threadPool) { - this.clusterService = clusterService; - this.threadPool = threadPool; - } - - public void initialize() { - clusterService.addListener(listener); - } - - private void clusterChanged(ClusterChangedEvent event) { - if (upgraded.get()) { - return; - } - - if (event.localNodeMaster()) { - // only want one doing this at a time, assume it succeeds and reset if not. - if (upgraded.compareAndSet(false, true)) { - final Executor executor = threadPool.generic(); - executor.execute(() -> maybeUpgradeIndices(event.state())); - } - } - } - - private void maybeUpgradeIndices(ClusterState state) { - // 99% of the time, this will be a noop, so precheck that before adding a cluster state update. - if (needsUpgrade(state)) { - logger.info("Upgrading partial searchable snapshots to use frozen shard limit group"); - submitUnbatchedTask("searchable-snapshot-index-upgrader", new ClusterStateUpdateTask() { - @Override - public ClusterState execute(ClusterState currentState) throws Exception { - return upgradeIndices(currentState); - } - - @Override - public void clusterStateProcessed(ClusterState oldState, ClusterState newState) { - clusterService.removeListener(listener); - } - - @Override - public void onFailure(Exception e) { - logger.warn( - "upgrading frozen indices to have frozen shard limit group failed, will retry on the next cluster state update", - e - ); - // let us try again later. - upgraded.set(false); - } - }); - } else { - clusterService.removeListener(listener); - } - } - - @SuppressForbidden(reason = "legacy usage of unbatched task") // TODO add support for batching here - private void submitUnbatchedTask(@SuppressWarnings("SameParameterValue") String source, ClusterStateUpdateTask task) { - clusterService.submitUnbatchedStateUpdateTask(source, task); - } - - static boolean needsUpgrade(ClusterState state) { - return state.metadata() - .stream() - .filter( - imd -> imd.getCompatibilityVersion().onOrAfter(IndexVersions.V_7_12_0) - && imd.getCompatibilityVersion().before(IndexVersions.V_8_0_0) - ) - .filter(IndexMetadata::isPartialSearchableSnapshot) - .map(IndexMetadata::getSettings) - .anyMatch(SearchableSnapshotIndexMetadataUpgrader::notFrozenShardLimitGroup); - } - - static ClusterState upgradeIndices(ClusterState currentState) { - if (needsUpgrade(currentState) == false) { - return currentState; - } - Metadata.Builder builder = Metadata.builder(currentState.metadata()); - currentState.metadata() - .stream() - .filter( - imd -> imd.getCompatibilityVersion().onOrAfter(IndexVersions.V_7_12_0) - && imd.getCompatibilityVersion().before(IndexVersions.V_8_0_0) - ) - .filter(imd -> imd.isPartialSearchableSnapshot() && notFrozenShardLimitGroup(imd.getSettings())) - .map(SearchableSnapshotIndexMetadataUpgrader::setShardLimitGroupFrozen) - .forEach(imd -> builder.put(imd, true)); - return ClusterState.builder(currentState).metadata(builder).build(); - } - - private static boolean notFrozenShardLimitGroup(org.elasticsearch.common.settings.Settings settings) { - return ShardLimitValidator.FROZEN_GROUP.equals(ShardLimitValidator.INDEX_SETTING_SHARD_LIMIT_GROUP.get(settings)) == false; - } - - private static IndexMetadata setShardLimitGroupFrozen(IndexMetadata indexMetadata) { - return IndexMetadata.builder(indexMetadata) - .settings( - Settings.builder() - .put(indexMetadata.getSettings()) - .put(ShardLimitValidator.INDEX_SETTING_SHARD_LIMIT_GROUP.getKey(), ShardLimitValidator.FROZEN_GROUP) - ) - .settingsVersion(indexMetadata.getSettingsVersion() + 1) - .build(); - } -} diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/upgrade/SearchableSnapshotIndexMetadataUpgraderTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/upgrade/SearchableSnapshotIndexMetadataUpgraderTests.java deleted file mode 100644 index 8de5a37102846..0000000000000 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/upgrade/SearchableSnapshotIndexMetadataUpgraderTests.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.searchablesnapshots.upgrade; - -import org.apache.lucene.tests.util.LuceneTestCase; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.UpdateForV9; -import org.elasticsearch.index.IndexModule; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.IndexVersions; -import org.elasticsearch.indices.ShardLimitValidator; -import org.elasticsearch.snapshots.SearchableSnapshotsSettings; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.index.IndexVersionUtils; - -import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.sameInstance; - -@UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) -@LuceneTestCase.AwaitsFix(bugUrl = "this testing a number of pre 8.0 upgrade scenarios so needs updating or removal for 9.0") -public class SearchableSnapshotIndexMetadataUpgraderTests extends ESTestCase { - - public void testNoUpgradeNeeded() { - Metadata.Builder metadataBuilder = randomMetadata( - normal(), - full(), - partial_8plusNoShardLimit(), - shardLimitGroupFrozen(partial_7_13plus()), - shardLimitGroupFrozen(partialNeedsUpgrade()) - ); - assertThat(needsUpgrade(metadataBuilder), is(false)); - } - - public void testNeedsUpgrade() { - assertThat( - needsUpgrade( - addIndex( - partialNeedsUpgrade(), - randomMetadata( - normal(), - full(), - partial_7_13plus(), - partialNeedsUpgrade(), - shardLimitGroupFrozen(partialNeedsUpgrade()) - ) - ) - ), - is(true) - ); - } - - public void testUpgradeIndices() { - Metadata.Builder metadataBuilder = addIndex( - partialNeedsUpgrade(), - randomMetadata(normal(), full(), partial_7_13plus(), partialNeedsUpgrade(), shardLimitGroupFrozen(partialNeedsUpgrade())) - ); - - ClusterState originalState = clusterState(metadataBuilder); - ClusterState upgradedState = SearchableSnapshotIndexMetadataUpgrader.upgradeIndices(originalState); - - assertThat(upgradedState, not(sameInstance(originalState))); - assertThat(upgradedState.metadata().indices().size(), equalTo(originalState.metadata().indices().size())); - - assertTrue(upgradedState.metadata().stream().anyMatch(upgraded -> { - IndexMetadata original = originalState.metadata().index(upgraded.getIndex()); - assertThat(original, notNullValue()); - if (upgraded.isPartialSearchableSnapshot() == false - || ShardLimitValidator.INDEX_SETTING_SHARD_LIMIT_GROUP.get(original.getSettings()) - .equals(ShardLimitValidator.FROZEN_GROUP)) { - assertThat(upgraded, sameInstance(original)); - return false; - } else { - assertThat(upgraded.isPartialSearchableSnapshot(), is(original.isPartialSearchableSnapshot())); - assertThat(upgraded.getNumberOfShards(), equalTo(original.getNumberOfShards())); - assertThat(upgraded.getNumberOfReplicas(), equalTo(original.getNumberOfReplicas())); - assertThat( - ShardLimitValidator.INDEX_SETTING_SHARD_LIMIT_GROUP.get(upgraded.getSettings()), - equalTo(ShardLimitValidator.FROZEN_GROUP) - ); - assertThat(upgraded.getSettingsVersion(), equalTo(original.getSettingsVersion() + 1)); - return true; - } - })); - - assertThat(SearchableSnapshotIndexMetadataUpgrader.needsUpgrade(upgradedState), is(false)); - } - - public void testNoopUpgrade() { - Metadata.Builder metadataBuilder = randomMetadata( - normal(), - full(), - partial_7_13plus(), - shardLimitGroupFrozen(partialNeedsUpgrade()), - partial_8plusNoShardLimit() - ); - ClusterState originalState = clusterState(metadataBuilder); - ClusterState upgradedState = SearchableSnapshotIndexMetadataUpgrader.upgradeIndices(originalState); - assertThat(upgradedState, sameInstance(originalState)); - } - - private Settings normal() { - return settings(IndexVersionUtils.randomVersion(random())).build(); - } - - /** - * Simulate an index mounted with no shard limit group. Notice that due to not applying the group during rolling upgrades, we can see - * other than 7.12 versions here, but not 8.0 (since a rolling upgrade to 8.0 requires an upgrade to 7.latest first). - */ - private Settings partialNeedsUpgrade() { - return searchableSnapshotSettings( - IndexVersionUtils.randomVersionBetween( - random(), - IndexVersions.V_7_12_0, - IndexVersionUtils.getPreviousVersion(IndexVersions.V_8_0_0) - ), - true - ); - } - - /** - * Simulate a 7.13plus mounted index with shard limit. - */ - private Settings partial_7_13plus() { - return shardLimitGroupFrozen( - searchableSnapshotSettings( - IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_7_13_0, IndexVersion.current()), - true - ) - ); - } - - /** - * This is an illegal state, but we simulate it to capture that we do the version check - */ - private Settings partial_8plusNoShardLimit() { - return searchableSnapshotSettings( - IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_8_0_0, IndexVersion.current()), - true - ); - } - - private Settings full() { - return searchableSnapshotSettings(IndexVersionUtils.randomVersion(random()), false); - } - - private Settings searchableSnapshotSettings(IndexVersion version, boolean partial) { - Settings.Builder settings = settings(version); - settings.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), SEARCHABLE_SNAPSHOT_STORE_TYPE); - if (partial || randomBoolean()) { - settings.put(SearchableSnapshotsSettings.SNAPSHOT_PARTIAL_SETTING.getKey(), partial); - } - return settings.build(); - } - - private Settings shardLimitGroupFrozen(Settings settings) { - return Settings.builder() - .put(settings) - .put(ShardLimitValidator.INDEX_SETTING_SHARD_LIMIT_GROUP.getKey(), ShardLimitValidator.FROZEN_GROUP) - .build(); - } - - private Metadata.Builder addIndex(Settings settings, Metadata.Builder builder) { - builder.put( - IndexMetadata.builder(randomAlphaOfLength(10)) - .settings(settings) - .numberOfShards(between(1, 10)) - .numberOfReplicas(between(0, 10)) - .build(), - false - ); - return builder; - } - - private Metadata.Builder randomMetadata(Settings... indexSettingsList) { - Metadata.Builder builder = new Metadata.Builder(); - for (Settings settings : indexSettingsList) { - for (int i = 0; i < between(0, 10); ++i) { - addIndex(settings, builder); - } - } - return builder; - } - - private boolean needsUpgrade(Metadata.Builder metadataBuilder) { - return SearchableSnapshotIndexMetadataUpgrader.needsUpgrade(clusterState(metadataBuilder)); - } - - private ClusterState clusterState(Metadata.Builder metadataBuilder) { - return ClusterState.builder(ClusterName.DEFAULT).metadata(metadataBuilder).build(); - } - -} From ce07060dce69f961c0906079529e91c7dd7d4b48 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 8 Oct 2024 13:09:39 +0200 Subject: [PATCH 166/194] Remove role mapping block check (#114223) This method is used in multiple contexts that may not all handle cluster-block exceptions gracefully, esp. since some types of cluster blocks are retryable. Removing this, and may follow up with handling the cluster state block for the affected transport actions instead. --- .../xpack/core/security/authz/RoleMappingMetadata.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java index 8f78fdbccd923..da6ff6ad24c34 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java @@ -12,7 +12,6 @@ import org.elasticsearch.cluster.AbstractNamedDiffable; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.NamedDiff; -import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; @@ -58,7 +57,6 @@ public final class RoleMappingMetadata extends AbstractNamedDiffable Date: Tue, 8 Oct 2024 22:16:13 +1100 Subject: [PATCH 167/194] Mute org.elasticsearch.index.SearchSlowLogTests testLevelPrecedence #114300 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 93893d7103afb..81ec1c414b0ca 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -370,6 +370,9 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/114188 - class: org.elasticsearch.ingest.geoip.IpinfoIpDataLookupsTests issue: https://github.com/elastic/elasticsearch/issues/114266 +- class: org.elasticsearch.index.SearchSlowLogTests + method: testLevelPrecedence + issue: https://github.com/elastic/elasticsearch/issues/114300 # Examples: # From ffea594158ed39afd62d94c88334cca5992632c7 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Tue, 8 Oct 2024 22:16:31 +1100 Subject: [PATCH 168/194] Mute org.elasticsearch.index.SearchSlowLogTests testTwoLoggersDifferentLevel #114301 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 81ec1c414b0ca..d6d5d47af3a76 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -373,6 +373,9 @@ tests: - class: org.elasticsearch.index.SearchSlowLogTests method: testLevelPrecedence issue: https://github.com/elastic/elasticsearch/issues/114300 +- class: org.elasticsearch.index.SearchSlowLogTests + method: testTwoLoggersDifferentLevel + issue: https://github.com/elastic/elasticsearch/issues/114301 # Examples: # From b80272a1a4d457ba0a950c5f01975c8b280da58c Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Tue, 8 Oct 2024 13:51:02 +0200 Subject: [PATCH 169/194] [DOCS] Update URL (#114292) --- docs/reference/intro.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/intro.asciidoc b/docs/reference/intro.asciidoc index 21b20a7e2f5d0..2908c55789bab 100644 --- a/docs/reference/intro.asciidoc +++ b/docs/reference/intro.asciidoc @@ -204,7 +204,7 @@ For general content, you have the following options for adding data to {es} indi If you're building a website or app, then you can call Elasticsearch APIs using an https://www.elastic.co/guide/en/elasticsearch/client/index.html[{es} client] in the programming language of your choice. If you use the Python client, then check out the `elasticsearch-labs` repo for various https://github.com/elastic/elasticsearch-labs/tree/main/notebooks/search/python-examples[example notebooks]. * {kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[File upload]: Use the {kib} file uploader to index single files for one-off testing and exploration. The GUI guides you through setting up your index and field mappings. * https://github.com/elastic/crawler[Web crawler]: Extract and index web page content into {es} documents. -* {enterprise-search-ref}/connectors.html[Connectors]: Sync data from various third-party data sources to create searchable, read-only replicas in {es}. +* <>: Sync data from various third-party data sources to create searchable, read-only replicas in {es}. [discrete] [[es-ingestion-overview-timestamped]] @@ -492,4 +492,4 @@ and restrictions. You can review the following guides to learn how to tune your * <> Many {es} options come with different performance considerations and trade-offs. The best way to determine the -optimal configuration for your use case is through https://www.elastic.co/elasticon/conf/2016/sf/quantitative-cluster-sizing[testing with your own data and queries]. \ No newline at end of file +optimal configuration for your use case is through https://www.elastic.co/elasticon/conf/2016/sf/quantitative-cluster-sizing[testing with your own data and queries]. From 10f6f255066590f9425081249a59ef72d16b1d8d Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:08:47 +0200 Subject: [PATCH 170/194] [DOCS] Update re-ranking intro to remove confusion about stages (#114302) --- docs/reference/reranking/index.asciidoc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/reference/reranking/index.asciidoc b/docs/reference/reranking/index.asciidoc index cc6f4a9007424..3171be7c872d4 100644 --- a/docs/reference/reranking/index.asciidoc +++ b/docs/reference/reranking/index.asciidoc @@ -1,12 +1,12 @@ [[re-ranking-overview]] = Re-ranking -Many search systems are built on two-stage retrieval pipelines. +Many search systems are built on multi-stage retrieval pipelines. -The first stage uses cheap, fast algorithms to find a broad set of possible matches. +Earlier stages use cheap, fast algorithms to find a broad set of possible matches. -The second stage uses a more powerful model, often machine learning-based, to reorder the documents. -This second step is called re-ranking. +Later stages use more powerful models, often machine learning-based, to reorder the documents. +This step is called re-ranking. Because the resource-intensive model is only applied to the smaller set of pre-filtered results, this approach returns more relevant results while still optimizing for search performance and computational costs. {es} supports various ranking and re-ranking techniques to optimize search relevance and performance. @@ -18,7 +18,7 @@ Because the resource-intensive model is only applied to the smaller set of pre-f [float] [[re-ranking-first-stage-pipeline]] -=== First stage: initial retrieval +=== Initial retrieval [float] [[re-ranking-ranking-overview-bm25]] @@ -45,7 +45,7 @@ Hybrid search techniques combine results from full-text and vector search pipeli [float] [[re-ranking-overview-second-stage]] -=== Second stage: Re-ranking +=== Re-ranking When using the following advanced re-ranking pipelines, first-stage retrieval mechanisms effectively generate a set of candidates. These candidates are funneled into the re-ranker to perform more computationally expensive re-ranking tasks. @@ -67,4 +67,4 @@ Learning To Rank involves training a machine learning model to build a ranking f LTR is best suited for when you have ample training data and need highly customized relevance tuning. include::semantic-reranking.asciidoc[] -include::learning-to-rank.asciidoc[] \ No newline at end of file +include::learning-to-rank.asciidoc[] From 6f518d437f528d794715db049a3b0f155e5317f9 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 8 Oct 2024 14:14:35 +0100 Subject: [PATCH 171/194] Remove unnecessary test overrides (#114291) These test overrides were introduced so that we had somewhere to hang an `@AwaitsFix` annotation, but now the tests are unmuted again there's no need for the overrides. Relates #108336 --- .../AzureStorageCleanupThirdPartyTests.java | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java index 7d280f31ecf19..abd4f506a0bb3 100644 --- a/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java +++ b/modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java @@ -60,31 +60,6 @@ public class AzureStorageCleanupThirdPartyTests extends AbstractThirdPartyReposi AzureHttpFixture.sharedKeyForAccountPredicate(AZURE_ACCOUNT) ); - @Override - public void testCreateSnapshot() { - super.testCreateSnapshot(); - } - - @Override - public void testIndexLatest() throws Exception { - super.testIndexLatest(); - } - - @Override - public void testListChildren() { - super.testListChildren(); - } - - @Override - public void testCleanup() throws Exception { - super.testCleanup(); - } - - @Override - public void testReadFromPositionWithLength() { - super.testReadFromPositionWithLength(); - } - @Override protected Collection> getPlugins() { return pluginList(AzureRepositoryPlugin.class); From d9dc165db24a9a99664fbde49ac8b321f414b918 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Tue, 8 Oct 2024 14:32:57 +0100 Subject: [PATCH 172/194] Further conversions to ChunkedXContentBuilder (#114237) --- .../reroute/ClusterRerouteResponse.java | 29 ++--- .../elasticsearch/cluster/ClusterState.java | 6 +- .../xcontent/ChunkedToXContentBuilder.java | 20 +++ .../xcontent/ChunkedToXContentHelper.java | 16 --- .../script/ScriptCacheStats.java | 57 +++------ .../reroute/ClusterRerouteResponseTests.java | 14 ++- .../ChunkedToXContentBuilderTests.java | 88 +++++++++++++ .../ChunkedToXContentHelperTests.java | 119 ------------------ .../xpack/esql/action/EsqlQueryResponse.java | 77 ++++-------- .../esql/action/EsqlQueryResponseTests.java | 8 +- .../shutdown/SingleNodeShutdownStatus.java | 51 ++++---- 11 files changed, 203 insertions(+), 282 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilderTests.java delete mode 100644 server/src/test/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelperTests.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java index b0ec0968f8d1d..7b344a4c25a1b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java @@ -13,12 +13,11 @@ import org.elasticsearch.action.support.master.IsAcknowledgedSupplier; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.allocation.RoutingExplanations; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.UpdateForV10; @@ -26,7 +25,6 @@ import org.elasticsearch.xcontent.ToXContent; import java.io.IOException; -import java.util.Collections; import java.util.Iterator; import java.util.Objects; @@ -98,20 +96,15 @@ public Iterator toXContentChunked(ToXContent.Params outerP } @Override - public Iterator toXContentChunkedV7(ToXContent.Params outerParams) { - return Iterators.concat( - Iterators.single((builder, params) -> builder.startObject().field(ACKNOWLEDGED_KEY, isAcknowledged())), - emitState(outerParams) - ? ChunkedToXContentHelper.wrapWithObject("state", state.toXContentChunked(outerParams)) - : Collections.emptyIterator(), - Iterators.single((builder, params) -> { - if (params.paramAsBoolean("explain", false)) { - explanations.toXContent(builder, params); - } - - builder.endObject(); - return builder; - }) - ); + public Iterator toXContentChunkedV7(ToXContent.Params params) { + return ChunkedToXContent.builder(params).object(b -> { + b.field(ACKNOWLEDGED_KEY, isAcknowledged()); + if (emitState(params)) { + b.xContentObject("state", state); + } + if (params.paramAsBoolean("explain", false)) { + b.append(explanations); + } + }); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index cafda93dda9a5..64df6e77326e4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -759,10 +759,8 @@ public Iterator toXContentChunked(ToXContent.Params outerP // customs metrics.contains(Metric.CUSTOMS) - ? Iterators.flatMap( - customs.entrySet().iterator(), - cursor -> ChunkedToXContentHelper.wrapWithObject(cursor.getKey(), cursor.getValue().toXContentChunked(outerParams)) - ) + ? ChunkedToXContent.builder(outerParams) + .forEach(customs.entrySet().iterator(), (b, e) -> b.xContentObject(e.getKey(), e.getValue())) : Collections.emptyIterator() ); } diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilder.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilder.java index 0868a7fa303ae..0102e58c7c1dc 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilder.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilder.java @@ -38,6 +38,10 @@ private void addChunk(ToXContent content) { builder.add(Objects.requireNonNull(content)); } + public ToXContent.Params params() { + return params; + } + private void startObject() { addChunk((b, p) -> b.startObject()); } @@ -259,6 +263,16 @@ public ChunkedToXContentBuilder array(Iterator items, BiConsumer items) { + startArray(); + items.forEachRemaining(this::append); + endArray(); + return this; + } + /** * Creates an array, with the contents set by appending together * the return values of {@code create} called on each item returned by {@code items} @@ -351,6 +365,12 @@ public ChunkedToXContentBuilder field(String name, ToXContent value) { return this; } + public ChunkedToXContentBuilder field(String name, ChunkedToXContent value) { + addChunk((b, p) -> b.field(name)); + append(value); + return this; + } + public ChunkedToXContentBuilder field(String name, Object value) { addChunk((b, p) -> b.field(name, value)); return this; diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java index 940d4495ae909..fcbe0ac2b2edb 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java @@ -53,26 +53,10 @@ public static Iterator field(String name, String value) { return Iterators.single(((builder, params) -> builder.field(name, value))); } - /** - * Creates an Iterator to serialize a named field where the value is represented by a {@link ChunkedToXContentObject}. - * Chunked equivalent for {@code XContentBuilder field(String name, ToXContent value)} - * @param name name of the field - * @param value value for this field - * @param params params to propagate for XContent serialization - * @return Iterator composing field name and value serialization - */ - public static Iterator field(String name, ChunkedToXContentObject value, ToXContent.Params params) { - return Iterators.concat(Iterators.single((builder, innerParam) -> builder.field(name)), value.toXContentChunked(params)); - } - public static Iterator array(String name, Iterator contents) { return Iterators.concat(ChunkedToXContentHelper.startArray(name), contents, ChunkedToXContentHelper.endArray()); } - public static Iterator wrapWithObject(String name, Iterator iterator) { - return Iterators.concat(startObject(name), iterator, endObject()); - } - /** * Creates an Iterator of a single ToXContent object that serializes the given object as a single chunk. Just wraps {@link * Iterators#single}, but still useful because it avoids any type ambiguity. diff --git a/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java b/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java index a9a89a3fa7610..adc1f65b88732 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptCacheStats.java @@ -21,15 +21,8 @@ import java.util.Iterator; import java.util.Map; import java.util.Objects; +import java.util.function.Function; -import static org.elasticsearch.common.collect.Iterators.concat; -import static org.elasticsearch.common.collect.Iterators.flatMap; -import static org.elasticsearch.common.collect.Iterators.single; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.endArray; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.endObject; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.field; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.startArray; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.startObject; import static org.elasticsearch.script.ScriptCacheStats.Fields.SCRIPT_CACHE_STATS; // This class is deprecated in favor of ScriptStats and ScriptContextStats @@ -76,35 +69,25 @@ public void writeTo(StreamOutput out) throws IOException { @Override public Iterator toXContentChunked(ToXContent.Params outerParams) { - return concat( - startObject(SCRIPT_CACHE_STATS), - startObject(Fields.SUM), - general != null - ? concat( - field(ScriptStats.Fields.COMPILATIONS, general.getCompilations()), - field(ScriptStats.Fields.CACHE_EVICTIONS, general.getCacheEvictions()), - field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, general.getCompilationLimitTriggered()), - endObject(), - endObject() - ) - : concat(single((builder, params) -> { - var sum = sum(); - return builder.field(ScriptStats.Fields.COMPILATIONS, sum.getCompilations()) - .field(ScriptStats.Fields.CACHE_EVICTIONS, sum.getCacheEvictions()) - .field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, sum.getCompilationLimitTriggered()) - .endObject(); - }), startArray(Fields.CONTEXTS), flatMap(context.keySet().stream().sorted().iterator(), ctx -> { - var stats = context.get(ctx); - return concat( - startObject(), - field(Fields.CONTEXT, ctx), - field(ScriptStats.Fields.COMPILATIONS, stats.getCompilations()), - field(ScriptStats.Fields.CACHE_EVICTIONS, stats.getCacheEvictions()), - field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, stats.getCompilationLimitTriggered()), - endObject() - ); - }), endArray(), endObject()) - ); + Function statsFields = s -> (b, p) -> b.field(ScriptStats.Fields.COMPILATIONS, s.getCompilations()) + .field(ScriptStats.Fields.CACHE_EVICTIONS, s.getCacheEvictions()) + .field(ScriptStats.Fields.COMPILATION_LIMIT_TRIGGERED, s.getCompilationLimitTriggered()); + + return ChunkedToXContent.builder(outerParams).object(SCRIPT_CACHE_STATS, sb -> { + if (general != null) { + sb.xContentObject(Fields.SUM, statsFields.apply(general)); + } else { + sb.xContentObject(Fields.SUM, statsFields.apply(sum())); + sb.array( + Fields.CONTEXTS, + context.entrySet().stream().sorted(Map.Entry.comparingByKey()).iterator(), + (eb, e) -> eb.object(ebo -> { + ebo.field(Fields.CONTEXT, e.getKey()); + ebo.append(statsFields.apply(e.getValue())); + }) + ); + } + }); } /** diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java index 8adf18cd82f5c..67e5f30f023c9 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java @@ -313,11 +313,15 @@ private void assertXContent( fail(e); } - final var expectedChunks = Objects.equals(params.param("metric"), "none") - ? 2 - : 4 + ClusterStateTests.expectedChunkCount(params, response.getState()); + int[] expectedChunks = new int[] { 3 }; + if (Objects.equals(params.param("metric"), "none") == false) { + expectedChunks[0] += 2 + ClusterStateTests.expectedChunkCount(params, response.getState()); + } + if (params.paramAsBoolean("explain", false)) { + expectedChunks[0]++; + } - AbstractChunkedSerializingTestCase.assertChunkCount(response, params, ignored -> expectedChunks); + AbstractChunkedSerializingTestCase.assertChunkCount(response, params, o -> expectedChunks[0]); assertCriticalWarnings(criticalDeprecationWarnings); // check the v7 API too @@ -331,7 +335,7 @@ public Iterator toXContentChunked(ToXContent.Params outerP public boolean isFragment() { return response.isFragment(); } - }, params, ignored -> expectedChunks); + }, params, o -> expectedChunks[0]++); // the v7 API should not emit any deprecation warnings assertCriticalWarnings(); } diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilderTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilderTests.java new file mode 100644 index 0000000000000..ff811f5d6d736 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/xcontent/ChunkedToXContentBuilderTests.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.common.xcontent; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.ToXContent; + +import java.util.function.IntFunction; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.equalTo; + +public class ChunkedToXContentBuilderTests extends ESTestCase { + + public void testFieldWithInnerChunkedObject() { + + ToXContent innerXContent = (b, p) -> { + b.startObject(); + b.field("field1", 10); + b.field("field2", "aaa"); + b.endObject(); + return b; + }; + + ToXContent outerXContent = (b, p) -> b.field("field3", 10).field("field4", innerXContent); + + String expectedContent = Strings.toString(outerXContent); + + ChunkedToXContentObject innerChunkedContent = params -> new ChunkedToXContentBuilder(params).object( + o -> o.field("field1", 10).field("field2", "aaa") + ); + + ChunkedToXContent outerChunkedContent = params -> new ChunkedToXContentBuilder(params).field("field3", 10) + .field("field4", innerChunkedContent); + + assertThat(Strings.toString(outerChunkedContent), equalTo(expectedContent)); + } + + public void testFieldWithInnerChunkedArray() { + + ToXContent innerXContent = (b, p) -> { + b.startArray(); + b.value(10); + b.value(20); + b.endArray(); + return b; + }; + + ToXContent outerXContent = (b, p) -> b.field("field3", 10).field("field4", innerXContent); + + String expectedContent = Strings.toString(outerXContent); + + IntFunction value = v -> (b, p) -> b.value(v); + + ChunkedToXContentObject innerChunkedContent = params -> new ChunkedToXContentBuilder(params).array( + IntStream.of(10, 20).mapToObj(value).iterator() + ); + + ChunkedToXContent outerChunkedContent = params -> new ChunkedToXContentBuilder(params).field("field3", 10) + .field("field4", innerChunkedContent); + + assertThat(Strings.toString(outerChunkedContent), equalTo(expectedContent)); + } + + public void testFieldWithInnerChunkedField() { + + ToXContent innerXContent = (b, p) -> b.value(10); + ToXContent outerXContent = (b, p) -> b.field("field3", 10).field("field4", innerXContent); + + String expectedContent = Strings.toString(outerXContent); + + ChunkedToXContentObject innerChunkedContent = params -> Iterators.single((b, p) -> b.value(10)); + + ChunkedToXContent outerChunkedContent = params -> new ChunkedToXContentBuilder(params).field("field3", 10) + .field("field4", innerChunkedContent); + + assertThat(Strings.toString(outerChunkedContent), equalTo(expectedContent)); + } +} diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelperTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelperTests.java deleted file mode 100644 index 353725fbd0756..0000000000000 --- a/server/src/test/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelperTests.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.common.xcontent; - -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Iterators; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xcontent.ToXContent; - -import java.util.Iterator; -import java.util.function.IntFunction; - -import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; -import static org.hamcrest.Matchers.equalTo; - -public class ChunkedToXContentHelperTests extends ESTestCase { - - public void testFieldWithInnerChunkedObject() { - - ToXContent innerXContent = (builder, p) -> { - builder.startObject(); - builder.field("field1", 10); - builder.field("field2", "aaa"); - builder.endObject(); - return builder; - }; - - ToXContent outerXContent = (builder, p) -> { - builder.field("field3", 10); - builder.field("field4", innerXContent); - return builder; - }; - - var expectedContent = Strings.toString(outerXContent); - - ChunkedToXContentObject innerChunkedContent = params -> Iterators.concat( - ChunkedToXContentHelper.startObject(), - ChunkedToXContentHelper.field("field1", 10), - ChunkedToXContentHelper.field("field2", "aaa"), - ChunkedToXContentHelper.endObject() - ); - - ChunkedToXContent outerChunkedContent = params -> Iterators.concat( - ChunkedToXContentHelper.field("field3", 10), - ChunkedToXContentHelper.field("field4", innerChunkedContent, EMPTY_PARAMS) - ); - - assertThat(Strings.toString(outerChunkedContent), equalTo(expectedContent)); - } - - public void testFieldWithInnerChunkedArray() { - - ToXContent innerXContent = (builder, p) -> { - builder.startArray(); - builder.value(10); - builder.value(20); - builder.endArray(); - return builder; - }; - - ToXContent outerXContent = (builder, p) -> { - builder.field("field3", 10); - builder.field("field4", innerXContent); - return builder; - }; - - var expectedContent = Strings.toString(outerXContent); - - IntFunction> value = v -> Iterators.single(((builder, p) -> builder.value(v))); - - ChunkedToXContentObject innerChunkedContent = params -> Iterators.concat( - ChunkedToXContentHelper.startArray(), - value.apply(10), - value.apply(20), - ChunkedToXContentHelper.endArray() - ); - - ChunkedToXContent outerChunkedContent = params -> Iterators.concat( - ChunkedToXContentHelper.field("field3", 10), - ChunkedToXContentHelper.field("field4", innerChunkedContent, EMPTY_PARAMS) - ); - - assertThat(Strings.toString(outerChunkedContent), equalTo(expectedContent)); - } - - public void testFieldWithInnerChunkedField() { - - ToXContent innerXContent = (builder, p) -> { - builder.value(10); - return builder; - }; - - ToXContent outerXContent = (builder, p) -> { - builder.field("field3", 10); - builder.field("field4", innerXContent); - return builder; - }; - - var expectedContent = Strings.toString(outerXContent); - - IntFunction> value = v -> Iterators.single(((builder, p) -> builder.value(v))); - - ChunkedToXContentObject innerChunkedContent = params -> Iterators.single((builder, p) -> builder.value(10)); - - ChunkedToXContent outerChunkedContent = params -> Iterators.concat( - ChunkedToXContentHelper.field("field3", 10), - ChunkedToXContentHelper.field("field4", innerChunkedContent, EMPTY_PARAMS) - ); - - assertThat(Strings.toString(outerChunkedContent), equalTo(expectedContent)); - } -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java index 8e4da3f138a6f..3232f3a9118d4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.ChunkedToXContentBuilder; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BlockStreamInput; @@ -30,7 +29,6 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import java.io.IOException; -import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Objects; @@ -186,58 +184,35 @@ public EsqlExecutionInfo getExecutionInfo() { return executionInfo; } - private Iterator asyncPropertiesOrEmpty() { - if (isAsync) { - return ChunkedToXContentHelper.singleChunk((builder, params) -> { - if (asyncExecutionId != null) { - builder.field("id", asyncExecutionId); - } - builder.field("is_running", isRunning); - return builder; - }); - } else { - return Collections.emptyIterator(); - } - } - @Override public Iterator toXContentChunked(ToXContent.Params params) { - boolean dropNullColumns = params.paramAsBoolean(DROP_NULL_COLUMNS_OPTION, false); - boolean[] nullColumns = dropNullColumns ? nullColumns() : null; - - Iterator tookTime; - if (executionInfo != null && executionInfo.overallTook() != null) { - tookTime = ChunkedToXContentHelper.singleChunk((builder, p) -> { - builder.field("took", executionInfo.overallTook().millis()); - return builder; - }); - } else { - tookTime = Collections.emptyIterator(); - } + return ChunkedToXContent.builder(params).object(b -> { + boolean dropNullColumns = b.params().paramAsBoolean(DROP_NULL_COLUMNS_OPTION, false); + boolean[] nullColumns = dropNullColumns ? nullColumns() : null; - Iterator columnHeadings = dropNullColumns - ? Iterators.concat( - ResponseXContentUtils.allColumns(columns, "all_columns"), - ResponseXContentUtils.nonNullColumns(columns, nullColumns, "columns") - ) - : ResponseXContentUtils.allColumns(columns, "columns"); - Iterator valuesIt = ResponseXContentUtils.columnValues(this.columns, this.pages, columnar, nullColumns); - Iterator profileRender = profile == null - ? List.of().iterator() - : ChunkedToXContentHelper.field("profile", profile, params); - Iterator executionInfoRender = executionInfo == null || executionInfo.isCrossClusterSearch() == false - ? List.of().iterator() - : ChunkedToXContentHelper.field("_clusters", executionInfo, params); - return Iterators.concat( - ChunkedToXContentHelper.startObject(), - asyncPropertiesOrEmpty(), - tookTime, - columnHeadings, - ChunkedToXContentHelper.array("values", valuesIt), - executionInfoRender, - profileRender, - ChunkedToXContentHelper.endObject() - ); + if (isAsync) { + if (asyncExecutionId != null) { + b.field("id", asyncExecutionId); + } + b.field("is_running", isRunning); + } + if (executionInfo != null && executionInfo.overallTook() != null) { + b.field("took", executionInfo.overallTook().millis()); + } + if (dropNullColumns) { + b.append(ResponseXContentUtils.allColumns(columns, "all_columns")) + .append(ResponseXContentUtils.nonNullColumns(columns, nullColumns, "columns")); + } else { + b.append(ResponseXContentUtils.allColumns(columns, "columns")); + } + b.array("values", ResponseXContentUtils.columnValues(this.columns, this.pages, columnar, nullColumns)); + if (executionInfo != null && executionInfo.isCrossClusterSearch()) { + b.field("_clusters", executionInfo); + } + if (profile != null) { + b.field("profile", profile); + } + }); } private boolean[] nullColumns() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index 7c0b6e6a2eaa3..abf03d4fe06dd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -525,19 +525,19 @@ public void testChunkResponseSizeColumnar() { try (EsqlQueryResponse resp = randomResponseAsync(true, null, true)) { int columnCount = resp.pages().get(0).getBlockCount(); int bodySize = resp.pages().stream().mapToInt(p -> p.getPositionCount() * p.getBlockCount()).sum() + columnCount * 2; - assertChunkCount(resp, r -> 6 + sizeClusterDetails + bodySize); // is_running + assertChunkCount(resp, r -> 7 + sizeClusterDetails + bodySize); // is_running } } public void testChunkResponseSizeRows() { int sizeClusterDetails = 14; try (EsqlQueryResponse resp = randomResponse(false, null)) { - int bodySize = resp.pages().stream().mapToInt(p -> p.getPositionCount()).sum(); + int bodySize = resp.pages().stream().mapToInt(Page::getPositionCount).sum(); assertChunkCount(resp, r -> 5 + sizeClusterDetails + bodySize); } try (EsqlQueryResponse resp = randomResponseAsync(false, null, true)) { - int bodySize = resp.pages().stream().mapToInt(p -> p.getPositionCount()).sum(); - assertChunkCount(resp, r -> 6 + sizeClusterDetails + bodySize); + int bodySize = resp.pages().stream().mapToInt(Page::getPositionCount).sum(); + assertChunkCount(resp, r -> 7 + sizeClusterDetails + bodySize); } } diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java index 39bf9e78b3b01..810bd8f6e9ceb 100644 --- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java +++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java @@ -12,11 +12,10 @@ import org.elasticsearch.cluster.metadata.ShutdownShardMigrationStatus; import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContent; @@ -25,10 +24,6 @@ import java.util.Iterator; import java.util.Objects; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.endObject; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.singleChunk; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.startObject; - public class SingleNodeShutdownStatus implements Writeable, ChunkedToXContentObject { private final SingleNodeShutdownMetadata metadata; @@ -116,26 +111,27 @@ public String toString() { @Override public Iterator toXContentChunked(ToXContent.Params params) { - return Iterators.concat(startObject(), singleChunk((builder, p) -> { - builder.field(SingleNodeShutdownMetadata.NODE_ID_FIELD.getPreferredName(), metadata.getNodeId()); - builder.field(SingleNodeShutdownMetadata.TYPE_FIELD.getPreferredName(), metadata.getType()); - builder.field(SingleNodeShutdownMetadata.REASON_FIELD.getPreferredName(), metadata.getReason()); - if (metadata.getAllocationDelay() != null) { - builder.field( - SingleNodeShutdownMetadata.ALLOCATION_DELAY_FIELD.getPreferredName(), - metadata.getAllocationDelay().getStringRep() + return ChunkedToXContent.builder(params).object(b -> { + b.append((builder, p) -> { + builder.field(SingleNodeShutdownMetadata.NODE_ID_FIELD.getPreferredName(), metadata.getNodeId()); + builder.field(SingleNodeShutdownMetadata.TYPE_FIELD.getPreferredName(), metadata.getType()); + builder.field(SingleNodeShutdownMetadata.REASON_FIELD.getPreferredName(), metadata.getReason()); + if (metadata.getAllocationDelay() != null) { + builder.field( + SingleNodeShutdownMetadata.ALLOCATION_DELAY_FIELD.getPreferredName(), + metadata.getAllocationDelay().getStringRep() + ); + } + builder.timeField( + SingleNodeShutdownMetadata.STARTED_AT_MILLIS_FIELD.getPreferredName(), + SingleNodeShutdownMetadata.STARTED_AT_READABLE_FIELD, + metadata.getStartedAtMillis() ); - } - builder.timeField( - SingleNodeShutdownMetadata.STARTED_AT_MILLIS_FIELD.getPreferredName(), - SingleNodeShutdownMetadata.STARTED_AT_READABLE_FIELD, - metadata.getStartedAtMillis() - ); - builder.field(STATUS.getPreferredName(), overallStatus()); - return builder; - }), - ChunkedToXContentHelper.field(SHARD_MIGRATION_FIELD.getPreferredName(), shardMigrationStatus, params), - singleChunk((builder, p) -> { + builder.field(STATUS.getPreferredName(), overallStatus()); + return builder; + }); + b.field(SHARD_MIGRATION_FIELD.getPreferredName(), shardMigrationStatus); + b.append((builder, p) -> { builder.field(PERSISTENT_TASKS_FIELD.getPreferredName(), persistentTasksStatus); builder.field(PLUGINS_STATUS.getPreferredName(), pluginsStatus); if (metadata.getTargetNodeName() != null) { @@ -148,8 +144,7 @@ public Iterator toXContentChunked(ToXContent.Params params ); } return builder; - }), - endObject() - ); + }); + }); } } From b5d6fa0130dd987cea0c215751553478dc38f632 Mon Sep 17 00:00:00 2001 From: Dan Rubinstein Date: Tue, 8 Oct 2024 09:57:46 -0400 Subject: [PATCH 173/194] Add chunking settings configuration to CohereService, AmazonBedrockService, and AzureOpenAiService (#113897) * Add chunking settings configuration to CohereService, AmazonBedrockService, and AzureOpenAiService * Update docs/changelog/113897.yaml * Run spotlessApply * Updating CohereServiceMixedIT to account for clusters without chunking settings in index mapping --------- Co-authored-by: Elastic Machine --- docs/changelog/113897.yaml | 6 + .../qa/mixed/CohereServiceMixedIT.java | 31 +- .../amazonbedrock/AmazonBedrockService.java | 41 ++- .../AmazonBedrockEmbeddingsModel.java | 6 +- .../azureopenai/AzureOpenAiService.java | 49 ++- .../AzureOpenAiEmbeddingsModel.java | 6 +- .../services/cohere/CohereService.java | 48 ++- .../embeddings/CohereEmbeddingsModel.java | 6 +- .../AmazonBedrockServiceTests.java | 338 +++++++++++++++++- .../AmazonBedrockEmbeddingsModelTests.java | 61 ++++ .../azureopenai/AzureOpenAiServiceTests.java | 296 ++++++++++++++- .../AzureOpenAiEmbeddingsModelTests.java | 26 ++ .../services/cohere/CohereServiceTests.java | 314 +++++++++++++++- .../CohereEmbeddingsModelTests.java | 27 ++ 14 files changed, 1215 insertions(+), 40 deletions(-) create mode 100644 docs/changelog/113897.yaml diff --git a/docs/changelog/113897.yaml b/docs/changelog/113897.yaml new file mode 100644 index 0000000000000..db0c53518613c --- /dev/null +++ b/docs/changelog/113897.yaml @@ -0,0 +1,6 @@ +pr: 113897 +summary: "Add chunking settings configuration to `CohereService,` `AmazonBedrockService,`\ + \ and `AzureOpenAiService`" +area: Machine Learning +type: enhancement +issues: [] diff --git a/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/CohereServiceMixedIT.java b/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/CohereServiceMixedIT.java index 69274b46d75c1..8cb37ad645358 100644 --- a/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/CohereServiceMixedIT.java +++ b/x-pack/plugin/inference/qa/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/inference/qa/mixed/CohereServiceMixedIT.java @@ -22,6 +22,7 @@ import java.util.Map; import static org.elasticsearch.xpack.inference.qa.mixed.MixedClusterSpecTestCase.bwcVersion; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasSize; @@ -32,6 +33,7 @@ public class CohereServiceMixedIT extends BaseMixedTestCase { private static final String COHERE_EMBEDDINGS_ADDED = "8.13.0"; private static final String COHERE_RERANK_ADDED = "8.14.0"; + private static final String COHERE_EMBEDDINGS_CHUNKING_SETTINGS_ADDED = "8.16.0"; private static final String BYTE_ALIAS_FOR_INT8_ADDED = "8.14.0"; private static final String MINIMUM_SUPPORTED_VERSION = "8.15.0"; @@ -65,13 +67,28 @@ public void testCohereEmbeddings() throws IOException { final String inferenceIdInt8 = "mixed-cluster-cohere-embeddings-int8"; final String inferenceIdFloat = "mixed-cluster-cohere-embeddings-float"; - // queue a response as PUT will call the service - cohereEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponseByte())); - put(inferenceIdInt8, embeddingConfigInt8(getUrl(cohereEmbeddingsServer)), TaskType.TEXT_EMBEDDING); - - // float model - cohereEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponseFloat())); - put(inferenceIdFloat, embeddingConfigFloat(getUrl(cohereEmbeddingsServer)), TaskType.TEXT_EMBEDDING); + try { + // queue a response as PUT will call the service + cohereEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponseByte())); + put(inferenceIdInt8, embeddingConfigInt8(getUrl(cohereEmbeddingsServer)), TaskType.TEXT_EMBEDDING); + + // float model + cohereEmbeddingsServer.enqueue(new MockResponse().setResponseCode(200).setBody(embeddingResponseFloat())); + put(inferenceIdFloat, embeddingConfigFloat(getUrl(cohereEmbeddingsServer)), TaskType.TEXT_EMBEDDING); + } catch (Exception e) { + if (bwcVersion.before(Version.fromString(COHERE_EMBEDDINGS_CHUNKING_SETTINGS_ADDED))) { + // Chunking settings were added in 8.16.0. if the version is before that, an exception will be thrown if the index mapping + // was created based on a mapping from an old node + assertThat( + e.getMessage(), + containsString( + "One or more nodes in your cluster does not support chunking_settings. " + + "Please update all nodes in your cluster to the latest version to use chunking_settings." + ) + ); + return; + } + } var configs = (List>) get(TaskType.TEXT_EMBEDDING, inferenceIdInt8).get("endpoints"); assertEquals("cohere", configs.get(0).get("service")); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java index bc0d10279ae44..c7c073660624d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockService.java @@ -16,6 +16,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.inference.ChunkingOptions; +import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; @@ -23,6 +24,8 @@ import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.core.inference.ChunkingSettingsFeatureFlag; +import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; import org.elasticsearch.xpack.inference.external.action.amazonbedrock.AmazonBedrockActionCreator; import org.elasticsearch.xpack.inference.external.amazonbedrock.AmazonBedrockRequestSender; @@ -99,8 +102,20 @@ protected void doChunkedInfer( var actionCreator = new AmazonBedrockActionCreator(amazonBedrockSender, this.getServiceComponents(), timeout); if (model instanceof AmazonBedrockModel baseAmazonBedrockModel) { var maxBatchSize = getEmbeddingsMaxBatchSize(baseAmazonBedrockModel.provider()); - var batchedRequests = new EmbeddingRequestChunker(inputs.getInputs(), maxBatchSize, EmbeddingRequestChunker.EmbeddingType.FLOAT) - .batchRequestsWithListeners(listener); + + List batchedRequests; + if (ChunkingSettingsFeatureFlag.isEnabled()) { + batchedRequests = new EmbeddingRequestChunker( + inputs.getInputs(), + maxBatchSize, + EmbeddingRequestChunker.EmbeddingType.FLOAT, + baseAmazonBedrockModel.getConfigurations().getChunkingSettings() + ).batchRequestsWithListeners(listener); + } else { + batchedRequests = new EmbeddingRequestChunker(inputs.getInputs(), maxBatchSize, EmbeddingRequestChunker.EmbeddingType.FLOAT) + .batchRequestsWithListeners(listener); + } + for (var request : batchedRequests) { var action = baseAmazonBedrockModel.accept(actionCreator, taskSettings); action.execute(new DocumentsOnlyInput(request.batch().inputs()), timeout, request.listener()); @@ -126,11 +141,19 @@ public void parseRequestConfig( Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); Map taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS); + ChunkingSettings chunkingSettings = null; + if (ChunkingSettingsFeatureFlag.isEnabled() && TaskType.TEXT_EMBEDDING.equals(taskType)) { + chunkingSettings = ChunkingSettingsBuilder.fromMap( + removeFromMapOrDefaultEmpty(config, ModelConfigurations.CHUNKING_SETTINGS) + ); + } + AmazonBedrockModel model = createModel( modelId, taskType, serviceSettingsMap, taskSettingsMap, + chunkingSettings, serviceSettingsMap, TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), ConfigurationParseContext.REQUEST @@ -157,11 +180,17 @@ public Model parsePersistedConfigWithSecrets( Map taskSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.TASK_SETTINGS); Map secretSettingsMap = removeFromMapOrDefaultEmpty(secrets, ModelSecrets.SECRET_SETTINGS); + ChunkingSettings chunkingSettings = null; + if (ChunkingSettingsFeatureFlag.isEnabled() && TaskType.TEXT_EMBEDDING.equals(taskType)) { + chunkingSettings = ChunkingSettingsBuilder.fromMap(removeFromMapOrDefaultEmpty(config, ModelConfigurations.CHUNKING_SETTINGS)); + } + return createModel( modelId, taskType, serviceSettingsMap, taskSettingsMap, + chunkingSettings, secretSettingsMap, parsePersistedConfigErrorMsg(modelId, NAME), ConfigurationParseContext.PERSISTENT @@ -173,11 +202,17 @@ public Model parsePersistedConfig(String modelId, TaskType taskType, Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); Map taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS); + ChunkingSettings chunkingSettings = null; + if (ChunkingSettingsFeatureFlag.isEnabled() && TaskType.TEXT_EMBEDDING.equals(taskType)) { + chunkingSettings = ChunkingSettingsBuilder.fromMap(removeFromMapOrDefaultEmpty(config, ModelConfigurations.CHUNKING_SETTINGS)); + } + return createModel( modelId, taskType, serviceSettingsMap, taskSettingsMap, + chunkingSettings, null, parsePersistedConfigErrorMsg(modelId, NAME), ConfigurationParseContext.PERSISTENT @@ -189,6 +224,7 @@ private static AmazonBedrockModel createModel( TaskType taskType, Map serviceSettings, Map taskSettings, + ChunkingSettings chunkingSettings, @Nullable Map secretSettings, String failureMessage, ConfigurationParseContext context @@ -201,6 +237,7 @@ private static AmazonBedrockModel createModel( NAME, serviceSettings, taskSettings, + chunkingSettings, secretSettings, context ); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/embeddings/AmazonBedrockEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/embeddings/AmazonBedrockEmbeddingsModel.java index 0e3a954a03279..186d977d20672 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/embeddings/AmazonBedrockEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/amazonbedrock/embeddings/AmazonBedrockEmbeddingsModel.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.inference.services.amazonbedrock.embeddings; import org.elasticsearch.common.ValidationException; +import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.EmptyTaskSettings; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ModelConfigurations; @@ -42,6 +43,7 @@ public AmazonBedrockEmbeddingsModel( String service, Map serviceSettings, Map taskSettings, + ChunkingSettings chunkingSettings, Map secretSettings, ConfigurationParseContext context ) { @@ -51,6 +53,7 @@ public AmazonBedrockEmbeddingsModel( service, AmazonBedrockEmbeddingsServiceSettings.fromMap(serviceSettings, context), new EmptyTaskSettings(), + chunkingSettings, AmazonBedrockSecretSettings.fromMap(secretSettings) ); } @@ -61,10 +64,11 @@ public AmazonBedrockEmbeddingsModel( String service, AmazonBedrockEmbeddingsServiceSettings serviceSettings, TaskSettings taskSettings, + ChunkingSettings chunkingSettings, AmazonBedrockSecretSettings secrets ) { super( - new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, new EmptyTaskSettings()), + new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, new EmptyTaskSettings(), chunkingSettings), new ModelSecrets(secrets) ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java index 96399bb954cd2..07708ee072099 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java @@ -16,6 +16,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.inference.ChunkingOptions; +import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; @@ -24,6 +25,8 @@ import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.core.inference.ChunkingSettingsFeatureFlag; +import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; import org.elasticsearch.xpack.inference.external.action.azureopenai.AzureOpenAiActionCreator; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; @@ -70,11 +73,19 @@ public void parseRequestConfig( Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); Map taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS); + ChunkingSettings chunkingSettings = null; + if (ChunkingSettingsFeatureFlag.isEnabled() && TaskType.TEXT_EMBEDDING.equals(taskType)) { + chunkingSettings = ChunkingSettingsBuilder.fromMap( + removeFromMapOrDefaultEmpty(config, ModelConfigurations.CHUNKING_SETTINGS) + ); + } + AzureOpenAiModel model = createModel( inferenceEntityId, taskType, serviceSettingsMap, taskSettingsMap, + chunkingSettings, serviceSettingsMap, TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), ConfigurationParseContext.REQUEST @@ -95,6 +106,7 @@ private static AzureOpenAiModel createModelFromPersistent( TaskType taskType, Map serviceSettings, Map taskSettings, + ChunkingSettings chunkingSettings, @Nullable Map secretSettings, String failureMessage ) { @@ -103,6 +115,7 @@ private static AzureOpenAiModel createModelFromPersistent( taskType, serviceSettings, taskSettings, + chunkingSettings, secretSettings, failureMessage, ConfigurationParseContext.PERSISTENT @@ -114,6 +127,7 @@ private static AzureOpenAiModel createModel( TaskType taskType, Map serviceSettings, Map taskSettings, + ChunkingSettings chunkingSettings, @Nullable Map secretSettings, String failureMessage, ConfigurationParseContext context @@ -126,6 +140,7 @@ private static AzureOpenAiModel createModel( NAME, serviceSettings, taskSettings, + chunkingSettings, secretSettings, context ); @@ -156,11 +171,17 @@ public AzureOpenAiModel parsePersistedConfigWithSecrets( Map taskSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.TASK_SETTINGS); Map secretSettingsMap = removeFromMapOrDefaultEmpty(secrets, ModelSecrets.SECRET_SETTINGS); + ChunkingSettings chunkingSettings = null; + if (ChunkingSettingsFeatureFlag.isEnabled() && TaskType.TEXT_EMBEDDING.equals(taskType)) { + chunkingSettings = ChunkingSettingsBuilder.fromMap(removeFromMapOrDefaultEmpty(config, ModelConfigurations.CHUNKING_SETTINGS)); + } + return createModelFromPersistent( inferenceEntityId, taskType, serviceSettingsMap, taskSettingsMap, + chunkingSettings, secretSettingsMap, parsePersistedConfigErrorMsg(inferenceEntityId, NAME) ); @@ -171,11 +192,17 @@ public AzureOpenAiModel parsePersistedConfig(String inferenceEntityId, TaskType Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); Map taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS); + ChunkingSettings chunkingSettings = null; + if (ChunkingSettingsFeatureFlag.isEnabled() && TaskType.TEXT_EMBEDDING.equals(taskType)) { + chunkingSettings = ChunkingSettingsBuilder.fromMap(removeFromMapOrDefaultEmpty(config, ModelConfigurations.CHUNKING_SETTINGS)); + } + return createModelFromPersistent( inferenceEntityId, taskType, serviceSettingsMap, taskSettingsMap, + chunkingSettings, null, parsePersistedConfigErrorMsg(inferenceEntityId, NAME) ); @@ -218,11 +245,23 @@ protected void doChunkedInfer( } AzureOpenAiModel azureOpenAiModel = (AzureOpenAiModel) model; var actionCreator = new AzureOpenAiActionCreator(getSender(), getServiceComponents()); - var batchedRequests = new EmbeddingRequestChunker( - inputs.getInputs(), - EMBEDDING_MAX_BATCH_SIZE, - EmbeddingRequestChunker.EmbeddingType.FLOAT - ).batchRequestsWithListeners(listener); + + List batchedRequests; + if (ChunkingSettingsFeatureFlag.isEnabled()) { + batchedRequests = new EmbeddingRequestChunker( + inputs.getInputs(), + EMBEDDING_MAX_BATCH_SIZE, + EmbeddingRequestChunker.EmbeddingType.FLOAT, + azureOpenAiModel.getConfigurations().getChunkingSettings() + ).batchRequestsWithListeners(listener); + } else { + batchedRequests = new EmbeddingRequestChunker( + inputs.getInputs(), + EMBEDDING_MAX_BATCH_SIZE, + EmbeddingRequestChunker.EmbeddingType.FLOAT + ).batchRequestsWithListeners(listener); + } + for (var request : batchedRequests) { var action = azureOpenAiModel.accept(actionCreator, taskSettings); action.execute(new DocumentsOnlyInput(request.batch().inputs()), timeout, request.listener()); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModel.java index 377bb33f58619..7b83d5322a696 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModel.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.inference.services.azureopenai.embeddings; import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.TaskType; @@ -38,6 +39,7 @@ public AzureOpenAiEmbeddingsModel( String service, Map serviceSettings, Map taskSettings, + ChunkingSettings chunkingSettings, @Nullable Map secrets, ConfigurationParseContext context ) { @@ -47,6 +49,7 @@ public AzureOpenAiEmbeddingsModel( service, AzureOpenAiEmbeddingsServiceSettings.fromMap(serviceSettings, context), AzureOpenAiEmbeddingsTaskSettings.fromMap(taskSettings), + chunkingSettings, AzureOpenAiSecretSettings.fromMap(secrets) ); } @@ -58,10 +61,11 @@ public AzureOpenAiEmbeddingsModel( String service, AzureOpenAiEmbeddingsServiceSettings serviceSettings, AzureOpenAiEmbeddingsTaskSettings taskSettings, + ChunkingSettings chunkingSettings, @Nullable AzureOpenAiSecretSettings secrets ) { super( - new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings), + new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings, chunkingSettings), new ModelSecrets(secrets), serviceSettings ); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java index 3ba93dd8d1b66..1804d3bdf5936 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java @@ -15,6 +15,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.inference.ChunkingOptions; +import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; @@ -23,6 +24,8 @@ import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.core.inference.ChunkingSettingsFeatureFlag; +import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsBuilder; import org.elasticsearch.xpack.inference.chunking.EmbeddingRequestChunker; import org.elasticsearch.xpack.inference.external.action.cohere.CohereActionCreator; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; @@ -76,11 +79,19 @@ public void parseRequestConfig( Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); Map taskSettingsMap = removeFromMapOrDefaultEmpty(config, ModelConfigurations.TASK_SETTINGS); + ChunkingSettings chunkingSettings = null; + if (ChunkingSettingsFeatureFlag.isEnabled() && TaskType.TEXT_EMBEDDING.equals(taskType)) { + chunkingSettings = ChunkingSettingsBuilder.fromMap( + removeFromMapOrDefaultEmpty(config, ModelConfigurations.CHUNKING_SETTINGS) + ); + } + CohereModel model = createModel( inferenceEntityId, taskType, serviceSettingsMap, taskSettingsMap, + chunkingSettings, serviceSettingsMap, TaskType.unsupportedTaskTypeErrorMsg(taskType, NAME), ConfigurationParseContext.REQUEST @@ -101,6 +112,7 @@ private static CohereModel createModelWithoutLoggingDeprecations( TaskType taskType, Map serviceSettings, Map taskSettings, + ChunkingSettings chunkingSettings, @Nullable Map secretSettings, String failureMessage ) { @@ -109,6 +121,7 @@ private static CohereModel createModelWithoutLoggingDeprecations( taskType, serviceSettings, taskSettings, + chunkingSettings, secretSettings, failureMessage, ConfigurationParseContext.PERSISTENT @@ -120,6 +133,7 @@ private static CohereModel createModel( TaskType taskType, Map serviceSettings, Map taskSettings, + ChunkingSettings chunkingSettings, @Nullable Map secretSettings, String failureMessage, ConfigurationParseContext context @@ -131,6 +145,7 @@ private static CohereModel createModel( NAME, serviceSettings, taskSettings, + chunkingSettings, secretSettings, context ); @@ -159,11 +174,17 @@ public CohereModel parsePersistedConfigWithSecrets( Map taskSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.TASK_SETTINGS); Map secretSettingsMap = removeFromMapOrThrowIfNull(secrets, ModelSecrets.SECRET_SETTINGS); + ChunkingSettings chunkingSettings = null; + if (ChunkingSettingsFeatureFlag.isEnabled() && TaskType.TEXT_EMBEDDING.equals(taskType)) { + chunkingSettings = ChunkingSettingsBuilder.fromMap(removeFromMapOrDefaultEmpty(config, ModelConfigurations.CHUNKING_SETTINGS)); + } + return createModelWithoutLoggingDeprecations( inferenceEntityId, taskType, serviceSettingsMap, taskSettingsMap, + chunkingSettings, secretSettingsMap, parsePersistedConfigErrorMsg(inferenceEntityId, NAME) ); @@ -174,11 +195,17 @@ public CohereModel parsePersistedConfig(String inferenceEntityId, TaskType taskT Map serviceSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.SERVICE_SETTINGS); Map taskSettingsMap = removeFromMapOrThrowIfNull(config, ModelConfigurations.TASK_SETTINGS); + ChunkingSettings chunkingSettings = null; + if (ChunkingSettingsFeatureFlag.isEnabled() && TaskType.TEXT_EMBEDDING.equals(taskType)) { + chunkingSettings = ChunkingSettingsBuilder.fromMap(removeFromMapOrDefaultEmpty(config, ModelConfigurations.CHUNKING_SETTINGS)); + } + return createModelWithoutLoggingDeprecations( inferenceEntityId, taskType, serviceSettingsMap, taskSettingsMap, + chunkingSettings, null, parsePersistedConfigErrorMsg(inferenceEntityId, NAME) ); @@ -223,11 +250,22 @@ protected void doChunkedInfer( CohereModel cohereModel = (CohereModel) model; var actionCreator = new CohereActionCreator(getSender(), getServiceComponents()); - var batchedRequests = new EmbeddingRequestChunker( - inputs.getInputs(), - EMBEDDING_MAX_BATCH_SIZE, - EmbeddingRequestChunker.EmbeddingType.fromDenseVectorElementType(model.getServiceSettings().elementType()) - ).batchRequestsWithListeners(listener); + List batchedRequests; + if (ChunkingSettingsFeatureFlag.isEnabled()) { + batchedRequests = new EmbeddingRequestChunker( + inputs.getInputs(), + EMBEDDING_MAX_BATCH_SIZE, + EmbeddingRequestChunker.EmbeddingType.fromDenseVectorElementType(model.getServiceSettings().elementType()), + cohereModel.getConfigurations().getChunkingSettings() + ).batchRequestsWithListeners(listener); + } else { + batchedRequests = new EmbeddingRequestChunker( + inputs.getInputs(), + EMBEDDING_MAX_BATCH_SIZE, + EmbeddingRequestChunker.EmbeddingType.fromDenseVectorElementType(model.getServiceSettings().elementType()) + ).batchRequestsWithListeners(listener); + } + for (var request : batchedRequests) { var action = cohereModel.accept(actionCreator, taskSettings, inputType); action.execute(new DocumentsOnlyInput(request.batch().inputs()), timeout, request.listener()); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java index fea5226bf9c6f..0f62ab51145f4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.inference.services.cohere.embeddings; import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; @@ -33,6 +34,7 @@ public CohereEmbeddingsModel( String service, Map serviceSettings, Map taskSettings, + ChunkingSettings chunkingSettings, @Nullable Map secrets, ConfigurationParseContext context ) { @@ -42,6 +44,7 @@ public CohereEmbeddingsModel( service, CohereEmbeddingsServiceSettings.fromMap(serviceSettings, context), CohereEmbeddingsTaskSettings.fromMap(taskSettings), + chunkingSettings, DefaultSecretSettings.fromMap(secrets) ); } @@ -53,10 +56,11 @@ public CohereEmbeddingsModel( String service, CohereEmbeddingsServiceSettings serviceSettings, CohereEmbeddingsTaskSettings taskSettings, + ChunkingSettings chunkingSettings, @Nullable DefaultSecretSettings secretSettings ) { super( - new ModelConfigurations(modelId, taskType, service, serviceSettings, taskSettings), + new ModelConfigurations(modelId, taskType, service, serviceSettings, taskSettings, chunkingSettings), new ModelSecrets(secretSettings), secretSettings, serviceSettings.getCommonSettings() diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java index 0b3cf533d818f..9c746e7c2aed9 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/AmazonBedrockServiceTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.inference.ChunkingOptions; +import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; @@ -25,6 +26,7 @@ import org.elasticsearch.inference.TaskType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.inference.ChunkingSettingsFeatureFlag; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.results.ChatCompletionResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; @@ -57,6 +59,8 @@ import static org.elasticsearch.xpack.inference.Utils.getInvalidModel; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests.createRandomChunkingSettings; +import static org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests.createRandomChunkingSettingsMap; import static org.elasticsearch.xpack.inference.results.ChatCompletionResultsTests.buildExpectationCompletion; import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationFloat; import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings; @@ -305,6 +309,93 @@ public void testParseRequestConfig_MovesModel() throws IOException { } } + public void testParseRequestConfig_ThrowsElasticsearchStatusExceptionWhenChunkingSettingsProvidedAndFeatureFlagDisabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is disabled", ChunkingSettingsFeatureFlag.isEnabled() == false); + try (var service = createAmazonBedrockService()) { + ActionListener modelVerificationListener = ActionListener.wrap( + model -> fail("Expected exception, but got model: " + model), + exception -> { + assertThat(exception, instanceOf(ElasticsearchStatusException.class)); + assertThat(exception.getMessage(), containsString("Model configuration contains settings")); + } + ); + + service.parseRequestConfig( + "id", + TaskType.TEXT_EMBEDDING, + getRequestConfigMap( + createEmbeddingsRequestSettingsMap("region", "model", "amazontitan", null, null, null, null), + Map.of(), + createRandomChunkingSettingsMap(), + getAmazonBedrockSecretSettingsMap("access", "secret") + ), + modelVerificationListener + ); + } + } + + public void testParseRequestConfig_CreatesAnAmazonBedrockEmbeddingsModelWhenChunkingSettingsProvidedAndFeatureFlagEnabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + try (var service = createAmazonBedrockService()) { + ActionListener modelVerificationListener = ActionListener.wrap(model -> { + assertThat(model, instanceOf(AmazonBedrockEmbeddingsModel.class)); + + var settings = (AmazonBedrockEmbeddingsServiceSettings) model.getServiceSettings(); + assertThat(settings.region(), is("region")); + assertThat(settings.modelId(), is("model")); + assertThat(settings.provider(), is(AmazonBedrockProvider.AMAZONTITAN)); + var secretSettings = (AmazonBedrockSecretSettings) model.getSecretSettings(); + assertThat(secretSettings.accessKey.toString(), is("access")); + assertThat(secretSettings.secretKey.toString(), is("secret")); + assertThat(model.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + }, exception -> fail("Unexpected exception: " + exception)); + + service.parseRequestConfig( + "id", + TaskType.TEXT_EMBEDDING, + getRequestConfigMap( + createEmbeddingsRequestSettingsMap("region", "model", "amazontitan", null, null, null, null), + Map.of(), + createRandomChunkingSettingsMap(), + getAmazonBedrockSecretSettingsMap("access", "secret") + ), + modelVerificationListener + ); + } + } + + public void testParseRequestConfig_CreatesAnAmazonBedrockEmbeddingsModelWhenChunkingSettingsNotProvidedAndFeatureFlagEnabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + try (var service = createAmazonBedrockService()) { + ActionListener modelVerificationListener = ActionListener.wrap(model -> { + assertThat(model, instanceOf(AmazonBedrockEmbeddingsModel.class)); + + var settings = (AmazonBedrockEmbeddingsServiceSettings) model.getServiceSettings(); + assertThat(settings.region(), is("region")); + assertThat(settings.modelId(), is("model")); + assertThat(settings.provider(), is(AmazonBedrockProvider.AMAZONTITAN)); + var secretSettings = (AmazonBedrockSecretSettings) model.getSecretSettings(); + assertThat(secretSettings.accessKey.toString(), is("access")); + assertThat(secretSettings.secretKey.toString(), is("secret")); + assertThat(model.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + }, exception -> fail("Unexpected exception: " + exception)); + + service.parseRequestConfig( + "id", + TaskType.TEXT_EMBEDDING, + getRequestConfigMap( + createEmbeddingsRequestSettingsMap("region", "model", "amazontitan", null, null, null, null), + Map.of(), + getAmazonBedrockSecretSettingsMap("access", "secret") + ), + modelVerificationListener + ); + } + } + public void testCreateModel_ForEmbeddingsTask_DimensionsIsNotAllowed() throws IOException { try (var service = createAmazonBedrockService()) { ActionListener modelVerificationListener = ActionListener.wrap( @@ -354,6 +445,100 @@ public void testParsePersistedConfigWithSecrets_CreatesAnAmazonBedrockEmbeddings } } + public void testParsePersistedConfigWithSecrets_CreatesAnAmazonBedrockEmbeddingsModelWithoutChunkingSettingsWhenFeatureFlagDisabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is disabled", ChunkingSettingsFeatureFlag.isEnabled() == false); + try (var service = createAmazonBedrockService()) { + var settingsMap = createEmbeddingsRequestSettingsMap("region", "model", "amazontitan", null, false, null, null); + var secretSettingsMap = getAmazonBedrockSecretSettingsMap("access", "secret"); + + var persistedConfig = getPersistedConfigMap(settingsMap, new HashMap(Map.of()), secretSettingsMap); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + assertThat(model, instanceOf(AmazonBedrockEmbeddingsModel.class)); + + var settings = (AmazonBedrockEmbeddingsServiceSettings) model.getServiceSettings(); + assertThat(settings.region(), is("region")); + assertThat(settings.modelId(), is("model")); + assertThat(settings.provider(), is(AmazonBedrockProvider.AMAZONTITAN)); + assertNull(model.getConfigurations().getChunkingSettings()); + var secretSettings = (AmazonBedrockSecretSettings) model.getSecretSettings(); + assertThat(secretSettings.accessKey.toString(), is("access")); + assertThat(secretSettings.secretKey.toString(), is("secret")); + } + } + + public void testParsePersistedConfigWithSecrets_CreatesAnAmazonBedrockEmbeddingsModelWhenChunkingSettingsProvidedAndFeatureFlagEnabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + try (var service = createAmazonBedrockService()) { + var settingsMap = createEmbeddingsRequestSettingsMap("region", "model", "amazontitan", null, false, null, null); + var secretSettingsMap = getAmazonBedrockSecretSettingsMap("access", "secret"); + + var persistedConfig = getPersistedConfigMap( + settingsMap, + new HashMap(Map.of()), + createRandomChunkingSettingsMap(), + secretSettingsMap + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + assertThat(model, instanceOf(AmazonBedrockEmbeddingsModel.class)); + + var settings = (AmazonBedrockEmbeddingsServiceSettings) model.getServiceSettings(); + assertThat(settings.region(), is("region")); + assertThat(settings.modelId(), is("model")); + assertThat(settings.provider(), is(AmazonBedrockProvider.AMAZONTITAN)); + assertThat(model.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + var secretSettings = (AmazonBedrockSecretSettings) model.getSecretSettings(); + assertThat(secretSettings.accessKey.toString(), is("access")); + assertThat(secretSettings.secretKey.toString(), is("secret")); + } + } + + public + void + testParsePersistedConfigWithSecrets_CreatesAnAmazonBedrockEmbeddingsModelWhenChunkingSettingsNotProvidedAndFeatureFlagEnabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + try (var service = createAmazonBedrockService()) { + var settingsMap = createEmbeddingsRequestSettingsMap("region", "model", "amazontitan", null, false, null, null); + var secretSettingsMap = getAmazonBedrockSecretSettingsMap("access", "secret"); + + var persistedConfig = getPersistedConfigMap(settingsMap, new HashMap(Map.of()), secretSettingsMap); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + assertThat(model, instanceOf(AmazonBedrockEmbeddingsModel.class)); + + var settings = (AmazonBedrockEmbeddingsServiceSettings) model.getServiceSettings(); + assertThat(settings.region(), is("region")); + assertThat(settings.modelId(), is("model")); + assertThat(settings.provider(), is(AmazonBedrockProvider.AMAZONTITAN)); + assertThat(model.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + var secretSettings = (AmazonBedrockSecretSettings) model.getSecretSettings(); + assertThat(secretSettings.accessKey.toString(), is("access")); + assertThat(secretSettings.secretKey.toString(), is("secret")); + } + } + public void testParsePersistedConfigWithSecrets_ThrowsErrorTryingToParseInvalidModel() throws IOException { try (var service = createAmazonBedrockService()) { var settingsMap = createChatCompletionRequestSettingsMap("region", "model", "amazontitan"); @@ -538,6 +723,84 @@ public void testParsePersistedConfig_CreatesAnAmazonBedrockEmbeddingsModel() thr } } + public + void + testParsePersistedConfig_CreatesAnAmazonBedrockEmbeddingsModelWithoutChunkingSettingsWhenChunkingSettingsFeatureFlagDisabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is disabled", ChunkingSettingsFeatureFlag.isEnabled() == false); + try (var service = createAmazonBedrockService()) { + var settingsMap = createEmbeddingsRequestSettingsMap("region", "model", "amazontitan", null, false, null, null); + var secretSettingsMap = getAmazonBedrockSecretSettingsMap("access", "secret"); + + var persistedConfig = getPersistedConfigMap( + settingsMap, + new HashMap(Map.of()), + createRandomChunkingSettingsMap(), + secretSettingsMap + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + assertThat(model, instanceOf(AmazonBedrockEmbeddingsModel.class)); + + var settings = (AmazonBedrockEmbeddingsServiceSettings) model.getServiceSettings(); + assertThat(settings.region(), is("region")); + assertThat(settings.modelId(), is("model")); + assertThat(settings.provider(), is(AmazonBedrockProvider.AMAZONTITAN)); + assertNull(model.getConfigurations().getChunkingSettings()); + assertNull(model.getSecretSettings()); + } + } + + public void testParsePersistedConfig_CreatesAnAmazonBedrockEmbeddingsModelWhenChunkingSettingsProvidedAndFeatureFlagEnabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + try (var service = createAmazonBedrockService()) { + var settingsMap = createEmbeddingsRequestSettingsMap("region", "model", "amazontitan", null, false, null, null); + var secretSettingsMap = getAmazonBedrockSecretSettingsMap("access", "secret"); + + var persistedConfig = getPersistedConfigMap( + settingsMap, + new HashMap(Map.of()), + createRandomChunkingSettingsMap(), + secretSettingsMap + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + assertThat(model, instanceOf(AmazonBedrockEmbeddingsModel.class)); + + var settings = (AmazonBedrockEmbeddingsServiceSettings) model.getServiceSettings(); + assertThat(settings.region(), is("region")); + assertThat(settings.modelId(), is("model")); + assertThat(settings.provider(), is(AmazonBedrockProvider.AMAZONTITAN)); + assertThat(model.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + assertNull(model.getSecretSettings()); + } + } + + public void testParsePersistedConfig_CreatesAnAmazonBedrockEmbeddingsModelWhenChunkingSettingsNotProvidedAndFeatureFlagEnabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + try (var service = createAmazonBedrockService()) { + var settingsMap = createEmbeddingsRequestSettingsMap("region", "model", "amazontitan", null, false, null, null); + var secretSettingsMap = getAmazonBedrockSecretSettingsMap("access", "secret"); + + var persistedConfig = getPersistedConfigMap(settingsMap, new HashMap(Map.of()), secretSettingsMap); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + assertThat(model, instanceOf(AmazonBedrockEmbeddingsModel.class)); + + var settings = (AmazonBedrockEmbeddingsServiceSettings) model.getServiceSettings(); + assertThat(settings.region(), is("region")); + assertThat(settings.modelId(), is("model")); + assertThat(settings.provider(), is(AmazonBedrockProvider.AMAZONTITAN)); + assertThat(model.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + assertNull(model.getSecretSettings()); + } + } + public void testParsePersistedConfig_CreatesAnAmazonBedrockChatCompletionModel() throws IOException { try (var service = createAmazonBedrockService()) { var settingsMap = createChatCompletionRequestSettingsMap("region", "model", "anthropic"); @@ -1034,6 +1297,49 @@ public void testInfer_UnauthorizedResponse() throws IOException { } public void testChunkedInfer_CallsInfer_ConvertsFloatResponse_ForEmbeddings() throws IOException { + var model = AmazonBedrockEmbeddingsModelTests.createModel( + "id", + "region", + "model", + AmazonBedrockProvider.AMAZONTITAN, + "access", + "secret" + ); + + testChunkedInfer(model); + } + + public void testChunkedInfer_ChunkingSettingsSetAndFeatureFlagEnabled() throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + var model = AmazonBedrockEmbeddingsModelTests.createModel( + "id", + "region", + "model", + AmazonBedrockProvider.AMAZONTITAN, + createRandomChunkingSettings(), + "access", + "secret" + ); + + testChunkedInfer(model); + } + + public void testChunkedInfer_ChunkingSettingsNotSetAndFeatureFlagEnabled() throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + var model = AmazonBedrockEmbeddingsModelTests.createModel( + "id", + "region", + "model", + AmazonBedrockProvider.AMAZONTITAN, + null, + "access", + "secret" + ); + + testChunkedInfer(model); + } + + private void testChunkedInfer(AmazonBedrockEmbeddingsModel model) throws IOException { var sender = mock(Sender.class); var factory = mock(HttpRequestSender.Factory.class); when(factory.createSender()).thenReturn(sender); @@ -1058,14 +1364,6 @@ public void testChunkedInfer_CallsInfer_ConvertsFloatResponse_ForEmbeddings() th requestSender.enqueue(mockResults2); } - var model = AmazonBedrockEmbeddingsModelTests.createModel( - "id", - "region", - "model", - AmazonBedrockProvider.AMAZONTITAN, - "access", - "secret" - ); PlainActionFuture> listener = new PlainActionFuture<>(); service.chunkedInfer( model, @@ -1106,6 +1404,18 @@ private AmazonBedrockService createAmazonBedrockService() { return new AmazonBedrockService(mock(HttpRequestSender.Factory.class), amazonBedrockFactory, createWithEmptySettings(threadPool)); } + private Map getRequestConfigMap( + Map serviceSettings, + Map taskSettings, + Map chunkingSettings, + Map secretSettings + ) { + var requestConfigMap = getRequestConfigMap(serviceSettings, taskSettings, secretSettings); + requestConfigMap.put(ModelConfigurations.CHUNKING_SETTINGS, chunkingSettings); + + return requestConfigMap; + } + private Map getRequestConfigMap( Map serviceSettings, Map taskSettings, @@ -1120,6 +1430,18 @@ private Map getRequestConfigMap( ); } + private Utils.PersistedConfig getPersistedConfigMap( + Map serviceSettings, + Map taskSettings, + Map chunkingSettings, + Map secretSettings + ) { + var persistedConfigMap = getPersistedConfigMap(serviceSettings, taskSettings, secretSettings); + persistedConfigMap.config().put(ModelConfigurations.CHUNKING_SETTINGS, chunkingSettings); + + return persistedConfigMap; + } + private Utils.PersistedConfig getPersistedConfigMap( Map serviceSettings, Map taskSettings, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/embeddings/AmazonBedrockEmbeddingsModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/embeddings/AmazonBedrockEmbeddingsModelTests.java index 711e3cbb5a511..72dc696ddd81b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/embeddings/AmazonBedrockEmbeddingsModelTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/amazonbedrock/embeddings/AmazonBedrockEmbeddingsModelTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.EmptyTaskSettings; import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; @@ -47,6 +48,65 @@ public static AmazonBedrockEmbeddingsModel createModel( return createModel(inferenceId, region, model, provider, null, false, null, null, new RateLimitSettings(240), accessKey, secretKey); } + public static AmazonBedrockEmbeddingsModel createModel( + String inferenceId, + String region, + String model, + AmazonBedrockProvider provider, + ChunkingSettings chunkingSettings, + String accessKey, + String secretKey + ) { + return createModel( + inferenceId, + region, + model, + provider, + null, + false, + null, + null, + new RateLimitSettings(240), + chunkingSettings, + accessKey, + secretKey + ); + } + + public static AmazonBedrockEmbeddingsModel createModel( + String inferenceId, + String region, + String model, + AmazonBedrockProvider provider, + @Nullable Integer dimensions, + boolean dimensionsSetByUser, + @Nullable Integer maxTokens, + @Nullable SimilarityMeasure similarity, + RateLimitSettings rateLimitSettings, + ChunkingSettings chunkingSettings, + String accessKey, + String secretKey + ) { + return new AmazonBedrockEmbeddingsModel( + inferenceId, + TaskType.TEXT_EMBEDDING, + "amazonbedrock", + new AmazonBedrockEmbeddingsServiceSettings( + region, + model, + provider, + dimensions, + dimensionsSetByUser, + maxTokens, + similarity, + rateLimitSettings + ), + new EmptyTaskSettings(), + chunkingSettings, + new AmazonBedrockSecretSettings(new SecureString(accessKey), new SecureString(secretKey)) + ); + } + public static AmazonBedrockEmbeddingsModel createModel( String inferenceId, String region, @@ -75,6 +135,7 @@ public static AmazonBedrockEmbeddingsModel createModel( rateLimitSettings ), new EmptyTaskSettings(), + null, new AmazonBedrockSecretSettings(new SecureString(accessKey), new SecureString(secretKey)) ); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java index 098e41b72ea8f..0fc8f3f2b0eb3 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.inference.ChunkingOptions; +import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; @@ -29,6 +30,7 @@ import org.elasticsearch.test.http.MockWebServer; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.ChunkingSettingsFeatureFlag; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; @@ -56,6 +58,8 @@ import static org.elasticsearch.xpack.inference.Utils.getPersistedConfigMap; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests.createRandomChunkingSettings; +import static org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests.createRandomChunkingSettingsMap; import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; import static org.elasticsearch.xpack.inference.external.request.azureopenai.AzureOpenAiUtils.API_KEY_HEADER; @@ -122,6 +126,88 @@ public void testParseRequestConfig_CreatesAnOpenAiEmbeddingsModel() throws IOExc } } + public void testParseRequestConfig_ThrowsElasticsearchStatusExceptionWhenChunkingSettingsProvidedAndFeatureFlagDisabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is disabled", ChunkingSettingsFeatureFlag.isEnabled() == false); + try (var service = createAzureOpenAiService()) { + var config = getRequestConfigMap( + getRequestAzureOpenAiServiceSettingsMap("resource_name", "deployment_id", "api_version", null, null), + getAzureOpenAiRequestTaskSettingsMap("user"), + createRandomChunkingSettingsMap(), + getAzureOpenAiSecretSettingsMap("secret", null) + ); + + ActionListener modelVerificationListener = ActionListener.wrap( + model -> fail("Expected exception, but got model: " + model), + exception -> { + assertThat(exception, instanceOf(ElasticsearchStatusException.class)); + assertThat(exception.getMessage(), containsString("Model configuration contains settings")); + } + ); + + service.parseRequestConfig("id", TaskType.TEXT_EMBEDDING, config, modelVerificationListener); + } + } + + public void testParseRequestConfig_CreatesAnOpenAiEmbeddingsModelWhenChunkingSettingsProvidedAndFeatureFlagEnabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + try (var service = createAzureOpenAiService()) { + ActionListener modelVerificationListener = ActionListener.wrap(model -> { + assertThat(model, instanceOf(AzureOpenAiEmbeddingsModel.class)); + + var embeddingsModel = (AzureOpenAiEmbeddingsModel) model; + assertThat(embeddingsModel.getServiceSettings().resourceName(), is("resource_name")); + assertThat(embeddingsModel.getServiceSettings().deploymentId(), is("deployment_id")); + assertThat(embeddingsModel.getServiceSettings().apiVersion(), is("api_version")); + assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + assertThat(embeddingsModel.getTaskSettings().user(), is("user")); + assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + }, exception -> fail("Unexpected exception: " + exception)); + + service.parseRequestConfig( + "id", + TaskType.TEXT_EMBEDDING, + getRequestConfigMap( + getRequestAzureOpenAiServiceSettingsMap("resource_name", "deployment_id", "api_version", null, null), + getAzureOpenAiRequestTaskSettingsMap("user"), + createRandomChunkingSettingsMap(), + getAzureOpenAiSecretSettingsMap("secret", null) + ), + modelVerificationListener + ); + } + } + + public void testParseRequestConfig_CreatesAnOpenAiEmbeddingsModelWhenChunkingSettingsNotProvidedAndFeatureFlagEnabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + try (var service = createAzureOpenAiService()) { + ActionListener modelVerificationListener = ActionListener.wrap(model -> { + assertThat(model, instanceOf(AzureOpenAiEmbeddingsModel.class)); + + var embeddingsModel = (AzureOpenAiEmbeddingsModel) model; + assertThat(embeddingsModel.getServiceSettings().resourceName(), is("resource_name")); + assertThat(embeddingsModel.getServiceSettings().deploymentId(), is("deployment_id")); + assertThat(embeddingsModel.getServiceSettings().apiVersion(), is("api_version")); + assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + assertThat(embeddingsModel.getTaskSettings().user(), is("user")); + assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + }, exception -> fail("Unexpected exception: " + exception)); + + service.parseRequestConfig( + "id", + TaskType.TEXT_EMBEDDING, + getRequestConfigMap( + getRequestAzureOpenAiServiceSettingsMap("resource_name", "deployment_id", "api_version", null, null), + getAzureOpenAiRequestTaskSettingsMap("user"), + getAzureOpenAiSecretSettingsMap("secret", null) + ), + modelVerificationListener + ); + } + } + public void testParseRequestConfig_ThrowsUnsupportedModelType() throws IOException { try (var service = createAzureOpenAiService()) { ActionListener modelVerificationListener = ActionListener.wrap( @@ -298,6 +384,103 @@ public void testParsePersistedConfigWithSecrets_CreatesAnAzureOpenAiEmbeddingsMo } } + public + void + testParsePersistedConfigWithSecrets_CreatesAnOpenAiEmbeddingsModelWithoutChunkingSettingsWhenChunkingSettingsFeatureFlagDisabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is disabled", ChunkingSettingsFeatureFlag.isEnabled() == false); + try (var service = createAzureOpenAiService()) { + var persistedConfig = getPersistedConfigMap( + getPersistentAzureOpenAiServiceSettingsMap("resource_name", "deployment_id", "api_version", 100, 512), + getAzureOpenAiRequestTaskSettingsMap("user"), + createRandomChunkingSettingsMap(), + getAzureOpenAiSecretSettingsMap("secret", null) + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + assertThat(model, instanceOf(AzureOpenAiEmbeddingsModel.class)); + + var embeddingsModel = (AzureOpenAiEmbeddingsModel) model; + assertThat(embeddingsModel.getServiceSettings().resourceName(), is("resource_name")); + assertThat(embeddingsModel.getServiceSettings().deploymentId(), is("deployment_id")); + assertThat(embeddingsModel.getServiceSettings().apiVersion(), is("api_version")); + assertThat(embeddingsModel.getServiceSettings().dimensions(), is(100)); + assertThat(embeddingsModel.getServiceSettings().maxInputTokens(), is(512)); + assertThat(embeddingsModel.getTaskSettings().user(), is("user")); + assertNull(embeddingsModel.getConfigurations().getChunkingSettings()); + assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } + } + + public void testParsePersistedConfigWithSecrets_CreatesAnOpenAiEmbeddingsModelWhenChunkingSettingsProvidedAndFeatureFlagEnabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + try (var service = createAzureOpenAiService()) { + var persistedConfig = getPersistedConfigMap( + getPersistentAzureOpenAiServiceSettingsMap("resource_name", "deployment_id", "api_version", 100, 512), + getAzureOpenAiRequestTaskSettingsMap("user"), + createRandomChunkingSettingsMap(), + getAzureOpenAiSecretSettingsMap("secret", null) + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + assertThat(model, instanceOf(AzureOpenAiEmbeddingsModel.class)); + + var embeddingsModel = (AzureOpenAiEmbeddingsModel) model; + assertThat(embeddingsModel.getServiceSettings().resourceName(), is("resource_name")); + assertThat(embeddingsModel.getServiceSettings().deploymentId(), is("deployment_id")); + assertThat(embeddingsModel.getServiceSettings().apiVersion(), is("api_version")); + assertThat(embeddingsModel.getServiceSettings().dimensions(), is(100)); + assertThat(embeddingsModel.getServiceSettings().maxInputTokens(), is(512)); + assertThat(embeddingsModel.getTaskSettings().user(), is("user")); + assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } + } + + public void testParsePersistedConfigWithSecrets_CreatesAnOpenAiEmbeddingsModelWhenChunkingSettingsNotProvidedAndFeatureFlagEnabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + try (var service = createAzureOpenAiService()) { + var persistedConfig = getPersistedConfigMap( + getPersistentAzureOpenAiServiceSettingsMap("resource_name", "deployment_id", "api_version", 100, 512), + getAzureOpenAiRequestTaskSettingsMap("user"), + getAzureOpenAiSecretSettingsMap("secret", null) + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + assertThat(model, instanceOf(AzureOpenAiEmbeddingsModel.class)); + + var embeddingsModel = (AzureOpenAiEmbeddingsModel) model; + assertThat(embeddingsModel.getServiceSettings().resourceName(), is("resource_name")); + assertThat(embeddingsModel.getServiceSettings().deploymentId(), is("deployment_id")); + assertThat(embeddingsModel.getServiceSettings().apiVersion(), is("api_version")); + assertThat(embeddingsModel.getServiceSettings().dimensions(), is(100)); + assertThat(embeddingsModel.getServiceSettings().maxInputTokens(), is(512)); + assertThat(embeddingsModel.getTaskSettings().user(), is("user")); + assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } + } + public void testParsePersistedConfigWithSecrets_ThrowsErrorTryingToParseInvalidModel() throws IOException { try (var service = createAzureOpenAiService()) { var persistedConfig = getPersistedConfigMap( @@ -494,6 +677,77 @@ public void testParsePersistedConfig_CreatesAnAzureOpenAiEmbeddingsModel() throw } } + public void testParsePersistedConfig_CreatesAnAzureOpenAiEmbeddingsModelWithoutChunkingSettingsWhenChunkingSettingsFeatureFlagDisabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is disabled", ChunkingSettingsFeatureFlag.isEnabled() == false); + try (var service = createAzureOpenAiService()) { + var persistedConfig = getPersistedConfigMap( + getPersistentAzureOpenAiServiceSettingsMap("resource_name", "deployment_id", "api_version", null, null), + getAzureOpenAiRequestTaskSettingsMap("user"), + createRandomChunkingSettingsMap() + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + assertThat(model, instanceOf(AzureOpenAiEmbeddingsModel.class)); + + var embeddingsModel = (AzureOpenAiEmbeddingsModel) model; + assertThat(embeddingsModel.getServiceSettings().resourceName(), is("resource_name")); + assertThat(embeddingsModel.getServiceSettings().deploymentId(), is("deployment_id")); + assertThat(embeddingsModel.getServiceSettings().apiVersion(), is("api_version")); + assertThat(embeddingsModel.getTaskSettings().user(), is("user")); + assertNull(embeddingsModel.getConfigurations().getChunkingSettings()); + assertNull(embeddingsModel.getSecretSettings()); + } + } + + public void testParsePersistedConfig_CreatesAnAzureOpenAiEmbeddingsModelWhenChunkingSettingsProvidedAndFeatureFlagEnabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + try (var service = createAzureOpenAiService()) { + var persistedConfig = getPersistedConfigMap( + getPersistentAzureOpenAiServiceSettingsMap("resource_name", "deployment_id", "api_version", null, null), + getAzureOpenAiRequestTaskSettingsMap("user"), + createRandomChunkingSettingsMap() + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + assertThat(model, instanceOf(AzureOpenAiEmbeddingsModel.class)); + + var embeddingsModel = (AzureOpenAiEmbeddingsModel) model; + assertThat(embeddingsModel.getServiceSettings().resourceName(), is("resource_name")); + assertThat(embeddingsModel.getServiceSettings().deploymentId(), is("deployment_id")); + assertThat(embeddingsModel.getServiceSettings().apiVersion(), is("api_version")); + assertThat(embeddingsModel.getTaskSettings().user(), is("user")); + assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + assertNull(embeddingsModel.getSecretSettings()); + } + } + + public void testParsePersistedConfig_CreatesAnOpenAiEmbeddingsModelWhenChunkingSettingsNotProvidedAndFeatureFlagEnabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + try (var service = createAzureOpenAiService()) { + var persistedConfig = getPersistedConfigMap( + getPersistentAzureOpenAiServiceSettingsMap("resource_name", "deployment_id", "api_version", null, null), + getAzureOpenAiRequestTaskSettingsMap("user") + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + assertThat(model, instanceOf(AzureOpenAiEmbeddingsModel.class)); + + var embeddingsModel = (AzureOpenAiEmbeddingsModel) model; + assertThat(embeddingsModel.getServiceSettings().resourceName(), is("resource_name")); + assertThat(embeddingsModel.getServiceSettings().deploymentId(), is("deployment_id")); + assertThat(embeddingsModel.getServiceSettings().apiVersion(), is("api_version")); + assertThat(embeddingsModel.getTaskSettings().user(), is("user")); + assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + assertNull(embeddingsModel.getSecretSettings()); + } + } + public void testParsePersistedConfig_ThrowsErrorTryingToParseInvalidModel() throws IOException { try (var service = createAzureOpenAiService()) { var persistedConfig = getPersistedConfigMap( @@ -1064,6 +1318,35 @@ public void testInfer_UnauthorisedResponse() throws IOException, URISyntaxExcept } public void testChunkedInfer_CallsInfer_ConvertsFloatResponse() throws IOException, URISyntaxException { + var model = AzureOpenAiEmbeddingsModelTests.createModel("resource", "deployment", "apiversion", "user", "apikey", null, "id"); + + testChunkedInfer(model); + } + + public void testChunkedInfer_ChunkingSettingsSetAndFeatureFlagEnabled() throws IOException, URISyntaxException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + var model = AzureOpenAiEmbeddingsModelTests.createModel( + "resource", + "deployment", + "apiversion", + "user", + createRandomChunkingSettings(), + "apikey", + null, + "id" + ); + + testChunkedInfer(model); + } + + public void testChunkedInfer_ChunkingSettingsNotSetAndFeatureFlagEnabled() throws IOException, URISyntaxException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + var model = AzureOpenAiEmbeddingsModelTests.createModel("resource", "deployment", "apiversion", "user", null, "apikey", null, "id"); + + testChunkedInfer(model); + } + + private void testChunkedInfer(AzureOpenAiEmbeddingsModel model) throws IOException, URISyntaxException { var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); try (var service = new AzureOpenAiService(senderFactory, createWithEmptySettings(threadPool))) { @@ -1098,7 +1381,6 @@ public void testChunkedInfer_CallsInfer_ConvertsFloatResponse() throws IOExcepti """; webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); - var model = AzureOpenAiEmbeddingsModelTests.createModel("resource", "deployment", "apiversion", "user", "apikey", null, "id"); model.setUri(new URI(getUrl(webServer))); PlainActionFuture> listener = new PlainActionFuture<>(); service.chunkedInfer( @@ -1145,6 +1427,18 @@ private AzureOpenAiService createAzureOpenAiService() { return new AzureOpenAiService(mock(HttpRequestSender.Factory.class), createWithEmptySettings(threadPool)); } + private Map getRequestConfigMap( + Map serviceSettings, + Map taskSettings, + Map chunkingSettings, + Map secretSettings + ) { + var requestConfigMap = getRequestConfigMap(serviceSettings, taskSettings, secretSettings); + requestConfigMap.put(ModelConfigurations.CHUNKING_SETTINGS, chunkingSettings); + + return requestConfigMap; + } + private Map getRequestConfigMap( Map serviceSettings, Map taskSettings, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModelTests.java index 1747155623a98..2f6760cb36e9f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModelTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModelTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; import org.elasticsearch.test.ESTestCase; @@ -100,6 +101,7 @@ public static AzureOpenAiEmbeddingsModel createModel( String deploymentId, String apiVersion, String user, + ChunkingSettings chunkingSettings, @Nullable String apiKey, @Nullable String entraId, String inferenceEntityId @@ -112,6 +114,29 @@ public static AzureOpenAiEmbeddingsModel createModel( "service", new AzureOpenAiEmbeddingsServiceSettings(resourceName, deploymentId, apiVersion, null, false, null, null, null), new AzureOpenAiEmbeddingsTaskSettings(user), + chunkingSettings, + new AzureOpenAiSecretSettings(secureApiKey, secureEntraId) + ); + } + + public static AzureOpenAiEmbeddingsModel createModel( + String resourceName, + String deploymentId, + String apiVersion, + String user, + @Nullable String apiKey, + @Nullable String entraId, + String inferenceEntityId + ) { + var secureApiKey = apiKey != null ? new SecureString(apiKey.toCharArray()) : null; + var secureEntraId = entraId != null ? new SecureString(entraId.toCharArray()) : null; + return new AzureOpenAiEmbeddingsModel( + inferenceEntityId, + TaskType.TEXT_EMBEDDING, + "service", + new AzureOpenAiEmbeddingsServiceSettings(resourceName, deploymentId, apiVersion, null, false, null, null, null), + new AzureOpenAiEmbeddingsTaskSettings(user), + null, new AzureOpenAiSecretSettings(secureApiKey, secureEntraId) ); } @@ -147,6 +172,7 @@ public static AzureOpenAiEmbeddingsModel createModel( null ), new AzureOpenAiEmbeddingsTaskSettings(user), + null, new AzureOpenAiSecretSettings(secureApiKey, secureEntraId) ); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java index 420a635963a29..758c38166778b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.inference.ChunkingOptions; +import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; @@ -30,6 +31,7 @@ import org.elasticsearch.test.http.MockWebServer; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.ChunkingSettingsFeatureFlag; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingByteResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; @@ -62,6 +64,8 @@ import static org.elasticsearch.xpack.inference.Utils.getPersistedConfigMap; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests.createRandomChunkingSettings; +import static org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests.createRandomChunkingSettingsMap; import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; import static org.elasticsearch.xpack.inference.results.TextEmbeddingResultsTests.buildExpectationFloat; @@ -130,6 +134,95 @@ public void testParseRequestConfig_CreatesACohereEmbeddingsModel() throws IOExce } } + public void testParseRequestConfig_ThrowsElasticsearchStatusExceptionWhenChunkingSettingsProvidedAndFeatureFlagDisabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is disabled", ChunkingSettingsFeatureFlag.isEnabled() == false); + try (var service = createCohereService()) { + var serviceSettings = CohereEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model", null); + + var config = getRequestConfigMap( + serviceSettings, + getTaskSettingsMap(null, null), + createRandomChunkingSettingsMap(), + getSecretSettingsMap("secret") + ); + + var failureListener = ActionListener.wrap((model) -> fail("Model parsing should have failed"), e -> { + MatcherAssert.assertThat(e, instanceOf(ElasticsearchStatusException.class)); + MatcherAssert.assertThat(e.getMessage(), containsString("Model configuration contains settings")); + }); + service.parseRequestConfig("id", TaskType.TEXT_EMBEDDING, config, failureListener); + } + } + + public void testParseRequestConfig_CreatesACohereEmbeddingsModelWhenChunkingSettingsProvidedAndFeatureFlagEnabled() throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + try (var service = createCohereService()) { + ActionListener modelListener = ActionListener.wrap(model -> { + MatcherAssert.assertThat(model, instanceOf(CohereEmbeddingsModel.class)); + + var embeddingsModel = (CohereEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getEmbeddingType(), is(CohereEmbeddingType.FLOAT)); + MatcherAssert.assertThat( + embeddingsModel.getTaskSettings(), + is(new CohereEmbeddingsTaskSettings(InputType.INGEST, CohereTruncation.START)) + ); + MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + }, e -> fail("Model parsing should have succeeded " + e.getMessage())); + + service.parseRequestConfig( + "id", + TaskType.TEXT_EMBEDDING, + getRequestConfigMap( + CohereEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model", CohereEmbeddingType.FLOAT), + getTaskSettingsMap(InputType.INGEST, CohereTruncation.START), + createRandomChunkingSettingsMap(), + getSecretSettingsMap("secret") + ), + modelListener + ); + + } + } + + public void testParseRequestConfig_CreatesACohereEmbeddingsModelWhenChunkingSettingsNotProvidedAndFeatureFlagEnabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + try (var service = createCohereService()) { + ActionListener modelListener = ActionListener.wrap(model -> { + MatcherAssert.assertThat(model, instanceOf(CohereEmbeddingsModel.class)); + + var embeddingsModel = (CohereEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getEmbeddingType(), is(CohereEmbeddingType.FLOAT)); + MatcherAssert.assertThat( + embeddingsModel.getTaskSettings(), + is(new CohereEmbeddingsTaskSettings(InputType.INGEST, CohereTruncation.START)) + ); + MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + }, e -> fail("Model parsing should have succeeded " + e.getMessage())); + + service.parseRequestConfig( + "id", + TaskType.TEXT_EMBEDDING, + getRequestConfigMap( + CohereEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model", CohereEmbeddingType.FLOAT), + getTaskSettingsMap(InputType.INGEST, CohereTruncation.START), + getSecretSettingsMap("secret") + ), + modelListener + ); + + } + } + public void testParseRequestConfig_OptionalTaskSettings() throws IOException { try (var service = createCohereService()) { @@ -305,6 +398,92 @@ public void testParsePersistedConfigWithSecrets_CreatesACohereEmbeddingsModel() } } + public void testParsePersistedConfigWithSecrets_CreatesACohereEmbeddingsModelWithoutChunkingSettingsWhenFeatureFlagDisabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is disabled", ChunkingSettingsFeatureFlag.isEnabled() == false); + try (var service = createCohereService()) { + var persistedConfig = getPersistedConfigMap( + CohereEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model", null), + getTaskSettingsMap(null, null), + createRandomChunkingSettingsMap(), + getSecretSettingsMap("secret") + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(CohereEmbeddingsModel.class)); + + var embeddingsModel = (CohereEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new CohereEmbeddingsTaskSettings(null, null))); + assertNull(embeddingsModel.getConfigurations().getChunkingSettings()); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } + } + + public void testParsePersistedConfigWithSecrets_CreatesACohereEmbeddingsModelWhenChunkingSettingsProvidedAndFeatureFlagEnabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + try (var service = createCohereService()) { + var persistedConfig = getPersistedConfigMap( + CohereEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model", null), + getTaskSettingsMap(null, null), + createRandomChunkingSettingsMap(), + getSecretSettingsMap("secret") + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(CohereEmbeddingsModel.class)); + + var embeddingsModel = (CohereEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new CohereEmbeddingsTaskSettings(null, null))); + MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } + } + + public void testParsePersistedConfigWithSecrets_CreatesACohereEmbeddingsModelWhenChunkingSettingsNotProvidedAndFeatureFlagEnabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + try (var service = createCohereService()) { + var persistedConfig = getPersistedConfigMap( + CohereEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model", null), + getTaskSettingsMap(null, null), + getSecretSettingsMap("secret") + ); + + var model = service.parsePersistedConfigWithSecrets( + "id", + TaskType.TEXT_EMBEDDING, + persistedConfig.config(), + persistedConfig.secrets() + ); + + MatcherAssert.assertThat(model, instanceOf(CohereEmbeddingsModel.class)); + + var embeddingsModel = (CohereEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new CohereEmbeddingsTaskSettings(null, null))); + MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + MatcherAssert.assertThat(embeddingsModel.getSecretSettings().apiKey().toString(), is("secret")); + } + } + public void testParsePersistedConfigWithSecrets_ThrowsErrorTryingToParseInvalidModel() throws IOException { try (var service = createCohereService()) { var persistedConfig = getPersistedConfigMap( @@ -507,6 +686,74 @@ public void testParsePersistedConfig_CreatesACohereEmbeddingsModel() throws IOEx } } + public void testParsePersistedConfig_CreatesACohereEmbeddingsModelWithoutChunkingSettingsWhenChunkingSettingsFeatureFlagDisabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is disabled", ChunkingSettingsFeatureFlag.isEnabled() == false); + try (var service = createCohereService()) { + var persistedConfig = getPersistedConfigMap( + CohereEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model", null), + getTaskSettingsMap(null, CohereTruncation.NONE), + createRandomChunkingSettingsMap() + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + MatcherAssert.assertThat(model, instanceOf(CohereEmbeddingsModel.class)); + + var embeddingsModel = (CohereEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new CohereEmbeddingsTaskSettings(null, CohereTruncation.NONE))); + assertNull(embeddingsModel.getConfigurations().getChunkingSettings()); + assertNull(embeddingsModel.getSecretSettings()); + } + } + + public void testParsePersistedConfig_CreatesACohereEmbeddingsModelWhenChunkingSettingsProvidedAndFeatureFlagEnabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + try (var service = createCohereService()) { + var persistedConfig = getPersistedConfigMap( + CohereEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model", null), + getTaskSettingsMap(null, CohereTruncation.NONE), + createRandomChunkingSettingsMap() + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + MatcherAssert.assertThat(model, instanceOf(CohereEmbeddingsModel.class)); + + var embeddingsModel = (CohereEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new CohereEmbeddingsTaskSettings(null, CohereTruncation.NONE))); + MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + assertNull(embeddingsModel.getSecretSettings()); + } + } + + public void testParsePersistedConfig_CreatesACohereEmbeddingsModelWhenChunkingSettingsNotProvidedAndFeatureFlagEnabled() + throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + try (var service = createCohereService()) { + var persistedConfig = getPersistedConfigMap( + CohereEmbeddingsServiceSettingsTests.getServiceSettingsMap("url", "model", null), + getTaskSettingsMap(null, CohereTruncation.NONE) + ); + + var model = service.parsePersistedConfig("id", TaskType.TEXT_EMBEDDING, persistedConfig.config()); + + MatcherAssert.assertThat(model, instanceOf(CohereEmbeddingsModel.class)); + + var embeddingsModel = (CohereEmbeddingsModel) model; + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().uri().toString(), is("url")); + MatcherAssert.assertThat(embeddingsModel.getServiceSettings().getCommonSettings().modelId(), is("model")); + MatcherAssert.assertThat(embeddingsModel.getTaskSettings(), is(new CohereEmbeddingsTaskSettings(null, CohereTruncation.NONE))); + MatcherAssert.assertThat(embeddingsModel.getConfigurations().getChunkingSettings(), instanceOf(ChunkingSettings.class)); + assertNull(embeddingsModel.getSecretSettings()); + } + } + public void testParsePersistedConfig_ThrowsErrorTryingToParseInvalidModel() throws IOException { try (var service = createCohereService()) { var persistedConfig = getPersistedConfigMap( @@ -1164,6 +1411,52 @@ public void testInfer_DoesNotSetInputType_WhenNotPresentInTaskSettings_AndUnspec } public void testChunkedInfer_BatchesCalls() throws IOException { + var model = CohereEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + new CohereEmbeddingsTaskSettings(null, null), + 1024, + 1024, + "model", + null + ); + + testChunkedInfer(model); + } + + public void testChunkedInfer_BatchesCallsChunkingSettingsSetAndFeatureFlagEnabled() throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + var model = CohereEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + new CohereEmbeddingsTaskSettings(null, null), + createRandomChunkingSettings(), + 1024, + 1024, + "model", + null + ); + + testChunkedInfer(model); + } + + public void testChunkedInfer_ChunkingSettingsNotSetAndFeatureFlagEnabled() throws IOException { + assumeTrue("Only if 'inference_chunking_settings' feature flag is enabled", ChunkingSettingsFeatureFlag.isEnabled()); + var model = CohereEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + new CohereEmbeddingsTaskSettings(null, null), + null, + 1024, + 1024, + "model", + null + ); + + testChunkedInfer(model); + } + + private void testChunkedInfer(CohereEmbeddingsModel model) throws IOException { var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); try (var service = new CohereService(senderFactory, createWithEmptySettings(threadPool))) { @@ -1200,15 +1493,6 @@ public void testChunkedInfer_BatchesCalls() throws IOException { """; webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); - var model = CohereEmbeddingsModelTests.createModel( - getUrl(webServer), - "secret", - new CohereEmbeddingsTaskSettings(null, null), - 1024, - 1024, - "model", - null - ); PlainActionFuture> listener = new PlainActionFuture<>(); // 2 inputs service.chunkedInfer( @@ -1399,6 +1683,18 @@ public void testInfer_StreamRequest_ErrorResponse() throws Exception { .hasErrorContaining("how dare you"); } + private Map getRequestConfigMap( + Map serviceSettings, + Map taskSettings, + Map chunkingSettings, + Map secretSettings + ) { + var requestConfigMap = getRequestConfigMap(serviceSettings, taskSettings, secretSettings); + requestConfigMap.put(ModelConfigurations.CHUNKING_SETTINGS, chunkingSettings); + + return requestConfigMap; + } + private Map getRequestConfigMap( Map serviceSettings, Map taskSettings, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModelTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModelTests.java index 093283c0b37d6..670e63a85cf9d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModelTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModelTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.core.Nullable; +import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; @@ -208,6 +209,7 @@ public static CohereEmbeddingsModel createModel( String url, String apiKey, CohereEmbeddingsTaskSettings taskSettings, + ChunkingSettings chunkingSettings, @Nullable Integer tokenLimit, @Nullable Integer dimensions, @Nullable String model, @@ -222,6 +224,30 @@ public static CohereEmbeddingsModel createModel( Objects.requireNonNullElse(embeddingType, CohereEmbeddingType.FLOAT) ), taskSettings, + chunkingSettings, + new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) + ); + } + + public static CohereEmbeddingsModel createModel( + String url, + String apiKey, + CohereEmbeddingsTaskSettings taskSettings, + @Nullable Integer tokenLimit, + @Nullable Integer dimensions, + @Nullable String model, + @Nullable CohereEmbeddingType embeddingType + ) { + return new CohereEmbeddingsModel( + "id", + TaskType.TEXT_EMBEDDING, + "service", + new CohereEmbeddingsServiceSettings( + new CohereServiceSettings(url, SimilarityMeasure.DOT_PRODUCT, dimensions, tokenLimit, model, null), + Objects.requireNonNullElse(embeddingType, CohereEmbeddingType.FLOAT) + ), + taskSettings, + null, new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) ); } @@ -245,6 +271,7 @@ public static CohereEmbeddingsModel createModel( Objects.requireNonNullElse(embeddingType, CohereEmbeddingType.FLOAT) ), taskSettings, + null, new DefaultSecretSettings(new SecureString(apiKey.toCharArray())) ); } From a68fd70b15377c21421198d8dea9e26a7fb02861 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 9 Oct 2024 01:06:40 +1100 Subject: [PATCH 174/194] Mute org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT test {p0=aggregations/stats_metric_fail_formatting/fail formatting} #114320 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index d6d5d47af3a76..2687f7c2a1f50 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -376,6 +376,9 @@ tests: - class: org.elasticsearch.index.SearchSlowLogTests method: testTwoLoggersDifferentLevel issue: https://github.com/elastic/elasticsearch/issues/114301 +- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT + method: test {p0=aggregations/stats_metric_fail_formatting/fail formatting} + issue: https://github.com/elastic/elasticsearch/issues/114320 # Examples: # From 0a0a4cb9ad24740040c1fce34367506ece843279 Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Wed, 9 Oct 2024 01:39:19 +1100 Subject: [PATCH 175/194] Update docker.elastic.co/wolfi/chainguard-base:latest Docker digest to 90888b1 (#114284) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Update | Change | |---|---|---| | docker.elastic.co/wolfi/chainguard-base | digest | `c16d3ad` -> `90888b1` | --- ### Configuration 📅 **Schedule**: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Renovate Bot](https://togithub.com/renovatebot/renovate). --- build-tools-internal/build.gradle | 9 +++++++++ .../gradle/internal/DockerBase.java | 6 ++++-- renovate.json | 18 +++++++++++++++++- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index 949091816a72f..38d3c0cd326f9 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -384,6 +384,15 @@ tasks.named("jar") { exclude("classpath.index") } +spotless { + java { + // IDEs can sometimes run annotation processors that leave files in + // here, causing Spotless to complain. Even though this path ought not + // to exist, exclude it anyway in order to avoid spurious failures. + toggleOffOn() + } +} + def resolveMainWrapperVersion() { new URL("https://raw.githubusercontent.com/elastic/elasticsearch/main/build-tools-internal/src/main/resources/minimumGradleVersion").text.trim() } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index 793ff6049e10e..95f279bfa5162 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -29,11 +29,13 @@ public enum DockerBase { CLOUD_ESS(null, "-cloud-ess", "apt-get"), // Chainguard based wolfi image with latest jdk - WOLFI( - "docker.elastic.co/wolfi/chainguard-base:latest@sha256:c16d3ad6cebf387e8dd2ad769f54320c4819fbbaa21e729fad087c7ae223b4d0", + // This is usually updated via renovatebot + // spotless:off + WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:90888b190da54062f67f3fef1372eb0ae7d81ea55f5a1f56d748b13e4853d984", "-wolfi", "apk" ), + // spotless:on // Based on WOLFI above, with more extras. We don't set a base image because // we programmatically extend from the Wolfi image. diff --git a/renovate.json b/renovate.json index 7dde3a9440ed5..0a1d588e6332c 100644 --- a/renovate.json +++ b/renovate.json @@ -1,7 +1,23 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": [ - "github>elastic/renovate-config:only-chainguard" + "github>elastic/renovate-config:only-chainguard", + ":disableDependencyDashboard" + ], + "labels": [">non-issue", ":Delivery/Packaging", "Team:Delivery"], + "baseBranches": ["main", "8.x"], + "packageRules": [ + { + "groupName": "wolfi (versioned)", + "groupSlug": "wolfi-versioned", + "description": "Override the `groupSlug` to create a non-special-character branch name", + "matchDatasources": [ + "docker" + ], + "matchPackagePatterns": [ + "^docker.elastic.co/wolfi/chainguard-base$" + ] + } ], "customManagers": [ { From 0cc05448c3f1ae5e1e13ec7b4d110393982f7db4 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Tue, 8 Oct 2024 10:49:59 -0400 Subject: [PATCH 176/194] Use ELSER By Default For Semantic Text (#113563) Co-authored-by: David Kyle --- docs/changelog/113563.yaml | 5 + .../xpack/inference/InferenceFeatures.java | 18 ++- .../inference/mapper/SemanticTextField.java | 1 + .../mapper/SemanticTextFieldMapper.java | 31 +++- .../mapper/SemanticTextFieldMapperTests.java | 143 +++++++++++++++--- .../inference/30_semantic_text_inference.yml | 31 ++++ .../test/inference/40_semantic_text_query.yml | 35 +++++ 7 files changed, 229 insertions(+), 35 deletions(-) create mode 100644 docs/changelog/113563.yaml diff --git a/docs/changelog/113563.yaml b/docs/changelog/113563.yaml new file mode 100644 index 0000000000000..48484ead99d77 --- /dev/null +++ b/docs/changelog/113563.yaml @@ -0,0 +1,5 @@ +pr: 113563 +summary: Use ELSER By Default For Semantic Text +area: Mapping +type: enhancement +issues: [] diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java index a3f2105054639..87b7be717d31b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.inference.rank.random.RandomRankRetrieverBuilder; import org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankRetrieverBuilder; +import java.util.HashSet; import java.util.Set; /** @@ -23,13 +24,16 @@ public class InferenceFeatures implements FeatureSpecification { @Override public Set getFeatures() { - return Set.of( - TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_RETRIEVER_SUPPORTED, - RandomRankRetrieverBuilder.RANDOM_RERANKER_RETRIEVER_SUPPORTED, - SemanticTextFieldMapper.SEMANTIC_TEXT_SEARCH_INFERENCE_ID, - SemanticQueryBuilder.SEMANTIC_TEXT_INNER_HITS, - TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_COMPOSITION_SUPPORTED - ); + var features = new HashSet(); + features.add(TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_RETRIEVER_SUPPORTED); + features.add(RandomRankRetrieverBuilder.RANDOM_RERANKER_RETRIEVER_SUPPORTED); + features.add(SemanticTextFieldMapper.SEMANTIC_TEXT_SEARCH_INFERENCE_ID); + features.add(SemanticQueryBuilder.SEMANTIC_TEXT_INNER_HITS); + features.add(TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_COMPOSITION_SUPPORTED); + if (DefaultElserFeatureFlag.isEnabled()) { + features.add(SemanticTextFieldMapper.SEMANTIC_TEXT_DEFAULT_ELSER_2); + } + return Set.copyOf(features); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java index 0c807c1166608..e60e95b58770f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java @@ -58,6 +58,7 @@ public record SemanticTextField(String fieldName, List originalValues, I static final String TEXT_FIELD = "text"; static final String INFERENCE_FIELD = "inference"; static final String INFERENCE_ID_FIELD = "inference_id"; + static final String SEARCH_INFERENCE_ID_FIELD = "search_inference_id"; static final String CHUNKS_FIELD = "chunks"; static final String CHUNKED_EMBEDDINGS_FIELD = "embeddings"; static final String CHUNKED_TEXT_FIELD = "text"; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 0483296cd2c6a..a5702b38ea3f2 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -54,6 +54,7 @@ import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; +import org.elasticsearch.xpack.inference.DefaultElserFeatureFlag; import java.io.IOException; import java.util.ArrayList; @@ -71,18 +72,23 @@ import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKS_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.INFERENCE_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.INFERENCE_ID_FIELD; +import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.MODEL_SETTINGS_FIELD; +import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.SEARCH_INFERENCE_ID_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.TEXT_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getChunksFieldName; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getEmbeddingsFieldName; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getOriginalTextFieldName; +import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService.DEFAULT_ELSER_ID; /** * A {@link FieldMapper} for semantic text fields. */ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFieldMapper { public static final NodeFeature SEMANTIC_TEXT_SEARCH_INFERENCE_ID = new NodeFeature("semantic_text.search_inference_id"); + public static final NodeFeature SEMANTIC_TEXT_DEFAULT_ELSER_2 = new NodeFeature("semantic_text.default_elser_2"); public static final String CONTENT_TYPE = "semantic_text"; + public static final String DEFAULT_ELSER_2_INFERENCE_ID = DEFAULT_ELSER_ID; private final IndexSettings indexSettings; @@ -96,25 +102,37 @@ public static class Builder extends FieldMapper.Builder { private final IndexSettings indexSettings; private final Parameter inferenceId = Parameter.stringParam( - "inference_id", + INFERENCE_ID_FIELD, false, mapper -> ((SemanticTextFieldType) mapper.fieldType()).inferenceId, - null + DefaultElserFeatureFlag.isEnabled() ? DEFAULT_ELSER_2_INFERENCE_ID : null ).addValidator(v -> { if (Strings.isEmpty(v)) { - throw new IllegalArgumentException("field [inference_id] must be specified"); + // If the default ELSER feature flag is enabled, the only way we get here is if the user explicitly sets the param to an + // empty value. However, if the feature flag is disabled, we can get here if the user didn't set the param. + // Adjust the error message appropriately. + String message = DefaultElserFeatureFlag.isEnabled() + ? "[" + INFERENCE_ID_FIELD + "] on mapper [" + leafName() + "] of type [" + CONTENT_TYPE + "] must not be empty" + : "[" + INFERENCE_ID_FIELD + "] on mapper [" + leafName() + "] of type [" + CONTENT_TYPE + "] must be specified"; + throw new IllegalArgumentException(message); } }); private final Parameter searchInferenceId = Parameter.stringParam( - "search_inference_id", + SEARCH_INFERENCE_ID_FIELD, true, mapper -> ((SemanticTextFieldType) mapper.fieldType()).searchInferenceId, null - ).acceptsNull(); + ).acceptsNull().addValidator(v -> { + if (v != null && Strings.isEmpty(v)) { + throw new IllegalArgumentException( + "[" + SEARCH_INFERENCE_ID_FIELD + "] on mapper [" + leafName() + "] of type [" + CONTENT_TYPE + "] must not be empty" + ); + } + }); private final Parameter modelSettings = new Parameter<>( - "model_settings", + MODEL_SETTINGS_FIELD, true, () -> null, (n, c, o) -> SemanticTextField.parseModelSettingsFromMap(o), @@ -204,6 +222,7 @@ public SemanticTextFieldMapper build(MapperBuilderContext context) { } var childContext = context.createChildContext(leafName(), ObjectMapper.Dynamic.FALSE); final ObjectMapper inferenceField = inferenceFieldBuilder.apply(childContext); + return new SemanticTextFieldMapper( leafName(), new SemanticTextFieldType( diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 1697b33fedd92..7c8d1bbf9fb4d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -23,6 +23,7 @@ import org.apache.lucene.search.join.QueryBitSetProducer; import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; @@ -58,6 +59,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.inference.DefaultElserFeatureFlag; import org.elasticsearch.xpack.inference.InferencePlugin; import org.elasticsearch.xpack.inference.model.TestModel; import org.junit.AssumptionViolatedException; @@ -77,8 +79,10 @@ import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.INFERENCE_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.INFERENCE_ID_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.MODEL_SETTINGS_FIELD; +import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.SEARCH_INFERENCE_ID_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getChunksFieldName; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getEmbeddingsFieldName; +import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.DEFAULT_ELSER_2_INFERENCE_ID; import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomSemanticText; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -92,7 +96,10 @@ protected Collection getPlugins() { @Override protected void minimalMapping(XContentBuilder b) throws IOException { - b.field("type", "semantic_text").field("inference_id", "test_model"); + b.field("type", "semantic_text"); + if (DefaultElserFeatureFlag.isEnabled() == false) { + b.field("inference_id", "test_model"); + } } @Override @@ -155,8 +162,16 @@ protected void assertSearchable(MappedFieldType fieldType) { } public void testDefaults() throws Exception { - DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); - assertEquals(Strings.toString(fieldMapping(this::minimalMapping)), mapper.mappingSource().toString()); + final String fieldName = "field"; + final XContentBuilder fieldMapping = fieldMapping(this::minimalMapping); + + MapperService mapperService = createMapperService(fieldMapping); + DocumentMapper mapper = mapperService.documentMapper(); + assertEquals(Strings.toString(fieldMapping), mapper.mappingSource().toString()); + assertSemanticTextField(mapperService, fieldName, false); + if (DefaultElserFeatureFlag.isEnabled()) { + assertInferenceEndpoints(mapperService, fieldName, DEFAULT_ELSER_2_INFERENCE_ID, DEFAULT_ELSER_2_INFERENCE_ID); + } ParsedDocument doc1 = mapper.parse(source(this::writeField)); List fields = doc1.rootDoc().getFields("field"); @@ -172,12 +187,80 @@ public void testFieldHasValue() { assertTrue(fieldType.fieldHasValue(fieldInfos)); } - public void testInferenceIdNotPresent() { - Exception e = expectThrows( - MapperParsingException.class, - () -> createMapperService(fieldMapping(b -> b.field("type", "semantic_text"))) - ); - assertThat(e.getMessage(), containsString("field [inference_id] must be specified")); + public void testSetInferenceEndpoints() throws IOException { + final String fieldName = "field"; + final String inferenceId = "foo"; + final String searchInferenceId = "bar"; + + CheckedBiConsumer assertSerialization = (expectedMapping, mapperService) -> { + DocumentMapper mapper = mapperService.documentMapper(); + assertEquals(Strings.toString(expectedMapping), mapper.mappingSource().toString()); + }; + + { + final XContentBuilder fieldMapping = fieldMapping(b -> b.field("type", "semantic_text").field(INFERENCE_ID_FIELD, inferenceId)); + final MapperService mapperService = createMapperService(fieldMapping); + assertSemanticTextField(mapperService, fieldName, false); + assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); + assertSerialization.accept(fieldMapping, mapperService); + } + { + if (DefaultElserFeatureFlag.isEnabled()) { + final XContentBuilder fieldMapping = fieldMapping( + b -> b.field("type", "semantic_text").field(SEARCH_INFERENCE_ID_FIELD, searchInferenceId) + ); + final MapperService mapperService = createMapperService(fieldMapping); + assertSemanticTextField(mapperService, fieldName, false); + assertInferenceEndpoints(mapperService, fieldName, DEFAULT_ELSER_2_INFERENCE_ID, searchInferenceId); + assertSerialization.accept(fieldMapping, mapperService); + } + } + { + final XContentBuilder fieldMapping = fieldMapping( + b -> b.field("type", "semantic_text") + .field(INFERENCE_ID_FIELD, inferenceId) + .field(SEARCH_INFERENCE_ID_FIELD, searchInferenceId) + ); + MapperService mapperService = createMapperService(fieldMapping); + assertSemanticTextField(mapperService, fieldName, false); + assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId); + assertSerialization.accept(fieldMapping, mapperService); + } + } + + public void testInvalidInferenceEndpoints() { + { + Exception e = expectThrows( + MapperParsingException.class, + () -> createMapperService(fieldMapping(b -> b.field("type", "semantic_text").field(INFERENCE_ID_FIELD, (String) null))) + ); + assertThat( + e.getMessage(), + containsString("[inference_id] on mapper [field] of type [semantic_text] must not have a [null] value") + ); + } + { + final String expectedMessage = DefaultElserFeatureFlag.isEnabled() + ? "[inference_id] on mapper [field] of type [semantic_text] must not be empty" + : "[inference_id] on mapper [field] of type [semantic_text] must be specified"; + Exception e = expectThrows( + MapperParsingException.class, + () -> createMapperService(fieldMapping(b -> b.field("type", "semantic_text").field(INFERENCE_ID_FIELD, ""))) + ); + assertThat(e.getMessage(), containsString(expectedMessage)); + } + { + if (DefaultElserFeatureFlag.isEnabled()) { + Exception e = expectThrows( + MapperParsingException.class, + () -> createMapperService(fieldMapping(b -> b.field("type", "semantic_text").field(SEARCH_INFERENCE_ID_FIELD, ""))) + ); + assertThat( + e.getMessage(), + containsString("[search_inference_id] on mapper [field] of type [semantic_text] must not be empty") + ); + } + } } public void testCannotBeUsedInMultiFields() { @@ -221,7 +304,7 @@ public void testDynamicUpdate() throws IOException { new SemanticTextField.ModelSettings(TaskType.SPARSE_EMBEDDING, null, null, null) ); assertSemanticTextField(mapperService, fieldName, true); - assertSearchInferenceId(mapperService, fieldName, inferenceId); + assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); } { @@ -232,7 +315,7 @@ public void testDynamicUpdate() throws IOException { new SemanticTextField.ModelSettings(TaskType.SPARSE_EMBEDDING, null, null, null) ); assertSemanticTextField(mapperService, fieldName, true); - assertSearchInferenceId(mapperService, fieldName, searchInferenceId); + assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId); } } @@ -331,19 +414,19 @@ public void testUpdateSearchInferenceId() throws IOException { String fieldName = randomFieldName(depth); MapperService mapperService = createMapperService(buildMapping.apply(fieldName, null)); assertSemanticTextField(mapperService, fieldName, false); - assertSearchInferenceId(mapperService, fieldName, inferenceId); + assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); merge(mapperService, buildMapping.apply(fieldName, searchInferenceId1)); assertSemanticTextField(mapperService, fieldName, false); - assertSearchInferenceId(mapperService, fieldName, searchInferenceId1); + assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId1); merge(mapperService, buildMapping.apply(fieldName, searchInferenceId2)); assertSemanticTextField(mapperService, fieldName, false); - assertSearchInferenceId(mapperService, fieldName, searchInferenceId2); + assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId2); merge(mapperService, buildMapping.apply(fieldName, null)); assertSemanticTextField(mapperService, fieldName, false); - assertSearchInferenceId(mapperService, fieldName, inferenceId); + assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); mapperService = mapperServiceForFieldWithModelSettings( fieldName, @@ -351,19 +434,19 @@ public void testUpdateSearchInferenceId() throws IOException { new SemanticTextField.ModelSettings(TaskType.SPARSE_EMBEDDING, null, null, null) ); assertSemanticTextField(mapperService, fieldName, true); - assertSearchInferenceId(mapperService, fieldName, inferenceId); + assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); merge(mapperService, buildMapping.apply(fieldName, searchInferenceId1)); assertSemanticTextField(mapperService, fieldName, true); - assertSearchInferenceId(mapperService, fieldName, searchInferenceId1); + assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId1); merge(mapperService, buildMapping.apply(fieldName, searchInferenceId2)); assertSemanticTextField(mapperService, fieldName, true); - assertSearchInferenceId(mapperService, fieldName, searchInferenceId2); + assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId2); merge(mapperService, buildMapping.apply(fieldName, null)); assertSemanticTextField(mapperService, fieldName, true); - assertSearchInferenceId(mapperService, fieldName, inferenceId); + assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); } } @@ -409,11 +492,17 @@ private static void assertSemanticTextField(MapperService mapperService, String } } - private static void assertSearchInferenceId(MapperService mapperService, String fieldName, String expectedSearchInferenceId) { + private static void assertInferenceEndpoints( + MapperService mapperService, + String fieldName, + String expectedInferenceId, + String expectedSearchInferenceId + ) { var fieldType = mapperService.fieldType(fieldName); assertNotNull(fieldType); assertThat(fieldType, instanceOf(SemanticTextFieldMapper.SemanticTextFieldType.class)); SemanticTextFieldMapper.SemanticTextFieldType semanticTextFieldType = (SemanticTextFieldMapper.SemanticTextFieldType) fieldType; + assertEquals(expectedInferenceId, semanticTextFieldType.getInferenceId()); assertEquals(expectedSearchInferenceId, semanticTextFieldType.getSearchInferenceId()); } @@ -433,9 +522,19 @@ public void testSuccessfulParse() throws IOException { MapperService mapperService = createMapperService(mapping); assertSemanticTextField(mapperService, fieldName1, false); - assertSearchInferenceId(mapperService, fieldName1, setSearchInferenceId ? searchInferenceId : model1.getInferenceEntityId()); + assertInferenceEndpoints( + mapperService, + fieldName1, + model1.getInferenceEntityId(), + setSearchInferenceId ? searchInferenceId : model1.getInferenceEntityId() + ); assertSemanticTextField(mapperService, fieldName2, false); - assertSearchInferenceId(mapperService, fieldName2, setSearchInferenceId ? searchInferenceId : model2.getInferenceEntityId()); + assertInferenceEndpoints( + mapperService, + fieldName2, + model2.getInferenceEntityId(), + setSearchInferenceId ? searchInferenceId : model2.getInferenceEntityId() + ); DocumentMapper documentMapper = mapperService.documentMapper(); ParsedDocument doc = documentMapper.parse( diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml index f58a5c33fd85d..1795d754d2a9c 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/30_semantic_text_inference.yml @@ -547,3 +547,34 @@ setup: - match: { _source.dense_field.text: "another updated inference test" } - match: { _source.dense_field.inference.chunks.0.text: "another updated inference test" } - exists: _source.dense_field.inference.chunks.0.embeddings + +--- +"Calculates embeddings using the default ELSER 2 endpoint": + - requires: + cluster_features: "semantic_text.default_elser_2" + reason: semantic_text default ELSER 2 inference ID introduced in 8.16.0 + + - do: + indices.create: + index: test-elser-2-default-index + body: + mappings: + properties: + sparse_field: + type: semantic_text + + - do: + index: + index: test-elser-2-default-index + id: doc_1 + body: + sparse_field: "inference test" + + - do: + get: + index: test-elser-2-default-index + id: doc_1 + + - match: { _source.sparse_field.text: "inference test" } + - exists: _source.sparse_field.inference.chunks.0.embeddings + - match: { _source.sparse_field.inference.chunks.0.text: "inference test" } diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/40_semantic_text_query.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/40_semantic_text_query.yml index 2070b3752791a..10858acc0aff8 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/40_semantic_text_query.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/40_semantic_text_query.yml @@ -839,3 +839,38 @@ setup: - match: { error.type: "resource_not_found_exception" } - match: { error.reason: "Inference endpoint not found [invalid-inference-id]" } + +--- +"Query a field that uses the default ELSER 2 endpoint": + - requires: + cluster_features: "semantic_text.default_elser_2" + reason: semantic_text default ELSER 2 inference ID introduced in 8.16.0 + + - do: + indices.create: + index: test-elser-2-default-index + body: + mappings: + properties: + sparse_field: + type: semantic_text + + - do: + index: + index: test-elser-2-default-index + id: doc_1 + body: + sparse_field: "inference test" + refresh: true + + - do: + search: + index: test-elser-2-default-index + body: + query: + semantic: + field: "sparse_field" + query: "inference test" + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "doc_1" } From 72b7d5ecf8e4195957cee32257085dc20f14ce1e Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 9 Oct 2024 02:05:32 +1100 Subject: [PATCH 177/194] Mute org.elasticsearch.xpack.inference.services.cohere.CohereServiceTests testInfer_StreamRequest_ErrorResponse #114327 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 2687f7c2a1f50..bc573959fb2f6 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -379,6 +379,9 @@ tests: - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=aggregations/stats_metric_fail_formatting/fail formatting} issue: https://github.com/elastic/elasticsearch/issues/114320 +- class: org.elasticsearch.xpack.inference.services.cohere.CohereServiceTests + method: testInfer_StreamRequest_ErrorResponse + issue: https://github.com/elastic/elasticsearch/issues/114327 # Examples: # From ae38b9091ff2689331aac60986cfe155ef45e5b0 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 9 Oct 2024 02:30:01 +1100 Subject: [PATCH 178/194] Mute org.elasticsearch.xpack.rank.rrf.RRFRankClientYamlTestSuiteIT test {yaml=rrf/700_rrf_retriever_search_api_compatibility/rrf retriever with top-level collapse} #114331 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index bc573959fb2f6..302bbbf80c41a 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -382,6 +382,9 @@ tests: - class: org.elasticsearch.xpack.inference.services.cohere.CohereServiceTests method: testInfer_StreamRequest_ErrorResponse issue: https://github.com/elastic/elasticsearch/issues/114327 +- class: org.elasticsearch.xpack.rank.rrf.RRFRankClientYamlTestSuiteIT + method: test {yaml=rrf/700_rrf_retriever_search_api_compatibility/rrf retriever with top-level collapse} + issue: https://github.com/elastic/elasticsearch/issues/114331 # Examples: # From 3a83fcdef969fdd376720ff0b57953623ca0d8b4 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Tue, 8 Oct 2024 16:37:04 +0100 Subject: [PATCH 179/194] [ML] Remove scale to zero feature flag (#114323) --- .../test/cluster/FeatureFlag.java | 1 - .../AdaptiveAllocationsScaler.java | 3 +-- .../AdaptiveAllocationsScalerService.java | 3 +-- .../ScaleToZeroFeatureFlag.java | 20 ------------------- .../AdaptiveAllocationsScalerTests.java | 4 ---- 5 files changed, 2 insertions(+), 29 deletions(-) delete mode 100644 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/ScaleToZeroFeatureFlag.java diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java index 5adf01a2a0e7d..aa72d3248812e 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java @@ -19,7 +19,6 @@ public enum FeatureFlag { TIME_SERIES_MODE("es.index_mode_feature_flag_registered=true", Version.fromString("8.0.0"), null), FAILURE_STORE_ENABLED("es.failure_store_feature_flag_enabled=true", Version.fromString("8.12.0"), null), CHUNKING_SETTINGS_ENABLED("es.inference_chunking_settings_feature_flag_enabled=true", Version.fromString("8.16.0"), null), - INFERENCE_SCALE_TO_ZERO("es.inference_scale_to_zero_feature_flag_enabled=true", Version.fromString("8.16.0"), null), INFERENCE_DEFAULT_ELSER("es.inference_default_elser_feature_flag_enabled=true", Version.fromString("8.16.0"), null); public final String systemProperty; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java index 05e7202b8efe9..58259b87c6b00 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScaler.java @@ -170,8 +170,7 @@ Integer scale() { if (maxNumberOfAllocations != null) { numberOfAllocations = Math.min(numberOfAllocations, maxNumberOfAllocations); } - if (ScaleToZeroFeatureFlag.isEnabled() - && (minNumberOfAllocations == null || minNumberOfAllocations == 0) + if ((minNumberOfAllocations == null || minNumberOfAllocations == 0) && timeWithoutRequestsSeconds > SCALE_TO_ZERO_AFTER_NO_REQUESTS_TIME_SECONDS) { logger.debug("[{}] adaptive allocations scaler: scaling down to zero, because of no requests.", deploymentId); numberOfAllocations = 0; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java index 775279a6b2553..193fa9e7e07f9 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java @@ -421,8 +421,7 @@ private void processDeploymentStats(GetDeploymentStatsAction.Response statsRespo } public boolean maybeStartAllocation(TrainedModelAssignment assignment) { - if (ScaleToZeroFeatureFlag.isEnabled() - && assignment.getAdaptiveAllocationsSettings() != null + if (assignment.getAdaptiveAllocationsSettings() != null && assignment.getAdaptiveAllocationsSettings().getEnabled() == Boolean.TRUE) { lastScaleUpTimesMillis.put(assignment.getDeploymentId(), System.currentTimeMillis()); updateNumberOfAllocations(assignment.getDeploymentId(), 1); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/ScaleToZeroFeatureFlag.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/ScaleToZeroFeatureFlag.java deleted file mode 100644 index 072b8c5593c93..0000000000000 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/ScaleToZeroFeatureFlag.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.ml.inference.adaptiveallocations; - -import org.elasticsearch.common.util.FeatureFlag; - -public class ScaleToZeroFeatureFlag { - private ScaleToZeroFeatureFlag() {} - - private static final FeatureFlag FEATURE_FLAG = new FeatureFlag("inference_scale_to_zero"); - - public static boolean isEnabled() { - return FEATURE_FLAG.isEnabled(); - } -} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java index 7d98aaf67a7f3..1887ebe8050e0 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerTests.java @@ -148,8 +148,6 @@ public void testAutoscaling_maxAllocationsSafeguard() { } public void testAutoscaling_scaleDownToZeroAllocations() { - assumeTrue("Should only run if adaptive allocations feature flag is enabled", ScaleToZeroFeatureFlag.isEnabled()); - AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1); // 1 hour with 1 request per 1 seconds, so don't scale. for (int i = 0; i < 3600; i++) { @@ -180,8 +178,6 @@ public void testAutoscaling_scaleDownToZeroAllocations() { } public void testAutoscaling_dontScaleDownToZeroAllocationsWhenMinAllocationsIsSet() { - assumeTrue("Should only run if adaptive allocations feature flag is enabled", ScaleToZeroFeatureFlag.isEnabled()); - AdaptiveAllocationsScaler adaptiveAllocationsScaler = new AdaptiveAllocationsScaler("test-deployment", 1); adaptiveAllocationsScaler.setMinMaxNumberOfAllocations(1, null); From 965265a1a47507548640da3a7c179a6b78126ef3 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Tue, 8 Oct 2024 09:14:03 -0700 Subject: [PATCH 180/194] Don't generate invalid combination of subobjects parameter in logsdb tests (#114265) --- muted-tests.yml | 3 --- .../DefaultMappingParametersHandler.java | 19 ++++++++----------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 302bbbf80c41a..696d7a4496e66 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -365,9 +365,6 @@ tests: - class: org.elasticsearch.xpack.inference.services.openai.OpenAiServiceTests method: testInfer_StreamRequest issue: https://github.com/elastic/elasticsearch/issues/114232 -- class: org.elasticsearch.logsdb.datageneration.DataGeneratorTests - method: testDataGeneratorProducesValidMappingAndDocument - issue: https://github.com/elastic/elasticsearch/issues/114188 - class: org.elasticsearch.ingest.geoip.IpinfoIpDataLookupsTests issue: https://github.com/elastic/elasticsearch/issues/114266 - class: org.elasticsearch.index.SearchSlowLogTests diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java index 81bd80f464525..4b33f3fefcf1b 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -102,18 +102,12 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequ // TODO enable subobjects: auto // It is disabled because it currently does not have auto flattening and that results in asserts being triggered when using // copy_to. - if (ESTestCase.randomBoolean()) { - parameters.put( - "subobjects", - ESTestCase.randomValueOtherThan( - ObjectMapper.Subobjects.AUTO, - () -> ESTestCase.randomFrom(ObjectMapper.Subobjects.values()) - ).toString() - ); - } + var subobjects = ESTestCase.randomValueOtherThan( + ObjectMapper.Subobjects.AUTO, + () -> ESTestCase.randomFrom(ObjectMapper.Subobjects.values()) + ); - if (request.parentSubobjects() == ObjectMapper.Subobjects.DISABLED - || parameters.getOrDefault("subobjects", "true").equals("false")) { + if (request.parentSubobjects() == ObjectMapper.Subobjects.DISABLED || subobjects == ObjectMapper.Subobjects.DISABLED) { // "enabled: false" is not compatible with subobjects: false // changing "dynamic" from parent context is not compatible with subobjects: false // changing subobjects value is not compatible with subobjects: false @@ -124,6 +118,9 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequ return parameters; } + if (ESTestCase.randomBoolean()) { + parameters.put("subobjects", subobjects.toString()); + } if (ESTestCase.randomBoolean()) { parameters.put("dynamic", ESTestCase.randomFrom("true", "false", "strict", "runtime")); } From f633148d102a314bb8e3074ccf797f9444c94fd1 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 8 Oct 2024 12:17:56 -0400 Subject: [PATCH 181/194] Docs: ESQL doesn't preserve `null`s in a list (#114335) The doc values don't preserve `null`s in a list so ESQL doesn't either. Closes #114324 --- .../esql/multivalued-fields.asciidoc | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/reference/esql/multivalued-fields.asciidoc b/docs/reference/esql/multivalued-fields.asciidoc index 2dfda3060d3ea..562ea2a2e6b4a 100644 --- a/docs/reference/esql/multivalued-fields.asciidoc +++ b/docs/reference/esql/multivalued-fields.asciidoc @@ -177,6 +177,37 @@ POST /_query ---- // TESTRESPONSE[s/"took": 28/"took": "$body.took"/] +[discrete] +[[esql-multivalued-nulls]] +==== `null` in a list + +`null` values in a list are not preserved at the storage layer: + +[source,console,id=esql-multivalued-fields-multivalued-nulls] +---- +POST /mv/_doc?refresh +{ "a": [2, null, 1] } + +POST /_query +{ + "query": "FROM mv | LIMIT 1" +} +---- + +[source,console-result] +---- +{ + "took": 28, + "columns": [ + { "name": "a", "type": "long"}, + ], + "values": [ + [[1, 2]], + ] +} +---- +// TESTRESPONSE[s/"took": 28/"took": "$body.took"/] + [discrete] [[esql-multivalued-fields-functions]] ==== Functions From d3fa42cda096a4dadc9ac15a1b6dce9a235834b7 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 8 Oct 2024 12:37:55 -0400 Subject: [PATCH 182/194] ESQL: Entirely remove META FUNCTIONS (#113967) This removes the undocumented `META FUNCTIONS` command that emits descriptions for all functions. This shouldn't be used because we never told anyone about it. I'd have preferred if we'd have explicitly documented it as no public or if we'd have left it snapshot-only. But sometimes you make a mistake. I'm hopeful no one is relying on it. It was never reliable and not public..... --- .../src/main/resources/changelog-schema.json | 1 + docs/changelog/113967.yaml | 13 + docs/reference/rest-api/usage.asciidoc | 3 +- x-pack/plugin/build.gradle | 1 + .../esql/qa/mixed/MixedClusterEsqlSpecIT.java | 4 - .../xpack/esql/ccq/MultiClusterSpecIT.java | 4 - .../src/main/resources/meta.csv-spec | 552 ----- .../xpack/esql/action/EsqlActionIT.java | 24 - .../esql/src/main/antlr/EsqlBaseLexer.g4 | 21 - .../esql/src/main/antlr/EsqlBaseLexer.tokens | 329 ++- .../esql/src/main/antlr/EsqlBaseParser.g4 | 5 - .../esql/src/main/antlr/EsqlBaseParser.tokens | 329 ++- .../xpack/esql/action/EsqlCapabilities.java | 6 +- .../function/EsqlFunctionRegistry.java | 26 - .../xpack/esql/parser/EsqlBaseLexer.interp | 19 +- .../xpack/esql/parser/EsqlBaseLexer.java | 1979 ++++++++--------- .../xpack/esql/parser/EsqlBaseParser.interp | 13 +- .../xpack/esql/parser/EsqlBaseParser.java | 1507 ++++++------- .../parser/EsqlBaseParserBaseListener.java | 12 - .../parser/EsqlBaseParserBaseVisitor.java | 7 - .../esql/parser/EsqlBaseParserListener.java | 12 - .../esql/parser/EsqlBaseParserVisitor.java | 7 - .../xpack/esql/parser/LogicalPlanBuilder.java | 6 - .../esql/plan/logical/meta/MetaFunctions.java | 143 -- .../xpack/esql/planner/Mapper.java | 4 - .../xpack/esql/stats/FeatureMetric.java | 9 +- .../esql/parser/StatementParserTests.java | 1 - .../esql/stats/VerifierMetricsTests.java | 40 - .../rest-api-spec/test/esql/60_usage.yml | 12 +- 29 files changed, 2038 insertions(+), 3051 deletions(-) create mode 100644 docs/changelog/113967.yaml delete mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec delete mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/meta/MetaFunctions.java diff --git a/build-tools-internal/src/main/resources/changelog-schema.json b/build-tools-internal/src/main/resources/changelog-schema.json index a435305a8e3e2..451701d74d690 100644 --- a/build-tools-internal/src/main/resources/changelog-schema.json +++ b/build-tools-internal/src/main/resources/changelog-schema.json @@ -284,6 +284,7 @@ "Cluster and node setting", "Command line tool", "CRUD", + "ES|QL", "Index setting", "Ingest", "JVM option", diff --git a/docs/changelog/113967.yaml b/docs/changelog/113967.yaml new file mode 100644 index 0000000000000..58b72eba49deb --- /dev/null +++ b/docs/changelog/113967.yaml @@ -0,0 +1,13 @@ +pr: 113967 +summary: "ESQL: Entirely remove META FUNCTIONS" +area: ES|QL +type: breaking +issues: [] +breaking: + title: "ESQL: Entirely remove META FUNCTIONS" + area: ES|QL + details: | + Removes an undocumented syntax from ESQL: META FUNCTION. This was never + reliable or really useful. Consult the documentation instead. + impact: "Removes an undocumented syntax from ESQL: META FUNCTION" + notable: false diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc index 4dcf0d328e4f1..957f57ffc9105 100644 --- a/docs/reference/rest-api/usage.asciidoc +++ b/docs/reference/rest-api/usage.asciidoc @@ -239,8 +239,7 @@ GET /_xpack/usage "keep" : 0, "enrich" : 0, "from" : 0, - "row" : 0, - "meta" : 0 + "row" : 0 }, "queries" : { "rest" : { diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 158cccb1b6ea2..3e5aaea43a9b9 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -84,5 +84,6 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("security/10_forbidden/Test bulk response with invalid credentials", "warning does not exist for compatibility") task.skipTest("wildcard/30_ignore_above_synthetic_source/wildcard field type ignore_above", "Temporary until backported") task.skipTest("inference/inference_crud/Test get all", "Assertions on number of inference models break due to default configs") + task.skipTest("esql/60_usage/Basic ESQL usage output (telemetry)", "The telemetry output changed. We dropped a column. That's safe.") }) diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java index 08b4794b740d6..d0d6d5fa49c42 100644 --- a/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java +++ b/x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java @@ -72,10 +72,6 @@ public MixedClusterEsqlSpecIT( protected void shouldSkipTest(String testName) throws IOException { super.shouldSkipTest(testName); assumeTrue("Test " + testName + " is skipped on " + bwcVersion, isEnabled(testName, instructions, bwcVersion)); - assumeFalse( - "Skip META tests on mixed version clusters because we change it too quickly", - testCase.requiredCapabilities.contains("meta") - ); if (mode == ASYNC) { assumeTrue("Async is not supported on " + bwcVersion, supportsAsync()); } diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java index 8d54dc63598f0..3e799730f7269 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java @@ -112,10 +112,6 @@ protected void shouldSkipTest(String testName) throws IOException { ); assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains("inlinestats")); assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains("inlinestats_v2")); - assumeFalse( - "Skip META tests on mixed version clusters because we change it too quickly", - testCase.requiredCapabilities.contains("meta") - ); } private TestFeatureService remoteFeaturesService() throws IOException { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec deleted file mode 100644 index 6e8d5fba67cee..0000000000000 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec +++ /dev/null @@ -1,552 +0,0 @@ -metaFunctionsSynopsis -required_capability: date_nanos_type -required_capability: meta - -meta functions | keep synopsis; - -synopsis:keyword -"double|integer|long|unsigned_long abs(number:double|integer|long|unsigned_long)" -"double acos(number:double|integer|long|unsigned_long)" -"double asin(number:double|integer|long|unsigned_long)" -"double atan(number:double|integer|long|unsigned_long)" -"double atan2(y_coordinate:double|integer|long|unsigned_long, x_coordinate:double|integer|long|unsigned_long)" -"double avg(number:double|integer|long)" -"double|date bin(field:integer|long|double|date, buckets:integer|long|double|date_period|time_duration, ?from:integer|long|double|date|keyword|text, ?to:integer|long|double|date|keyword|text)" -"double|date bucket(field:integer|long|double|date, buckets:integer|long|double|date_period|time_duration, ?from:integer|long|double|date|keyword|text, ?to:integer|long|double|date|keyword|text)" -"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version case(condition:boolean, trueValue...:boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)" -"double cbrt(number:double|integer|long|unsigned_long)" -"double|integer|long|unsigned_long ceil(number:double|integer|long|unsigned_long)" -"boolean cidr_match(ip:ip, blockX...:keyword|text)" -"boolean|cartesian_point|cartesian_shape|date|geo_point|geo_shape|integer|ip|keyword|long|text|version coalesce(first:boolean|cartesian_point|cartesian_shape|date|geo_point|geo_shape|integer|ip|keyword|long|text|version, ?rest...:boolean|cartesian_point|cartesian_shape|date|geo_point|geo_shape|integer|ip|keyword|long|text|version)" -"keyword concat(string1:keyword|text, string2...:keyword|text)" -"double cos(angle:double|integer|long|unsigned_long)" -"double cosh(number:double|integer|long|unsigned_long)" -"long count(?field:boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|unsigned_long|version)" -"long count_distinct(field:boolean|date|double|integer|ip|keyword|long|text|version, ?precision:integer|long|unsigned_long)" -"integer date_diff(unit:keyword|text, startTimestamp:date, endTimestamp:date)" -"long date_extract(datePart:keyword|text, date:date)" -"keyword date_format(?dateFormat:keyword|text, date:date)" -"date date_parse(?datePattern:keyword|text, dateString:keyword|text)" -"date date_trunc(interval:date_period|time_duration, date:date)" -double e() -"boolean ends_with(str:keyword|text, suffix:keyword|text)" -"double exp(number:double|integer|long|unsigned_long)" -"double|integer|long|unsigned_long floor(number:double|integer|long|unsigned_long)" -"keyword from_base64(string:keyword|text)" -"boolean|date|double|integer|ip|keyword|long|text|version greatest(first:boolean|date|double|integer|ip|keyword|long|text|version, ?rest...:boolean|date|double|integer|ip|keyword|long|text|version)" -"ip ip_prefix(ip:ip, prefixLengthV4:integer, prefixLengthV6:integer)" -"boolean|date|double|integer|ip|keyword|long|text|version least(first:boolean|date|double|integer|ip|keyword|long|text|version, ?rest...:boolean|date|double|integer|ip|keyword|long|text|version)" -"keyword left(string:keyword|text, length:integer)" -"integer length(string:keyword|text)" -"integer locate(string:keyword|text, substring:keyword|text, ?start:integer)" -"double log(?base:integer|unsigned_long|long|double, number:integer|unsigned_long|long|double)" -"double log10(number:double|integer|long|unsigned_long)" -"keyword|text ltrim(string:keyword|text)" -"boolean|double|integer|long|date|ip|keyword|text|long|version max(field:boolean|double|integer|long|date|ip|keyword|text|long|version)" -"double median(number:double|integer|long)" -"double median_absolute_deviation(number:double|integer|long)" -"boolean|double|integer|long|date|ip|keyword|text|long|version min(field:boolean|double|integer|long|date|ip|keyword|text|long|version)" -"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version mv_append(field1:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version, field2:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version)" -"double mv_avg(number:double|integer|long|unsigned_long)" -"keyword mv_concat(string:text|keyword, delim:text|keyword)" -"integer mv_count(field:boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)" -"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version mv_dedupe(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version)" -"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version mv_first(field:boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)" -"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version mv_last(field:boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)" -"boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version mv_max(field:boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version)" -"double|integer|long|unsigned_long mv_median(number:double|integer|long|unsigned_long)" -"double|integer|long|unsigned_long mv_median_absolute_deviation(number:double|integer|long|unsigned_long)" -"boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version mv_min(field:boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version)" -"double|integer|long mv_percentile(number:double|integer|long, percentile:double|integer|long)" -"double mv_pseries_weighted_sum(number:double, p:double)" -"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version mv_slice(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version, start:integer, ?end:integer)" -"boolean|date|double|integer|ip|keyword|long|text|version mv_sort(field:boolean|date|double|integer|ip|keyword|long|text|version, ?order:keyword)" -"double|integer|long|unsigned_long mv_sum(number:double|integer|long|unsigned_long)" -"keyword mv_zip(string1:keyword|text, string2:keyword|text, ?delim:keyword|text)" -date now() -"double percentile(number:double|integer|long, percentile:double|integer|long)" -double pi() -"double pow(base:double|integer|long|unsigned_long, exponent:double|integer|long|unsigned_long)" -"keyword repeat(string:keyword|text, number:integer)" -"keyword replace(string:keyword|text, regex:keyword|text, newString:keyword|text)" -"keyword|text reverse(str:keyword|text)" -"keyword right(string:keyword|text, length:integer)" -"double|integer|long|unsigned_long round(number:double|integer|long|unsigned_long, ?decimals:integer)" -"keyword|text rtrim(string:keyword|text)" -"double signum(number:double|integer|long|unsigned_long)" -"double sin(angle:double|integer|long|unsigned_long)" -"double sinh(number:double|integer|long|unsigned_long)" -"keyword space(number:integer)" -"keyword split(string:keyword|text, delim:keyword|text)" -"double sqrt(number:double|integer|long|unsigned_long)" -"geo_point|cartesian_point st_centroid_agg(field:geo_point|cartesian_point)" -"boolean st_contains(geomA:geo_point|cartesian_point|geo_shape|cartesian_shape, geomB:geo_point|cartesian_point|geo_shape|cartesian_shape)" -"boolean st_disjoint(geomA:geo_point|cartesian_point|geo_shape|cartesian_shape, geomB:geo_point|cartesian_point|geo_shape|cartesian_shape)" -"double st_distance(geomA:geo_point|cartesian_point, geomB:geo_point|cartesian_point)" -"boolean st_intersects(geomA:geo_point|cartesian_point|geo_shape|cartesian_shape, geomB:geo_point|cartesian_point|geo_shape|cartesian_shape)" -"boolean st_within(geomA:geo_point|cartesian_point|geo_shape|cartesian_shape, geomB:geo_point|cartesian_point|geo_shape|cartesian_shape)" -"double st_x(point:geo_point|cartesian_point)" -"double st_y(point:geo_point|cartesian_point)" -"boolean starts_with(str:keyword|text, prefix:keyword|text)" -"keyword substring(string:keyword|text, start:integer, ?length:integer)" -"long|double sum(number:double|integer|long)" -"double tan(angle:double|integer|long|unsigned_long)" -"double tanh(number:double|integer|long|unsigned_long)" -double tau() -"keyword to_base64(string:keyword|text)" -"boolean to_bool(field:boolean|keyword|text|double|long|unsigned_long|integer)" -"boolean to_boolean(field:boolean|keyword|text|double|long|unsigned_long|integer)" -"cartesian_point to_cartesianpoint(field:cartesian_point|keyword|text)" -"cartesian_shape to_cartesianshape(field:cartesian_point|cartesian_shape|keyword|text)" -"date_nanos to_date_nanos(field:date|date_nanos|keyword|text|double|long|unsigned_long)" -"date_nanos to_datenanos(field:date|date_nanos|keyword|text|double|long|unsigned_long)" -"date_period to_dateperiod(field:date_period|keyword|text)" -"date to_datetime(field:date|date_nanos|keyword|text|double|long|unsigned_long|integer)" -"double to_dbl(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long)" -"double to_degrees(number:double|integer|long|unsigned_long)" -"double to_double(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long)" -"date to_dt(field:date|date_nanos|keyword|text|double|long|unsigned_long|integer)" -"geo_point to_geopoint(field:geo_point|keyword|text)" -"geo_shape to_geoshape(field:geo_point|geo_shape|keyword|text)" -"integer to_int(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer)" -"integer to_integer(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer)" -"ip to_ip(field:ip|keyword|text)" -"long to_long(field:boolean|date|date_nanos|keyword|text|double|long|unsigned_long|integer|counter_integer|counter_long)" -"keyword|text to_lower(str:keyword|text)" -"double to_radians(number:double|integer|long|unsigned_long)" -"keyword to_str(field:boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)" -"keyword to_string(field:boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)" -"time_duration to_timeduration(field:time_duration|keyword|text)" -"unsigned_long to_ul(field:boolean|date|keyword|text|double|long|unsigned_long|integer)" -"unsigned_long to_ulong(field:boolean|date|keyword|text|double|long|unsigned_long|integer)" -"unsigned_long to_unsigned_long(field:boolean|date|keyword|text|double|long|unsigned_long|integer)" -"keyword|text to_upper(str:keyword|text)" -"version to_ver(field:keyword|text|version)" -"version to_version(field:keyword|text|version)" -"boolean|double|integer|long|date|ip|keyword|text top(field:boolean|double|integer|long|date|ip|keyword|text, limit:integer, order:keyword)" -"keyword|text trim(string:keyword|text)" -"boolean|date|double|integer|ip|keyword|long|text|version values(field:boolean|date|double|integer|ip|keyword|long|text|version)" -"double weighted_avg(number:double|integer|long, weight:double|integer|long)" -; - -metaFunctionsArgs -required_capability: meta -required_capability: date_nanos_type - - META functions -| EVAL name = SUBSTRING(name, 0, 14) -| KEEP name, argNames, argTypes, argDescriptions; - - name:keyword | argNames:keyword | argTypes:keyword | argDescriptions:keyword -abs |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. -acos |number |"double|integer|long|unsigned_long" |Number between -1 and 1. If `null`, the function returns `null`. -asin |number |"double|integer|long|unsigned_long" |Number between -1 and 1. If `null`, the function returns `null`. -atan |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. -atan2 |[y_coordinate, x_coordinate] |["double|integer|long|unsigned_long", "double|integer|long|unsigned_long"] |[y coordinate. If `null`\, the function returns `null`., x coordinate. If `null`\, the function returns `null`.] -avg |number |"double|integer|long" |[""] -bin |[field, buckets, from, to] |["integer|long|double|date", "integer|long|double|date_period|time_duration", "integer|long|double|date|keyword|text", "integer|long|double|date|keyword|text"] |[Numeric or date expression from which to derive buckets., Target number of buckets\, or desired bucket size if `from` and `to` parameters are omitted., Start of the range. Can be a number\, a date or a date expressed as a string., End of the range. Can be a number\, a date or a date expressed as a string.] -bucket |[field, buckets, from, to] |["integer|long|double|date", "integer|long|double|date_period|time_duration", "integer|long|double|date|keyword|text", "integer|long|double|date|keyword|text"] |[Numeric or date expression from which to derive buckets., Target number of buckets\, or desired bucket size if `from` and `to` parameters are omitted., Start of the range. Can be a number\, a date or a date expressed as a string., End of the range. Can be a number\, a date or a date expressed as a string.] -case |[condition, trueValue] |[boolean, "boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"] |[A condition., The value that's returned when the corresponding condition is the first to evaluate to `true`. The default value is returned when no condition matches.] -cbrt |number |"double|integer|long|unsigned_long" |"Numeric expression. If `null`, the function returns `null`." -ceil |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. -cidr_match |[ip, blockX] |[ip, "keyword|text"] |[IP address of type `ip` (both IPv4 and IPv6 are supported)., CIDR block to test the IP against.] -coalesce |first |"boolean|cartesian_point|cartesian_shape|date|geo_point|geo_shape|integer|ip|keyword|long|text|version" |Expression to evaluate. -concat |[string1, string2] |["keyword|text", "keyword|text"] |[Strings to concatenate., Strings to concatenate.] -cos |angle |"double|integer|long|unsigned_long" |An angle, in radians. If `null`, the function returns `null`. -cosh |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. -count |field |"boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|unsigned_long|version" |Expression that outputs values to be counted. If omitted, equivalent to `COUNT(*)` (the number of rows). -count_distinct|[field, precision] |["boolean|date|double|integer|ip|keyword|long|text|version", "integer|long|unsigned_long"] |[Column or literal for which to count the number of distinct values., Precision threshold. Refer to <>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000.] -date_diff |[unit, startTimestamp, endTimestamp]|["keyword|text", date, date] |[Time difference unit, A string representing a start timestamp, A string representing an end timestamp] -date_extract |[datePart, date] |["keyword|text", date] |[Part of the date to extract. Can be: `aligned_day_of_week_in_month`\, `aligned_day_of_week_in_year`\, `aligned_week_of_month`\, `aligned_week_of_year`\, `ampm_of_day`\, `clock_hour_of_ampm`\, `clock_hour_of_day`\, `day_of_month`\, `day_of_week`\, `day_of_year`\, `epoch_day`\, `era`\, `hour_of_ampm`\, `hour_of_day`\, `instant_seconds`\, `micro_of_day`\, `micro_of_second`\, `milli_of_day`\, `milli_of_second`\, `minute_of_day`\, `minute_of_hour`\, `month_of_year`\, `nano_of_day`\, `nano_of_second`\, `offset_seconds`\, `proleptic_month`\, `second_of_day`\, `second_of_minute`\, `year`\, or `year_of_era`. Refer to https://docs.oracle.com/javase/8/docs/api/java/time/temporal/ChronoField.html[java.time.temporal.ChronoField] for a description of these values. If `null`\, the function returns `null`., Date expression. If `null`\, the function returns `null`.] -date_format |[dateFormat, date] |["keyword|text", date] |[Date format (optional). If no format is specified\, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used. If `null`\, the function returns `null`., Date expression. If `null`\, the function returns `null`.] -date_parse |[datePattern, dateString] |["keyword|text", "keyword|text"] |[The date format. Refer to the https://docs.oracle.com/en/java/javase/14/docs/api/java.base/java/time/format/DateTimeFormatter.html[`DateTimeFormatter` documentation] for the syntax. If `null`\, the function returns `null`., Date expression as a string. If `null` or an empty string\, the function returns `null`.] -date_trunc |[interval, date] |["date_period|time_duration", date] |[Interval; expressed using the timespan literal syntax., Date expression] -e |null |null |null -ends_with |[str, suffix] |["keyword|text", "keyword|text"] |[String expression. If `null`\, the function returns `null`., String expression. If `null`\, the function returns `null`.] -exp |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. -floor |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. -from_base64 |string |"keyword|text" |A base64 string. -greatest |first |"boolean|date|double|integer|ip|keyword|long|text|version" |First of the columns to evaluate. -ip_prefix |[ip, prefixLengthV4, prefixLengthV6]|[ip, integer, integer] |[IP address of type `ip` (both IPv4 and IPv6 are supported)., Prefix length for IPv4 addresses., Prefix length for IPv6 addresses.] -least |first |"boolean|date|double|integer|ip|keyword|long|text|version" |First of the columns to evaluate. -left |[string, length] |["keyword|text", integer] |[The string from which to return a substring., The number of characters to return.] -length |string |"keyword|text" |String expression. If `null`, the function returns `null`. -locate |[string, substring, start] |["keyword|text", "keyword|text", "integer"] |[An input string, A substring to locate in the input string, The start index] -log |[base, number] |["integer|unsigned_long|long|double", "integer|unsigned_long|long|double"] |["Base of logarithm. If `null`\, the function returns `null`. If not provided\, this function returns the natural logarithm (base e) of a value.", "Numeric expression. If `null`\, the function returns `null`."] -log10 |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. -ltrim |string |"keyword|text" |String expression. If `null`, the function returns `null`. -max |field |"boolean|double|integer|long|date|ip|keyword|text|long|version" |[""] -median |number |"double|integer|long" |[""] -median_absolut|number |"double|integer|long" |[""] -min |field |"boolean|double|integer|long|date|ip|keyword|text|long|version" |[""] -mv_append |[field1, field2] |["boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version", "boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version"] | ["", ""] -mv_avg |number |"double|integer|long|unsigned_long" |Multivalue expression. -mv_concat |[string, delim] |["text|keyword", "text|keyword"] |[Multivalue expression., Delimiter.] -mv_count |field |"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version" |Multivalue expression. -mv_dedupe |field |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version" |Multivalue expression. -mv_first |field |"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version" |Multivalue expression. -mv_last |field |"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version" |Multivalue expression. -mv_max |field |"boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version" |Multivalue expression. -mv_median |number |"double|integer|long|unsigned_long" |Multivalue expression. -mv_median_abso|number |"double|integer|long|unsigned_long" |Multivalue expression. -mv_min |field |"boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version" |Multivalue expression. -mv_percentile |[number, percentile] |["double|integer|long", "double|integer|long"] |[Multivalue expression., The percentile to calculate. Must be a number between 0 and 100. Numbers out of range will return a null instead.] -mv_pseries_wei|[number, p] |[double, double] |[Multivalue expression., It is a constant number that represents the 'p' parameter in the P-Series. It impacts every element's contribution to the weighted sum.] -mv_slice |[field, start, end] |["boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version", integer, integer]|[Multivalue expression. If `null`\, the function returns `null`., Start position. If `null`\, the function returns `null`. The start argument can be negative. An index of -1 is used to specify the last value in the list., End position(included). Optional; if omitted\, the position at `start` is returned. The end argument can be negative. An index of -1 is used to specify the last value in the list.] -mv_sort |[field, order] |["boolean|date|double|integer|ip|keyword|long|text|version", keyword] |[Multivalue expression. If `null`\, the function returns `null`., Sort order. The valid options are ASC and DESC\, the default is ASC.] -mv_sum |number |"double|integer|long|unsigned_long" |Multivalue expression. -mv_zip |[string1, string2, delim] |["keyword|text", "keyword|text", "keyword|text"] |[Multivalue expression., Multivalue expression., Delimiter. Optional; if omitted\, `\,` is used as a default delimiter.] -now |null |null |null -percentile |[number, percentile] |["double|integer|long", "double|integer|long"] |[, ] -pi |null |null |null -pow |[base, exponent] |["double|integer|long|unsigned_long", "double|integer|long|unsigned_long"] |["Numeric expression for the base. If `null`\, the function returns `null`.", "Numeric expression for the exponent. If `null`\, the function returns `null`."] -repeat |[string, number] |["keyword|text", integer] |[String expression., Number times to repeat.] -replace |[string, regex, newString] |["keyword|text", "keyword|text", "keyword|text"] |[String expression., Regular expression., Replacement string.] -reverse |str |"keyword|text" |String expression. If `null`, the function returns `null`. -right |[string, length] |["keyword|text", integer] |[The string from which to returns a substring., The number of characters to return.] -round |[number, decimals] |["double|integer|long|unsigned_long", integer] |["The numeric value to round. If `null`\, the function returns `null`.", "The number of decimal places to round to. Defaults to 0. If `null`\, the function returns `null`."] -rtrim |string |"keyword|text" |String expression. If `null`, the function returns `null`. -signum |number |"double|integer|long|unsigned_long" |"Numeric expression. If `null`, the function returns `null`." -sin |angle |"double|integer|long|unsigned_long" |An angle, in radians. If `null`, the function returns `null`. -sinh |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. -space |number |"integer" |Number of spaces in result. -split |[string, delim] |["keyword|text", "keyword|text"] |[String expression. If `null`\, the function returns `null`., Delimiter. Only single byte delimiters are currently supported.] -sqrt |number |"double|integer|long|unsigned_long" |"Numeric expression. If `null`, the function returns `null`." -st_centroid_ag|field |"geo_point|cartesian_point" |[""] -st_contains |[geomA, geomB] |["geo_point|cartesian_point|geo_shape|cartesian_shape", "geo_point|cartesian_point|geo_shape|cartesian_shape"] |[Expression of type `geo_point`\, `cartesian_point`\, `geo_shape` or `cartesian_shape`. If `null`\, the function returns `null`., Expression of type `geo_point`\, `cartesian_point`\, `geo_shape` or `cartesian_shape`. If `null`\, the function returns `null`. The second parameter must also have the same coordinate system as the first. This means it is not possible to combine `geo_*` and `cartesian_*` parameters.] -st_disjoint |[geomA, geomB] |["geo_point|cartesian_point|geo_shape|cartesian_shape", "geo_point|cartesian_point|geo_shape|cartesian_shape"] |[Expression of type `geo_point`\, `cartesian_point`\, `geo_shape` or `cartesian_shape`. If `null`\, the function returns `null`., Expression of type `geo_point`\, `cartesian_point`\, `geo_shape` or `cartesian_shape`. If `null`\, the function returns `null`. The second parameter must also have the same coordinate system as the first. This means it is not possible to combine `geo_*` and `cartesian_*` parameters.] -st_distance |[geomA, geomB] |["geo_point|cartesian_point", "geo_point|cartesian_point"] |[Expression of type `geo_point` or `cartesian_point`. If `null`\, the function returns `null`., Expression of type `geo_point` or `cartesian_point`. If `null`\, the function returns `null`. The second parameter must also have the same coordinate system as the first. This means it is not possible to combine `geo_point` and `cartesian_point` parameters.] -st_intersects |[geomA, geomB] |["geo_point|cartesian_point|geo_shape|cartesian_shape", "geo_point|cartesian_point|geo_shape|cartesian_shape"] |[Expression of type `geo_point`\, `cartesian_point`\, `geo_shape` or `cartesian_shape`. If `null`\, the function returns `null`., Expression of type `geo_point`\, `cartesian_point`\, `geo_shape` or `cartesian_shape`. If `null`\, the function returns `null`. The second parameter must also have the same coordinate system as the first. This means it is not possible to combine `geo_*` and `cartesian_*` parameters.] -st_within |[geomA, geomB] |["geo_point|cartesian_point|geo_shape|cartesian_shape", "geo_point|cartesian_point|geo_shape|cartesian_shape"] |[Expression of type `geo_point`\, `cartesian_point`\, `geo_shape` or `cartesian_shape`. If `null`\, the function returns `null`., Expression of type `geo_point`\, `cartesian_point`\, `geo_shape` or `cartesian_shape`. If `null`\, the function returns `null`. The second parameter must also have the same coordinate system as the first. This means it is not possible to combine `geo_*` and `cartesian_*` parameters.] -st_x |point |"geo_point|cartesian_point" |Expression of type `geo_point` or `cartesian_point`. If `null`, the function returns `null`. -st_y |point |"geo_point|cartesian_point" |Expression of type `geo_point` or `cartesian_point`. If `null`, the function returns `null`. -starts_with |[str, prefix] |["keyword|text", "keyword|text"] |[String expression. If `null`\, the function returns `null`., String expression. If `null`\, the function returns `null`.] -substring |[string, start, length] |["keyword|text", integer, integer] |[String expression. If `null`\, the function returns `null`., Start position., Length of the substring from the start position. Optional; if omitted\, all positions after `start` are returned.] -sum |number |"double|integer|long" |[""] -tan |angle |"double|integer|long|unsigned_long" |An angle, in radians. If `null`, the function returns `null`. -tanh |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. -tau |null |null |null -to_base64 |string |"keyword|text" |A string. -to_bool |field |"boolean|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. -to_boolean |field |"boolean|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. -to_cartesianpo|field |"cartesian_point|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. -to_cartesiansh|field |"cartesian_point|cartesian_shape|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. -to_date_nanos |field |"date|date_nanos|keyword|text|double|long|unsigned_long" |Input value. The input can be a single- or multi-valued column or an expression. -to_datenanos |field |"date|date_nanos|keyword|text|double|long|unsigned_long" |Input value. The input can be a single- or multi-valued column or an expression. -to_dateperiod |field |"date_period|keyword|text" |Input value. The input is a valid constant date period expression. -to_datetime |field |"date|date_nanos|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. -to_dbl |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long" |Input value. The input can be a single- or multi-valued column or an expression. -to_degrees |number |"double|integer|long|unsigned_long" |Input value. The input can be a single- or multi-valued column or an expression. -to_double |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long" |Input value. The input can be a single- or multi-valued column or an expression. -to_dt |field |"date|date_nanos|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. -to_geopoint |field |"geo_point|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. -to_geoshape |field |"geo_point|geo_shape|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. -to_int |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer" |Input value. The input can be a single- or multi-valued column or an expression. -to_integer |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer" |Input value. The input can be a single- or multi-valued column or an expression. -to_ip |field |"ip|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. -to_long |field |"boolean|date|date_nanos|keyword|text|double|long|unsigned_long|integer|counter_integer|counter_long" |Input value. The input can be a single- or multi-valued column or an expression. -to_lower |str |"keyword|text" |String expression. If `null`, the function returns `null`. -to_radians |number |"double|integer|long|unsigned_long" |Input value. The input can be a single- or multi-valued column or an expression. -to_str |field |"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version" |Input value. The input can be a single- or multi-valued column or an expression. -to_string |field |"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version" |Input value. The input can be a single- or multi-valued column or an expression. -to_timeduratio|field |"time_duration|keyword|text" |Input value. The input is a valid constant time duration expression. -to_ul |field |"boolean|date|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. -to_ulong |field |"boolean|date|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. -to_unsigned_lo|field |"boolean|date|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. -to_upper |str |"keyword|text" |String expression. If `null`, the function returns `null`. -to_ver |field |"keyword|text|version" |Input value. The input can be a single- or multi-valued column or an expression. -to_version |field |"keyword|text|version" |Input value. The input can be a single- or multi-valued column or an expression. -top |[field, limit, order] |["boolean|double|integer|long|date|ip|keyword|text", integer, keyword] |[The field to collect the top values for.,The maximum number of values to collect.,The order to calculate the top values. Either `asc` or `desc`.] -trim |string |"keyword|text" |String expression. If `null`, the function returns `null`. -values |field |"boolean|date|double|integer|ip|keyword|long|text|version" |[""] -weighted_avg |[number, weight] |["double|integer|long", "double|integer|long"] |[A numeric value., A numeric weight.] -; - -metaFunctionsDescription -required_capability: meta - - META functions -| EVAL name = SUBSTRING(name, 0, 14) -| KEEP name, description -; - - name:keyword | description:keyword -abs |Returns the absolute value. -acos |Returns the {wikipedia}/Inverse_trigonometric_functions[arccosine] of `n` as an angle, expressed in radians. -asin |Returns the {wikipedia}/Inverse_trigonometric_functions[arcsine] of the input numeric expression as an angle, expressed in radians. -atan |Returns the {wikipedia}/Inverse_trigonometric_functions[arctangent] of the input numeric expression as an angle, expressed in radians. -atan2 |The {wikipedia}/Atan2[angle] between the positive x-axis and the ray from the origin to the point (x , y) in the Cartesian plane, expressed in radians. -avg |The average of a numeric field. -bin |Creates groups of values - buckets - out of a datetime or numeric input. The size of the buckets can either be provided directly, or chosen based on a recommended count and values range. -bucket |Creates groups of values - buckets - out of a datetime or numeric input. The size of the buckets can either be provided directly, or chosen based on a recommended count and values range. -case |Accepts pairs of conditions and values. The function returns the value that belongs to the first condition that evaluates to `true`. If the number of arguments is odd, the last argument is the default value which is returned when no condition matches. If the number of arguments is even, and no condition matches, the function returns `null`. -cbrt |Returns the cube root of a number. The input can be any numeric value, the return value is always a double. Cube roots of infinities are null. -ceil |Round a number up to the nearest integer. -cidr_match |Returns true if the provided IP is contained in one of the provided CIDR blocks. -coalesce |Returns the first of its arguments that is not null. If all arguments are null, it returns `null`. -concat |Concatenates two or more strings. -cos |Returns the {wikipedia}/Sine_and_cosine[cosine] of an angle. -cosh |Returns the {wikipedia}/Hyperbolic_functions[hyperbolic cosine] of a number. -count |Returns the total number (count) of input values. -count_distinct|Returns the approximate number of distinct values. -date_diff |Subtracts the `startTimestamp` from the `endTimestamp` and returns the difference in multiples of `unit`. If `startTimestamp` is later than the `endTimestamp`, negative values are returned. -date_extract |Extracts parts of a date, like year, month, day, hour. -date_format |Returns a string representation of a date, in the provided format. -date_parse |Returns a date by parsing the second argument using the format specified in the first argument. -date_trunc |Rounds down a date to the closest interval. -e |Returns {wikipedia}/E_(mathematical_constant)[Euler's number]. -ends_with |Returns a boolean that indicates whether a keyword string ends with another string. -exp |Returns the value of e raised to the power of the given number. -floor |Round a number down to the nearest integer. -from_base64 |Decode a base64 string. -greatest |Returns the maximum value from multiple columns. This is similar to <> except it is intended to run on multiple columns at once. -ip_prefix |Truncates an IP to a given prefix length. -least |Returns the minimum value from multiple columns. This is similar to <> except it is intended to run on multiple columns at once. -left |Returns the substring that extracts 'length' chars from 'string' starting from the left. -length |Returns the character length of a string. -locate |Returns an integer that indicates the position of a keyword substring within another string. Returns `0` if the substring cannot be found. Note that string positions start from `1`. -log |Returns the logarithm of a value to a base. The input can be any numeric value, the return value is always a double. Logs of zero, negative numbers, and base of one return `null` as well as a warning. -log10 |Returns the logarithm of a value to base 10. The input can be any numeric value, the return value is always a double. Logs of 0 and negative numbers return `null` as well as a warning. -ltrim |Removes leading whitespaces from a string. -max |The maximum value of a field. -median |The value that is greater than half of all values and less than half of all values, also known as the 50% <>. -median_absolut|"Returns the median absolute deviation, a measure of variability. It is a robust statistic, meaning that it is useful for describing data that may have outliers, or may not be normally distributed. For such data it can be more descriptive than standard deviation. It is calculated as the median of each data point's deviation from the median of the entire sample. That is, for a random variable `X`, the median absolute deviation is `median(|median(X) - X|)`." -min |The minimum value of a field. -mv_append |Concatenates values of two multi-value fields. -mv_avg |Converts a multivalued field into a single valued field containing the average of all of the values. -mv_concat |Converts a multivalued string expression into a single valued column containing the concatenation of all values separated by a delimiter. -mv_count |Converts a multivalued expression into a single valued column containing a count of the number of values. -mv_dedupe |Remove duplicate values from a multivalued field. -mv_first |Converts a multivalued expression into a single valued column containing the first value. This is most useful when reading from a function that emits multivalued columns in a known order like <>. The order that <> are read from underlying storage is not guaranteed. It is *frequently* ascending, but don't rely on that. If you need the minimum value use <> instead of `MV_FIRST`. `MV_MIN` has optimizations for sorted values so there isn't a performance benefit to `MV_FIRST`. -mv_last |Converts a multivalue expression into a single valued column containing the last value. This is most useful when reading from a function that emits multivalued columns in a known order like <>. The order that <> are read from underlying storage is not guaranteed. It is *frequently* ascending, but don't rely on that. If you need the maximum value use <> instead of `MV_LAST`. `MV_MAX` has optimizations for sorted values so there isn't a performance benefit to `MV_LAST`. -mv_max |Converts a multivalued expression into a single valued column containing the maximum value. -mv_median |Converts a multivalued field into a single valued field containing the median value. -mv_median_abso|"Converts a multivalued field into a single valued field containing the median absolute deviation. It is calculated as the median of each data point's deviation from the median of the entire sample. That is, for a random variable `X`, the median absolute deviation is `median(|median(X) - X|)`." -mv_min |Converts a multivalued expression into a single valued column containing the minimum value. -mv_percentile |Converts a multivalued field into a single valued field containing the value at which a certain percentage of observed values occur. -mv_pseries_wei|Converts a multivalued expression into a single-valued column by multiplying every element on the input list by its corresponding term in P-Series and computing the sum. -mv_slice |Returns a subset of the multivalued field using the start and end index values. -mv_sort |Sorts a multivalued field in lexicographical order. -mv_sum |Converts a multivalued field into a single valued field containing the sum of all of the values. -mv_zip |Combines the values from two multivalued fields with a delimiter that joins them together. -now |Returns current date and time. -percentile |Returns the value at which a certain percentage of observed values occur. For example, the 95th percentile is the value which is greater than 95% of the observed values and the 50th percentile is the `MEDIAN`. -pi |Returns {wikipedia}/Pi[Pi], the ratio of a circle's circumference to its diameter. -pow |Returns the value of `base` raised to the power of `exponent`. -repeat |Returns a string constructed by concatenating `string` with itself the specified `number` of times. -replace |The function substitutes in the string `str` any match of the regular expression `regex` with the replacement string `newStr`. -reverse |Returns a new string representing the input string in reverse order. -right |Return the substring that extracts 'length' chars from 'str' starting from the right. -round |Rounds a number to the specified number of decimal places. Defaults to 0, which returns the nearest integer. If the precision is a negative number, rounds to the number of digits left of the decimal point. -rtrim |Removes trailing whitespaces from a string. -signum |Returns the sign of the given number. It returns `-1` for negative numbers, `0` for `0` and `1` for positive numbers. -sin |Returns the {wikipedia}/Sine_and_cosine[sine] of an angle. -sinh |Returns the {wikipedia}/Hyperbolic_functions[hyperbolic sine] of a number. -space |Returns a string made of `number` spaces. -split |Split a single valued string into multiple strings. -sqrt |Returns the square root of a number. The input can be any numeric value, the return value is always a double. Square roots of negative numbers and infinities are null. -st_centroid_ag|Calculate the spatial centroid over a field with spatial point geometry type. -st_contains |Returns whether the first geometry contains the second geometry. This is the inverse of the <> function. -st_disjoint |Returns whether the two geometries or geometry columns are disjoint. This is the inverse of the <> function. In mathematical terms: ST_Disjoint(A, B) ⇔ A ⋂ B = ∅ -st_distance |Computes the distance between two points. For cartesian geometries, this is the pythagorean distance in the same units as the original coordinates. For geographic geometries, this is the circular distance along the great circle in meters. -st_intersects |Returns true if two geometries intersect. They intersect if they have any point in common, including their interior points (points along lines or within polygons). This is the inverse of the <> function. In mathematical terms: ST_Intersects(A, B) ⇔ A ⋂ B ≠ ∅ -st_within |Returns whether the first geometry is within the second geometry. This is the inverse of the <> function. -st_x |Extracts the `x` coordinate from the supplied point. If the points is of type `geo_point` this is equivalent to extracting the `longitude` value. -st_y |Extracts the `y` coordinate from the supplied point. If the points is of type `geo_point` this is equivalent to extracting the `latitude` value. -starts_with |Returns a boolean that indicates whether a keyword string starts with another string. -substring |Returns a substring of a string, specified by a start position and an optional length. -sum |The sum of a numeric expression. -tan |Returns the {wikipedia}/Sine_and_cosine[tangent] of an angle. -tanh |Returns the {wikipedia}/Hyperbolic_functions[hyperbolic tangent] of a number. -tau |Returns the https://tauday.com/tau-manifesto[ratio] of a circle's circumference to its radius. -to_base64 |Encode a string to a base64 string. -to_bool |Converts an input value to a boolean value. A string value of *true* will be case-insensitive converted to the Boolean *true*. For anything else, including the empty string, the function will return *false*. The numerical value of *0* will be converted to *false*, anything else will be converted to *true*. -to_boolean |Converts an input value to a boolean value. A string value of *true* will be case-insensitive converted to the Boolean *true*. For anything else, including the empty string, the function will return *false*. The numerical value of *0* will be converted to *false*, anything else will be converted to *true*. -to_cartesianpo|Converts an input value to a `cartesian_point` value. A string will only be successfully converted if it respects the {wikipedia}/Well-known_text_representation_of_geometry[WKT Point] format. -to_cartesiansh|Converts an input value to a `cartesian_shape` value. A string will only be successfully converted if it respects the {wikipedia}/Well-known_text_representation_of_geometry[WKT] format. -to_date_nanos |Converts an input to a nanosecond-resolution date value (aka date_nanos). -to_datenanos |Converts an input to a nanosecond-resolution date value (aka date_nanos). -to_dateperiod |Converts an input value into a `date_period` value. -to_datetime |Converts an input value to a date value. A string will only be successfully converted if it's respecting the format `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`. To convert dates in other formats, use <>. -to_dbl |Converts an input value to a double value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the {wikipedia}/Unix_time[Unix epoch], converted to double. Boolean *true* will be converted to double *1.0*, *false* to *0.0*. -to_degrees |Converts a number in {wikipedia}/Radian[radians] to {wikipedia}/Degree_(angle)[degrees]. -to_double |Converts an input value to a double value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the {wikipedia}/Unix_time[Unix epoch], converted to double. Boolean *true* will be converted to double *1.0*, *false* to *0.0*. -to_dt |Converts an input value to a date value. A string will only be successfully converted if it's respecting the format `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`. To convert dates in other formats, use <>. -to_geopoint |Converts an input value to a `geo_point` value. A string will only be successfully converted if it respects the {wikipedia}/Well-known_text_representation_of_geometry[WKT Point] format. -to_geoshape |Converts an input value to a `geo_shape` value. A string will only be successfully converted if it respects the {wikipedia}/Well-known_text_representation_of_geometry[WKT] format. -to_int |Converts an input value to an integer value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the {wikipedia}/Unix_time[Unix epoch], converted to integer. Boolean *true* will be converted to integer *1*, *false* to *0*. -to_integer |Converts an input value to an integer value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the {wikipedia}/Unix_time[Unix epoch], converted to integer. Boolean *true* will be converted to integer *1*, *false* to *0*. -to_ip |Converts an input string to an IP value. -to_long |Converts an input value to a long value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the {wikipedia}/Unix_time[Unix epoch], converted to long. Boolean *true* will be converted to long *1*, *false* to *0*. -to_lower |Returns a new string representing the input string converted to lower case. -to_radians |Converts a number in {wikipedia}/Degree_(angle)[degrees] to {wikipedia}/Radian[radians]. -to_str |Converts an input value into a string. -to_string |Converts an input value into a string. -to_timeduratio|Converts an input value into a `time_duration` value. -to_ul |Converts an input value to an unsigned long value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the {wikipedia}/Unix_time[Unix epoch], converted to unsigned long. Boolean *true* will be converted to unsigned long *1*, *false* to *0*. -to_ulong |Converts an input value to an unsigned long value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the {wikipedia}/Unix_time[Unix epoch], converted to unsigned long. Boolean *true* will be converted to unsigned long *1*, *false* to *0*. -to_unsigned_lo|Converts an input value to an unsigned long value. If the input parameter is of a date type, its value will be interpreted as milliseconds since the {wikipedia}/Unix_time[Unix epoch], converted to unsigned long. Boolean *true* will be converted to unsigned long *1*, *false* to *0*. -to_upper |Returns a new string representing the input string converted to upper case. -to_ver |Converts an input string to a version value. -to_version |Converts an input string to a version value. -top |Collects the top values for a field. Includes repeated values. -trim |Removes leading and trailing whitespaces from a string. -values |Returns all values in a group as a multivalued field. The order of the returned values isn't guaranteed. If you need the values returned in order use <>. -weighted_avg |The weighted average of a numeric expression. -; - -metaFunctionsRemaining -required_capability: meta -required_capability: date_nanos_type - - META functions -| EVAL name = SUBSTRING(name, 0, 14) -| KEEP name, * -| DROP synopsis, description, argNames, argTypes, argDescriptions -; - - name:keyword | returnType:keyword | optionalArgs:boolean |variadic:boolean|isAggregation:boolean -abs |"double|integer|long|unsigned_long" |false |false |false -acos |double |false |false |false -asin |double |false |false |false -atan |double |false |false |false -atan2 |double |[false, false] |false |false -avg |double |false |false |true -bin |"double|date" |[false, false, true, true] |false |false -bucket |"double|date" |[false, false, true, true] |false |false -case |"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version" |[false, false] |true |false -cbrt |double |false |false |false -ceil |"double|integer|long|unsigned_long" |false |false |false -cidr_match |boolean |[false, false] |true |false -coalesce |"boolean|cartesian_point|cartesian_shape|date|geo_point|geo_shape|integer|ip|keyword|long|text|version" |false |true |false -concat |keyword |[false, false] |true |false -cos |double |false |false |false -cosh |double |false |false |false -count |long |true |false |true -count_distinct|long |[false, true] |false |true -date_diff |integer |[false, false, false] |false |false -date_extract |long |[false, false] |false |false -date_format |keyword |[true, false] |false |false -date_parse |date |[true, false] |false |false -date_trunc |date |[false, false] |false |false -e |double |null |false |false -ends_with |boolean |[false, false] |false |false -exp |double |false |false |false -floor |"double|integer|long|unsigned_long" |false |false |false -from_base64 |keyword |false |false |false -greatest |"boolean|date|double|integer|ip|keyword|long|text|version" |false |true |false -ip_prefix |ip |[false, false, false] |false |false -least |"boolean|date|double|integer|ip|keyword|long|text|version" |false |true |false -left |keyword |[false, false] |false |false -length |integer |false |false |false -locate |integer |[false, false, true] |false |false -log |double |[true, false] |false |false -log10 |double |false |false |false -ltrim |"keyword|text" |false |false |false -max |"boolean|double|integer|long|date|ip|keyword|text|long|version" |false |false |true -median |double |false |false |true -median_absolut|double |false |false |true -min |"boolean|double|integer|long|date|ip|keyword|text|long|version" |false |false |true -mv_append |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version" |[false, false] |false |false -mv_avg |double |false |false |false -mv_concat |keyword |[false, false] |false |false -mv_count |integer |false |false |false -mv_dedupe |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version" |false |false |false -mv_first |"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"|false |false |false -mv_last |"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"|false |false |false -mv_max |"boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version" |false |false |false -mv_median |"double|integer|long|unsigned_long" |false |false |false -mv_median_abso|"double|integer|long|unsigned_long" |false |false |false -mv_min |"boolean|date|date_nanos|double|integer|ip|keyword|long|text|unsigned_long|version" |false |false |false -mv_percentile |"double|integer|long" |[false, false] |false |false -mv_pseries_wei|"double" |[false, false] |false |false -mv_slice |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version" |[false, false, true] |false |false -mv_sort |"boolean|date|double|integer|ip|keyword|long|text|version" |[false, true] |false |false -mv_sum |"double|integer|long|unsigned_long" |false |false |false -mv_zip |keyword |[false, false, true] |false |false -now |date |null |false |false -percentile |double |[false, false] |false |true -pi |double |null |false |false -pow |double |[false, false] |false |false -repeat |keyword |[false, false] |false |false -replace |keyword |[false, false, false] |false |false -reverse |"keyword|text" |false |false |false -right |keyword |[false, false] |false |false -round |"double|integer|long|unsigned_long" |[false, true] |false |false -rtrim |"keyword|text" |false |false |false -signum |double |false |false |false -sin |double |false |false |false -sinh |double |false |false |false -space |keyword |false |false |false -split |keyword |[false, false] |false |false -sqrt |double |false |false |false -st_centroid_ag|"geo_point|cartesian_point" |false |false |true -st_contains |boolean |[false, false] |false |false -st_disjoint |boolean |[false, false] |false |false -st_distance |double |[false, false] |false |false -st_intersects |boolean |[false, false] |false |false -st_within |boolean |[false, false] |false |false -st_x |double |false |false |false -st_y |double |false |false |false -starts_with |boolean |[false, false] |false |false -substring |keyword |[false, false, true] |false |false -sum |"long|double" |false |false |true -tan |double |false |false |false -tanh |double |false |false |false -tau |double |null |false |false -to_base64 |keyword |false |false |false -to_bool |boolean |false |false |false -to_boolean |boolean |false |false |false -to_cartesianpo|cartesian_point |false |false |false -to_cartesiansh|cartesian_shape |false |false |false -to_date_nanos |date_nanos |false |false |false -to_datenanos |date_nanos |false |false |false -to_dateperiod |date_period |false |false |false -to_datetime |date |false |false |false -to_dbl |double |false |false |false -to_degrees |double |false |false |false -to_double |double |false |false |false -to_dt |date |false |false |false -to_geopoint |geo_point |false |false |false -to_geoshape |geo_shape |false |false |false -to_int |integer |false |false |false -to_integer |integer |false |false |false -to_ip |ip |false |false |false -to_long |long |false |false |false -to_lower |"keyword|text" |false |false |false -to_radians |double |false |false |false -to_str |keyword |false |false |false -to_string |keyword |false |false |false -to_timeduratio|time_duration |false |false |false -to_ul |unsigned_long |false |false |false -to_ulong |unsigned_long |false |false |false -to_unsigned_lo|unsigned_long |false |false |false -to_upper |"keyword|text" |false |false |false -to_ver |version |false |false |false -to_version |version |false |false |false -top |"boolean|double|integer|long|date|ip|keyword|text" |[false, false, false] |false |true -trim |"keyword|text" |false |false |false -values |"boolean|date|double|integer|ip|keyword|long|text|version" |false |false |true -weighted_avg |"double" |[false, false] |false |true -; - -metaFunctionsFiltered -required_capability: meta - -META FUNCTIONS -| WHERE STARTS_WITH(name, "sin") -; - -name:keyword | synopsis:keyword |argNames:keyword | argTypes:keyword | argDescriptions:keyword | returnType:keyword | description:keyword | optionalArgs:boolean | variadic:boolean | isAggregation:boolean -sin |"double sin(angle:double|integer|long|unsigned_long)" |angle |"double|integer|long|unsigned_long" | "An angle, in radians. If `null`, the function returns `null`." | double | "Returns the {wikipedia}/Sine_and_cosine[sine] of an angle." | false | false | false -sinh |"double sinh(number:double|integer|long|unsigned_long)" |number |"double|integer|long|unsigned_long" | "Numeric expression. If `null`, the function returns `null`." | double | "Returns the {wikipedia}/Hyperbolic_functions[hyperbolic sine] of a number." | false | false | false -; - -countFunctions -required_capability: meta - -meta functions | stats a = count(*), b = count(*), c = count(*) | mv_expand c; - -a:long | b:long | c:long -122 | 122 | 122 -; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index e0bef22718d0d..147b13b36c44b 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -34,7 +34,6 @@ import org.elasticsearch.xpack.core.esql.action.ColumnInfo; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.parser.ParsingException; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.junit.Before; @@ -1038,29 +1037,6 @@ public void testShowInfo() { } } - public void testMetaFunctions() { - try (EsqlQueryResponse results = run("meta functions")) { - assertThat( - results.columns(), - equalTo( - List.of( - new ColumnInfoImpl("name", "keyword"), - new ColumnInfoImpl("synopsis", "keyword"), - new ColumnInfoImpl("argNames", "keyword"), - new ColumnInfoImpl("argTypes", "keyword"), - new ColumnInfoImpl("argDescriptions", "keyword"), - new ColumnInfoImpl("returnType", "keyword"), - new ColumnInfoImpl("description", "keyword"), - new ColumnInfoImpl("optionalArgs", "boolean"), - new ColumnInfoImpl("variadic", "boolean"), - new ColumnInfoImpl("isAggregation", "boolean") - ) - ) - ); - assertThat(getValuesList(results).size(), equalTo(new EsqlFunctionRegistry().listFunctions().size())); - } - } - public void testInWithNullValue() { try (EsqlQueryResponse results = run("from test | where null in (data, 2) | keep data")) { assertThat(results.columns(), equalTo(List.of(new ColumnInfoImpl("data", "long")))); diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 index 6570a25469971..e7f10d96c89ae 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4 @@ -66,7 +66,6 @@ FROM : 'from' -> pushMode(FROM_MODE); GROK : 'grok' -> pushMode(EXPRESSION_MODE); KEEP : 'keep' -> pushMode(PROJECT_MODE); LIMIT : 'limit' -> pushMode(EXPRESSION_MODE); -META : 'meta' -> pushMode(META_MODE); MV_EXPAND : 'mv_expand' -> pushMode(MVEXPAND_MODE); RENAME : 'rename' -> pushMode(RENAME_MODE); ROW : 'row' -> pushMode(EXPRESSION_MODE); @@ -467,26 +466,6 @@ SHOW_WS : WS -> channel(HIDDEN) ; -// -// META commands -// -mode META_MODE; -META_PIPE : PIPE -> type(PIPE), popMode; - -FUNCTIONS : 'functions'; - -META_LINE_COMMENT - : LINE_COMMENT -> channel(HIDDEN) - ; - -META_MULTILINE_COMMENT - : MULTILINE_COMMENT -> channel(HIDDEN) - ; - -META_WS - : WS -> channel(HIDDEN) - ; - mode SETTING_MODE; SETTING_CLOSING_BRACKET : CLOSING_BRACKET -> type(CLOSING_BRACKET), popMode; diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens index 747fbbc64cf5f..4fd37ab9900f2 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens @@ -7,122 +7,117 @@ FROM=6 GROK=7 KEEP=8 LIMIT=9 -META=10 -MV_EXPAND=11 -RENAME=12 -ROW=13 -SHOW=14 -SORT=15 -STATS=16 -WHERE=17 -DEV_INLINESTATS=18 -DEV_LOOKUP=19 -DEV_MATCH=20 -DEV_METRICS=21 -UNKNOWN_CMD=22 -LINE_COMMENT=23 -MULTILINE_COMMENT=24 -WS=25 -PIPE=26 -QUOTED_STRING=27 -INTEGER_LITERAL=28 -DECIMAL_LITERAL=29 -BY=30 -AND=31 -ASC=32 -ASSIGN=33 -CAST_OP=34 -COMMA=35 -DESC=36 -DOT=37 -FALSE=38 -FIRST=39 -IN=40 -IS=41 -LAST=42 -LIKE=43 -LP=44 -NOT=45 -NULL=46 -NULLS=47 -OR=48 -PARAM=49 -RLIKE=50 -RP=51 -TRUE=52 -EQ=53 -CIEQ=54 -NEQ=55 -LT=56 -LTE=57 -GT=58 -GTE=59 -PLUS=60 -MINUS=61 -ASTERISK=62 -SLASH=63 -PERCENT=64 -NAMED_OR_POSITIONAL_PARAM=65 -OPENING_BRACKET=66 -CLOSING_BRACKET=67 -UNQUOTED_IDENTIFIER=68 -QUOTED_IDENTIFIER=69 -EXPR_LINE_COMMENT=70 -EXPR_MULTILINE_COMMENT=71 -EXPR_WS=72 -EXPLAIN_WS=73 -EXPLAIN_LINE_COMMENT=74 -EXPLAIN_MULTILINE_COMMENT=75 -METADATA=76 -UNQUOTED_SOURCE=77 -FROM_LINE_COMMENT=78 -FROM_MULTILINE_COMMENT=79 -FROM_WS=80 -ID_PATTERN=81 -PROJECT_LINE_COMMENT=82 -PROJECT_MULTILINE_COMMENT=83 -PROJECT_WS=84 -AS=85 -RENAME_LINE_COMMENT=86 -RENAME_MULTILINE_COMMENT=87 -RENAME_WS=88 -ON=89 -WITH=90 -ENRICH_POLICY_NAME=91 -ENRICH_LINE_COMMENT=92 -ENRICH_MULTILINE_COMMENT=93 -ENRICH_WS=94 -ENRICH_FIELD_LINE_COMMENT=95 -ENRICH_FIELD_MULTILINE_COMMENT=96 -ENRICH_FIELD_WS=97 -MVEXPAND_LINE_COMMENT=98 -MVEXPAND_MULTILINE_COMMENT=99 -MVEXPAND_WS=100 -INFO=101 -SHOW_LINE_COMMENT=102 -SHOW_MULTILINE_COMMENT=103 -SHOW_WS=104 -FUNCTIONS=105 -META_LINE_COMMENT=106 -META_MULTILINE_COMMENT=107 -META_WS=108 -COLON=109 -SETTING=110 -SETTING_LINE_COMMENT=111 -SETTTING_MULTILINE_COMMENT=112 -SETTING_WS=113 -LOOKUP_LINE_COMMENT=114 -LOOKUP_MULTILINE_COMMENT=115 -LOOKUP_WS=116 -LOOKUP_FIELD_LINE_COMMENT=117 -LOOKUP_FIELD_MULTILINE_COMMENT=118 -LOOKUP_FIELD_WS=119 -METRICS_LINE_COMMENT=120 -METRICS_MULTILINE_COMMENT=121 -METRICS_WS=122 -CLOSING_METRICS_LINE_COMMENT=123 -CLOSING_METRICS_MULTILINE_COMMENT=124 -CLOSING_METRICS_WS=125 +MV_EXPAND=10 +RENAME=11 +ROW=12 +SHOW=13 +SORT=14 +STATS=15 +WHERE=16 +DEV_INLINESTATS=17 +DEV_LOOKUP=18 +DEV_MATCH=19 +DEV_METRICS=20 +UNKNOWN_CMD=21 +LINE_COMMENT=22 +MULTILINE_COMMENT=23 +WS=24 +PIPE=25 +QUOTED_STRING=26 +INTEGER_LITERAL=27 +DECIMAL_LITERAL=28 +BY=29 +AND=30 +ASC=31 +ASSIGN=32 +CAST_OP=33 +COMMA=34 +DESC=35 +DOT=36 +FALSE=37 +FIRST=38 +IN=39 +IS=40 +LAST=41 +LIKE=42 +LP=43 +NOT=44 +NULL=45 +NULLS=46 +OR=47 +PARAM=48 +RLIKE=49 +RP=50 +TRUE=51 +EQ=52 +CIEQ=53 +NEQ=54 +LT=55 +LTE=56 +GT=57 +GTE=58 +PLUS=59 +MINUS=60 +ASTERISK=61 +SLASH=62 +PERCENT=63 +NAMED_OR_POSITIONAL_PARAM=64 +OPENING_BRACKET=65 +CLOSING_BRACKET=66 +UNQUOTED_IDENTIFIER=67 +QUOTED_IDENTIFIER=68 +EXPR_LINE_COMMENT=69 +EXPR_MULTILINE_COMMENT=70 +EXPR_WS=71 +EXPLAIN_WS=72 +EXPLAIN_LINE_COMMENT=73 +EXPLAIN_MULTILINE_COMMENT=74 +METADATA=75 +UNQUOTED_SOURCE=76 +FROM_LINE_COMMENT=77 +FROM_MULTILINE_COMMENT=78 +FROM_WS=79 +ID_PATTERN=80 +PROJECT_LINE_COMMENT=81 +PROJECT_MULTILINE_COMMENT=82 +PROJECT_WS=83 +AS=84 +RENAME_LINE_COMMENT=85 +RENAME_MULTILINE_COMMENT=86 +RENAME_WS=87 +ON=88 +WITH=89 +ENRICH_POLICY_NAME=90 +ENRICH_LINE_COMMENT=91 +ENRICH_MULTILINE_COMMENT=92 +ENRICH_WS=93 +ENRICH_FIELD_LINE_COMMENT=94 +ENRICH_FIELD_MULTILINE_COMMENT=95 +ENRICH_FIELD_WS=96 +MVEXPAND_LINE_COMMENT=97 +MVEXPAND_MULTILINE_COMMENT=98 +MVEXPAND_WS=99 +INFO=100 +SHOW_LINE_COMMENT=101 +SHOW_MULTILINE_COMMENT=102 +SHOW_WS=103 +COLON=104 +SETTING=105 +SETTING_LINE_COMMENT=106 +SETTTING_MULTILINE_COMMENT=107 +SETTING_WS=108 +LOOKUP_LINE_COMMENT=109 +LOOKUP_MULTILINE_COMMENT=110 +LOOKUP_WS=111 +LOOKUP_FIELD_LINE_COMMENT=112 +LOOKUP_FIELD_MULTILINE_COMMENT=113 +LOOKUP_FIELD_WS=114 +METRICS_LINE_COMMENT=115 +METRICS_MULTILINE_COMMENT=116 +METRICS_WS=117 +CLOSING_METRICS_LINE_COMMENT=118 +CLOSING_METRICS_MULTILINE_COMMENT=119 +CLOSING_METRICS_WS=120 'dissect'=1 'drop'=2 'enrich'=3 @@ -132,55 +127,53 @@ CLOSING_METRICS_WS=125 'grok'=7 'keep'=8 'limit'=9 -'meta'=10 -'mv_expand'=11 -'rename'=12 -'row'=13 -'show'=14 -'sort'=15 -'stats'=16 -'where'=17 -'|'=26 -'by'=30 -'and'=31 -'asc'=32 -'='=33 -'::'=34 -','=35 -'desc'=36 -'.'=37 -'false'=38 -'first'=39 -'in'=40 -'is'=41 -'last'=42 -'like'=43 -'('=44 -'not'=45 -'null'=46 -'nulls'=47 -'or'=48 -'?'=49 -'rlike'=50 -')'=51 -'true'=52 -'=='=53 -'=~'=54 -'!='=55 -'<'=56 -'<='=57 -'>'=58 -'>='=59 -'+'=60 -'-'=61 -'*'=62 -'/'=63 -'%'=64 -']'=67 -'metadata'=76 -'as'=85 -'on'=89 -'with'=90 -'info'=101 -'functions'=105 -':'=109 +'mv_expand'=10 +'rename'=11 +'row'=12 +'show'=13 +'sort'=14 +'stats'=15 +'where'=16 +'|'=25 +'by'=29 +'and'=30 +'asc'=31 +'='=32 +'::'=33 +','=34 +'desc'=35 +'.'=36 +'false'=37 +'first'=38 +'in'=39 +'is'=40 +'last'=41 +'like'=42 +'('=43 +'not'=44 +'null'=45 +'nulls'=46 +'or'=47 +'?'=48 +'rlike'=49 +')'=50 +'true'=51 +'=='=52 +'=~'=53 +'!='=54 +'<'=55 +'<='=56 +'>'=57 +'>='=58 +'+'=59 +'-'=60 +'*'=61 +'/'=62 +'%'=63 +']'=66 +'metadata'=75 +'as'=84 +'on'=88 +'with'=89 +'info'=100 +':'=104 diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 index a5691a16ca50b..eefe352e9cdd9 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 @@ -32,7 +32,6 @@ query sourceCommand : explainCommand | fromCommand - | metaCommand | rowCommand | showCommand // in development @@ -289,10 +288,6 @@ showCommand : SHOW INFO #showInfo ; -metaCommand - : META FUNCTIONS #metaFunctions - ; - enrichCommand : ENRICH policyName=ENRICH_POLICY_NAME (ON matchField=qualifiedNamePattern)? (WITH enrichWithClause (COMMA enrichWithClause)*)? ; diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens index 747fbbc64cf5f..4fd37ab9900f2 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens @@ -7,122 +7,117 @@ FROM=6 GROK=7 KEEP=8 LIMIT=9 -META=10 -MV_EXPAND=11 -RENAME=12 -ROW=13 -SHOW=14 -SORT=15 -STATS=16 -WHERE=17 -DEV_INLINESTATS=18 -DEV_LOOKUP=19 -DEV_MATCH=20 -DEV_METRICS=21 -UNKNOWN_CMD=22 -LINE_COMMENT=23 -MULTILINE_COMMENT=24 -WS=25 -PIPE=26 -QUOTED_STRING=27 -INTEGER_LITERAL=28 -DECIMAL_LITERAL=29 -BY=30 -AND=31 -ASC=32 -ASSIGN=33 -CAST_OP=34 -COMMA=35 -DESC=36 -DOT=37 -FALSE=38 -FIRST=39 -IN=40 -IS=41 -LAST=42 -LIKE=43 -LP=44 -NOT=45 -NULL=46 -NULLS=47 -OR=48 -PARAM=49 -RLIKE=50 -RP=51 -TRUE=52 -EQ=53 -CIEQ=54 -NEQ=55 -LT=56 -LTE=57 -GT=58 -GTE=59 -PLUS=60 -MINUS=61 -ASTERISK=62 -SLASH=63 -PERCENT=64 -NAMED_OR_POSITIONAL_PARAM=65 -OPENING_BRACKET=66 -CLOSING_BRACKET=67 -UNQUOTED_IDENTIFIER=68 -QUOTED_IDENTIFIER=69 -EXPR_LINE_COMMENT=70 -EXPR_MULTILINE_COMMENT=71 -EXPR_WS=72 -EXPLAIN_WS=73 -EXPLAIN_LINE_COMMENT=74 -EXPLAIN_MULTILINE_COMMENT=75 -METADATA=76 -UNQUOTED_SOURCE=77 -FROM_LINE_COMMENT=78 -FROM_MULTILINE_COMMENT=79 -FROM_WS=80 -ID_PATTERN=81 -PROJECT_LINE_COMMENT=82 -PROJECT_MULTILINE_COMMENT=83 -PROJECT_WS=84 -AS=85 -RENAME_LINE_COMMENT=86 -RENAME_MULTILINE_COMMENT=87 -RENAME_WS=88 -ON=89 -WITH=90 -ENRICH_POLICY_NAME=91 -ENRICH_LINE_COMMENT=92 -ENRICH_MULTILINE_COMMENT=93 -ENRICH_WS=94 -ENRICH_FIELD_LINE_COMMENT=95 -ENRICH_FIELD_MULTILINE_COMMENT=96 -ENRICH_FIELD_WS=97 -MVEXPAND_LINE_COMMENT=98 -MVEXPAND_MULTILINE_COMMENT=99 -MVEXPAND_WS=100 -INFO=101 -SHOW_LINE_COMMENT=102 -SHOW_MULTILINE_COMMENT=103 -SHOW_WS=104 -FUNCTIONS=105 -META_LINE_COMMENT=106 -META_MULTILINE_COMMENT=107 -META_WS=108 -COLON=109 -SETTING=110 -SETTING_LINE_COMMENT=111 -SETTTING_MULTILINE_COMMENT=112 -SETTING_WS=113 -LOOKUP_LINE_COMMENT=114 -LOOKUP_MULTILINE_COMMENT=115 -LOOKUP_WS=116 -LOOKUP_FIELD_LINE_COMMENT=117 -LOOKUP_FIELD_MULTILINE_COMMENT=118 -LOOKUP_FIELD_WS=119 -METRICS_LINE_COMMENT=120 -METRICS_MULTILINE_COMMENT=121 -METRICS_WS=122 -CLOSING_METRICS_LINE_COMMENT=123 -CLOSING_METRICS_MULTILINE_COMMENT=124 -CLOSING_METRICS_WS=125 +MV_EXPAND=10 +RENAME=11 +ROW=12 +SHOW=13 +SORT=14 +STATS=15 +WHERE=16 +DEV_INLINESTATS=17 +DEV_LOOKUP=18 +DEV_MATCH=19 +DEV_METRICS=20 +UNKNOWN_CMD=21 +LINE_COMMENT=22 +MULTILINE_COMMENT=23 +WS=24 +PIPE=25 +QUOTED_STRING=26 +INTEGER_LITERAL=27 +DECIMAL_LITERAL=28 +BY=29 +AND=30 +ASC=31 +ASSIGN=32 +CAST_OP=33 +COMMA=34 +DESC=35 +DOT=36 +FALSE=37 +FIRST=38 +IN=39 +IS=40 +LAST=41 +LIKE=42 +LP=43 +NOT=44 +NULL=45 +NULLS=46 +OR=47 +PARAM=48 +RLIKE=49 +RP=50 +TRUE=51 +EQ=52 +CIEQ=53 +NEQ=54 +LT=55 +LTE=56 +GT=57 +GTE=58 +PLUS=59 +MINUS=60 +ASTERISK=61 +SLASH=62 +PERCENT=63 +NAMED_OR_POSITIONAL_PARAM=64 +OPENING_BRACKET=65 +CLOSING_BRACKET=66 +UNQUOTED_IDENTIFIER=67 +QUOTED_IDENTIFIER=68 +EXPR_LINE_COMMENT=69 +EXPR_MULTILINE_COMMENT=70 +EXPR_WS=71 +EXPLAIN_WS=72 +EXPLAIN_LINE_COMMENT=73 +EXPLAIN_MULTILINE_COMMENT=74 +METADATA=75 +UNQUOTED_SOURCE=76 +FROM_LINE_COMMENT=77 +FROM_MULTILINE_COMMENT=78 +FROM_WS=79 +ID_PATTERN=80 +PROJECT_LINE_COMMENT=81 +PROJECT_MULTILINE_COMMENT=82 +PROJECT_WS=83 +AS=84 +RENAME_LINE_COMMENT=85 +RENAME_MULTILINE_COMMENT=86 +RENAME_WS=87 +ON=88 +WITH=89 +ENRICH_POLICY_NAME=90 +ENRICH_LINE_COMMENT=91 +ENRICH_MULTILINE_COMMENT=92 +ENRICH_WS=93 +ENRICH_FIELD_LINE_COMMENT=94 +ENRICH_FIELD_MULTILINE_COMMENT=95 +ENRICH_FIELD_WS=96 +MVEXPAND_LINE_COMMENT=97 +MVEXPAND_MULTILINE_COMMENT=98 +MVEXPAND_WS=99 +INFO=100 +SHOW_LINE_COMMENT=101 +SHOW_MULTILINE_COMMENT=102 +SHOW_WS=103 +COLON=104 +SETTING=105 +SETTING_LINE_COMMENT=106 +SETTTING_MULTILINE_COMMENT=107 +SETTING_WS=108 +LOOKUP_LINE_COMMENT=109 +LOOKUP_MULTILINE_COMMENT=110 +LOOKUP_WS=111 +LOOKUP_FIELD_LINE_COMMENT=112 +LOOKUP_FIELD_MULTILINE_COMMENT=113 +LOOKUP_FIELD_WS=114 +METRICS_LINE_COMMENT=115 +METRICS_MULTILINE_COMMENT=116 +METRICS_WS=117 +CLOSING_METRICS_LINE_COMMENT=118 +CLOSING_METRICS_MULTILINE_COMMENT=119 +CLOSING_METRICS_WS=120 'dissect'=1 'drop'=2 'enrich'=3 @@ -132,55 +127,53 @@ CLOSING_METRICS_WS=125 'grok'=7 'keep'=8 'limit'=9 -'meta'=10 -'mv_expand'=11 -'rename'=12 -'row'=13 -'show'=14 -'sort'=15 -'stats'=16 -'where'=17 -'|'=26 -'by'=30 -'and'=31 -'asc'=32 -'='=33 -'::'=34 -','=35 -'desc'=36 -'.'=37 -'false'=38 -'first'=39 -'in'=40 -'is'=41 -'last'=42 -'like'=43 -'('=44 -'not'=45 -'null'=46 -'nulls'=47 -'or'=48 -'?'=49 -'rlike'=50 -')'=51 -'true'=52 -'=='=53 -'=~'=54 -'!='=55 -'<'=56 -'<='=57 -'>'=58 -'>='=59 -'+'=60 -'-'=61 -'*'=62 -'/'=63 -'%'=64 -']'=67 -'metadata'=76 -'as'=85 -'on'=89 -'with'=90 -'info'=101 -'functions'=105 -':'=109 +'mv_expand'=10 +'rename'=11 +'row'=12 +'show'=13 +'sort'=14 +'stats'=15 +'where'=16 +'|'=25 +'by'=29 +'and'=30 +'asc'=31 +'='=32 +'::'=33 +','=34 +'desc'=35 +'.'=36 +'false'=37 +'first'=38 +'in'=39 +'is'=40 +'last'=41 +'like'=42 +'('=43 +'not'=44 +'null'=45 +'nulls'=46 +'or'=47 +'?'=48 +'rlike'=49 +')'=50 +'true'=51 +'=='=52 +'=~'=53 +'!='=54 +'<'=55 +'<='=56 +'>'=57 +'>='=58 +'+'=59 +'-'=60 +'*'=61 +'/'=62 +'%'=63 +']'=66 +'metadata'=75 +'as'=84 +'on'=88 +'with'=89 +'info'=100 +':'=104 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index c39a2041a61be..5b4428ee24118 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -262,11 +262,9 @@ public enum Cap { MATCH_OPERATOR(true), /** - * Support for the {@code META} keyword. Tests with this tag are - * intentionally excluded from mixed version clusters because we - * continually add functions, so they constantly fail if we don't. + * Removing support for the {@code META} keyword. */ - META, + NO_META, /** * Add CombineBinaryComparisons rule. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 7c0f1fa3a8ad0..8e238f9ed760c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -133,7 +133,6 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToLower; import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Trim; -import org.elasticsearch.xpack.esql.plan.logical.meta.MetaFunctions; import org.elasticsearch.xpack.esql.session.Configuration; import java.lang.reflect.Constructor; @@ -450,31 +449,6 @@ public record FunctionDescription( boolean variadic, boolean isAggregation ) { - public String fullSignature() { - StringBuilder builder = new StringBuilder(); - builder.append(MetaFunctions.withPipes(returnType)); - builder.append(" "); - builder.append(name); - builder.append("("); - for (int i = 0; i < args.size(); i++) { - ArgSignature arg = args.get(i); - if (i > 0) { - builder.append(", "); - } - if (arg.optional()) { - builder.append("?"); - } - builder.append(arg.name()); - if (i == args.size() - 1 && variadic) { - builder.append("..."); - } - builder.append(":"); - builder.append(MetaFunctions.withPipes(arg.type())); - } - builder.append(")"); - return builder.toString(); - } - /** * The name of every argument. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp index 8122a56884280..4f3a843ee09d1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp @@ -9,7 +9,6 @@ null 'grok' 'keep' 'limit' -'meta' 'mv_expand' 'rename' 'row' @@ -104,10 +103,6 @@ null null null null -'functions' -null -null -null ':' null null @@ -137,7 +132,6 @@ FROM GROK KEEP LIMIT -META MV_EXPAND RENAME ROW @@ -232,10 +226,6 @@ INFO SHOW_LINE_COMMENT SHOW_MULTILINE_COMMENT SHOW_WS -FUNCTIONS -META_LINE_COMMENT -META_MULTILINE_COMMENT -META_WS COLON SETTING SETTING_LINE_COMMENT @@ -264,7 +254,6 @@ FROM GROK KEEP LIMIT -META MV_EXPAND RENAME ROW @@ -408,11 +397,6 @@ INFO SHOW_LINE_COMMENT SHOW_MULTILINE_COMMENT SHOW_WS -META_PIPE -FUNCTIONS -META_LINE_COMMENT -META_MULTILINE_COMMENT -META_WS SETTING_CLOSING_BRACKET COLON SETTING @@ -467,7 +451,6 @@ ENRICH_MODE ENRICH_FIELD_MODE MVEXPAND_MODE SHOW_MODE -META_MODE SETTING_MODE LOOKUP_MODE LOOKUP_FIELD_MODE @@ -475,4 +458,4 @@ METRICS_MODE CLOSING_METRICS_MODE atn: -[4, 0, 125, 1474, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 2, 189, 7, 189, 2, 190, 7, 190, 2, 191, 7, 191, 2, 192, 7, 192, 2, 193, 7, 193, 2, 194, 7, 194, 2, 195, 7, 195, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 21, 4, 21, 591, 8, 21, 11, 21, 12, 21, 592, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 601, 8, 22, 10, 22, 12, 22, 604, 9, 22, 1, 22, 3, 22, 607, 8, 22, 1, 22, 3, 22, 610, 8, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 5, 23, 619, 8, 23, 10, 23, 12, 23, 622, 9, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 4, 24, 630, 8, 24, 11, 24, 12, 24, 631, 1, 24, 1, 24, 1, 25, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 1, 27, 1, 27, 1, 28, 1, 28, 1, 28, 1, 29, 1, 29, 1, 30, 1, 30, 3, 30, 651, 8, 30, 1, 30, 4, 30, 654, 8, 30, 11, 30, 12, 30, 655, 1, 31, 1, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 3, 33, 665, 8, 33, 1, 34, 1, 34, 1, 35, 1, 35, 1, 35, 3, 35, 672, 8, 35, 1, 36, 1, 36, 1, 36, 5, 36, 677, 8, 36, 10, 36, 12, 36, 680, 9, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 5, 36, 688, 8, 36, 10, 36, 12, 36, 691, 9, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 3, 36, 698, 8, 36, 1, 36, 3, 36, 701, 8, 36, 3, 36, 703, 8, 36, 1, 37, 4, 37, 706, 8, 37, 11, 37, 12, 37, 707, 1, 38, 4, 38, 711, 8, 38, 11, 38, 12, 38, 712, 1, 38, 1, 38, 5, 38, 717, 8, 38, 10, 38, 12, 38, 720, 9, 38, 1, 38, 1, 38, 4, 38, 724, 8, 38, 11, 38, 12, 38, 725, 1, 38, 4, 38, 729, 8, 38, 11, 38, 12, 38, 730, 1, 38, 1, 38, 5, 38, 735, 8, 38, 10, 38, 12, 38, 738, 9, 38, 3, 38, 740, 8, 38, 1, 38, 1, 38, 1, 38, 1, 38, 4, 38, 746, 8, 38, 11, 38, 12, 38, 747, 1, 38, 1, 38, 3, 38, 752, 8, 38, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 64, 1, 64, 1, 64, 1, 65, 1, 65, 1, 66, 1, 66, 1, 66, 1, 67, 1, 67, 1, 68, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 73, 1, 73, 1, 74, 1, 74, 1, 74, 1, 74, 1, 74, 1, 75, 1, 75, 1, 75, 3, 75, 879, 8, 75, 1, 75, 5, 75, 882, 8, 75, 10, 75, 12, 75, 885, 9, 75, 1, 75, 1, 75, 4, 75, 889, 8, 75, 11, 75, 12, 75, 890, 3, 75, 893, 8, 75, 1, 76, 1, 76, 1, 76, 1, 76, 1, 76, 1, 77, 1, 77, 1, 77, 1, 77, 1, 77, 1, 78, 1, 78, 5, 78, 907, 8, 78, 10, 78, 12, 78, 910, 9, 78, 1, 78, 1, 78, 3, 78, 914, 8, 78, 1, 78, 4, 78, 917, 8, 78, 11, 78, 12, 78, 918, 3, 78, 921, 8, 78, 1, 79, 1, 79, 4, 79, 925, 8, 79, 11, 79, 12, 79, 926, 1, 79, 1, 79, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 1, 95, 1, 95, 1, 95, 1, 95, 1, 95, 1, 95, 1, 96, 1, 96, 1, 96, 3, 96, 1004, 8, 96, 1, 97, 4, 97, 1007, 8, 97, 11, 97, 12, 97, 1008, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 1, 106, 1, 106, 1, 106, 1, 106, 3, 106, 1048, 8, 106, 1, 107, 1, 107, 3, 107, 1052, 8, 107, 1, 107, 5, 107, 1055, 8, 107, 10, 107, 12, 107, 1058, 9, 107, 1, 107, 1, 107, 3, 107, 1062, 8, 107, 1, 107, 4, 107, 1065, 8, 107, 11, 107, 12, 107, 1066, 3, 107, 1069, 8, 107, 1, 108, 1, 108, 4, 108, 1073, 8, 108, 11, 108, 12, 108, 1074, 1, 109, 1, 109, 1, 109, 1, 109, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 124, 1, 124, 1, 124, 1, 124, 1, 124, 1, 125, 1, 125, 1, 126, 4, 126, 1150, 8, 126, 11, 126, 12, 126, 1151, 1, 126, 1, 126, 3, 126, 1156, 8, 126, 1, 126, 4, 126, 1159, 8, 126, 11, 126, 12, 126, 1160, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 128, 1, 128, 1, 129, 1, 129, 1, 129, 1, 129, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 153, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 154, 1, 154, 1, 154, 1, 154, 1, 154, 1, 154, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 160, 4, 160, 1311, 8, 160, 11, 160, 12, 160, 1312, 1, 161, 1, 161, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 173, 1, 173, 1, 173, 1, 173, 1, 174, 1, 174, 1, 174, 1, 174, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 188, 1, 188, 1, 189, 1, 189, 1, 189, 1, 189, 1, 190, 1, 190, 1, 190, 1, 190, 1, 191, 1, 191, 1, 191, 1, 191, 1, 192, 1, 192, 1, 192, 1, 192, 1, 192, 1, 192, 1, 193, 1, 193, 1, 193, 1, 193, 1, 193, 1, 193, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 194, 1, 195, 1, 195, 1, 195, 1, 195, 1, 195, 2, 620, 689, 0, 196, 16, 1, 18, 2, 20, 3, 22, 4, 24, 5, 26, 6, 28, 7, 30, 8, 32, 9, 34, 10, 36, 11, 38, 12, 40, 13, 42, 14, 44, 15, 46, 16, 48, 17, 50, 18, 52, 19, 54, 20, 56, 21, 58, 22, 60, 23, 62, 24, 64, 25, 66, 26, 68, 0, 70, 0, 72, 0, 74, 0, 76, 0, 78, 0, 80, 0, 82, 0, 84, 0, 86, 0, 88, 27, 90, 28, 92, 29, 94, 30, 96, 31, 98, 32, 100, 33, 102, 34, 104, 35, 106, 36, 108, 37, 110, 38, 112, 39, 114, 40, 116, 41, 118, 42, 120, 43, 122, 44, 124, 45, 126, 46, 128, 47, 130, 48, 132, 49, 134, 50, 136, 51, 138, 52, 140, 53, 142, 54, 144, 55, 146, 56, 148, 57, 150, 58, 152, 59, 154, 60, 156, 61, 158, 62, 160, 63, 162, 64, 164, 0, 166, 65, 168, 66, 170, 67, 172, 68, 174, 0, 176, 69, 178, 70, 180, 71, 182, 72, 184, 0, 186, 0, 188, 73, 190, 74, 192, 75, 194, 0, 196, 0, 198, 0, 200, 0, 202, 0, 204, 0, 206, 76, 208, 0, 210, 77, 212, 0, 214, 0, 216, 78, 218, 79, 220, 80, 222, 0, 224, 0, 226, 0, 228, 0, 230, 0, 232, 81, 234, 82, 236, 83, 238, 84, 240, 0, 242, 0, 244, 0, 246, 0, 248, 85, 250, 0, 252, 86, 254, 87, 256, 88, 258, 0, 260, 0, 262, 89, 264, 90, 266, 0, 268, 91, 270, 0, 272, 92, 274, 93, 276, 94, 278, 0, 280, 0, 282, 0, 284, 0, 286, 0, 288, 0, 290, 0, 292, 95, 294, 96, 296, 97, 298, 0, 300, 0, 302, 0, 304, 0, 306, 98, 308, 99, 310, 100, 312, 0, 314, 101, 316, 102, 318, 103, 320, 104, 322, 0, 324, 105, 326, 106, 328, 107, 330, 108, 332, 0, 334, 109, 336, 110, 338, 111, 340, 112, 342, 113, 344, 0, 346, 0, 348, 0, 350, 0, 352, 0, 354, 0, 356, 0, 358, 114, 360, 115, 362, 116, 364, 0, 366, 0, 368, 0, 370, 0, 372, 117, 374, 118, 376, 119, 378, 0, 380, 0, 382, 0, 384, 120, 386, 121, 388, 122, 390, 0, 392, 0, 394, 123, 396, 124, 398, 125, 400, 0, 402, 0, 404, 0, 406, 0, 16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 35, 2, 0, 68, 68, 100, 100, 2, 0, 73, 73, 105, 105, 2, 0, 83, 83, 115, 115, 2, 0, 69, 69, 101, 101, 2, 0, 67, 67, 99, 99, 2, 0, 84, 84, 116, 116, 2, 0, 82, 82, 114, 114, 2, 0, 79, 79, 111, 111, 2, 0, 80, 80, 112, 112, 2, 0, 78, 78, 110, 110, 2, 0, 72, 72, 104, 104, 2, 0, 86, 86, 118, 118, 2, 0, 65, 65, 97, 97, 2, 0, 76, 76, 108, 108, 2, 0, 88, 88, 120, 120, 2, 0, 70, 70, 102, 102, 2, 0, 77, 77, 109, 109, 2, 0, 71, 71, 103, 103, 2, 0, 75, 75, 107, 107, 2, 0, 87, 87, 119, 119, 2, 0, 85, 85, 117, 117, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 8, 0, 34, 34, 78, 78, 82, 82, 84, 84, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 2, 0, 66, 66, 98, 98, 2, 0, 89, 89, 121, 121, 11, 0, 9, 10, 13, 13, 32, 32, 34, 34, 44, 44, 47, 47, 58, 58, 61, 61, 91, 91, 93, 93, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 1501, 0, 16, 1, 0, 0, 0, 0, 18, 1, 0, 0, 0, 0, 20, 1, 0, 0, 0, 0, 22, 1, 0, 0, 0, 0, 24, 1, 0, 0, 0, 0, 26, 1, 0, 0, 0, 0, 28, 1, 0, 0, 0, 0, 30, 1, 0, 0, 0, 0, 32, 1, 0, 0, 0, 0, 34, 1, 0, 0, 0, 0, 36, 1, 0, 0, 0, 0, 38, 1, 0, 0, 0, 0, 40, 1, 0, 0, 0, 0, 42, 1, 0, 0, 0, 0, 44, 1, 0, 0, 0, 0, 46, 1, 0, 0, 0, 0, 48, 1, 0, 0, 0, 0, 50, 1, 0, 0, 0, 0, 52, 1, 0, 0, 0, 0, 54, 1, 0, 0, 0, 0, 56, 1, 0, 0, 0, 0, 58, 1, 0, 0, 0, 0, 60, 1, 0, 0, 0, 0, 62, 1, 0, 0, 0, 0, 64, 1, 0, 0, 0, 1, 66, 1, 0, 0, 0, 1, 88, 1, 0, 0, 0, 1, 90, 1, 0, 0, 0, 1, 92, 1, 0, 0, 0, 1, 94, 1, 0, 0, 0, 1, 96, 1, 0, 0, 0, 1, 98, 1, 0, 0, 0, 1, 100, 1, 0, 0, 0, 1, 102, 1, 0, 0, 0, 1, 104, 1, 0, 0, 0, 1, 106, 1, 0, 0, 0, 1, 108, 1, 0, 0, 0, 1, 110, 1, 0, 0, 0, 1, 112, 1, 0, 0, 0, 1, 114, 1, 0, 0, 0, 1, 116, 1, 0, 0, 0, 1, 118, 1, 0, 0, 0, 1, 120, 1, 0, 0, 0, 1, 122, 1, 0, 0, 0, 1, 124, 1, 0, 0, 0, 1, 126, 1, 0, 0, 0, 1, 128, 1, 0, 0, 0, 1, 130, 1, 0, 0, 0, 1, 132, 1, 0, 0, 0, 1, 134, 1, 0, 0, 0, 1, 136, 1, 0, 0, 0, 1, 138, 1, 0, 0, 0, 1, 140, 1, 0, 0, 0, 1, 142, 1, 0, 0, 0, 1, 144, 1, 0, 0, 0, 1, 146, 1, 0, 0, 0, 1, 148, 1, 0, 0, 0, 1, 150, 1, 0, 0, 0, 1, 152, 1, 0, 0, 0, 1, 154, 1, 0, 0, 0, 1, 156, 1, 0, 0, 0, 1, 158, 1, 0, 0, 0, 1, 160, 1, 0, 0, 0, 1, 162, 1, 0, 0, 0, 1, 164, 1, 0, 0, 0, 1, 166, 1, 0, 0, 0, 1, 168, 1, 0, 0, 0, 1, 170, 1, 0, 0, 0, 1, 172, 1, 0, 0, 0, 1, 176, 1, 0, 0, 0, 1, 178, 1, 0, 0, 0, 1, 180, 1, 0, 0, 0, 1, 182, 1, 0, 0, 0, 2, 184, 1, 0, 0, 0, 2, 186, 1, 0, 0, 0, 2, 188, 1, 0, 0, 0, 2, 190, 1, 0, 0, 0, 2, 192, 1, 0, 0, 0, 3, 194, 1, 0, 0, 0, 3, 196, 1, 0, 0, 0, 3, 198, 1, 0, 0, 0, 3, 200, 1, 0, 0, 0, 3, 202, 1, 0, 0, 0, 3, 204, 1, 0, 0, 0, 3, 206, 1, 0, 0, 0, 3, 210, 1, 0, 0, 0, 3, 212, 1, 0, 0, 0, 3, 214, 1, 0, 0, 0, 3, 216, 1, 0, 0, 0, 3, 218, 1, 0, 0, 0, 3, 220, 1, 0, 0, 0, 4, 222, 1, 0, 0, 0, 4, 224, 1, 0, 0, 0, 4, 226, 1, 0, 0, 0, 4, 232, 1, 0, 0, 0, 4, 234, 1, 0, 0, 0, 4, 236, 1, 0, 0, 0, 4, 238, 1, 0, 0, 0, 5, 240, 1, 0, 0, 0, 5, 242, 1, 0, 0, 0, 5, 244, 1, 0, 0, 0, 5, 246, 1, 0, 0, 0, 5, 248, 1, 0, 0, 0, 5, 250, 1, 0, 0, 0, 5, 252, 1, 0, 0, 0, 5, 254, 1, 0, 0, 0, 5, 256, 1, 0, 0, 0, 6, 258, 1, 0, 0, 0, 6, 260, 1, 0, 0, 0, 6, 262, 1, 0, 0, 0, 6, 264, 1, 0, 0, 0, 6, 268, 1, 0, 0, 0, 6, 270, 1, 0, 0, 0, 6, 272, 1, 0, 0, 0, 6, 274, 1, 0, 0, 0, 6, 276, 1, 0, 0, 0, 7, 278, 1, 0, 0, 0, 7, 280, 1, 0, 0, 0, 7, 282, 1, 0, 0, 0, 7, 284, 1, 0, 0, 0, 7, 286, 1, 0, 0, 0, 7, 288, 1, 0, 0, 0, 7, 290, 1, 0, 0, 0, 7, 292, 1, 0, 0, 0, 7, 294, 1, 0, 0, 0, 7, 296, 1, 0, 0, 0, 8, 298, 1, 0, 0, 0, 8, 300, 1, 0, 0, 0, 8, 302, 1, 0, 0, 0, 8, 304, 1, 0, 0, 0, 8, 306, 1, 0, 0, 0, 8, 308, 1, 0, 0, 0, 8, 310, 1, 0, 0, 0, 9, 312, 1, 0, 0, 0, 9, 314, 1, 0, 0, 0, 9, 316, 1, 0, 0, 0, 9, 318, 1, 0, 0, 0, 9, 320, 1, 0, 0, 0, 10, 322, 1, 0, 0, 0, 10, 324, 1, 0, 0, 0, 10, 326, 1, 0, 0, 0, 10, 328, 1, 0, 0, 0, 10, 330, 1, 0, 0, 0, 11, 332, 1, 0, 0, 0, 11, 334, 1, 0, 0, 0, 11, 336, 1, 0, 0, 0, 11, 338, 1, 0, 0, 0, 11, 340, 1, 0, 0, 0, 11, 342, 1, 0, 0, 0, 12, 344, 1, 0, 0, 0, 12, 346, 1, 0, 0, 0, 12, 348, 1, 0, 0, 0, 12, 350, 1, 0, 0, 0, 12, 352, 1, 0, 0, 0, 12, 354, 1, 0, 0, 0, 12, 356, 1, 0, 0, 0, 12, 358, 1, 0, 0, 0, 12, 360, 1, 0, 0, 0, 12, 362, 1, 0, 0, 0, 13, 364, 1, 0, 0, 0, 13, 366, 1, 0, 0, 0, 13, 368, 1, 0, 0, 0, 13, 370, 1, 0, 0, 0, 13, 372, 1, 0, 0, 0, 13, 374, 1, 0, 0, 0, 13, 376, 1, 0, 0, 0, 14, 378, 1, 0, 0, 0, 14, 380, 1, 0, 0, 0, 14, 382, 1, 0, 0, 0, 14, 384, 1, 0, 0, 0, 14, 386, 1, 0, 0, 0, 14, 388, 1, 0, 0, 0, 15, 390, 1, 0, 0, 0, 15, 392, 1, 0, 0, 0, 15, 394, 1, 0, 0, 0, 15, 396, 1, 0, 0, 0, 15, 398, 1, 0, 0, 0, 15, 400, 1, 0, 0, 0, 15, 402, 1, 0, 0, 0, 15, 404, 1, 0, 0, 0, 15, 406, 1, 0, 0, 0, 16, 408, 1, 0, 0, 0, 18, 418, 1, 0, 0, 0, 20, 425, 1, 0, 0, 0, 22, 434, 1, 0, 0, 0, 24, 441, 1, 0, 0, 0, 26, 451, 1, 0, 0, 0, 28, 458, 1, 0, 0, 0, 30, 465, 1, 0, 0, 0, 32, 472, 1, 0, 0, 0, 34, 480, 1, 0, 0, 0, 36, 487, 1, 0, 0, 0, 38, 499, 1, 0, 0, 0, 40, 508, 1, 0, 0, 0, 42, 514, 1, 0, 0, 0, 44, 521, 1, 0, 0, 0, 46, 528, 1, 0, 0, 0, 48, 536, 1, 0, 0, 0, 50, 544, 1, 0, 0, 0, 52, 559, 1, 0, 0, 0, 54, 569, 1, 0, 0, 0, 56, 578, 1, 0, 0, 0, 58, 590, 1, 0, 0, 0, 60, 596, 1, 0, 0, 0, 62, 613, 1, 0, 0, 0, 64, 629, 1, 0, 0, 0, 66, 635, 1, 0, 0, 0, 68, 639, 1, 0, 0, 0, 70, 641, 1, 0, 0, 0, 72, 643, 1, 0, 0, 0, 74, 646, 1, 0, 0, 0, 76, 648, 1, 0, 0, 0, 78, 657, 1, 0, 0, 0, 80, 659, 1, 0, 0, 0, 82, 664, 1, 0, 0, 0, 84, 666, 1, 0, 0, 0, 86, 671, 1, 0, 0, 0, 88, 702, 1, 0, 0, 0, 90, 705, 1, 0, 0, 0, 92, 751, 1, 0, 0, 0, 94, 753, 1, 0, 0, 0, 96, 756, 1, 0, 0, 0, 98, 760, 1, 0, 0, 0, 100, 764, 1, 0, 0, 0, 102, 766, 1, 0, 0, 0, 104, 769, 1, 0, 0, 0, 106, 771, 1, 0, 0, 0, 108, 776, 1, 0, 0, 0, 110, 778, 1, 0, 0, 0, 112, 784, 1, 0, 0, 0, 114, 790, 1, 0, 0, 0, 116, 793, 1, 0, 0, 0, 118, 796, 1, 0, 0, 0, 120, 801, 1, 0, 0, 0, 122, 806, 1, 0, 0, 0, 124, 808, 1, 0, 0, 0, 126, 812, 1, 0, 0, 0, 128, 817, 1, 0, 0, 0, 130, 823, 1, 0, 0, 0, 132, 826, 1, 0, 0, 0, 134, 828, 1, 0, 0, 0, 136, 834, 1, 0, 0, 0, 138, 836, 1, 0, 0, 0, 140, 841, 1, 0, 0, 0, 142, 844, 1, 0, 0, 0, 144, 847, 1, 0, 0, 0, 146, 850, 1, 0, 0, 0, 148, 852, 1, 0, 0, 0, 150, 855, 1, 0, 0, 0, 152, 857, 1, 0, 0, 0, 154, 860, 1, 0, 0, 0, 156, 862, 1, 0, 0, 0, 158, 864, 1, 0, 0, 0, 160, 866, 1, 0, 0, 0, 162, 868, 1, 0, 0, 0, 164, 870, 1, 0, 0, 0, 166, 892, 1, 0, 0, 0, 168, 894, 1, 0, 0, 0, 170, 899, 1, 0, 0, 0, 172, 920, 1, 0, 0, 0, 174, 922, 1, 0, 0, 0, 176, 930, 1, 0, 0, 0, 178, 932, 1, 0, 0, 0, 180, 936, 1, 0, 0, 0, 182, 940, 1, 0, 0, 0, 184, 944, 1, 0, 0, 0, 186, 949, 1, 0, 0, 0, 188, 954, 1, 0, 0, 0, 190, 958, 1, 0, 0, 0, 192, 962, 1, 0, 0, 0, 194, 966, 1, 0, 0, 0, 196, 971, 1, 0, 0, 0, 198, 975, 1, 0, 0, 0, 200, 979, 1, 0, 0, 0, 202, 983, 1, 0, 0, 0, 204, 987, 1, 0, 0, 0, 206, 991, 1, 0, 0, 0, 208, 1003, 1, 0, 0, 0, 210, 1006, 1, 0, 0, 0, 212, 1010, 1, 0, 0, 0, 214, 1014, 1, 0, 0, 0, 216, 1018, 1, 0, 0, 0, 218, 1022, 1, 0, 0, 0, 220, 1026, 1, 0, 0, 0, 222, 1030, 1, 0, 0, 0, 224, 1035, 1, 0, 0, 0, 226, 1039, 1, 0, 0, 0, 228, 1047, 1, 0, 0, 0, 230, 1068, 1, 0, 0, 0, 232, 1072, 1, 0, 0, 0, 234, 1076, 1, 0, 0, 0, 236, 1080, 1, 0, 0, 0, 238, 1084, 1, 0, 0, 0, 240, 1088, 1, 0, 0, 0, 242, 1093, 1, 0, 0, 0, 244, 1097, 1, 0, 0, 0, 246, 1101, 1, 0, 0, 0, 248, 1105, 1, 0, 0, 0, 250, 1108, 1, 0, 0, 0, 252, 1112, 1, 0, 0, 0, 254, 1116, 1, 0, 0, 0, 256, 1120, 1, 0, 0, 0, 258, 1124, 1, 0, 0, 0, 260, 1129, 1, 0, 0, 0, 262, 1134, 1, 0, 0, 0, 264, 1139, 1, 0, 0, 0, 266, 1146, 1, 0, 0, 0, 268, 1155, 1, 0, 0, 0, 270, 1162, 1, 0, 0, 0, 272, 1166, 1, 0, 0, 0, 274, 1170, 1, 0, 0, 0, 276, 1174, 1, 0, 0, 0, 278, 1178, 1, 0, 0, 0, 280, 1184, 1, 0, 0, 0, 282, 1188, 1, 0, 0, 0, 284, 1192, 1, 0, 0, 0, 286, 1196, 1, 0, 0, 0, 288, 1200, 1, 0, 0, 0, 290, 1204, 1, 0, 0, 0, 292, 1208, 1, 0, 0, 0, 294, 1212, 1, 0, 0, 0, 296, 1216, 1, 0, 0, 0, 298, 1220, 1, 0, 0, 0, 300, 1225, 1, 0, 0, 0, 302, 1229, 1, 0, 0, 0, 304, 1233, 1, 0, 0, 0, 306, 1237, 1, 0, 0, 0, 308, 1241, 1, 0, 0, 0, 310, 1245, 1, 0, 0, 0, 312, 1249, 1, 0, 0, 0, 314, 1254, 1, 0, 0, 0, 316, 1259, 1, 0, 0, 0, 318, 1263, 1, 0, 0, 0, 320, 1267, 1, 0, 0, 0, 322, 1271, 1, 0, 0, 0, 324, 1276, 1, 0, 0, 0, 326, 1286, 1, 0, 0, 0, 328, 1290, 1, 0, 0, 0, 330, 1294, 1, 0, 0, 0, 332, 1298, 1, 0, 0, 0, 334, 1303, 1, 0, 0, 0, 336, 1310, 1, 0, 0, 0, 338, 1314, 1, 0, 0, 0, 340, 1318, 1, 0, 0, 0, 342, 1322, 1, 0, 0, 0, 344, 1326, 1, 0, 0, 0, 346, 1331, 1, 0, 0, 0, 348, 1335, 1, 0, 0, 0, 350, 1339, 1, 0, 0, 0, 352, 1343, 1, 0, 0, 0, 354, 1348, 1, 0, 0, 0, 356, 1352, 1, 0, 0, 0, 358, 1356, 1, 0, 0, 0, 360, 1360, 1, 0, 0, 0, 362, 1364, 1, 0, 0, 0, 364, 1368, 1, 0, 0, 0, 366, 1374, 1, 0, 0, 0, 368, 1378, 1, 0, 0, 0, 370, 1382, 1, 0, 0, 0, 372, 1386, 1, 0, 0, 0, 374, 1390, 1, 0, 0, 0, 376, 1394, 1, 0, 0, 0, 378, 1398, 1, 0, 0, 0, 380, 1403, 1, 0, 0, 0, 382, 1409, 1, 0, 0, 0, 384, 1415, 1, 0, 0, 0, 386, 1419, 1, 0, 0, 0, 388, 1423, 1, 0, 0, 0, 390, 1427, 1, 0, 0, 0, 392, 1433, 1, 0, 0, 0, 394, 1439, 1, 0, 0, 0, 396, 1443, 1, 0, 0, 0, 398, 1447, 1, 0, 0, 0, 400, 1451, 1, 0, 0, 0, 402, 1457, 1, 0, 0, 0, 404, 1463, 1, 0, 0, 0, 406, 1469, 1, 0, 0, 0, 408, 409, 7, 0, 0, 0, 409, 410, 7, 1, 0, 0, 410, 411, 7, 2, 0, 0, 411, 412, 7, 2, 0, 0, 412, 413, 7, 3, 0, 0, 413, 414, 7, 4, 0, 0, 414, 415, 7, 5, 0, 0, 415, 416, 1, 0, 0, 0, 416, 417, 6, 0, 0, 0, 417, 17, 1, 0, 0, 0, 418, 419, 7, 0, 0, 0, 419, 420, 7, 6, 0, 0, 420, 421, 7, 7, 0, 0, 421, 422, 7, 8, 0, 0, 422, 423, 1, 0, 0, 0, 423, 424, 6, 1, 1, 0, 424, 19, 1, 0, 0, 0, 425, 426, 7, 3, 0, 0, 426, 427, 7, 9, 0, 0, 427, 428, 7, 6, 0, 0, 428, 429, 7, 1, 0, 0, 429, 430, 7, 4, 0, 0, 430, 431, 7, 10, 0, 0, 431, 432, 1, 0, 0, 0, 432, 433, 6, 2, 2, 0, 433, 21, 1, 0, 0, 0, 434, 435, 7, 3, 0, 0, 435, 436, 7, 11, 0, 0, 436, 437, 7, 12, 0, 0, 437, 438, 7, 13, 0, 0, 438, 439, 1, 0, 0, 0, 439, 440, 6, 3, 0, 0, 440, 23, 1, 0, 0, 0, 441, 442, 7, 3, 0, 0, 442, 443, 7, 14, 0, 0, 443, 444, 7, 8, 0, 0, 444, 445, 7, 13, 0, 0, 445, 446, 7, 12, 0, 0, 446, 447, 7, 1, 0, 0, 447, 448, 7, 9, 0, 0, 448, 449, 1, 0, 0, 0, 449, 450, 6, 4, 3, 0, 450, 25, 1, 0, 0, 0, 451, 452, 7, 15, 0, 0, 452, 453, 7, 6, 0, 0, 453, 454, 7, 7, 0, 0, 454, 455, 7, 16, 0, 0, 455, 456, 1, 0, 0, 0, 456, 457, 6, 5, 4, 0, 457, 27, 1, 0, 0, 0, 458, 459, 7, 17, 0, 0, 459, 460, 7, 6, 0, 0, 460, 461, 7, 7, 0, 0, 461, 462, 7, 18, 0, 0, 462, 463, 1, 0, 0, 0, 463, 464, 6, 6, 0, 0, 464, 29, 1, 0, 0, 0, 465, 466, 7, 18, 0, 0, 466, 467, 7, 3, 0, 0, 467, 468, 7, 3, 0, 0, 468, 469, 7, 8, 0, 0, 469, 470, 1, 0, 0, 0, 470, 471, 6, 7, 1, 0, 471, 31, 1, 0, 0, 0, 472, 473, 7, 13, 0, 0, 473, 474, 7, 1, 0, 0, 474, 475, 7, 16, 0, 0, 475, 476, 7, 1, 0, 0, 476, 477, 7, 5, 0, 0, 477, 478, 1, 0, 0, 0, 478, 479, 6, 8, 0, 0, 479, 33, 1, 0, 0, 0, 480, 481, 7, 16, 0, 0, 481, 482, 7, 3, 0, 0, 482, 483, 7, 5, 0, 0, 483, 484, 7, 12, 0, 0, 484, 485, 1, 0, 0, 0, 485, 486, 6, 9, 5, 0, 486, 35, 1, 0, 0, 0, 487, 488, 7, 16, 0, 0, 488, 489, 7, 11, 0, 0, 489, 490, 5, 95, 0, 0, 490, 491, 7, 3, 0, 0, 491, 492, 7, 14, 0, 0, 492, 493, 7, 8, 0, 0, 493, 494, 7, 12, 0, 0, 494, 495, 7, 9, 0, 0, 495, 496, 7, 0, 0, 0, 496, 497, 1, 0, 0, 0, 497, 498, 6, 10, 6, 0, 498, 37, 1, 0, 0, 0, 499, 500, 7, 6, 0, 0, 500, 501, 7, 3, 0, 0, 501, 502, 7, 9, 0, 0, 502, 503, 7, 12, 0, 0, 503, 504, 7, 16, 0, 0, 504, 505, 7, 3, 0, 0, 505, 506, 1, 0, 0, 0, 506, 507, 6, 11, 7, 0, 507, 39, 1, 0, 0, 0, 508, 509, 7, 6, 0, 0, 509, 510, 7, 7, 0, 0, 510, 511, 7, 19, 0, 0, 511, 512, 1, 0, 0, 0, 512, 513, 6, 12, 0, 0, 513, 41, 1, 0, 0, 0, 514, 515, 7, 2, 0, 0, 515, 516, 7, 10, 0, 0, 516, 517, 7, 7, 0, 0, 517, 518, 7, 19, 0, 0, 518, 519, 1, 0, 0, 0, 519, 520, 6, 13, 8, 0, 520, 43, 1, 0, 0, 0, 521, 522, 7, 2, 0, 0, 522, 523, 7, 7, 0, 0, 523, 524, 7, 6, 0, 0, 524, 525, 7, 5, 0, 0, 525, 526, 1, 0, 0, 0, 526, 527, 6, 14, 0, 0, 527, 45, 1, 0, 0, 0, 528, 529, 7, 2, 0, 0, 529, 530, 7, 5, 0, 0, 530, 531, 7, 12, 0, 0, 531, 532, 7, 5, 0, 0, 532, 533, 7, 2, 0, 0, 533, 534, 1, 0, 0, 0, 534, 535, 6, 15, 0, 0, 535, 47, 1, 0, 0, 0, 536, 537, 7, 19, 0, 0, 537, 538, 7, 10, 0, 0, 538, 539, 7, 3, 0, 0, 539, 540, 7, 6, 0, 0, 540, 541, 7, 3, 0, 0, 541, 542, 1, 0, 0, 0, 542, 543, 6, 16, 0, 0, 543, 49, 1, 0, 0, 0, 544, 545, 4, 17, 0, 0, 545, 546, 7, 1, 0, 0, 546, 547, 7, 9, 0, 0, 547, 548, 7, 13, 0, 0, 548, 549, 7, 1, 0, 0, 549, 550, 7, 9, 0, 0, 550, 551, 7, 3, 0, 0, 551, 552, 7, 2, 0, 0, 552, 553, 7, 5, 0, 0, 553, 554, 7, 12, 0, 0, 554, 555, 7, 5, 0, 0, 555, 556, 7, 2, 0, 0, 556, 557, 1, 0, 0, 0, 557, 558, 6, 17, 0, 0, 558, 51, 1, 0, 0, 0, 559, 560, 4, 18, 1, 0, 560, 561, 7, 13, 0, 0, 561, 562, 7, 7, 0, 0, 562, 563, 7, 7, 0, 0, 563, 564, 7, 18, 0, 0, 564, 565, 7, 20, 0, 0, 565, 566, 7, 8, 0, 0, 566, 567, 1, 0, 0, 0, 567, 568, 6, 18, 9, 0, 568, 53, 1, 0, 0, 0, 569, 570, 4, 19, 2, 0, 570, 571, 7, 16, 0, 0, 571, 572, 7, 12, 0, 0, 572, 573, 7, 5, 0, 0, 573, 574, 7, 4, 0, 0, 574, 575, 7, 10, 0, 0, 575, 576, 1, 0, 0, 0, 576, 577, 6, 19, 0, 0, 577, 55, 1, 0, 0, 0, 578, 579, 4, 20, 3, 0, 579, 580, 7, 16, 0, 0, 580, 581, 7, 3, 0, 0, 581, 582, 7, 5, 0, 0, 582, 583, 7, 6, 0, 0, 583, 584, 7, 1, 0, 0, 584, 585, 7, 4, 0, 0, 585, 586, 7, 2, 0, 0, 586, 587, 1, 0, 0, 0, 587, 588, 6, 20, 10, 0, 588, 57, 1, 0, 0, 0, 589, 591, 8, 21, 0, 0, 590, 589, 1, 0, 0, 0, 591, 592, 1, 0, 0, 0, 592, 590, 1, 0, 0, 0, 592, 593, 1, 0, 0, 0, 593, 594, 1, 0, 0, 0, 594, 595, 6, 21, 0, 0, 595, 59, 1, 0, 0, 0, 596, 597, 5, 47, 0, 0, 597, 598, 5, 47, 0, 0, 598, 602, 1, 0, 0, 0, 599, 601, 8, 22, 0, 0, 600, 599, 1, 0, 0, 0, 601, 604, 1, 0, 0, 0, 602, 600, 1, 0, 0, 0, 602, 603, 1, 0, 0, 0, 603, 606, 1, 0, 0, 0, 604, 602, 1, 0, 0, 0, 605, 607, 5, 13, 0, 0, 606, 605, 1, 0, 0, 0, 606, 607, 1, 0, 0, 0, 607, 609, 1, 0, 0, 0, 608, 610, 5, 10, 0, 0, 609, 608, 1, 0, 0, 0, 609, 610, 1, 0, 0, 0, 610, 611, 1, 0, 0, 0, 611, 612, 6, 22, 11, 0, 612, 61, 1, 0, 0, 0, 613, 614, 5, 47, 0, 0, 614, 615, 5, 42, 0, 0, 615, 620, 1, 0, 0, 0, 616, 619, 3, 62, 23, 0, 617, 619, 9, 0, 0, 0, 618, 616, 1, 0, 0, 0, 618, 617, 1, 0, 0, 0, 619, 622, 1, 0, 0, 0, 620, 621, 1, 0, 0, 0, 620, 618, 1, 0, 0, 0, 621, 623, 1, 0, 0, 0, 622, 620, 1, 0, 0, 0, 623, 624, 5, 42, 0, 0, 624, 625, 5, 47, 0, 0, 625, 626, 1, 0, 0, 0, 626, 627, 6, 23, 11, 0, 627, 63, 1, 0, 0, 0, 628, 630, 7, 23, 0, 0, 629, 628, 1, 0, 0, 0, 630, 631, 1, 0, 0, 0, 631, 629, 1, 0, 0, 0, 631, 632, 1, 0, 0, 0, 632, 633, 1, 0, 0, 0, 633, 634, 6, 24, 11, 0, 634, 65, 1, 0, 0, 0, 635, 636, 5, 124, 0, 0, 636, 637, 1, 0, 0, 0, 637, 638, 6, 25, 12, 0, 638, 67, 1, 0, 0, 0, 639, 640, 7, 24, 0, 0, 640, 69, 1, 0, 0, 0, 641, 642, 7, 25, 0, 0, 642, 71, 1, 0, 0, 0, 643, 644, 5, 92, 0, 0, 644, 645, 7, 26, 0, 0, 645, 73, 1, 0, 0, 0, 646, 647, 8, 27, 0, 0, 647, 75, 1, 0, 0, 0, 648, 650, 7, 3, 0, 0, 649, 651, 7, 28, 0, 0, 650, 649, 1, 0, 0, 0, 650, 651, 1, 0, 0, 0, 651, 653, 1, 0, 0, 0, 652, 654, 3, 68, 26, 0, 653, 652, 1, 0, 0, 0, 654, 655, 1, 0, 0, 0, 655, 653, 1, 0, 0, 0, 655, 656, 1, 0, 0, 0, 656, 77, 1, 0, 0, 0, 657, 658, 5, 64, 0, 0, 658, 79, 1, 0, 0, 0, 659, 660, 5, 96, 0, 0, 660, 81, 1, 0, 0, 0, 661, 665, 8, 29, 0, 0, 662, 663, 5, 96, 0, 0, 663, 665, 5, 96, 0, 0, 664, 661, 1, 0, 0, 0, 664, 662, 1, 0, 0, 0, 665, 83, 1, 0, 0, 0, 666, 667, 5, 95, 0, 0, 667, 85, 1, 0, 0, 0, 668, 672, 3, 70, 27, 0, 669, 672, 3, 68, 26, 0, 670, 672, 3, 84, 34, 0, 671, 668, 1, 0, 0, 0, 671, 669, 1, 0, 0, 0, 671, 670, 1, 0, 0, 0, 672, 87, 1, 0, 0, 0, 673, 678, 5, 34, 0, 0, 674, 677, 3, 72, 28, 0, 675, 677, 3, 74, 29, 0, 676, 674, 1, 0, 0, 0, 676, 675, 1, 0, 0, 0, 677, 680, 1, 0, 0, 0, 678, 676, 1, 0, 0, 0, 678, 679, 1, 0, 0, 0, 679, 681, 1, 0, 0, 0, 680, 678, 1, 0, 0, 0, 681, 703, 5, 34, 0, 0, 682, 683, 5, 34, 0, 0, 683, 684, 5, 34, 0, 0, 684, 685, 5, 34, 0, 0, 685, 689, 1, 0, 0, 0, 686, 688, 8, 22, 0, 0, 687, 686, 1, 0, 0, 0, 688, 691, 1, 0, 0, 0, 689, 690, 1, 0, 0, 0, 689, 687, 1, 0, 0, 0, 690, 692, 1, 0, 0, 0, 691, 689, 1, 0, 0, 0, 692, 693, 5, 34, 0, 0, 693, 694, 5, 34, 0, 0, 694, 695, 5, 34, 0, 0, 695, 697, 1, 0, 0, 0, 696, 698, 5, 34, 0, 0, 697, 696, 1, 0, 0, 0, 697, 698, 1, 0, 0, 0, 698, 700, 1, 0, 0, 0, 699, 701, 5, 34, 0, 0, 700, 699, 1, 0, 0, 0, 700, 701, 1, 0, 0, 0, 701, 703, 1, 0, 0, 0, 702, 673, 1, 0, 0, 0, 702, 682, 1, 0, 0, 0, 703, 89, 1, 0, 0, 0, 704, 706, 3, 68, 26, 0, 705, 704, 1, 0, 0, 0, 706, 707, 1, 0, 0, 0, 707, 705, 1, 0, 0, 0, 707, 708, 1, 0, 0, 0, 708, 91, 1, 0, 0, 0, 709, 711, 3, 68, 26, 0, 710, 709, 1, 0, 0, 0, 711, 712, 1, 0, 0, 0, 712, 710, 1, 0, 0, 0, 712, 713, 1, 0, 0, 0, 713, 714, 1, 0, 0, 0, 714, 718, 3, 108, 46, 0, 715, 717, 3, 68, 26, 0, 716, 715, 1, 0, 0, 0, 717, 720, 1, 0, 0, 0, 718, 716, 1, 0, 0, 0, 718, 719, 1, 0, 0, 0, 719, 752, 1, 0, 0, 0, 720, 718, 1, 0, 0, 0, 721, 723, 3, 108, 46, 0, 722, 724, 3, 68, 26, 0, 723, 722, 1, 0, 0, 0, 724, 725, 1, 0, 0, 0, 725, 723, 1, 0, 0, 0, 725, 726, 1, 0, 0, 0, 726, 752, 1, 0, 0, 0, 727, 729, 3, 68, 26, 0, 728, 727, 1, 0, 0, 0, 729, 730, 1, 0, 0, 0, 730, 728, 1, 0, 0, 0, 730, 731, 1, 0, 0, 0, 731, 739, 1, 0, 0, 0, 732, 736, 3, 108, 46, 0, 733, 735, 3, 68, 26, 0, 734, 733, 1, 0, 0, 0, 735, 738, 1, 0, 0, 0, 736, 734, 1, 0, 0, 0, 736, 737, 1, 0, 0, 0, 737, 740, 1, 0, 0, 0, 738, 736, 1, 0, 0, 0, 739, 732, 1, 0, 0, 0, 739, 740, 1, 0, 0, 0, 740, 741, 1, 0, 0, 0, 741, 742, 3, 76, 30, 0, 742, 752, 1, 0, 0, 0, 743, 745, 3, 108, 46, 0, 744, 746, 3, 68, 26, 0, 745, 744, 1, 0, 0, 0, 746, 747, 1, 0, 0, 0, 747, 745, 1, 0, 0, 0, 747, 748, 1, 0, 0, 0, 748, 749, 1, 0, 0, 0, 749, 750, 3, 76, 30, 0, 750, 752, 1, 0, 0, 0, 751, 710, 1, 0, 0, 0, 751, 721, 1, 0, 0, 0, 751, 728, 1, 0, 0, 0, 751, 743, 1, 0, 0, 0, 752, 93, 1, 0, 0, 0, 753, 754, 7, 30, 0, 0, 754, 755, 7, 31, 0, 0, 755, 95, 1, 0, 0, 0, 756, 757, 7, 12, 0, 0, 757, 758, 7, 9, 0, 0, 758, 759, 7, 0, 0, 0, 759, 97, 1, 0, 0, 0, 760, 761, 7, 12, 0, 0, 761, 762, 7, 2, 0, 0, 762, 763, 7, 4, 0, 0, 763, 99, 1, 0, 0, 0, 764, 765, 5, 61, 0, 0, 765, 101, 1, 0, 0, 0, 766, 767, 5, 58, 0, 0, 767, 768, 5, 58, 0, 0, 768, 103, 1, 0, 0, 0, 769, 770, 5, 44, 0, 0, 770, 105, 1, 0, 0, 0, 771, 772, 7, 0, 0, 0, 772, 773, 7, 3, 0, 0, 773, 774, 7, 2, 0, 0, 774, 775, 7, 4, 0, 0, 775, 107, 1, 0, 0, 0, 776, 777, 5, 46, 0, 0, 777, 109, 1, 0, 0, 0, 778, 779, 7, 15, 0, 0, 779, 780, 7, 12, 0, 0, 780, 781, 7, 13, 0, 0, 781, 782, 7, 2, 0, 0, 782, 783, 7, 3, 0, 0, 783, 111, 1, 0, 0, 0, 784, 785, 7, 15, 0, 0, 785, 786, 7, 1, 0, 0, 786, 787, 7, 6, 0, 0, 787, 788, 7, 2, 0, 0, 788, 789, 7, 5, 0, 0, 789, 113, 1, 0, 0, 0, 790, 791, 7, 1, 0, 0, 791, 792, 7, 9, 0, 0, 792, 115, 1, 0, 0, 0, 793, 794, 7, 1, 0, 0, 794, 795, 7, 2, 0, 0, 795, 117, 1, 0, 0, 0, 796, 797, 7, 13, 0, 0, 797, 798, 7, 12, 0, 0, 798, 799, 7, 2, 0, 0, 799, 800, 7, 5, 0, 0, 800, 119, 1, 0, 0, 0, 801, 802, 7, 13, 0, 0, 802, 803, 7, 1, 0, 0, 803, 804, 7, 18, 0, 0, 804, 805, 7, 3, 0, 0, 805, 121, 1, 0, 0, 0, 806, 807, 5, 40, 0, 0, 807, 123, 1, 0, 0, 0, 808, 809, 7, 9, 0, 0, 809, 810, 7, 7, 0, 0, 810, 811, 7, 5, 0, 0, 811, 125, 1, 0, 0, 0, 812, 813, 7, 9, 0, 0, 813, 814, 7, 20, 0, 0, 814, 815, 7, 13, 0, 0, 815, 816, 7, 13, 0, 0, 816, 127, 1, 0, 0, 0, 817, 818, 7, 9, 0, 0, 818, 819, 7, 20, 0, 0, 819, 820, 7, 13, 0, 0, 820, 821, 7, 13, 0, 0, 821, 822, 7, 2, 0, 0, 822, 129, 1, 0, 0, 0, 823, 824, 7, 7, 0, 0, 824, 825, 7, 6, 0, 0, 825, 131, 1, 0, 0, 0, 826, 827, 5, 63, 0, 0, 827, 133, 1, 0, 0, 0, 828, 829, 7, 6, 0, 0, 829, 830, 7, 13, 0, 0, 830, 831, 7, 1, 0, 0, 831, 832, 7, 18, 0, 0, 832, 833, 7, 3, 0, 0, 833, 135, 1, 0, 0, 0, 834, 835, 5, 41, 0, 0, 835, 137, 1, 0, 0, 0, 836, 837, 7, 5, 0, 0, 837, 838, 7, 6, 0, 0, 838, 839, 7, 20, 0, 0, 839, 840, 7, 3, 0, 0, 840, 139, 1, 0, 0, 0, 841, 842, 5, 61, 0, 0, 842, 843, 5, 61, 0, 0, 843, 141, 1, 0, 0, 0, 844, 845, 5, 61, 0, 0, 845, 846, 5, 126, 0, 0, 846, 143, 1, 0, 0, 0, 847, 848, 5, 33, 0, 0, 848, 849, 5, 61, 0, 0, 849, 145, 1, 0, 0, 0, 850, 851, 5, 60, 0, 0, 851, 147, 1, 0, 0, 0, 852, 853, 5, 60, 0, 0, 853, 854, 5, 61, 0, 0, 854, 149, 1, 0, 0, 0, 855, 856, 5, 62, 0, 0, 856, 151, 1, 0, 0, 0, 857, 858, 5, 62, 0, 0, 858, 859, 5, 61, 0, 0, 859, 153, 1, 0, 0, 0, 860, 861, 5, 43, 0, 0, 861, 155, 1, 0, 0, 0, 862, 863, 5, 45, 0, 0, 863, 157, 1, 0, 0, 0, 864, 865, 5, 42, 0, 0, 865, 159, 1, 0, 0, 0, 866, 867, 5, 47, 0, 0, 867, 161, 1, 0, 0, 0, 868, 869, 5, 37, 0, 0, 869, 163, 1, 0, 0, 0, 870, 871, 4, 74, 4, 0, 871, 872, 3, 54, 19, 0, 872, 873, 1, 0, 0, 0, 873, 874, 6, 74, 13, 0, 874, 165, 1, 0, 0, 0, 875, 878, 3, 132, 58, 0, 876, 879, 3, 70, 27, 0, 877, 879, 3, 84, 34, 0, 878, 876, 1, 0, 0, 0, 878, 877, 1, 0, 0, 0, 879, 883, 1, 0, 0, 0, 880, 882, 3, 86, 35, 0, 881, 880, 1, 0, 0, 0, 882, 885, 1, 0, 0, 0, 883, 881, 1, 0, 0, 0, 883, 884, 1, 0, 0, 0, 884, 893, 1, 0, 0, 0, 885, 883, 1, 0, 0, 0, 886, 888, 3, 132, 58, 0, 887, 889, 3, 68, 26, 0, 888, 887, 1, 0, 0, 0, 889, 890, 1, 0, 0, 0, 890, 888, 1, 0, 0, 0, 890, 891, 1, 0, 0, 0, 891, 893, 1, 0, 0, 0, 892, 875, 1, 0, 0, 0, 892, 886, 1, 0, 0, 0, 893, 167, 1, 0, 0, 0, 894, 895, 5, 91, 0, 0, 895, 896, 1, 0, 0, 0, 896, 897, 6, 76, 0, 0, 897, 898, 6, 76, 0, 0, 898, 169, 1, 0, 0, 0, 899, 900, 5, 93, 0, 0, 900, 901, 1, 0, 0, 0, 901, 902, 6, 77, 12, 0, 902, 903, 6, 77, 12, 0, 903, 171, 1, 0, 0, 0, 904, 908, 3, 70, 27, 0, 905, 907, 3, 86, 35, 0, 906, 905, 1, 0, 0, 0, 907, 910, 1, 0, 0, 0, 908, 906, 1, 0, 0, 0, 908, 909, 1, 0, 0, 0, 909, 921, 1, 0, 0, 0, 910, 908, 1, 0, 0, 0, 911, 914, 3, 84, 34, 0, 912, 914, 3, 78, 31, 0, 913, 911, 1, 0, 0, 0, 913, 912, 1, 0, 0, 0, 914, 916, 1, 0, 0, 0, 915, 917, 3, 86, 35, 0, 916, 915, 1, 0, 0, 0, 917, 918, 1, 0, 0, 0, 918, 916, 1, 0, 0, 0, 918, 919, 1, 0, 0, 0, 919, 921, 1, 0, 0, 0, 920, 904, 1, 0, 0, 0, 920, 913, 1, 0, 0, 0, 921, 173, 1, 0, 0, 0, 922, 924, 3, 80, 32, 0, 923, 925, 3, 82, 33, 0, 924, 923, 1, 0, 0, 0, 925, 926, 1, 0, 0, 0, 926, 924, 1, 0, 0, 0, 926, 927, 1, 0, 0, 0, 927, 928, 1, 0, 0, 0, 928, 929, 3, 80, 32, 0, 929, 175, 1, 0, 0, 0, 930, 931, 3, 174, 79, 0, 931, 177, 1, 0, 0, 0, 932, 933, 3, 60, 22, 0, 933, 934, 1, 0, 0, 0, 934, 935, 6, 81, 11, 0, 935, 179, 1, 0, 0, 0, 936, 937, 3, 62, 23, 0, 937, 938, 1, 0, 0, 0, 938, 939, 6, 82, 11, 0, 939, 181, 1, 0, 0, 0, 940, 941, 3, 64, 24, 0, 941, 942, 1, 0, 0, 0, 942, 943, 6, 83, 11, 0, 943, 183, 1, 0, 0, 0, 944, 945, 3, 168, 76, 0, 945, 946, 1, 0, 0, 0, 946, 947, 6, 84, 14, 0, 947, 948, 6, 84, 15, 0, 948, 185, 1, 0, 0, 0, 949, 950, 3, 66, 25, 0, 950, 951, 1, 0, 0, 0, 951, 952, 6, 85, 16, 0, 952, 953, 6, 85, 12, 0, 953, 187, 1, 0, 0, 0, 954, 955, 3, 64, 24, 0, 955, 956, 1, 0, 0, 0, 956, 957, 6, 86, 11, 0, 957, 189, 1, 0, 0, 0, 958, 959, 3, 60, 22, 0, 959, 960, 1, 0, 0, 0, 960, 961, 6, 87, 11, 0, 961, 191, 1, 0, 0, 0, 962, 963, 3, 62, 23, 0, 963, 964, 1, 0, 0, 0, 964, 965, 6, 88, 11, 0, 965, 193, 1, 0, 0, 0, 966, 967, 3, 66, 25, 0, 967, 968, 1, 0, 0, 0, 968, 969, 6, 89, 16, 0, 969, 970, 6, 89, 12, 0, 970, 195, 1, 0, 0, 0, 971, 972, 3, 168, 76, 0, 972, 973, 1, 0, 0, 0, 973, 974, 6, 90, 14, 0, 974, 197, 1, 0, 0, 0, 975, 976, 3, 170, 77, 0, 976, 977, 1, 0, 0, 0, 977, 978, 6, 91, 17, 0, 978, 199, 1, 0, 0, 0, 979, 980, 3, 334, 159, 0, 980, 981, 1, 0, 0, 0, 981, 982, 6, 92, 18, 0, 982, 201, 1, 0, 0, 0, 983, 984, 3, 104, 44, 0, 984, 985, 1, 0, 0, 0, 985, 986, 6, 93, 19, 0, 986, 203, 1, 0, 0, 0, 987, 988, 3, 100, 42, 0, 988, 989, 1, 0, 0, 0, 989, 990, 6, 94, 20, 0, 990, 205, 1, 0, 0, 0, 991, 992, 7, 16, 0, 0, 992, 993, 7, 3, 0, 0, 993, 994, 7, 5, 0, 0, 994, 995, 7, 12, 0, 0, 995, 996, 7, 0, 0, 0, 996, 997, 7, 12, 0, 0, 997, 998, 7, 5, 0, 0, 998, 999, 7, 12, 0, 0, 999, 207, 1, 0, 0, 0, 1000, 1004, 8, 32, 0, 0, 1001, 1002, 5, 47, 0, 0, 1002, 1004, 8, 33, 0, 0, 1003, 1000, 1, 0, 0, 0, 1003, 1001, 1, 0, 0, 0, 1004, 209, 1, 0, 0, 0, 1005, 1007, 3, 208, 96, 0, 1006, 1005, 1, 0, 0, 0, 1007, 1008, 1, 0, 0, 0, 1008, 1006, 1, 0, 0, 0, 1008, 1009, 1, 0, 0, 0, 1009, 211, 1, 0, 0, 0, 1010, 1011, 3, 210, 97, 0, 1011, 1012, 1, 0, 0, 0, 1012, 1013, 6, 98, 21, 0, 1013, 213, 1, 0, 0, 0, 1014, 1015, 3, 88, 36, 0, 1015, 1016, 1, 0, 0, 0, 1016, 1017, 6, 99, 22, 0, 1017, 215, 1, 0, 0, 0, 1018, 1019, 3, 60, 22, 0, 1019, 1020, 1, 0, 0, 0, 1020, 1021, 6, 100, 11, 0, 1021, 217, 1, 0, 0, 0, 1022, 1023, 3, 62, 23, 0, 1023, 1024, 1, 0, 0, 0, 1024, 1025, 6, 101, 11, 0, 1025, 219, 1, 0, 0, 0, 1026, 1027, 3, 64, 24, 0, 1027, 1028, 1, 0, 0, 0, 1028, 1029, 6, 102, 11, 0, 1029, 221, 1, 0, 0, 0, 1030, 1031, 3, 66, 25, 0, 1031, 1032, 1, 0, 0, 0, 1032, 1033, 6, 103, 16, 0, 1033, 1034, 6, 103, 12, 0, 1034, 223, 1, 0, 0, 0, 1035, 1036, 3, 108, 46, 0, 1036, 1037, 1, 0, 0, 0, 1037, 1038, 6, 104, 23, 0, 1038, 225, 1, 0, 0, 0, 1039, 1040, 3, 104, 44, 0, 1040, 1041, 1, 0, 0, 0, 1041, 1042, 6, 105, 19, 0, 1042, 227, 1, 0, 0, 0, 1043, 1048, 3, 70, 27, 0, 1044, 1048, 3, 68, 26, 0, 1045, 1048, 3, 84, 34, 0, 1046, 1048, 3, 158, 71, 0, 1047, 1043, 1, 0, 0, 0, 1047, 1044, 1, 0, 0, 0, 1047, 1045, 1, 0, 0, 0, 1047, 1046, 1, 0, 0, 0, 1048, 229, 1, 0, 0, 0, 1049, 1052, 3, 70, 27, 0, 1050, 1052, 3, 158, 71, 0, 1051, 1049, 1, 0, 0, 0, 1051, 1050, 1, 0, 0, 0, 1052, 1056, 1, 0, 0, 0, 1053, 1055, 3, 228, 106, 0, 1054, 1053, 1, 0, 0, 0, 1055, 1058, 1, 0, 0, 0, 1056, 1054, 1, 0, 0, 0, 1056, 1057, 1, 0, 0, 0, 1057, 1069, 1, 0, 0, 0, 1058, 1056, 1, 0, 0, 0, 1059, 1062, 3, 84, 34, 0, 1060, 1062, 3, 78, 31, 0, 1061, 1059, 1, 0, 0, 0, 1061, 1060, 1, 0, 0, 0, 1062, 1064, 1, 0, 0, 0, 1063, 1065, 3, 228, 106, 0, 1064, 1063, 1, 0, 0, 0, 1065, 1066, 1, 0, 0, 0, 1066, 1064, 1, 0, 0, 0, 1066, 1067, 1, 0, 0, 0, 1067, 1069, 1, 0, 0, 0, 1068, 1051, 1, 0, 0, 0, 1068, 1061, 1, 0, 0, 0, 1069, 231, 1, 0, 0, 0, 1070, 1073, 3, 230, 107, 0, 1071, 1073, 3, 174, 79, 0, 1072, 1070, 1, 0, 0, 0, 1072, 1071, 1, 0, 0, 0, 1073, 1074, 1, 0, 0, 0, 1074, 1072, 1, 0, 0, 0, 1074, 1075, 1, 0, 0, 0, 1075, 233, 1, 0, 0, 0, 1076, 1077, 3, 60, 22, 0, 1077, 1078, 1, 0, 0, 0, 1078, 1079, 6, 109, 11, 0, 1079, 235, 1, 0, 0, 0, 1080, 1081, 3, 62, 23, 0, 1081, 1082, 1, 0, 0, 0, 1082, 1083, 6, 110, 11, 0, 1083, 237, 1, 0, 0, 0, 1084, 1085, 3, 64, 24, 0, 1085, 1086, 1, 0, 0, 0, 1086, 1087, 6, 111, 11, 0, 1087, 239, 1, 0, 0, 0, 1088, 1089, 3, 66, 25, 0, 1089, 1090, 1, 0, 0, 0, 1090, 1091, 6, 112, 16, 0, 1091, 1092, 6, 112, 12, 0, 1092, 241, 1, 0, 0, 0, 1093, 1094, 3, 100, 42, 0, 1094, 1095, 1, 0, 0, 0, 1095, 1096, 6, 113, 20, 0, 1096, 243, 1, 0, 0, 0, 1097, 1098, 3, 104, 44, 0, 1098, 1099, 1, 0, 0, 0, 1099, 1100, 6, 114, 19, 0, 1100, 245, 1, 0, 0, 0, 1101, 1102, 3, 108, 46, 0, 1102, 1103, 1, 0, 0, 0, 1103, 1104, 6, 115, 23, 0, 1104, 247, 1, 0, 0, 0, 1105, 1106, 7, 12, 0, 0, 1106, 1107, 7, 2, 0, 0, 1107, 249, 1, 0, 0, 0, 1108, 1109, 3, 232, 108, 0, 1109, 1110, 1, 0, 0, 0, 1110, 1111, 6, 117, 24, 0, 1111, 251, 1, 0, 0, 0, 1112, 1113, 3, 60, 22, 0, 1113, 1114, 1, 0, 0, 0, 1114, 1115, 6, 118, 11, 0, 1115, 253, 1, 0, 0, 0, 1116, 1117, 3, 62, 23, 0, 1117, 1118, 1, 0, 0, 0, 1118, 1119, 6, 119, 11, 0, 1119, 255, 1, 0, 0, 0, 1120, 1121, 3, 64, 24, 0, 1121, 1122, 1, 0, 0, 0, 1122, 1123, 6, 120, 11, 0, 1123, 257, 1, 0, 0, 0, 1124, 1125, 3, 66, 25, 0, 1125, 1126, 1, 0, 0, 0, 1126, 1127, 6, 121, 16, 0, 1127, 1128, 6, 121, 12, 0, 1128, 259, 1, 0, 0, 0, 1129, 1130, 3, 168, 76, 0, 1130, 1131, 1, 0, 0, 0, 1131, 1132, 6, 122, 14, 0, 1132, 1133, 6, 122, 25, 0, 1133, 261, 1, 0, 0, 0, 1134, 1135, 7, 7, 0, 0, 1135, 1136, 7, 9, 0, 0, 1136, 1137, 1, 0, 0, 0, 1137, 1138, 6, 123, 26, 0, 1138, 263, 1, 0, 0, 0, 1139, 1140, 7, 19, 0, 0, 1140, 1141, 7, 1, 0, 0, 1141, 1142, 7, 5, 0, 0, 1142, 1143, 7, 10, 0, 0, 1143, 1144, 1, 0, 0, 0, 1144, 1145, 6, 124, 26, 0, 1145, 265, 1, 0, 0, 0, 1146, 1147, 8, 34, 0, 0, 1147, 267, 1, 0, 0, 0, 1148, 1150, 3, 266, 125, 0, 1149, 1148, 1, 0, 0, 0, 1150, 1151, 1, 0, 0, 0, 1151, 1149, 1, 0, 0, 0, 1151, 1152, 1, 0, 0, 0, 1152, 1153, 1, 0, 0, 0, 1153, 1154, 3, 334, 159, 0, 1154, 1156, 1, 0, 0, 0, 1155, 1149, 1, 0, 0, 0, 1155, 1156, 1, 0, 0, 0, 1156, 1158, 1, 0, 0, 0, 1157, 1159, 3, 266, 125, 0, 1158, 1157, 1, 0, 0, 0, 1159, 1160, 1, 0, 0, 0, 1160, 1158, 1, 0, 0, 0, 1160, 1161, 1, 0, 0, 0, 1161, 269, 1, 0, 0, 0, 1162, 1163, 3, 268, 126, 0, 1163, 1164, 1, 0, 0, 0, 1164, 1165, 6, 127, 27, 0, 1165, 271, 1, 0, 0, 0, 1166, 1167, 3, 60, 22, 0, 1167, 1168, 1, 0, 0, 0, 1168, 1169, 6, 128, 11, 0, 1169, 273, 1, 0, 0, 0, 1170, 1171, 3, 62, 23, 0, 1171, 1172, 1, 0, 0, 0, 1172, 1173, 6, 129, 11, 0, 1173, 275, 1, 0, 0, 0, 1174, 1175, 3, 64, 24, 0, 1175, 1176, 1, 0, 0, 0, 1176, 1177, 6, 130, 11, 0, 1177, 277, 1, 0, 0, 0, 1178, 1179, 3, 66, 25, 0, 1179, 1180, 1, 0, 0, 0, 1180, 1181, 6, 131, 16, 0, 1181, 1182, 6, 131, 12, 0, 1182, 1183, 6, 131, 12, 0, 1183, 279, 1, 0, 0, 0, 1184, 1185, 3, 100, 42, 0, 1185, 1186, 1, 0, 0, 0, 1186, 1187, 6, 132, 20, 0, 1187, 281, 1, 0, 0, 0, 1188, 1189, 3, 104, 44, 0, 1189, 1190, 1, 0, 0, 0, 1190, 1191, 6, 133, 19, 0, 1191, 283, 1, 0, 0, 0, 1192, 1193, 3, 108, 46, 0, 1193, 1194, 1, 0, 0, 0, 1194, 1195, 6, 134, 23, 0, 1195, 285, 1, 0, 0, 0, 1196, 1197, 3, 264, 124, 0, 1197, 1198, 1, 0, 0, 0, 1198, 1199, 6, 135, 28, 0, 1199, 287, 1, 0, 0, 0, 1200, 1201, 3, 232, 108, 0, 1201, 1202, 1, 0, 0, 0, 1202, 1203, 6, 136, 24, 0, 1203, 289, 1, 0, 0, 0, 1204, 1205, 3, 176, 80, 0, 1205, 1206, 1, 0, 0, 0, 1206, 1207, 6, 137, 29, 0, 1207, 291, 1, 0, 0, 0, 1208, 1209, 3, 60, 22, 0, 1209, 1210, 1, 0, 0, 0, 1210, 1211, 6, 138, 11, 0, 1211, 293, 1, 0, 0, 0, 1212, 1213, 3, 62, 23, 0, 1213, 1214, 1, 0, 0, 0, 1214, 1215, 6, 139, 11, 0, 1215, 295, 1, 0, 0, 0, 1216, 1217, 3, 64, 24, 0, 1217, 1218, 1, 0, 0, 0, 1218, 1219, 6, 140, 11, 0, 1219, 297, 1, 0, 0, 0, 1220, 1221, 3, 66, 25, 0, 1221, 1222, 1, 0, 0, 0, 1222, 1223, 6, 141, 16, 0, 1223, 1224, 6, 141, 12, 0, 1224, 299, 1, 0, 0, 0, 1225, 1226, 3, 108, 46, 0, 1226, 1227, 1, 0, 0, 0, 1227, 1228, 6, 142, 23, 0, 1228, 301, 1, 0, 0, 0, 1229, 1230, 3, 176, 80, 0, 1230, 1231, 1, 0, 0, 0, 1231, 1232, 6, 143, 29, 0, 1232, 303, 1, 0, 0, 0, 1233, 1234, 3, 172, 78, 0, 1234, 1235, 1, 0, 0, 0, 1235, 1236, 6, 144, 30, 0, 1236, 305, 1, 0, 0, 0, 1237, 1238, 3, 60, 22, 0, 1238, 1239, 1, 0, 0, 0, 1239, 1240, 6, 145, 11, 0, 1240, 307, 1, 0, 0, 0, 1241, 1242, 3, 62, 23, 0, 1242, 1243, 1, 0, 0, 0, 1243, 1244, 6, 146, 11, 0, 1244, 309, 1, 0, 0, 0, 1245, 1246, 3, 64, 24, 0, 1246, 1247, 1, 0, 0, 0, 1247, 1248, 6, 147, 11, 0, 1248, 311, 1, 0, 0, 0, 1249, 1250, 3, 66, 25, 0, 1250, 1251, 1, 0, 0, 0, 1251, 1252, 6, 148, 16, 0, 1252, 1253, 6, 148, 12, 0, 1253, 313, 1, 0, 0, 0, 1254, 1255, 7, 1, 0, 0, 1255, 1256, 7, 9, 0, 0, 1256, 1257, 7, 15, 0, 0, 1257, 1258, 7, 7, 0, 0, 1258, 315, 1, 0, 0, 0, 1259, 1260, 3, 60, 22, 0, 1260, 1261, 1, 0, 0, 0, 1261, 1262, 6, 150, 11, 0, 1262, 317, 1, 0, 0, 0, 1263, 1264, 3, 62, 23, 0, 1264, 1265, 1, 0, 0, 0, 1265, 1266, 6, 151, 11, 0, 1266, 319, 1, 0, 0, 0, 1267, 1268, 3, 64, 24, 0, 1268, 1269, 1, 0, 0, 0, 1269, 1270, 6, 152, 11, 0, 1270, 321, 1, 0, 0, 0, 1271, 1272, 3, 66, 25, 0, 1272, 1273, 1, 0, 0, 0, 1273, 1274, 6, 153, 16, 0, 1274, 1275, 6, 153, 12, 0, 1275, 323, 1, 0, 0, 0, 1276, 1277, 7, 15, 0, 0, 1277, 1278, 7, 20, 0, 0, 1278, 1279, 7, 9, 0, 0, 1279, 1280, 7, 4, 0, 0, 1280, 1281, 7, 5, 0, 0, 1281, 1282, 7, 1, 0, 0, 1282, 1283, 7, 7, 0, 0, 1283, 1284, 7, 9, 0, 0, 1284, 1285, 7, 2, 0, 0, 1285, 325, 1, 0, 0, 0, 1286, 1287, 3, 60, 22, 0, 1287, 1288, 1, 0, 0, 0, 1288, 1289, 6, 155, 11, 0, 1289, 327, 1, 0, 0, 0, 1290, 1291, 3, 62, 23, 0, 1291, 1292, 1, 0, 0, 0, 1292, 1293, 6, 156, 11, 0, 1293, 329, 1, 0, 0, 0, 1294, 1295, 3, 64, 24, 0, 1295, 1296, 1, 0, 0, 0, 1296, 1297, 6, 157, 11, 0, 1297, 331, 1, 0, 0, 0, 1298, 1299, 3, 170, 77, 0, 1299, 1300, 1, 0, 0, 0, 1300, 1301, 6, 158, 17, 0, 1301, 1302, 6, 158, 12, 0, 1302, 333, 1, 0, 0, 0, 1303, 1304, 5, 58, 0, 0, 1304, 335, 1, 0, 0, 0, 1305, 1311, 3, 78, 31, 0, 1306, 1311, 3, 68, 26, 0, 1307, 1311, 3, 108, 46, 0, 1308, 1311, 3, 70, 27, 0, 1309, 1311, 3, 84, 34, 0, 1310, 1305, 1, 0, 0, 0, 1310, 1306, 1, 0, 0, 0, 1310, 1307, 1, 0, 0, 0, 1310, 1308, 1, 0, 0, 0, 1310, 1309, 1, 0, 0, 0, 1311, 1312, 1, 0, 0, 0, 1312, 1310, 1, 0, 0, 0, 1312, 1313, 1, 0, 0, 0, 1313, 337, 1, 0, 0, 0, 1314, 1315, 3, 60, 22, 0, 1315, 1316, 1, 0, 0, 0, 1316, 1317, 6, 161, 11, 0, 1317, 339, 1, 0, 0, 0, 1318, 1319, 3, 62, 23, 0, 1319, 1320, 1, 0, 0, 0, 1320, 1321, 6, 162, 11, 0, 1321, 341, 1, 0, 0, 0, 1322, 1323, 3, 64, 24, 0, 1323, 1324, 1, 0, 0, 0, 1324, 1325, 6, 163, 11, 0, 1325, 343, 1, 0, 0, 0, 1326, 1327, 3, 66, 25, 0, 1327, 1328, 1, 0, 0, 0, 1328, 1329, 6, 164, 16, 0, 1329, 1330, 6, 164, 12, 0, 1330, 345, 1, 0, 0, 0, 1331, 1332, 3, 334, 159, 0, 1332, 1333, 1, 0, 0, 0, 1333, 1334, 6, 165, 18, 0, 1334, 347, 1, 0, 0, 0, 1335, 1336, 3, 104, 44, 0, 1336, 1337, 1, 0, 0, 0, 1337, 1338, 6, 166, 19, 0, 1338, 349, 1, 0, 0, 0, 1339, 1340, 3, 108, 46, 0, 1340, 1341, 1, 0, 0, 0, 1341, 1342, 6, 167, 23, 0, 1342, 351, 1, 0, 0, 0, 1343, 1344, 3, 262, 123, 0, 1344, 1345, 1, 0, 0, 0, 1345, 1346, 6, 168, 31, 0, 1346, 1347, 6, 168, 32, 0, 1347, 353, 1, 0, 0, 0, 1348, 1349, 3, 210, 97, 0, 1349, 1350, 1, 0, 0, 0, 1350, 1351, 6, 169, 21, 0, 1351, 355, 1, 0, 0, 0, 1352, 1353, 3, 88, 36, 0, 1353, 1354, 1, 0, 0, 0, 1354, 1355, 6, 170, 22, 0, 1355, 357, 1, 0, 0, 0, 1356, 1357, 3, 60, 22, 0, 1357, 1358, 1, 0, 0, 0, 1358, 1359, 6, 171, 11, 0, 1359, 359, 1, 0, 0, 0, 1360, 1361, 3, 62, 23, 0, 1361, 1362, 1, 0, 0, 0, 1362, 1363, 6, 172, 11, 0, 1363, 361, 1, 0, 0, 0, 1364, 1365, 3, 64, 24, 0, 1365, 1366, 1, 0, 0, 0, 1366, 1367, 6, 173, 11, 0, 1367, 363, 1, 0, 0, 0, 1368, 1369, 3, 66, 25, 0, 1369, 1370, 1, 0, 0, 0, 1370, 1371, 6, 174, 16, 0, 1371, 1372, 6, 174, 12, 0, 1372, 1373, 6, 174, 12, 0, 1373, 365, 1, 0, 0, 0, 1374, 1375, 3, 104, 44, 0, 1375, 1376, 1, 0, 0, 0, 1376, 1377, 6, 175, 19, 0, 1377, 367, 1, 0, 0, 0, 1378, 1379, 3, 108, 46, 0, 1379, 1380, 1, 0, 0, 0, 1380, 1381, 6, 176, 23, 0, 1381, 369, 1, 0, 0, 0, 1382, 1383, 3, 232, 108, 0, 1383, 1384, 1, 0, 0, 0, 1384, 1385, 6, 177, 24, 0, 1385, 371, 1, 0, 0, 0, 1386, 1387, 3, 60, 22, 0, 1387, 1388, 1, 0, 0, 0, 1388, 1389, 6, 178, 11, 0, 1389, 373, 1, 0, 0, 0, 1390, 1391, 3, 62, 23, 0, 1391, 1392, 1, 0, 0, 0, 1392, 1393, 6, 179, 11, 0, 1393, 375, 1, 0, 0, 0, 1394, 1395, 3, 64, 24, 0, 1395, 1396, 1, 0, 0, 0, 1396, 1397, 6, 180, 11, 0, 1397, 377, 1, 0, 0, 0, 1398, 1399, 3, 66, 25, 0, 1399, 1400, 1, 0, 0, 0, 1400, 1401, 6, 181, 16, 0, 1401, 1402, 6, 181, 12, 0, 1402, 379, 1, 0, 0, 0, 1403, 1404, 3, 210, 97, 0, 1404, 1405, 1, 0, 0, 0, 1405, 1406, 6, 182, 21, 0, 1406, 1407, 6, 182, 12, 0, 1407, 1408, 6, 182, 33, 0, 1408, 381, 1, 0, 0, 0, 1409, 1410, 3, 88, 36, 0, 1410, 1411, 1, 0, 0, 0, 1411, 1412, 6, 183, 22, 0, 1412, 1413, 6, 183, 12, 0, 1413, 1414, 6, 183, 33, 0, 1414, 383, 1, 0, 0, 0, 1415, 1416, 3, 60, 22, 0, 1416, 1417, 1, 0, 0, 0, 1417, 1418, 6, 184, 11, 0, 1418, 385, 1, 0, 0, 0, 1419, 1420, 3, 62, 23, 0, 1420, 1421, 1, 0, 0, 0, 1421, 1422, 6, 185, 11, 0, 1422, 387, 1, 0, 0, 0, 1423, 1424, 3, 64, 24, 0, 1424, 1425, 1, 0, 0, 0, 1425, 1426, 6, 186, 11, 0, 1426, 389, 1, 0, 0, 0, 1427, 1428, 3, 334, 159, 0, 1428, 1429, 1, 0, 0, 0, 1429, 1430, 6, 187, 18, 0, 1430, 1431, 6, 187, 12, 0, 1431, 1432, 6, 187, 10, 0, 1432, 391, 1, 0, 0, 0, 1433, 1434, 3, 104, 44, 0, 1434, 1435, 1, 0, 0, 0, 1435, 1436, 6, 188, 19, 0, 1436, 1437, 6, 188, 12, 0, 1437, 1438, 6, 188, 10, 0, 1438, 393, 1, 0, 0, 0, 1439, 1440, 3, 60, 22, 0, 1440, 1441, 1, 0, 0, 0, 1441, 1442, 6, 189, 11, 0, 1442, 395, 1, 0, 0, 0, 1443, 1444, 3, 62, 23, 0, 1444, 1445, 1, 0, 0, 0, 1445, 1446, 6, 190, 11, 0, 1446, 397, 1, 0, 0, 0, 1447, 1448, 3, 64, 24, 0, 1448, 1449, 1, 0, 0, 0, 1449, 1450, 6, 191, 11, 0, 1450, 399, 1, 0, 0, 0, 1451, 1452, 3, 176, 80, 0, 1452, 1453, 1, 0, 0, 0, 1453, 1454, 6, 192, 12, 0, 1454, 1455, 6, 192, 0, 0, 1455, 1456, 6, 192, 29, 0, 1456, 401, 1, 0, 0, 0, 1457, 1458, 3, 172, 78, 0, 1458, 1459, 1, 0, 0, 0, 1459, 1460, 6, 193, 12, 0, 1460, 1461, 6, 193, 0, 0, 1461, 1462, 6, 193, 30, 0, 1462, 403, 1, 0, 0, 0, 1463, 1464, 3, 94, 39, 0, 1464, 1465, 1, 0, 0, 0, 1465, 1466, 6, 194, 12, 0, 1466, 1467, 6, 194, 0, 0, 1467, 1468, 6, 194, 34, 0, 1468, 405, 1, 0, 0, 0, 1469, 1470, 3, 66, 25, 0, 1470, 1471, 1, 0, 0, 0, 1471, 1472, 6, 195, 16, 0, 1472, 1473, 6, 195, 12, 0, 1473, 407, 1, 0, 0, 0, 66, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 592, 602, 606, 609, 618, 620, 631, 650, 655, 664, 671, 676, 678, 689, 697, 700, 702, 707, 712, 718, 725, 730, 736, 739, 747, 751, 878, 883, 890, 892, 908, 913, 918, 920, 926, 1003, 1008, 1047, 1051, 1056, 1061, 1066, 1068, 1072, 1074, 1151, 1155, 1160, 1310, 1312, 35, 5, 1, 0, 5, 4, 0, 5, 6, 0, 5, 2, 0, 5, 3, 0, 5, 10, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 5, 12, 0, 5, 14, 0, 0, 1, 0, 4, 0, 0, 7, 20, 0, 7, 66, 0, 5, 0, 0, 7, 26, 0, 7, 67, 0, 7, 109, 0, 7, 35, 0, 7, 33, 0, 7, 77, 0, 7, 27, 0, 7, 37, 0, 7, 81, 0, 5, 11, 0, 5, 7, 0, 7, 91, 0, 7, 90, 0, 7, 69, 0, 7, 68, 0, 7, 89, 0, 5, 13, 0, 5, 15, 0, 7, 30, 0] \ No newline at end of file +[4, 0, 120, 1427, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67, 2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2, 73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78, 7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 2, 81, 7, 81, 2, 82, 7, 82, 2, 83, 7, 83, 2, 84, 7, 84, 2, 85, 7, 85, 2, 86, 7, 86, 2, 87, 7, 87, 2, 88, 7, 88, 2, 89, 7, 89, 2, 90, 7, 90, 2, 91, 7, 91, 2, 92, 7, 92, 2, 93, 7, 93, 2, 94, 7, 94, 2, 95, 7, 95, 2, 96, 7, 96, 2, 97, 7, 97, 2, 98, 7, 98, 2, 99, 7, 99, 2, 100, 7, 100, 2, 101, 7, 101, 2, 102, 7, 102, 2, 103, 7, 103, 2, 104, 7, 104, 2, 105, 7, 105, 2, 106, 7, 106, 2, 107, 7, 107, 2, 108, 7, 108, 2, 109, 7, 109, 2, 110, 7, 110, 2, 111, 7, 111, 2, 112, 7, 112, 2, 113, 7, 113, 2, 114, 7, 114, 2, 115, 7, 115, 2, 116, 7, 116, 2, 117, 7, 117, 2, 118, 7, 118, 2, 119, 7, 119, 2, 120, 7, 120, 2, 121, 7, 121, 2, 122, 7, 122, 2, 123, 7, 123, 2, 124, 7, 124, 2, 125, 7, 125, 2, 126, 7, 126, 2, 127, 7, 127, 2, 128, 7, 128, 2, 129, 7, 129, 2, 130, 7, 130, 2, 131, 7, 131, 2, 132, 7, 132, 2, 133, 7, 133, 2, 134, 7, 134, 2, 135, 7, 135, 2, 136, 7, 136, 2, 137, 7, 137, 2, 138, 7, 138, 2, 139, 7, 139, 2, 140, 7, 140, 2, 141, 7, 141, 2, 142, 7, 142, 2, 143, 7, 143, 2, 144, 7, 144, 2, 145, 7, 145, 2, 146, 7, 146, 2, 147, 7, 147, 2, 148, 7, 148, 2, 149, 7, 149, 2, 150, 7, 150, 2, 151, 7, 151, 2, 152, 7, 152, 2, 153, 7, 153, 2, 154, 7, 154, 2, 155, 7, 155, 2, 156, 7, 156, 2, 157, 7, 157, 2, 158, 7, 158, 2, 159, 7, 159, 2, 160, 7, 160, 2, 161, 7, 161, 2, 162, 7, 162, 2, 163, 7, 163, 2, 164, 7, 164, 2, 165, 7, 165, 2, 166, 7, 166, 2, 167, 7, 167, 2, 168, 7, 168, 2, 169, 7, 169, 2, 170, 7, 170, 2, 171, 7, 171, 2, 172, 7, 172, 2, 173, 7, 173, 2, 174, 7, 174, 2, 175, 7, 175, 2, 176, 7, 176, 2, 177, 7, 177, 2, 178, 7, 178, 2, 179, 7, 179, 2, 180, 7, 180, 2, 181, 7, 181, 2, 182, 7, 182, 2, 183, 7, 183, 2, 184, 7, 184, 2, 185, 7, 185, 2, 186, 7, 186, 2, 187, 7, 187, 2, 188, 7, 188, 2, 189, 7, 189, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 19, 1, 20, 4, 20, 571, 8, 20, 11, 20, 12, 20, 572, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 581, 8, 21, 10, 21, 12, 21, 584, 9, 21, 1, 21, 3, 21, 587, 8, 21, 1, 21, 3, 21, 590, 8, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 599, 8, 22, 10, 22, 12, 22, 602, 9, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 4, 23, 610, 8, 23, 11, 23, 12, 23, 611, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 28, 1, 28, 1, 29, 1, 29, 3, 29, 631, 8, 29, 1, 29, 4, 29, 634, 8, 29, 11, 29, 12, 29, 635, 1, 30, 1, 30, 1, 31, 1, 31, 1, 32, 1, 32, 1, 32, 3, 32, 645, 8, 32, 1, 33, 1, 33, 1, 34, 1, 34, 1, 34, 3, 34, 652, 8, 34, 1, 35, 1, 35, 1, 35, 5, 35, 657, 8, 35, 10, 35, 12, 35, 660, 9, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 5, 35, 668, 8, 35, 10, 35, 12, 35, 671, 9, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 3, 35, 678, 8, 35, 1, 35, 3, 35, 681, 8, 35, 3, 35, 683, 8, 35, 1, 36, 4, 36, 686, 8, 36, 11, 36, 12, 36, 687, 1, 37, 4, 37, 691, 8, 37, 11, 37, 12, 37, 692, 1, 37, 1, 37, 5, 37, 697, 8, 37, 10, 37, 12, 37, 700, 9, 37, 1, 37, 1, 37, 4, 37, 704, 8, 37, 11, 37, 12, 37, 705, 1, 37, 4, 37, 709, 8, 37, 11, 37, 12, 37, 710, 1, 37, 1, 37, 5, 37, 715, 8, 37, 10, 37, 12, 37, 718, 9, 37, 3, 37, 720, 8, 37, 1, 37, 1, 37, 1, 37, 1, 37, 4, 37, 726, 8, 37, 11, 37, 12, 37, 727, 1, 37, 1, 37, 3, 37, 732, 8, 37, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 1, 64, 1, 64, 1, 65, 1, 65, 1, 65, 1, 66, 1, 66, 1, 67, 1, 67, 1, 67, 1, 68, 1, 68, 1, 69, 1, 69, 1, 70, 1, 70, 1, 71, 1, 71, 1, 72, 1, 72, 1, 73, 1, 73, 1, 73, 1, 73, 1, 73, 1, 74, 1, 74, 1, 74, 3, 74, 859, 8, 74, 1, 74, 5, 74, 862, 8, 74, 10, 74, 12, 74, 865, 9, 74, 1, 74, 1, 74, 4, 74, 869, 8, 74, 11, 74, 12, 74, 870, 3, 74, 873, 8, 74, 1, 75, 1, 75, 1, 75, 1, 75, 1, 75, 1, 76, 1, 76, 1, 76, 1, 76, 1, 76, 1, 77, 1, 77, 5, 77, 887, 8, 77, 10, 77, 12, 77, 890, 9, 77, 1, 77, 1, 77, 3, 77, 894, 8, 77, 1, 77, 4, 77, 897, 8, 77, 11, 77, 12, 77, 898, 3, 77, 901, 8, 77, 1, 78, 1, 78, 4, 78, 905, 8, 78, 11, 78, 12, 78, 906, 1, 78, 1, 78, 1, 79, 1, 79, 1, 80, 1, 80, 1, 80, 1, 80, 1, 81, 1, 81, 1, 81, 1, 81, 1, 82, 1, 82, 1, 82, 1, 82, 1, 83, 1, 83, 1, 83, 1, 83, 1, 83, 1, 84, 1, 84, 1, 84, 1, 84, 1, 84, 1, 85, 1, 85, 1, 85, 1, 85, 1, 86, 1, 86, 1, 86, 1, 86, 1, 87, 1, 87, 1, 87, 1, 87, 1, 88, 1, 88, 1, 88, 1, 88, 1, 88, 1, 89, 1, 89, 1, 89, 1, 89, 1, 90, 1, 90, 1, 90, 1, 90, 1, 91, 1, 91, 1, 91, 1, 91, 1, 92, 1, 92, 1, 92, 1, 92, 1, 93, 1, 93, 1, 93, 1, 93, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 94, 1, 95, 1, 95, 1, 95, 3, 95, 984, 8, 95, 1, 96, 4, 96, 987, 8, 96, 11, 96, 12, 96, 988, 1, 97, 1, 97, 1, 97, 1, 97, 1, 98, 1, 98, 1, 98, 1, 98, 1, 99, 1, 99, 1, 99, 1, 99, 1, 100, 1, 100, 1, 100, 1, 100, 1, 101, 1, 101, 1, 101, 1, 101, 1, 102, 1, 102, 1, 102, 1, 102, 1, 102, 1, 103, 1, 103, 1, 103, 1, 103, 1, 104, 1, 104, 1, 104, 1, 104, 1, 105, 1, 105, 1, 105, 1, 105, 3, 105, 1028, 8, 105, 1, 106, 1, 106, 3, 106, 1032, 8, 106, 1, 106, 5, 106, 1035, 8, 106, 10, 106, 12, 106, 1038, 9, 106, 1, 106, 1, 106, 3, 106, 1042, 8, 106, 1, 106, 4, 106, 1045, 8, 106, 11, 106, 12, 106, 1046, 3, 106, 1049, 8, 106, 1, 107, 1, 107, 4, 107, 1053, 8, 107, 11, 107, 12, 107, 1054, 1, 108, 1, 108, 1, 108, 1, 108, 1, 109, 1, 109, 1, 109, 1, 109, 1, 110, 1, 110, 1, 110, 1, 110, 1, 111, 1, 111, 1, 111, 1, 111, 1, 111, 1, 112, 1, 112, 1, 112, 1, 112, 1, 113, 1, 113, 1, 113, 1, 113, 1, 114, 1, 114, 1, 114, 1, 114, 1, 115, 1, 115, 1, 115, 1, 116, 1, 116, 1, 116, 1, 116, 1, 117, 1, 117, 1, 117, 1, 117, 1, 118, 1, 118, 1, 118, 1, 118, 1, 119, 1, 119, 1, 119, 1, 119, 1, 120, 1, 120, 1, 120, 1, 120, 1, 120, 1, 121, 1, 121, 1, 121, 1, 121, 1, 121, 1, 122, 1, 122, 1, 122, 1, 122, 1, 122, 1, 123, 1, 123, 1, 123, 1, 123, 1, 123, 1, 123, 1, 123, 1, 124, 1, 124, 1, 125, 4, 125, 1130, 8, 125, 11, 125, 12, 125, 1131, 1, 125, 1, 125, 3, 125, 1136, 8, 125, 1, 125, 4, 125, 1139, 8, 125, 11, 125, 12, 125, 1140, 1, 126, 1, 126, 1, 126, 1, 126, 1, 127, 1, 127, 1, 127, 1, 127, 1, 128, 1, 128, 1, 128, 1, 128, 1, 129, 1, 129, 1, 129, 1, 129, 1, 130, 1, 130, 1, 130, 1, 130, 1, 130, 1, 130, 1, 131, 1, 131, 1, 131, 1, 131, 1, 132, 1, 132, 1, 132, 1, 132, 1, 133, 1, 133, 1, 133, 1, 133, 1, 134, 1, 134, 1, 134, 1, 134, 1, 135, 1, 135, 1, 135, 1, 135, 1, 136, 1, 136, 1, 136, 1, 136, 1, 137, 1, 137, 1, 137, 1, 137, 1, 138, 1, 138, 1, 138, 1, 138, 1, 139, 1, 139, 1, 139, 1, 139, 1, 140, 1, 140, 1, 140, 1, 140, 1, 140, 1, 141, 1, 141, 1, 141, 1, 141, 1, 142, 1, 142, 1, 142, 1, 142, 1, 143, 1, 143, 1, 143, 1, 143, 1, 144, 1, 144, 1, 144, 1, 144, 1, 145, 1, 145, 1, 145, 1, 145, 1, 146, 1, 146, 1, 146, 1, 146, 1, 147, 1, 147, 1, 147, 1, 147, 1, 147, 1, 148, 1, 148, 1, 148, 1, 148, 1, 148, 1, 149, 1, 149, 1, 149, 1, 149, 1, 150, 1, 150, 1, 150, 1, 150, 1, 151, 1, 151, 1, 151, 1, 151, 1, 152, 1, 152, 1, 152, 1, 152, 1, 152, 1, 153, 1, 153, 1, 154, 1, 154, 1, 154, 1, 154, 1, 154, 4, 154, 1264, 8, 154, 11, 154, 12, 154, 1265, 1, 155, 1, 155, 1, 155, 1, 155, 1, 156, 1, 156, 1, 156, 1, 156, 1, 157, 1, 157, 1, 157, 1, 157, 1, 158, 1, 158, 1, 158, 1, 158, 1, 158, 1, 159, 1, 159, 1, 159, 1, 159, 1, 160, 1, 160, 1, 160, 1, 160, 1, 161, 1, 161, 1, 161, 1, 161, 1, 162, 1, 162, 1, 162, 1, 162, 1, 162, 1, 163, 1, 163, 1, 163, 1, 163, 1, 164, 1, 164, 1, 164, 1, 164, 1, 165, 1, 165, 1, 165, 1, 165, 1, 166, 1, 166, 1, 166, 1, 166, 1, 167, 1, 167, 1, 167, 1, 167, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 168, 1, 169, 1, 169, 1, 169, 1, 169, 1, 170, 1, 170, 1, 170, 1, 170, 1, 171, 1, 171, 1, 171, 1, 171, 1, 172, 1, 172, 1, 172, 1, 172, 1, 173, 1, 173, 1, 173, 1, 173, 1, 174, 1, 174, 1, 174, 1, 174, 1, 175, 1, 175, 1, 175, 1, 175, 1, 175, 1, 176, 1, 176, 1, 176, 1, 176, 1, 176, 1, 176, 1, 177, 1, 177, 1, 177, 1, 177, 1, 177, 1, 177, 1, 178, 1, 178, 1, 178, 1, 178, 1, 179, 1, 179, 1, 179, 1, 179, 1, 180, 1, 180, 1, 180, 1, 180, 1, 181, 1, 181, 1, 181, 1, 181, 1, 181, 1, 181, 1, 182, 1, 182, 1, 182, 1, 182, 1, 182, 1, 182, 1, 183, 1, 183, 1, 183, 1, 183, 1, 184, 1, 184, 1, 184, 1, 184, 1, 185, 1, 185, 1, 185, 1, 185, 1, 186, 1, 186, 1, 186, 1, 186, 1, 186, 1, 186, 1, 187, 1, 187, 1, 187, 1, 187, 1, 187, 1, 187, 1, 188, 1, 188, 1, 188, 1, 188, 1, 188, 1, 188, 1, 189, 1, 189, 1, 189, 1, 189, 1, 189, 2, 600, 669, 0, 190, 15, 1, 17, 2, 19, 3, 21, 4, 23, 5, 25, 6, 27, 7, 29, 8, 31, 9, 33, 10, 35, 11, 37, 12, 39, 13, 41, 14, 43, 15, 45, 16, 47, 17, 49, 18, 51, 19, 53, 20, 55, 21, 57, 22, 59, 23, 61, 24, 63, 25, 65, 0, 67, 0, 69, 0, 71, 0, 73, 0, 75, 0, 77, 0, 79, 0, 81, 0, 83, 0, 85, 26, 87, 27, 89, 28, 91, 29, 93, 30, 95, 31, 97, 32, 99, 33, 101, 34, 103, 35, 105, 36, 107, 37, 109, 38, 111, 39, 113, 40, 115, 41, 117, 42, 119, 43, 121, 44, 123, 45, 125, 46, 127, 47, 129, 48, 131, 49, 133, 50, 135, 51, 137, 52, 139, 53, 141, 54, 143, 55, 145, 56, 147, 57, 149, 58, 151, 59, 153, 60, 155, 61, 157, 62, 159, 63, 161, 0, 163, 64, 165, 65, 167, 66, 169, 67, 171, 0, 173, 68, 175, 69, 177, 70, 179, 71, 181, 0, 183, 0, 185, 72, 187, 73, 189, 74, 191, 0, 193, 0, 195, 0, 197, 0, 199, 0, 201, 0, 203, 75, 205, 0, 207, 76, 209, 0, 211, 0, 213, 77, 215, 78, 217, 79, 219, 0, 221, 0, 223, 0, 225, 0, 227, 0, 229, 80, 231, 81, 233, 82, 235, 83, 237, 0, 239, 0, 241, 0, 243, 0, 245, 84, 247, 0, 249, 85, 251, 86, 253, 87, 255, 0, 257, 0, 259, 88, 261, 89, 263, 0, 265, 90, 267, 0, 269, 91, 271, 92, 273, 93, 275, 0, 277, 0, 279, 0, 281, 0, 283, 0, 285, 0, 287, 0, 289, 94, 291, 95, 293, 96, 295, 0, 297, 0, 299, 0, 301, 0, 303, 97, 305, 98, 307, 99, 309, 0, 311, 100, 313, 101, 315, 102, 317, 103, 319, 0, 321, 104, 323, 105, 325, 106, 327, 107, 329, 108, 331, 0, 333, 0, 335, 0, 337, 0, 339, 0, 341, 0, 343, 0, 345, 109, 347, 110, 349, 111, 351, 0, 353, 0, 355, 0, 357, 0, 359, 112, 361, 113, 363, 114, 365, 0, 367, 0, 369, 0, 371, 115, 373, 116, 375, 117, 377, 0, 379, 0, 381, 118, 383, 119, 385, 120, 387, 0, 389, 0, 391, 0, 393, 0, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 35, 2, 0, 68, 68, 100, 100, 2, 0, 73, 73, 105, 105, 2, 0, 83, 83, 115, 115, 2, 0, 69, 69, 101, 101, 2, 0, 67, 67, 99, 99, 2, 0, 84, 84, 116, 116, 2, 0, 82, 82, 114, 114, 2, 0, 79, 79, 111, 111, 2, 0, 80, 80, 112, 112, 2, 0, 78, 78, 110, 110, 2, 0, 72, 72, 104, 104, 2, 0, 86, 86, 118, 118, 2, 0, 65, 65, 97, 97, 2, 0, 76, 76, 108, 108, 2, 0, 88, 88, 120, 120, 2, 0, 70, 70, 102, 102, 2, 0, 77, 77, 109, 109, 2, 0, 71, 71, 103, 103, 2, 0, 75, 75, 107, 107, 2, 0, 87, 87, 119, 119, 2, 0, 85, 85, 117, 117, 6, 0, 9, 10, 13, 13, 32, 32, 47, 47, 91, 91, 93, 93, 2, 0, 10, 10, 13, 13, 3, 0, 9, 10, 13, 13, 32, 32, 1, 0, 48, 57, 2, 0, 65, 90, 97, 122, 8, 0, 34, 34, 78, 78, 82, 82, 84, 84, 92, 92, 110, 110, 114, 114, 116, 116, 4, 0, 10, 10, 13, 13, 34, 34, 92, 92, 2, 0, 43, 43, 45, 45, 1, 0, 96, 96, 2, 0, 66, 66, 98, 98, 2, 0, 89, 89, 121, 121, 11, 0, 9, 10, 13, 13, 32, 32, 34, 34, 44, 44, 47, 47, 58, 58, 61, 61, 91, 91, 93, 93, 124, 124, 2, 0, 42, 42, 47, 47, 11, 0, 9, 10, 13, 13, 32, 32, 34, 35, 44, 44, 47, 47, 58, 58, 60, 60, 62, 63, 92, 92, 124, 124, 1455, 0, 15, 1, 0, 0, 0, 0, 17, 1, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25, 1, 0, 0, 0, 0, 27, 1, 0, 0, 0, 0, 29, 1, 0, 0, 0, 0, 31, 1, 0, 0, 0, 0, 33, 1, 0, 0, 0, 0, 35, 1, 0, 0, 0, 0, 37, 1, 0, 0, 0, 0, 39, 1, 0, 0, 0, 0, 41, 1, 0, 0, 0, 0, 43, 1, 0, 0, 0, 0, 45, 1, 0, 0, 0, 0, 47, 1, 0, 0, 0, 0, 49, 1, 0, 0, 0, 0, 51, 1, 0, 0, 0, 0, 53, 1, 0, 0, 0, 0, 55, 1, 0, 0, 0, 0, 57, 1, 0, 0, 0, 0, 59, 1, 0, 0, 0, 0, 61, 1, 0, 0, 0, 1, 63, 1, 0, 0, 0, 1, 85, 1, 0, 0, 0, 1, 87, 1, 0, 0, 0, 1, 89, 1, 0, 0, 0, 1, 91, 1, 0, 0, 0, 1, 93, 1, 0, 0, 0, 1, 95, 1, 0, 0, 0, 1, 97, 1, 0, 0, 0, 1, 99, 1, 0, 0, 0, 1, 101, 1, 0, 0, 0, 1, 103, 1, 0, 0, 0, 1, 105, 1, 0, 0, 0, 1, 107, 1, 0, 0, 0, 1, 109, 1, 0, 0, 0, 1, 111, 1, 0, 0, 0, 1, 113, 1, 0, 0, 0, 1, 115, 1, 0, 0, 0, 1, 117, 1, 0, 0, 0, 1, 119, 1, 0, 0, 0, 1, 121, 1, 0, 0, 0, 1, 123, 1, 0, 0, 0, 1, 125, 1, 0, 0, 0, 1, 127, 1, 0, 0, 0, 1, 129, 1, 0, 0, 0, 1, 131, 1, 0, 0, 0, 1, 133, 1, 0, 0, 0, 1, 135, 1, 0, 0, 0, 1, 137, 1, 0, 0, 0, 1, 139, 1, 0, 0, 0, 1, 141, 1, 0, 0, 0, 1, 143, 1, 0, 0, 0, 1, 145, 1, 0, 0, 0, 1, 147, 1, 0, 0, 0, 1, 149, 1, 0, 0, 0, 1, 151, 1, 0, 0, 0, 1, 153, 1, 0, 0, 0, 1, 155, 1, 0, 0, 0, 1, 157, 1, 0, 0, 0, 1, 159, 1, 0, 0, 0, 1, 161, 1, 0, 0, 0, 1, 163, 1, 0, 0, 0, 1, 165, 1, 0, 0, 0, 1, 167, 1, 0, 0, 0, 1, 169, 1, 0, 0, 0, 1, 173, 1, 0, 0, 0, 1, 175, 1, 0, 0, 0, 1, 177, 1, 0, 0, 0, 1, 179, 1, 0, 0, 0, 2, 181, 1, 0, 0, 0, 2, 183, 1, 0, 0, 0, 2, 185, 1, 0, 0, 0, 2, 187, 1, 0, 0, 0, 2, 189, 1, 0, 0, 0, 3, 191, 1, 0, 0, 0, 3, 193, 1, 0, 0, 0, 3, 195, 1, 0, 0, 0, 3, 197, 1, 0, 0, 0, 3, 199, 1, 0, 0, 0, 3, 201, 1, 0, 0, 0, 3, 203, 1, 0, 0, 0, 3, 207, 1, 0, 0, 0, 3, 209, 1, 0, 0, 0, 3, 211, 1, 0, 0, 0, 3, 213, 1, 0, 0, 0, 3, 215, 1, 0, 0, 0, 3, 217, 1, 0, 0, 0, 4, 219, 1, 0, 0, 0, 4, 221, 1, 0, 0, 0, 4, 223, 1, 0, 0, 0, 4, 229, 1, 0, 0, 0, 4, 231, 1, 0, 0, 0, 4, 233, 1, 0, 0, 0, 4, 235, 1, 0, 0, 0, 5, 237, 1, 0, 0, 0, 5, 239, 1, 0, 0, 0, 5, 241, 1, 0, 0, 0, 5, 243, 1, 0, 0, 0, 5, 245, 1, 0, 0, 0, 5, 247, 1, 0, 0, 0, 5, 249, 1, 0, 0, 0, 5, 251, 1, 0, 0, 0, 5, 253, 1, 0, 0, 0, 6, 255, 1, 0, 0, 0, 6, 257, 1, 0, 0, 0, 6, 259, 1, 0, 0, 0, 6, 261, 1, 0, 0, 0, 6, 265, 1, 0, 0, 0, 6, 267, 1, 0, 0, 0, 6, 269, 1, 0, 0, 0, 6, 271, 1, 0, 0, 0, 6, 273, 1, 0, 0, 0, 7, 275, 1, 0, 0, 0, 7, 277, 1, 0, 0, 0, 7, 279, 1, 0, 0, 0, 7, 281, 1, 0, 0, 0, 7, 283, 1, 0, 0, 0, 7, 285, 1, 0, 0, 0, 7, 287, 1, 0, 0, 0, 7, 289, 1, 0, 0, 0, 7, 291, 1, 0, 0, 0, 7, 293, 1, 0, 0, 0, 8, 295, 1, 0, 0, 0, 8, 297, 1, 0, 0, 0, 8, 299, 1, 0, 0, 0, 8, 301, 1, 0, 0, 0, 8, 303, 1, 0, 0, 0, 8, 305, 1, 0, 0, 0, 8, 307, 1, 0, 0, 0, 9, 309, 1, 0, 0, 0, 9, 311, 1, 0, 0, 0, 9, 313, 1, 0, 0, 0, 9, 315, 1, 0, 0, 0, 9, 317, 1, 0, 0, 0, 10, 319, 1, 0, 0, 0, 10, 321, 1, 0, 0, 0, 10, 323, 1, 0, 0, 0, 10, 325, 1, 0, 0, 0, 10, 327, 1, 0, 0, 0, 10, 329, 1, 0, 0, 0, 11, 331, 1, 0, 0, 0, 11, 333, 1, 0, 0, 0, 11, 335, 1, 0, 0, 0, 11, 337, 1, 0, 0, 0, 11, 339, 1, 0, 0, 0, 11, 341, 1, 0, 0, 0, 11, 343, 1, 0, 0, 0, 11, 345, 1, 0, 0, 0, 11, 347, 1, 0, 0, 0, 11, 349, 1, 0, 0, 0, 12, 351, 1, 0, 0, 0, 12, 353, 1, 0, 0, 0, 12, 355, 1, 0, 0, 0, 12, 357, 1, 0, 0, 0, 12, 359, 1, 0, 0, 0, 12, 361, 1, 0, 0, 0, 12, 363, 1, 0, 0, 0, 13, 365, 1, 0, 0, 0, 13, 367, 1, 0, 0, 0, 13, 369, 1, 0, 0, 0, 13, 371, 1, 0, 0, 0, 13, 373, 1, 0, 0, 0, 13, 375, 1, 0, 0, 0, 14, 377, 1, 0, 0, 0, 14, 379, 1, 0, 0, 0, 14, 381, 1, 0, 0, 0, 14, 383, 1, 0, 0, 0, 14, 385, 1, 0, 0, 0, 14, 387, 1, 0, 0, 0, 14, 389, 1, 0, 0, 0, 14, 391, 1, 0, 0, 0, 14, 393, 1, 0, 0, 0, 15, 395, 1, 0, 0, 0, 17, 405, 1, 0, 0, 0, 19, 412, 1, 0, 0, 0, 21, 421, 1, 0, 0, 0, 23, 428, 1, 0, 0, 0, 25, 438, 1, 0, 0, 0, 27, 445, 1, 0, 0, 0, 29, 452, 1, 0, 0, 0, 31, 459, 1, 0, 0, 0, 33, 467, 1, 0, 0, 0, 35, 479, 1, 0, 0, 0, 37, 488, 1, 0, 0, 0, 39, 494, 1, 0, 0, 0, 41, 501, 1, 0, 0, 0, 43, 508, 1, 0, 0, 0, 45, 516, 1, 0, 0, 0, 47, 524, 1, 0, 0, 0, 49, 539, 1, 0, 0, 0, 51, 549, 1, 0, 0, 0, 53, 558, 1, 0, 0, 0, 55, 570, 1, 0, 0, 0, 57, 576, 1, 0, 0, 0, 59, 593, 1, 0, 0, 0, 61, 609, 1, 0, 0, 0, 63, 615, 1, 0, 0, 0, 65, 619, 1, 0, 0, 0, 67, 621, 1, 0, 0, 0, 69, 623, 1, 0, 0, 0, 71, 626, 1, 0, 0, 0, 73, 628, 1, 0, 0, 0, 75, 637, 1, 0, 0, 0, 77, 639, 1, 0, 0, 0, 79, 644, 1, 0, 0, 0, 81, 646, 1, 0, 0, 0, 83, 651, 1, 0, 0, 0, 85, 682, 1, 0, 0, 0, 87, 685, 1, 0, 0, 0, 89, 731, 1, 0, 0, 0, 91, 733, 1, 0, 0, 0, 93, 736, 1, 0, 0, 0, 95, 740, 1, 0, 0, 0, 97, 744, 1, 0, 0, 0, 99, 746, 1, 0, 0, 0, 101, 749, 1, 0, 0, 0, 103, 751, 1, 0, 0, 0, 105, 756, 1, 0, 0, 0, 107, 758, 1, 0, 0, 0, 109, 764, 1, 0, 0, 0, 111, 770, 1, 0, 0, 0, 113, 773, 1, 0, 0, 0, 115, 776, 1, 0, 0, 0, 117, 781, 1, 0, 0, 0, 119, 786, 1, 0, 0, 0, 121, 788, 1, 0, 0, 0, 123, 792, 1, 0, 0, 0, 125, 797, 1, 0, 0, 0, 127, 803, 1, 0, 0, 0, 129, 806, 1, 0, 0, 0, 131, 808, 1, 0, 0, 0, 133, 814, 1, 0, 0, 0, 135, 816, 1, 0, 0, 0, 137, 821, 1, 0, 0, 0, 139, 824, 1, 0, 0, 0, 141, 827, 1, 0, 0, 0, 143, 830, 1, 0, 0, 0, 145, 832, 1, 0, 0, 0, 147, 835, 1, 0, 0, 0, 149, 837, 1, 0, 0, 0, 151, 840, 1, 0, 0, 0, 153, 842, 1, 0, 0, 0, 155, 844, 1, 0, 0, 0, 157, 846, 1, 0, 0, 0, 159, 848, 1, 0, 0, 0, 161, 850, 1, 0, 0, 0, 163, 872, 1, 0, 0, 0, 165, 874, 1, 0, 0, 0, 167, 879, 1, 0, 0, 0, 169, 900, 1, 0, 0, 0, 171, 902, 1, 0, 0, 0, 173, 910, 1, 0, 0, 0, 175, 912, 1, 0, 0, 0, 177, 916, 1, 0, 0, 0, 179, 920, 1, 0, 0, 0, 181, 924, 1, 0, 0, 0, 183, 929, 1, 0, 0, 0, 185, 934, 1, 0, 0, 0, 187, 938, 1, 0, 0, 0, 189, 942, 1, 0, 0, 0, 191, 946, 1, 0, 0, 0, 193, 951, 1, 0, 0, 0, 195, 955, 1, 0, 0, 0, 197, 959, 1, 0, 0, 0, 199, 963, 1, 0, 0, 0, 201, 967, 1, 0, 0, 0, 203, 971, 1, 0, 0, 0, 205, 983, 1, 0, 0, 0, 207, 986, 1, 0, 0, 0, 209, 990, 1, 0, 0, 0, 211, 994, 1, 0, 0, 0, 213, 998, 1, 0, 0, 0, 215, 1002, 1, 0, 0, 0, 217, 1006, 1, 0, 0, 0, 219, 1010, 1, 0, 0, 0, 221, 1015, 1, 0, 0, 0, 223, 1019, 1, 0, 0, 0, 225, 1027, 1, 0, 0, 0, 227, 1048, 1, 0, 0, 0, 229, 1052, 1, 0, 0, 0, 231, 1056, 1, 0, 0, 0, 233, 1060, 1, 0, 0, 0, 235, 1064, 1, 0, 0, 0, 237, 1068, 1, 0, 0, 0, 239, 1073, 1, 0, 0, 0, 241, 1077, 1, 0, 0, 0, 243, 1081, 1, 0, 0, 0, 245, 1085, 1, 0, 0, 0, 247, 1088, 1, 0, 0, 0, 249, 1092, 1, 0, 0, 0, 251, 1096, 1, 0, 0, 0, 253, 1100, 1, 0, 0, 0, 255, 1104, 1, 0, 0, 0, 257, 1109, 1, 0, 0, 0, 259, 1114, 1, 0, 0, 0, 261, 1119, 1, 0, 0, 0, 263, 1126, 1, 0, 0, 0, 265, 1135, 1, 0, 0, 0, 267, 1142, 1, 0, 0, 0, 269, 1146, 1, 0, 0, 0, 271, 1150, 1, 0, 0, 0, 273, 1154, 1, 0, 0, 0, 275, 1158, 1, 0, 0, 0, 277, 1164, 1, 0, 0, 0, 279, 1168, 1, 0, 0, 0, 281, 1172, 1, 0, 0, 0, 283, 1176, 1, 0, 0, 0, 285, 1180, 1, 0, 0, 0, 287, 1184, 1, 0, 0, 0, 289, 1188, 1, 0, 0, 0, 291, 1192, 1, 0, 0, 0, 293, 1196, 1, 0, 0, 0, 295, 1200, 1, 0, 0, 0, 297, 1205, 1, 0, 0, 0, 299, 1209, 1, 0, 0, 0, 301, 1213, 1, 0, 0, 0, 303, 1217, 1, 0, 0, 0, 305, 1221, 1, 0, 0, 0, 307, 1225, 1, 0, 0, 0, 309, 1229, 1, 0, 0, 0, 311, 1234, 1, 0, 0, 0, 313, 1239, 1, 0, 0, 0, 315, 1243, 1, 0, 0, 0, 317, 1247, 1, 0, 0, 0, 319, 1251, 1, 0, 0, 0, 321, 1256, 1, 0, 0, 0, 323, 1263, 1, 0, 0, 0, 325, 1267, 1, 0, 0, 0, 327, 1271, 1, 0, 0, 0, 329, 1275, 1, 0, 0, 0, 331, 1279, 1, 0, 0, 0, 333, 1284, 1, 0, 0, 0, 335, 1288, 1, 0, 0, 0, 337, 1292, 1, 0, 0, 0, 339, 1296, 1, 0, 0, 0, 341, 1301, 1, 0, 0, 0, 343, 1305, 1, 0, 0, 0, 345, 1309, 1, 0, 0, 0, 347, 1313, 1, 0, 0, 0, 349, 1317, 1, 0, 0, 0, 351, 1321, 1, 0, 0, 0, 353, 1327, 1, 0, 0, 0, 355, 1331, 1, 0, 0, 0, 357, 1335, 1, 0, 0, 0, 359, 1339, 1, 0, 0, 0, 361, 1343, 1, 0, 0, 0, 363, 1347, 1, 0, 0, 0, 365, 1351, 1, 0, 0, 0, 367, 1356, 1, 0, 0, 0, 369, 1362, 1, 0, 0, 0, 371, 1368, 1, 0, 0, 0, 373, 1372, 1, 0, 0, 0, 375, 1376, 1, 0, 0, 0, 377, 1380, 1, 0, 0, 0, 379, 1386, 1, 0, 0, 0, 381, 1392, 1, 0, 0, 0, 383, 1396, 1, 0, 0, 0, 385, 1400, 1, 0, 0, 0, 387, 1404, 1, 0, 0, 0, 389, 1410, 1, 0, 0, 0, 391, 1416, 1, 0, 0, 0, 393, 1422, 1, 0, 0, 0, 395, 396, 7, 0, 0, 0, 396, 397, 7, 1, 0, 0, 397, 398, 7, 2, 0, 0, 398, 399, 7, 2, 0, 0, 399, 400, 7, 3, 0, 0, 400, 401, 7, 4, 0, 0, 401, 402, 7, 5, 0, 0, 402, 403, 1, 0, 0, 0, 403, 404, 6, 0, 0, 0, 404, 16, 1, 0, 0, 0, 405, 406, 7, 0, 0, 0, 406, 407, 7, 6, 0, 0, 407, 408, 7, 7, 0, 0, 408, 409, 7, 8, 0, 0, 409, 410, 1, 0, 0, 0, 410, 411, 6, 1, 1, 0, 411, 18, 1, 0, 0, 0, 412, 413, 7, 3, 0, 0, 413, 414, 7, 9, 0, 0, 414, 415, 7, 6, 0, 0, 415, 416, 7, 1, 0, 0, 416, 417, 7, 4, 0, 0, 417, 418, 7, 10, 0, 0, 418, 419, 1, 0, 0, 0, 419, 420, 6, 2, 2, 0, 420, 20, 1, 0, 0, 0, 421, 422, 7, 3, 0, 0, 422, 423, 7, 11, 0, 0, 423, 424, 7, 12, 0, 0, 424, 425, 7, 13, 0, 0, 425, 426, 1, 0, 0, 0, 426, 427, 6, 3, 0, 0, 427, 22, 1, 0, 0, 0, 428, 429, 7, 3, 0, 0, 429, 430, 7, 14, 0, 0, 430, 431, 7, 8, 0, 0, 431, 432, 7, 13, 0, 0, 432, 433, 7, 12, 0, 0, 433, 434, 7, 1, 0, 0, 434, 435, 7, 9, 0, 0, 435, 436, 1, 0, 0, 0, 436, 437, 6, 4, 3, 0, 437, 24, 1, 0, 0, 0, 438, 439, 7, 15, 0, 0, 439, 440, 7, 6, 0, 0, 440, 441, 7, 7, 0, 0, 441, 442, 7, 16, 0, 0, 442, 443, 1, 0, 0, 0, 443, 444, 6, 5, 4, 0, 444, 26, 1, 0, 0, 0, 445, 446, 7, 17, 0, 0, 446, 447, 7, 6, 0, 0, 447, 448, 7, 7, 0, 0, 448, 449, 7, 18, 0, 0, 449, 450, 1, 0, 0, 0, 450, 451, 6, 6, 0, 0, 451, 28, 1, 0, 0, 0, 452, 453, 7, 18, 0, 0, 453, 454, 7, 3, 0, 0, 454, 455, 7, 3, 0, 0, 455, 456, 7, 8, 0, 0, 456, 457, 1, 0, 0, 0, 457, 458, 6, 7, 1, 0, 458, 30, 1, 0, 0, 0, 459, 460, 7, 13, 0, 0, 460, 461, 7, 1, 0, 0, 461, 462, 7, 16, 0, 0, 462, 463, 7, 1, 0, 0, 463, 464, 7, 5, 0, 0, 464, 465, 1, 0, 0, 0, 465, 466, 6, 8, 0, 0, 466, 32, 1, 0, 0, 0, 467, 468, 7, 16, 0, 0, 468, 469, 7, 11, 0, 0, 469, 470, 5, 95, 0, 0, 470, 471, 7, 3, 0, 0, 471, 472, 7, 14, 0, 0, 472, 473, 7, 8, 0, 0, 473, 474, 7, 12, 0, 0, 474, 475, 7, 9, 0, 0, 475, 476, 7, 0, 0, 0, 476, 477, 1, 0, 0, 0, 477, 478, 6, 9, 5, 0, 478, 34, 1, 0, 0, 0, 479, 480, 7, 6, 0, 0, 480, 481, 7, 3, 0, 0, 481, 482, 7, 9, 0, 0, 482, 483, 7, 12, 0, 0, 483, 484, 7, 16, 0, 0, 484, 485, 7, 3, 0, 0, 485, 486, 1, 0, 0, 0, 486, 487, 6, 10, 6, 0, 487, 36, 1, 0, 0, 0, 488, 489, 7, 6, 0, 0, 489, 490, 7, 7, 0, 0, 490, 491, 7, 19, 0, 0, 491, 492, 1, 0, 0, 0, 492, 493, 6, 11, 0, 0, 493, 38, 1, 0, 0, 0, 494, 495, 7, 2, 0, 0, 495, 496, 7, 10, 0, 0, 496, 497, 7, 7, 0, 0, 497, 498, 7, 19, 0, 0, 498, 499, 1, 0, 0, 0, 499, 500, 6, 12, 7, 0, 500, 40, 1, 0, 0, 0, 501, 502, 7, 2, 0, 0, 502, 503, 7, 7, 0, 0, 503, 504, 7, 6, 0, 0, 504, 505, 7, 5, 0, 0, 505, 506, 1, 0, 0, 0, 506, 507, 6, 13, 0, 0, 507, 42, 1, 0, 0, 0, 508, 509, 7, 2, 0, 0, 509, 510, 7, 5, 0, 0, 510, 511, 7, 12, 0, 0, 511, 512, 7, 5, 0, 0, 512, 513, 7, 2, 0, 0, 513, 514, 1, 0, 0, 0, 514, 515, 6, 14, 0, 0, 515, 44, 1, 0, 0, 0, 516, 517, 7, 19, 0, 0, 517, 518, 7, 10, 0, 0, 518, 519, 7, 3, 0, 0, 519, 520, 7, 6, 0, 0, 520, 521, 7, 3, 0, 0, 521, 522, 1, 0, 0, 0, 522, 523, 6, 15, 0, 0, 523, 46, 1, 0, 0, 0, 524, 525, 4, 16, 0, 0, 525, 526, 7, 1, 0, 0, 526, 527, 7, 9, 0, 0, 527, 528, 7, 13, 0, 0, 528, 529, 7, 1, 0, 0, 529, 530, 7, 9, 0, 0, 530, 531, 7, 3, 0, 0, 531, 532, 7, 2, 0, 0, 532, 533, 7, 5, 0, 0, 533, 534, 7, 12, 0, 0, 534, 535, 7, 5, 0, 0, 535, 536, 7, 2, 0, 0, 536, 537, 1, 0, 0, 0, 537, 538, 6, 16, 0, 0, 538, 48, 1, 0, 0, 0, 539, 540, 4, 17, 1, 0, 540, 541, 7, 13, 0, 0, 541, 542, 7, 7, 0, 0, 542, 543, 7, 7, 0, 0, 543, 544, 7, 18, 0, 0, 544, 545, 7, 20, 0, 0, 545, 546, 7, 8, 0, 0, 546, 547, 1, 0, 0, 0, 547, 548, 6, 17, 8, 0, 548, 50, 1, 0, 0, 0, 549, 550, 4, 18, 2, 0, 550, 551, 7, 16, 0, 0, 551, 552, 7, 12, 0, 0, 552, 553, 7, 5, 0, 0, 553, 554, 7, 4, 0, 0, 554, 555, 7, 10, 0, 0, 555, 556, 1, 0, 0, 0, 556, 557, 6, 18, 0, 0, 557, 52, 1, 0, 0, 0, 558, 559, 4, 19, 3, 0, 559, 560, 7, 16, 0, 0, 560, 561, 7, 3, 0, 0, 561, 562, 7, 5, 0, 0, 562, 563, 7, 6, 0, 0, 563, 564, 7, 1, 0, 0, 564, 565, 7, 4, 0, 0, 565, 566, 7, 2, 0, 0, 566, 567, 1, 0, 0, 0, 567, 568, 6, 19, 9, 0, 568, 54, 1, 0, 0, 0, 569, 571, 8, 21, 0, 0, 570, 569, 1, 0, 0, 0, 571, 572, 1, 0, 0, 0, 572, 570, 1, 0, 0, 0, 572, 573, 1, 0, 0, 0, 573, 574, 1, 0, 0, 0, 574, 575, 6, 20, 0, 0, 575, 56, 1, 0, 0, 0, 576, 577, 5, 47, 0, 0, 577, 578, 5, 47, 0, 0, 578, 582, 1, 0, 0, 0, 579, 581, 8, 22, 0, 0, 580, 579, 1, 0, 0, 0, 581, 584, 1, 0, 0, 0, 582, 580, 1, 0, 0, 0, 582, 583, 1, 0, 0, 0, 583, 586, 1, 0, 0, 0, 584, 582, 1, 0, 0, 0, 585, 587, 5, 13, 0, 0, 586, 585, 1, 0, 0, 0, 586, 587, 1, 0, 0, 0, 587, 589, 1, 0, 0, 0, 588, 590, 5, 10, 0, 0, 589, 588, 1, 0, 0, 0, 589, 590, 1, 0, 0, 0, 590, 591, 1, 0, 0, 0, 591, 592, 6, 21, 10, 0, 592, 58, 1, 0, 0, 0, 593, 594, 5, 47, 0, 0, 594, 595, 5, 42, 0, 0, 595, 600, 1, 0, 0, 0, 596, 599, 3, 59, 22, 0, 597, 599, 9, 0, 0, 0, 598, 596, 1, 0, 0, 0, 598, 597, 1, 0, 0, 0, 599, 602, 1, 0, 0, 0, 600, 601, 1, 0, 0, 0, 600, 598, 1, 0, 0, 0, 601, 603, 1, 0, 0, 0, 602, 600, 1, 0, 0, 0, 603, 604, 5, 42, 0, 0, 604, 605, 5, 47, 0, 0, 605, 606, 1, 0, 0, 0, 606, 607, 6, 22, 10, 0, 607, 60, 1, 0, 0, 0, 608, 610, 7, 23, 0, 0, 609, 608, 1, 0, 0, 0, 610, 611, 1, 0, 0, 0, 611, 609, 1, 0, 0, 0, 611, 612, 1, 0, 0, 0, 612, 613, 1, 0, 0, 0, 613, 614, 6, 23, 10, 0, 614, 62, 1, 0, 0, 0, 615, 616, 5, 124, 0, 0, 616, 617, 1, 0, 0, 0, 617, 618, 6, 24, 11, 0, 618, 64, 1, 0, 0, 0, 619, 620, 7, 24, 0, 0, 620, 66, 1, 0, 0, 0, 621, 622, 7, 25, 0, 0, 622, 68, 1, 0, 0, 0, 623, 624, 5, 92, 0, 0, 624, 625, 7, 26, 0, 0, 625, 70, 1, 0, 0, 0, 626, 627, 8, 27, 0, 0, 627, 72, 1, 0, 0, 0, 628, 630, 7, 3, 0, 0, 629, 631, 7, 28, 0, 0, 630, 629, 1, 0, 0, 0, 630, 631, 1, 0, 0, 0, 631, 633, 1, 0, 0, 0, 632, 634, 3, 65, 25, 0, 633, 632, 1, 0, 0, 0, 634, 635, 1, 0, 0, 0, 635, 633, 1, 0, 0, 0, 635, 636, 1, 0, 0, 0, 636, 74, 1, 0, 0, 0, 637, 638, 5, 64, 0, 0, 638, 76, 1, 0, 0, 0, 639, 640, 5, 96, 0, 0, 640, 78, 1, 0, 0, 0, 641, 645, 8, 29, 0, 0, 642, 643, 5, 96, 0, 0, 643, 645, 5, 96, 0, 0, 644, 641, 1, 0, 0, 0, 644, 642, 1, 0, 0, 0, 645, 80, 1, 0, 0, 0, 646, 647, 5, 95, 0, 0, 647, 82, 1, 0, 0, 0, 648, 652, 3, 67, 26, 0, 649, 652, 3, 65, 25, 0, 650, 652, 3, 81, 33, 0, 651, 648, 1, 0, 0, 0, 651, 649, 1, 0, 0, 0, 651, 650, 1, 0, 0, 0, 652, 84, 1, 0, 0, 0, 653, 658, 5, 34, 0, 0, 654, 657, 3, 69, 27, 0, 655, 657, 3, 71, 28, 0, 656, 654, 1, 0, 0, 0, 656, 655, 1, 0, 0, 0, 657, 660, 1, 0, 0, 0, 658, 656, 1, 0, 0, 0, 658, 659, 1, 0, 0, 0, 659, 661, 1, 0, 0, 0, 660, 658, 1, 0, 0, 0, 661, 683, 5, 34, 0, 0, 662, 663, 5, 34, 0, 0, 663, 664, 5, 34, 0, 0, 664, 665, 5, 34, 0, 0, 665, 669, 1, 0, 0, 0, 666, 668, 8, 22, 0, 0, 667, 666, 1, 0, 0, 0, 668, 671, 1, 0, 0, 0, 669, 670, 1, 0, 0, 0, 669, 667, 1, 0, 0, 0, 670, 672, 1, 0, 0, 0, 671, 669, 1, 0, 0, 0, 672, 673, 5, 34, 0, 0, 673, 674, 5, 34, 0, 0, 674, 675, 5, 34, 0, 0, 675, 677, 1, 0, 0, 0, 676, 678, 5, 34, 0, 0, 677, 676, 1, 0, 0, 0, 677, 678, 1, 0, 0, 0, 678, 680, 1, 0, 0, 0, 679, 681, 5, 34, 0, 0, 680, 679, 1, 0, 0, 0, 680, 681, 1, 0, 0, 0, 681, 683, 1, 0, 0, 0, 682, 653, 1, 0, 0, 0, 682, 662, 1, 0, 0, 0, 683, 86, 1, 0, 0, 0, 684, 686, 3, 65, 25, 0, 685, 684, 1, 0, 0, 0, 686, 687, 1, 0, 0, 0, 687, 685, 1, 0, 0, 0, 687, 688, 1, 0, 0, 0, 688, 88, 1, 0, 0, 0, 689, 691, 3, 65, 25, 0, 690, 689, 1, 0, 0, 0, 691, 692, 1, 0, 0, 0, 692, 690, 1, 0, 0, 0, 692, 693, 1, 0, 0, 0, 693, 694, 1, 0, 0, 0, 694, 698, 3, 105, 45, 0, 695, 697, 3, 65, 25, 0, 696, 695, 1, 0, 0, 0, 697, 700, 1, 0, 0, 0, 698, 696, 1, 0, 0, 0, 698, 699, 1, 0, 0, 0, 699, 732, 1, 0, 0, 0, 700, 698, 1, 0, 0, 0, 701, 703, 3, 105, 45, 0, 702, 704, 3, 65, 25, 0, 703, 702, 1, 0, 0, 0, 704, 705, 1, 0, 0, 0, 705, 703, 1, 0, 0, 0, 705, 706, 1, 0, 0, 0, 706, 732, 1, 0, 0, 0, 707, 709, 3, 65, 25, 0, 708, 707, 1, 0, 0, 0, 709, 710, 1, 0, 0, 0, 710, 708, 1, 0, 0, 0, 710, 711, 1, 0, 0, 0, 711, 719, 1, 0, 0, 0, 712, 716, 3, 105, 45, 0, 713, 715, 3, 65, 25, 0, 714, 713, 1, 0, 0, 0, 715, 718, 1, 0, 0, 0, 716, 714, 1, 0, 0, 0, 716, 717, 1, 0, 0, 0, 717, 720, 1, 0, 0, 0, 718, 716, 1, 0, 0, 0, 719, 712, 1, 0, 0, 0, 719, 720, 1, 0, 0, 0, 720, 721, 1, 0, 0, 0, 721, 722, 3, 73, 29, 0, 722, 732, 1, 0, 0, 0, 723, 725, 3, 105, 45, 0, 724, 726, 3, 65, 25, 0, 725, 724, 1, 0, 0, 0, 726, 727, 1, 0, 0, 0, 727, 725, 1, 0, 0, 0, 727, 728, 1, 0, 0, 0, 728, 729, 1, 0, 0, 0, 729, 730, 3, 73, 29, 0, 730, 732, 1, 0, 0, 0, 731, 690, 1, 0, 0, 0, 731, 701, 1, 0, 0, 0, 731, 708, 1, 0, 0, 0, 731, 723, 1, 0, 0, 0, 732, 90, 1, 0, 0, 0, 733, 734, 7, 30, 0, 0, 734, 735, 7, 31, 0, 0, 735, 92, 1, 0, 0, 0, 736, 737, 7, 12, 0, 0, 737, 738, 7, 9, 0, 0, 738, 739, 7, 0, 0, 0, 739, 94, 1, 0, 0, 0, 740, 741, 7, 12, 0, 0, 741, 742, 7, 2, 0, 0, 742, 743, 7, 4, 0, 0, 743, 96, 1, 0, 0, 0, 744, 745, 5, 61, 0, 0, 745, 98, 1, 0, 0, 0, 746, 747, 5, 58, 0, 0, 747, 748, 5, 58, 0, 0, 748, 100, 1, 0, 0, 0, 749, 750, 5, 44, 0, 0, 750, 102, 1, 0, 0, 0, 751, 752, 7, 0, 0, 0, 752, 753, 7, 3, 0, 0, 753, 754, 7, 2, 0, 0, 754, 755, 7, 4, 0, 0, 755, 104, 1, 0, 0, 0, 756, 757, 5, 46, 0, 0, 757, 106, 1, 0, 0, 0, 758, 759, 7, 15, 0, 0, 759, 760, 7, 12, 0, 0, 760, 761, 7, 13, 0, 0, 761, 762, 7, 2, 0, 0, 762, 763, 7, 3, 0, 0, 763, 108, 1, 0, 0, 0, 764, 765, 7, 15, 0, 0, 765, 766, 7, 1, 0, 0, 766, 767, 7, 6, 0, 0, 767, 768, 7, 2, 0, 0, 768, 769, 7, 5, 0, 0, 769, 110, 1, 0, 0, 0, 770, 771, 7, 1, 0, 0, 771, 772, 7, 9, 0, 0, 772, 112, 1, 0, 0, 0, 773, 774, 7, 1, 0, 0, 774, 775, 7, 2, 0, 0, 775, 114, 1, 0, 0, 0, 776, 777, 7, 13, 0, 0, 777, 778, 7, 12, 0, 0, 778, 779, 7, 2, 0, 0, 779, 780, 7, 5, 0, 0, 780, 116, 1, 0, 0, 0, 781, 782, 7, 13, 0, 0, 782, 783, 7, 1, 0, 0, 783, 784, 7, 18, 0, 0, 784, 785, 7, 3, 0, 0, 785, 118, 1, 0, 0, 0, 786, 787, 5, 40, 0, 0, 787, 120, 1, 0, 0, 0, 788, 789, 7, 9, 0, 0, 789, 790, 7, 7, 0, 0, 790, 791, 7, 5, 0, 0, 791, 122, 1, 0, 0, 0, 792, 793, 7, 9, 0, 0, 793, 794, 7, 20, 0, 0, 794, 795, 7, 13, 0, 0, 795, 796, 7, 13, 0, 0, 796, 124, 1, 0, 0, 0, 797, 798, 7, 9, 0, 0, 798, 799, 7, 20, 0, 0, 799, 800, 7, 13, 0, 0, 800, 801, 7, 13, 0, 0, 801, 802, 7, 2, 0, 0, 802, 126, 1, 0, 0, 0, 803, 804, 7, 7, 0, 0, 804, 805, 7, 6, 0, 0, 805, 128, 1, 0, 0, 0, 806, 807, 5, 63, 0, 0, 807, 130, 1, 0, 0, 0, 808, 809, 7, 6, 0, 0, 809, 810, 7, 13, 0, 0, 810, 811, 7, 1, 0, 0, 811, 812, 7, 18, 0, 0, 812, 813, 7, 3, 0, 0, 813, 132, 1, 0, 0, 0, 814, 815, 5, 41, 0, 0, 815, 134, 1, 0, 0, 0, 816, 817, 7, 5, 0, 0, 817, 818, 7, 6, 0, 0, 818, 819, 7, 20, 0, 0, 819, 820, 7, 3, 0, 0, 820, 136, 1, 0, 0, 0, 821, 822, 5, 61, 0, 0, 822, 823, 5, 61, 0, 0, 823, 138, 1, 0, 0, 0, 824, 825, 5, 61, 0, 0, 825, 826, 5, 126, 0, 0, 826, 140, 1, 0, 0, 0, 827, 828, 5, 33, 0, 0, 828, 829, 5, 61, 0, 0, 829, 142, 1, 0, 0, 0, 830, 831, 5, 60, 0, 0, 831, 144, 1, 0, 0, 0, 832, 833, 5, 60, 0, 0, 833, 834, 5, 61, 0, 0, 834, 146, 1, 0, 0, 0, 835, 836, 5, 62, 0, 0, 836, 148, 1, 0, 0, 0, 837, 838, 5, 62, 0, 0, 838, 839, 5, 61, 0, 0, 839, 150, 1, 0, 0, 0, 840, 841, 5, 43, 0, 0, 841, 152, 1, 0, 0, 0, 842, 843, 5, 45, 0, 0, 843, 154, 1, 0, 0, 0, 844, 845, 5, 42, 0, 0, 845, 156, 1, 0, 0, 0, 846, 847, 5, 47, 0, 0, 847, 158, 1, 0, 0, 0, 848, 849, 5, 37, 0, 0, 849, 160, 1, 0, 0, 0, 850, 851, 4, 73, 4, 0, 851, 852, 3, 51, 18, 0, 852, 853, 1, 0, 0, 0, 853, 854, 6, 73, 12, 0, 854, 162, 1, 0, 0, 0, 855, 858, 3, 129, 57, 0, 856, 859, 3, 67, 26, 0, 857, 859, 3, 81, 33, 0, 858, 856, 1, 0, 0, 0, 858, 857, 1, 0, 0, 0, 859, 863, 1, 0, 0, 0, 860, 862, 3, 83, 34, 0, 861, 860, 1, 0, 0, 0, 862, 865, 1, 0, 0, 0, 863, 861, 1, 0, 0, 0, 863, 864, 1, 0, 0, 0, 864, 873, 1, 0, 0, 0, 865, 863, 1, 0, 0, 0, 866, 868, 3, 129, 57, 0, 867, 869, 3, 65, 25, 0, 868, 867, 1, 0, 0, 0, 869, 870, 1, 0, 0, 0, 870, 868, 1, 0, 0, 0, 870, 871, 1, 0, 0, 0, 871, 873, 1, 0, 0, 0, 872, 855, 1, 0, 0, 0, 872, 866, 1, 0, 0, 0, 873, 164, 1, 0, 0, 0, 874, 875, 5, 91, 0, 0, 875, 876, 1, 0, 0, 0, 876, 877, 6, 75, 0, 0, 877, 878, 6, 75, 0, 0, 878, 166, 1, 0, 0, 0, 879, 880, 5, 93, 0, 0, 880, 881, 1, 0, 0, 0, 881, 882, 6, 76, 11, 0, 882, 883, 6, 76, 11, 0, 883, 168, 1, 0, 0, 0, 884, 888, 3, 67, 26, 0, 885, 887, 3, 83, 34, 0, 886, 885, 1, 0, 0, 0, 887, 890, 1, 0, 0, 0, 888, 886, 1, 0, 0, 0, 888, 889, 1, 0, 0, 0, 889, 901, 1, 0, 0, 0, 890, 888, 1, 0, 0, 0, 891, 894, 3, 81, 33, 0, 892, 894, 3, 75, 30, 0, 893, 891, 1, 0, 0, 0, 893, 892, 1, 0, 0, 0, 894, 896, 1, 0, 0, 0, 895, 897, 3, 83, 34, 0, 896, 895, 1, 0, 0, 0, 897, 898, 1, 0, 0, 0, 898, 896, 1, 0, 0, 0, 898, 899, 1, 0, 0, 0, 899, 901, 1, 0, 0, 0, 900, 884, 1, 0, 0, 0, 900, 893, 1, 0, 0, 0, 901, 170, 1, 0, 0, 0, 902, 904, 3, 77, 31, 0, 903, 905, 3, 79, 32, 0, 904, 903, 1, 0, 0, 0, 905, 906, 1, 0, 0, 0, 906, 904, 1, 0, 0, 0, 906, 907, 1, 0, 0, 0, 907, 908, 1, 0, 0, 0, 908, 909, 3, 77, 31, 0, 909, 172, 1, 0, 0, 0, 910, 911, 3, 171, 78, 0, 911, 174, 1, 0, 0, 0, 912, 913, 3, 57, 21, 0, 913, 914, 1, 0, 0, 0, 914, 915, 6, 80, 10, 0, 915, 176, 1, 0, 0, 0, 916, 917, 3, 59, 22, 0, 917, 918, 1, 0, 0, 0, 918, 919, 6, 81, 10, 0, 919, 178, 1, 0, 0, 0, 920, 921, 3, 61, 23, 0, 921, 922, 1, 0, 0, 0, 922, 923, 6, 82, 10, 0, 923, 180, 1, 0, 0, 0, 924, 925, 3, 165, 75, 0, 925, 926, 1, 0, 0, 0, 926, 927, 6, 83, 13, 0, 927, 928, 6, 83, 14, 0, 928, 182, 1, 0, 0, 0, 929, 930, 3, 63, 24, 0, 930, 931, 1, 0, 0, 0, 931, 932, 6, 84, 15, 0, 932, 933, 6, 84, 11, 0, 933, 184, 1, 0, 0, 0, 934, 935, 3, 61, 23, 0, 935, 936, 1, 0, 0, 0, 936, 937, 6, 85, 10, 0, 937, 186, 1, 0, 0, 0, 938, 939, 3, 57, 21, 0, 939, 940, 1, 0, 0, 0, 940, 941, 6, 86, 10, 0, 941, 188, 1, 0, 0, 0, 942, 943, 3, 59, 22, 0, 943, 944, 1, 0, 0, 0, 944, 945, 6, 87, 10, 0, 945, 190, 1, 0, 0, 0, 946, 947, 3, 63, 24, 0, 947, 948, 1, 0, 0, 0, 948, 949, 6, 88, 15, 0, 949, 950, 6, 88, 11, 0, 950, 192, 1, 0, 0, 0, 951, 952, 3, 165, 75, 0, 952, 953, 1, 0, 0, 0, 953, 954, 6, 89, 13, 0, 954, 194, 1, 0, 0, 0, 955, 956, 3, 167, 76, 0, 956, 957, 1, 0, 0, 0, 957, 958, 6, 90, 16, 0, 958, 196, 1, 0, 0, 0, 959, 960, 3, 321, 153, 0, 960, 961, 1, 0, 0, 0, 961, 962, 6, 91, 17, 0, 962, 198, 1, 0, 0, 0, 963, 964, 3, 101, 43, 0, 964, 965, 1, 0, 0, 0, 965, 966, 6, 92, 18, 0, 966, 200, 1, 0, 0, 0, 967, 968, 3, 97, 41, 0, 968, 969, 1, 0, 0, 0, 969, 970, 6, 93, 19, 0, 970, 202, 1, 0, 0, 0, 971, 972, 7, 16, 0, 0, 972, 973, 7, 3, 0, 0, 973, 974, 7, 5, 0, 0, 974, 975, 7, 12, 0, 0, 975, 976, 7, 0, 0, 0, 976, 977, 7, 12, 0, 0, 977, 978, 7, 5, 0, 0, 978, 979, 7, 12, 0, 0, 979, 204, 1, 0, 0, 0, 980, 984, 8, 32, 0, 0, 981, 982, 5, 47, 0, 0, 982, 984, 8, 33, 0, 0, 983, 980, 1, 0, 0, 0, 983, 981, 1, 0, 0, 0, 984, 206, 1, 0, 0, 0, 985, 987, 3, 205, 95, 0, 986, 985, 1, 0, 0, 0, 987, 988, 1, 0, 0, 0, 988, 986, 1, 0, 0, 0, 988, 989, 1, 0, 0, 0, 989, 208, 1, 0, 0, 0, 990, 991, 3, 207, 96, 0, 991, 992, 1, 0, 0, 0, 992, 993, 6, 97, 20, 0, 993, 210, 1, 0, 0, 0, 994, 995, 3, 85, 35, 0, 995, 996, 1, 0, 0, 0, 996, 997, 6, 98, 21, 0, 997, 212, 1, 0, 0, 0, 998, 999, 3, 57, 21, 0, 999, 1000, 1, 0, 0, 0, 1000, 1001, 6, 99, 10, 0, 1001, 214, 1, 0, 0, 0, 1002, 1003, 3, 59, 22, 0, 1003, 1004, 1, 0, 0, 0, 1004, 1005, 6, 100, 10, 0, 1005, 216, 1, 0, 0, 0, 1006, 1007, 3, 61, 23, 0, 1007, 1008, 1, 0, 0, 0, 1008, 1009, 6, 101, 10, 0, 1009, 218, 1, 0, 0, 0, 1010, 1011, 3, 63, 24, 0, 1011, 1012, 1, 0, 0, 0, 1012, 1013, 6, 102, 15, 0, 1013, 1014, 6, 102, 11, 0, 1014, 220, 1, 0, 0, 0, 1015, 1016, 3, 105, 45, 0, 1016, 1017, 1, 0, 0, 0, 1017, 1018, 6, 103, 22, 0, 1018, 222, 1, 0, 0, 0, 1019, 1020, 3, 101, 43, 0, 1020, 1021, 1, 0, 0, 0, 1021, 1022, 6, 104, 18, 0, 1022, 224, 1, 0, 0, 0, 1023, 1028, 3, 67, 26, 0, 1024, 1028, 3, 65, 25, 0, 1025, 1028, 3, 81, 33, 0, 1026, 1028, 3, 155, 70, 0, 1027, 1023, 1, 0, 0, 0, 1027, 1024, 1, 0, 0, 0, 1027, 1025, 1, 0, 0, 0, 1027, 1026, 1, 0, 0, 0, 1028, 226, 1, 0, 0, 0, 1029, 1032, 3, 67, 26, 0, 1030, 1032, 3, 155, 70, 0, 1031, 1029, 1, 0, 0, 0, 1031, 1030, 1, 0, 0, 0, 1032, 1036, 1, 0, 0, 0, 1033, 1035, 3, 225, 105, 0, 1034, 1033, 1, 0, 0, 0, 1035, 1038, 1, 0, 0, 0, 1036, 1034, 1, 0, 0, 0, 1036, 1037, 1, 0, 0, 0, 1037, 1049, 1, 0, 0, 0, 1038, 1036, 1, 0, 0, 0, 1039, 1042, 3, 81, 33, 0, 1040, 1042, 3, 75, 30, 0, 1041, 1039, 1, 0, 0, 0, 1041, 1040, 1, 0, 0, 0, 1042, 1044, 1, 0, 0, 0, 1043, 1045, 3, 225, 105, 0, 1044, 1043, 1, 0, 0, 0, 1045, 1046, 1, 0, 0, 0, 1046, 1044, 1, 0, 0, 0, 1046, 1047, 1, 0, 0, 0, 1047, 1049, 1, 0, 0, 0, 1048, 1031, 1, 0, 0, 0, 1048, 1041, 1, 0, 0, 0, 1049, 228, 1, 0, 0, 0, 1050, 1053, 3, 227, 106, 0, 1051, 1053, 3, 171, 78, 0, 1052, 1050, 1, 0, 0, 0, 1052, 1051, 1, 0, 0, 0, 1053, 1054, 1, 0, 0, 0, 1054, 1052, 1, 0, 0, 0, 1054, 1055, 1, 0, 0, 0, 1055, 230, 1, 0, 0, 0, 1056, 1057, 3, 57, 21, 0, 1057, 1058, 1, 0, 0, 0, 1058, 1059, 6, 108, 10, 0, 1059, 232, 1, 0, 0, 0, 1060, 1061, 3, 59, 22, 0, 1061, 1062, 1, 0, 0, 0, 1062, 1063, 6, 109, 10, 0, 1063, 234, 1, 0, 0, 0, 1064, 1065, 3, 61, 23, 0, 1065, 1066, 1, 0, 0, 0, 1066, 1067, 6, 110, 10, 0, 1067, 236, 1, 0, 0, 0, 1068, 1069, 3, 63, 24, 0, 1069, 1070, 1, 0, 0, 0, 1070, 1071, 6, 111, 15, 0, 1071, 1072, 6, 111, 11, 0, 1072, 238, 1, 0, 0, 0, 1073, 1074, 3, 97, 41, 0, 1074, 1075, 1, 0, 0, 0, 1075, 1076, 6, 112, 19, 0, 1076, 240, 1, 0, 0, 0, 1077, 1078, 3, 101, 43, 0, 1078, 1079, 1, 0, 0, 0, 1079, 1080, 6, 113, 18, 0, 1080, 242, 1, 0, 0, 0, 1081, 1082, 3, 105, 45, 0, 1082, 1083, 1, 0, 0, 0, 1083, 1084, 6, 114, 22, 0, 1084, 244, 1, 0, 0, 0, 1085, 1086, 7, 12, 0, 0, 1086, 1087, 7, 2, 0, 0, 1087, 246, 1, 0, 0, 0, 1088, 1089, 3, 229, 107, 0, 1089, 1090, 1, 0, 0, 0, 1090, 1091, 6, 116, 23, 0, 1091, 248, 1, 0, 0, 0, 1092, 1093, 3, 57, 21, 0, 1093, 1094, 1, 0, 0, 0, 1094, 1095, 6, 117, 10, 0, 1095, 250, 1, 0, 0, 0, 1096, 1097, 3, 59, 22, 0, 1097, 1098, 1, 0, 0, 0, 1098, 1099, 6, 118, 10, 0, 1099, 252, 1, 0, 0, 0, 1100, 1101, 3, 61, 23, 0, 1101, 1102, 1, 0, 0, 0, 1102, 1103, 6, 119, 10, 0, 1103, 254, 1, 0, 0, 0, 1104, 1105, 3, 63, 24, 0, 1105, 1106, 1, 0, 0, 0, 1106, 1107, 6, 120, 15, 0, 1107, 1108, 6, 120, 11, 0, 1108, 256, 1, 0, 0, 0, 1109, 1110, 3, 165, 75, 0, 1110, 1111, 1, 0, 0, 0, 1111, 1112, 6, 121, 13, 0, 1112, 1113, 6, 121, 24, 0, 1113, 258, 1, 0, 0, 0, 1114, 1115, 7, 7, 0, 0, 1115, 1116, 7, 9, 0, 0, 1116, 1117, 1, 0, 0, 0, 1117, 1118, 6, 122, 25, 0, 1118, 260, 1, 0, 0, 0, 1119, 1120, 7, 19, 0, 0, 1120, 1121, 7, 1, 0, 0, 1121, 1122, 7, 5, 0, 0, 1122, 1123, 7, 10, 0, 0, 1123, 1124, 1, 0, 0, 0, 1124, 1125, 6, 123, 25, 0, 1125, 262, 1, 0, 0, 0, 1126, 1127, 8, 34, 0, 0, 1127, 264, 1, 0, 0, 0, 1128, 1130, 3, 263, 124, 0, 1129, 1128, 1, 0, 0, 0, 1130, 1131, 1, 0, 0, 0, 1131, 1129, 1, 0, 0, 0, 1131, 1132, 1, 0, 0, 0, 1132, 1133, 1, 0, 0, 0, 1133, 1134, 3, 321, 153, 0, 1134, 1136, 1, 0, 0, 0, 1135, 1129, 1, 0, 0, 0, 1135, 1136, 1, 0, 0, 0, 1136, 1138, 1, 0, 0, 0, 1137, 1139, 3, 263, 124, 0, 1138, 1137, 1, 0, 0, 0, 1139, 1140, 1, 0, 0, 0, 1140, 1138, 1, 0, 0, 0, 1140, 1141, 1, 0, 0, 0, 1141, 266, 1, 0, 0, 0, 1142, 1143, 3, 265, 125, 0, 1143, 1144, 1, 0, 0, 0, 1144, 1145, 6, 126, 26, 0, 1145, 268, 1, 0, 0, 0, 1146, 1147, 3, 57, 21, 0, 1147, 1148, 1, 0, 0, 0, 1148, 1149, 6, 127, 10, 0, 1149, 270, 1, 0, 0, 0, 1150, 1151, 3, 59, 22, 0, 1151, 1152, 1, 0, 0, 0, 1152, 1153, 6, 128, 10, 0, 1153, 272, 1, 0, 0, 0, 1154, 1155, 3, 61, 23, 0, 1155, 1156, 1, 0, 0, 0, 1156, 1157, 6, 129, 10, 0, 1157, 274, 1, 0, 0, 0, 1158, 1159, 3, 63, 24, 0, 1159, 1160, 1, 0, 0, 0, 1160, 1161, 6, 130, 15, 0, 1161, 1162, 6, 130, 11, 0, 1162, 1163, 6, 130, 11, 0, 1163, 276, 1, 0, 0, 0, 1164, 1165, 3, 97, 41, 0, 1165, 1166, 1, 0, 0, 0, 1166, 1167, 6, 131, 19, 0, 1167, 278, 1, 0, 0, 0, 1168, 1169, 3, 101, 43, 0, 1169, 1170, 1, 0, 0, 0, 1170, 1171, 6, 132, 18, 0, 1171, 280, 1, 0, 0, 0, 1172, 1173, 3, 105, 45, 0, 1173, 1174, 1, 0, 0, 0, 1174, 1175, 6, 133, 22, 0, 1175, 282, 1, 0, 0, 0, 1176, 1177, 3, 261, 123, 0, 1177, 1178, 1, 0, 0, 0, 1178, 1179, 6, 134, 27, 0, 1179, 284, 1, 0, 0, 0, 1180, 1181, 3, 229, 107, 0, 1181, 1182, 1, 0, 0, 0, 1182, 1183, 6, 135, 23, 0, 1183, 286, 1, 0, 0, 0, 1184, 1185, 3, 173, 79, 0, 1185, 1186, 1, 0, 0, 0, 1186, 1187, 6, 136, 28, 0, 1187, 288, 1, 0, 0, 0, 1188, 1189, 3, 57, 21, 0, 1189, 1190, 1, 0, 0, 0, 1190, 1191, 6, 137, 10, 0, 1191, 290, 1, 0, 0, 0, 1192, 1193, 3, 59, 22, 0, 1193, 1194, 1, 0, 0, 0, 1194, 1195, 6, 138, 10, 0, 1195, 292, 1, 0, 0, 0, 1196, 1197, 3, 61, 23, 0, 1197, 1198, 1, 0, 0, 0, 1198, 1199, 6, 139, 10, 0, 1199, 294, 1, 0, 0, 0, 1200, 1201, 3, 63, 24, 0, 1201, 1202, 1, 0, 0, 0, 1202, 1203, 6, 140, 15, 0, 1203, 1204, 6, 140, 11, 0, 1204, 296, 1, 0, 0, 0, 1205, 1206, 3, 105, 45, 0, 1206, 1207, 1, 0, 0, 0, 1207, 1208, 6, 141, 22, 0, 1208, 298, 1, 0, 0, 0, 1209, 1210, 3, 173, 79, 0, 1210, 1211, 1, 0, 0, 0, 1211, 1212, 6, 142, 28, 0, 1212, 300, 1, 0, 0, 0, 1213, 1214, 3, 169, 77, 0, 1214, 1215, 1, 0, 0, 0, 1215, 1216, 6, 143, 29, 0, 1216, 302, 1, 0, 0, 0, 1217, 1218, 3, 57, 21, 0, 1218, 1219, 1, 0, 0, 0, 1219, 1220, 6, 144, 10, 0, 1220, 304, 1, 0, 0, 0, 1221, 1222, 3, 59, 22, 0, 1222, 1223, 1, 0, 0, 0, 1223, 1224, 6, 145, 10, 0, 1224, 306, 1, 0, 0, 0, 1225, 1226, 3, 61, 23, 0, 1226, 1227, 1, 0, 0, 0, 1227, 1228, 6, 146, 10, 0, 1228, 308, 1, 0, 0, 0, 1229, 1230, 3, 63, 24, 0, 1230, 1231, 1, 0, 0, 0, 1231, 1232, 6, 147, 15, 0, 1232, 1233, 6, 147, 11, 0, 1233, 310, 1, 0, 0, 0, 1234, 1235, 7, 1, 0, 0, 1235, 1236, 7, 9, 0, 0, 1236, 1237, 7, 15, 0, 0, 1237, 1238, 7, 7, 0, 0, 1238, 312, 1, 0, 0, 0, 1239, 1240, 3, 57, 21, 0, 1240, 1241, 1, 0, 0, 0, 1241, 1242, 6, 149, 10, 0, 1242, 314, 1, 0, 0, 0, 1243, 1244, 3, 59, 22, 0, 1244, 1245, 1, 0, 0, 0, 1245, 1246, 6, 150, 10, 0, 1246, 316, 1, 0, 0, 0, 1247, 1248, 3, 61, 23, 0, 1248, 1249, 1, 0, 0, 0, 1249, 1250, 6, 151, 10, 0, 1250, 318, 1, 0, 0, 0, 1251, 1252, 3, 167, 76, 0, 1252, 1253, 1, 0, 0, 0, 1253, 1254, 6, 152, 16, 0, 1254, 1255, 6, 152, 11, 0, 1255, 320, 1, 0, 0, 0, 1256, 1257, 5, 58, 0, 0, 1257, 322, 1, 0, 0, 0, 1258, 1264, 3, 75, 30, 0, 1259, 1264, 3, 65, 25, 0, 1260, 1264, 3, 105, 45, 0, 1261, 1264, 3, 67, 26, 0, 1262, 1264, 3, 81, 33, 0, 1263, 1258, 1, 0, 0, 0, 1263, 1259, 1, 0, 0, 0, 1263, 1260, 1, 0, 0, 0, 1263, 1261, 1, 0, 0, 0, 1263, 1262, 1, 0, 0, 0, 1264, 1265, 1, 0, 0, 0, 1265, 1263, 1, 0, 0, 0, 1265, 1266, 1, 0, 0, 0, 1266, 324, 1, 0, 0, 0, 1267, 1268, 3, 57, 21, 0, 1268, 1269, 1, 0, 0, 0, 1269, 1270, 6, 155, 10, 0, 1270, 326, 1, 0, 0, 0, 1271, 1272, 3, 59, 22, 0, 1272, 1273, 1, 0, 0, 0, 1273, 1274, 6, 156, 10, 0, 1274, 328, 1, 0, 0, 0, 1275, 1276, 3, 61, 23, 0, 1276, 1277, 1, 0, 0, 0, 1277, 1278, 6, 157, 10, 0, 1278, 330, 1, 0, 0, 0, 1279, 1280, 3, 63, 24, 0, 1280, 1281, 1, 0, 0, 0, 1281, 1282, 6, 158, 15, 0, 1282, 1283, 6, 158, 11, 0, 1283, 332, 1, 0, 0, 0, 1284, 1285, 3, 321, 153, 0, 1285, 1286, 1, 0, 0, 0, 1286, 1287, 6, 159, 17, 0, 1287, 334, 1, 0, 0, 0, 1288, 1289, 3, 101, 43, 0, 1289, 1290, 1, 0, 0, 0, 1290, 1291, 6, 160, 18, 0, 1291, 336, 1, 0, 0, 0, 1292, 1293, 3, 105, 45, 0, 1293, 1294, 1, 0, 0, 0, 1294, 1295, 6, 161, 22, 0, 1295, 338, 1, 0, 0, 0, 1296, 1297, 3, 259, 122, 0, 1297, 1298, 1, 0, 0, 0, 1298, 1299, 6, 162, 30, 0, 1299, 1300, 6, 162, 31, 0, 1300, 340, 1, 0, 0, 0, 1301, 1302, 3, 207, 96, 0, 1302, 1303, 1, 0, 0, 0, 1303, 1304, 6, 163, 20, 0, 1304, 342, 1, 0, 0, 0, 1305, 1306, 3, 85, 35, 0, 1306, 1307, 1, 0, 0, 0, 1307, 1308, 6, 164, 21, 0, 1308, 344, 1, 0, 0, 0, 1309, 1310, 3, 57, 21, 0, 1310, 1311, 1, 0, 0, 0, 1311, 1312, 6, 165, 10, 0, 1312, 346, 1, 0, 0, 0, 1313, 1314, 3, 59, 22, 0, 1314, 1315, 1, 0, 0, 0, 1315, 1316, 6, 166, 10, 0, 1316, 348, 1, 0, 0, 0, 1317, 1318, 3, 61, 23, 0, 1318, 1319, 1, 0, 0, 0, 1319, 1320, 6, 167, 10, 0, 1320, 350, 1, 0, 0, 0, 1321, 1322, 3, 63, 24, 0, 1322, 1323, 1, 0, 0, 0, 1323, 1324, 6, 168, 15, 0, 1324, 1325, 6, 168, 11, 0, 1325, 1326, 6, 168, 11, 0, 1326, 352, 1, 0, 0, 0, 1327, 1328, 3, 101, 43, 0, 1328, 1329, 1, 0, 0, 0, 1329, 1330, 6, 169, 18, 0, 1330, 354, 1, 0, 0, 0, 1331, 1332, 3, 105, 45, 0, 1332, 1333, 1, 0, 0, 0, 1333, 1334, 6, 170, 22, 0, 1334, 356, 1, 0, 0, 0, 1335, 1336, 3, 229, 107, 0, 1336, 1337, 1, 0, 0, 0, 1337, 1338, 6, 171, 23, 0, 1338, 358, 1, 0, 0, 0, 1339, 1340, 3, 57, 21, 0, 1340, 1341, 1, 0, 0, 0, 1341, 1342, 6, 172, 10, 0, 1342, 360, 1, 0, 0, 0, 1343, 1344, 3, 59, 22, 0, 1344, 1345, 1, 0, 0, 0, 1345, 1346, 6, 173, 10, 0, 1346, 362, 1, 0, 0, 0, 1347, 1348, 3, 61, 23, 0, 1348, 1349, 1, 0, 0, 0, 1349, 1350, 6, 174, 10, 0, 1350, 364, 1, 0, 0, 0, 1351, 1352, 3, 63, 24, 0, 1352, 1353, 1, 0, 0, 0, 1353, 1354, 6, 175, 15, 0, 1354, 1355, 6, 175, 11, 0, 1355, 366, 1, 0, 0, 0, 1356, 1357, 3, 207, 96, 0, 1357, 1358, 1, 0, 0, 0, 1358, 1359, 6, 176, 20, 0, 1359, 1360, 6, 176, 11, 0, 1360, 1361, 6, 176, 32, 0, 1361, 368, 1, 0, 0, 0, 1362, 1363, 3, 85, 35, 0, 1363, 1364, 1, 0, 0, 0, 1364, 1365, 6, 177, 21, 0, 1365, 1366, 6, 177, 11, 0, 1366, 1367, 6, 177, 32, 0, 1367, 370, 1, 0, 0, 0, 1368, 1369, 3, 57, 21, 0, 1369, 1370, 1, 0, 0, 0, 1370, 1371, 6, 178, 10, 0, 1371, 372, 1, 0, 0, 0, 1372, 1373, 3, 59, 22, 0, 1373, 1374, 1, 0, 0, 0, 1374, 1375, 6, 179, 10, 0, 1375, 374, 1, 0, 0, 0, 1376, 1377, 3, 61, 23, 0, 1377, 1378, 1, 0, 0, 0, 1378, 1379, 6, 180, 10, 0, 1379, 376, 1, 0, 0, 0, 1380, 1381, 3, 321, 153, 0, 1381, 1382, 1, 0, 0, 0, 1382, 1383, 6, 181, 17, 0, 1383, 1384, 6, 181, 11, 0, 1384, 1385, 6, 181, 9, 0, 1385, 378, 1, 0, 0, 0, 1386, 1387, 3, 101, 43, 0, 1387, 1388, 1, 0, 0, 0, 1388, 1389, 6, 182, 18, 0, 1389, 1390, 6, 182, 11, 0, 1390, 1391, 6, 182, 9, 0, 1391, 380, 1, 0, 0, 0, 1392, 1393, 3, 57, 21, 0, 1393, 1394, 1, 0, 0, 0, 1394, 1395, 6, 183, 10, 0, 1395, 382, 1, 0, 0, 0, 1396, 1397, 3, 59, 22, 0, 1397, 1398, 1, 0, 0, 0, 1398, 1399, 6, 184, 10, 0, 1399, 384, 1, 0, 0, 0, 1400, 1401, 3, 61, 23, 0, 1401, 1402, 1, 0, 0, 0, 1402, 1403, 6, 185, 10, 0, 1403, 386, 1, 0, 0, 0, 1404, 1405, 3, 173, 79, 0, 1405, 1406, 1, 0, 0, 0, 1406, 1407, 6, 186, 11, 0, 1407, 1408, 6, 186, 0, 0, 1408, 1409, 6, 186, 28, 0, 1409, 388, 1, 0, 0, 0, 1410, 1411, 3, 169, 77, 0, 1411, 1412, 1, 0, 0, 0, 1412, 1413, 6, 187, 11, 0, 1413, 1414, 6, 187, 0, 0, 1414, 1415, 6, 187, 29, 0, 1415, 390, 1, 0, 0, 0, 1416, 1417, 3, 91, 38, 0, 1417, 1418, 1, 0, 0, 0, 1418, 1419, 6, 188, 11, 0, 1419, 1420, 6, 188, 0, 0, 1420, 1421, 6, 188, 33, 0, 1421, 392, 1, 0, 0, 0, 1422, 1423, 3, 63, 24, 0, 1423, 1424, 1, 0, 0, 0, 1424, 1425, 6, 189, 15, 0, 1425, 1426, 6, 189, 11, 0, 1426, 394, 1, 0, 0, 0, 65, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 572, 582, 586, 589, 598, 600, 611, 630, 635, 644, 651, 656, 658, 669, 677, 680, 682, 687, 692, 698, 705, 710, 716, 719, 727, 731, 858, 863, 870, 872, 888, 893, 898, 900, 906, 983, 988, 1027, 1031, 1036, 1041, 1046, 1048, 1052, 1054, 1131, 1135, 1140, 1263, 1265, 34, 5, 1, 0, 5, 4, 0, 5, 6, 0, 5, 2, 0, 5, 3, 0, 5, 8, 0, 5, 5, 0, 5, 9, 0, 5, 11, 0, 5, 13, 0, 0, 1, 0, 4, 0, 0, 7, 19, 0, 7, 65, 0, 5, 0, 0, 7, 25, 0, 7, 66, 0, 7, 104, 0, 7, 34, 0, 7, 32, 0, 7, 76, 0, 7, 26, 0, 7, 36, 0, 7, 80, 0, 5, 10, 0, 5, 7, 0, 7, 90, 0, 7, 89, 0, 7, 68, 0, 7, 67, 0, 7, 88, 0, 5, 12, 0, 5, 14, 0, 7, 29, 0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java index a746a0d49004f..d3ad1d00d749e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java @@ -26,36 +26,34 @@ public class EsqlBaseLexer extends LexerConfig { new PredictionContextCache(); public static final int DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, KEEP=8, - LIMIT=9, META=10, MV_EXPAND=11, RENAME=12, ROW=13, SHOW=14, SORT=15, STATS=16, - WHERE=17, DEV_INLINESTATS=18, DEV_LOOKUP=19, DEV_MATCH=20, DEV_METRICS=21, - UNKNOWN_CMD=22, LINE_COMMENT=23, MULTILINE_COMMENT=24, WS=25, PIPE=26, - QUOTED_STRING=27, INTEGER_LITERAL=28, DECIMAL_LITERAL=29, BY=30, AND=31, - ASC=32, ASSIGN=33, CAST_OP=34, COMMA=35, DESC=36, DOT=37, FALSE=38, FIRST=39, - IN=40, IS=41, LAST=42, LIKE=43, LP=44, NOT=45, NULL=46, NULLS=47, OR=48, - PARAM=49, RLIKE=50, RP=51, TRUE=52, EQ=53, CIEQ=54, NEQ=55, LT=56, LTE=57, - GT=58, GTE=59, PLUS=60, MINUS=61, ASTERISK=62, SLASH=63, PERCENT=64, NAMED_OR_POSITIONAL_PARAM=65, - OPENING_BRACKET=66, CLOSING_BRACKET=67, UNQUOTED_IDENTIFIER=68, QUOTED_IDENTIFIER=69, - EXPR_LINE_COMMENT=70, EXPR_MULTILINE_COMMENT=71, EXPR_WS=72, EXPLAIN_WS=73, - EXPLAIN_LINE_COMMENT=74, EXPLAIN_MULTILINE_COMMENT=75, METADATA=76, UNQUOTED_SOURCE=77, - FROM_LINE_COMMENT=78, FROM_MULTILINE_COMMENT=79, FROM_WS=80, ID_PATTERN=81, - PROJECT_LINE_COMMENT=82, PROJECT_MULTILINE_COMMENT=83, PROJECT_WS=84, - AS=85, RENAME_LINE_COMMENT=86, RENAME_MULTILINE_COMMENT=87, RENAME_WS=88, - ON=89, WITH=90, ENRICH_POLICY_NAME=91, ENRICH_LINE_COMMENT=92, ENRICH_MULTILINE_COMMENT=93, - ENRICH_WS=94, ENRICH_FIELD_LINE_COMMENT=95, ENRICH_FIELD_MULTILINE_COMMENT=96, - ENRICH_FIELD_WS=97, MVEXPAND_LINE_COMMENT=98, MVEXPAND_MULTILINE_COMMENT=99, - MVEXPAND_WS=100, INFO=101, SHOW_LINE_COMMENT=102, SHOW_MULTILINE_COMMENT=103, - SHOW_WS=104, FUNCTIONS=105, META_LINE_COMMENT=106, META_MULTILINE_COMMENT=107, - META_WS=108, COLON=109, SETTING=110, SETTING_LINE_COMMENT=111, SETTTING_MULTILINE_COMMENT=112, - SETTING_WS=113, LOOKUP_LINE_COMMENT=114, LOOKUP_MULTILINE_COMMENT=115, - LOOKUP_WS=116, LOOKUP_FIELD_LINE_COMMENT=117, LOOKUP_FIELD_MULTILINE_COMMENT=118, - LOOKUP_FIELD_WS=119, METRICS_LINE_COMMENT=120, METRICS_MULTILINE_COMMENT=121, - METRICS_WS=122, CLOSING_METRICS_LINE_COMMENT=123, CLOSING_METRICS_MULTILINE_COMMENT=124, - CLOSING_METRICS_WS=125; + LIMIT=9, MV_EXPAND=10, RENAME=11, ROW=12, SHOW=13, SORT=14, STATS=15, + WHERE=16, DEV_INLINESTATS=17, DEV_LOOKUP=18, DEV_MATCH=19, DEV_METRICS=20, + UNKNOWN_CMD=21, LINE_COMMENT=22, MULTILINE_COMMENT=23, WS=24, PIPE=25, + QUOTED_STRING=26, INTEGER_LITERAL=27, DECIMAL_LITERAL=28, BY=29, AND=30, + ASC=31, ASSIGN=32, CAST_OP=33, COMMA=34, DESC=35, DOT=36, FALSE=37, FIRST=38, + IN=39, IS=40, LAST=41, LIKE=42, LP=43, NOT=44, NULL=45, NULLS=46, OR=47, + PARAM=48, RLIKE=49, RP=50, TRUE=51, EQ=52, CIEQ=53, NEQ=54, LT=55, LTE=56, + GT=57, GTE=58, PLUS=59, MINUS=60, ASTERISK=61, SLASH=62, PERCENT=63, NAMED_OR_POSITIONAL_PARAM=64, + OPENING_BRACKET=65, CLOSING_BRACKET=66, UNQUOTED_IDENTIFIER=67, QUOTED_IDENTIFIER=68, + EXPR_LINE_COMMENT=69, EXPR_MULTILINE_COMMENT=70, EXPR_WS=71, EXPLAIN_WS=72, + EXPLAIN_LINE_COMMENT=73, EXPLAIN_MULTILINE_COMMENT=74, METADATA=75, UNQUOTED_SOURCE=76, + FROM_LINE_COMMENT=77, FROM_MULTILINE_COMMENT=78, FROM_WS=79, ID_PATTERN=80, + PROJECT_LINE_COMMENT=81, PROJECT_MULTILINE_COMMENT=82, PROJECT_WS=83, + AS=84, RENAME_LINE_COMMENT=85, RENAME_MULTILINE_COMMENT=86, RENAME_WS=87, + ON=88, WITH=89, ENRICH_POLICY_NAME=90, ENRICH_LINE_COMMENT=91, ENRICH_MULTILINE_COMMENT=92, + ENRICH_WS=93, ENRICH_FIELD_LINE_COMMENT=94, ENRICH_FIELD_MULTILINE_COMMENT=95, + ENRICH_FIELD_WS=96, MVEXPAND_LINE_COMMENT=97, MVEXPAND_MULTILINE_COMMENT=98, + MVEXPAND_WS=99, INFO=100, SHOW_LINE_COMMENT=101, SHOW_MULTILINE_COMMENT=102, + SHOW_WS=103, COLON=104, SETTING=105, SETTING_LINE_COMMENT=106, SETTTING_MULTILINE_COMMENT=107, + SETTING_WS=108, LOOKUP_LINE_COMMENT=109, LOOKUP_MULTILINE_COMMENT=110, + LOOKUP_WS=111, LOOKUP_FIELD_LINE_COMMENT=112, LOOKUP_FIELD_MULTILINE_COMMENT=113, + LOOKUP_FIELD_WS=114, METRICS_LINE_COMMENT=115, METRICS_MULTILINE_COMMENT=116, + METRICS_WS=117, CLOSING_METRICS_LINE_COMMENT=118, CLOSING_METRICS_MULTILINE_COMMENT=119, + CLOSING_METRICS_WS=120; public static final int EXPRESSION_MODE=1, EXPLAIN_MODE=2, FROM_MODE=3, PROJECT_MODE=4, RENAME_MODE=5, - ENRICH_MODE=6, ENRICH_FIELD_MODE=7, MVEXPAND_MODE=8, SHOW_MODE=9, META_MODE=10, - SETTING_MODE=11, LOOKUP_MODE=12, LOOKUP_FIELD_MODE=13, METRICS_MODE=14, - CLOSING_METRICS_MODE=15; + ENRICH_MODE=6, ENRICH_FIELD_MODE=7, MVEXPAND_MODE=8, SHOW_MODE=9, SETTING_MODE=10, + LOOKUP_MODE=11, LOOKUP_FIELD_MODE=12, METRICS_MODE=13, CLOSING_METRICS_MODE=14; public static String[] channelNames = { "DEFAULT_TOKEN_CHANNEL", "HIDDEN" }; @@ -63,18 +61,17 @@ public class EsqlBaseLexer extends LexerConfig { public static String[] modeNames = { "DEFAULT_MODE", "EXPRESSION_MODE", "EXPLAIN_MODE", "FROM_MODE", "PROJECT_MODE", "RENAME_MODE", "ENRICH_MODE", "ENRICH_FIELD_MODE", "MVEXPAND_MODE", "SHOW_MODE", - "META_MODE", "SETTING_MODE", "LOOKUP_MODE", "LOOKUP_FIELD_MODE", "METRICS_MODE", - "CLOSING_METRICS_MODE" + "SETTING_MODE", "LOOKUP_MODE", "LOOKUP_FIELD_MODE", "METRICS_MODE", "CLOSING_METRICS_MODE" }; private static String[] makeRuleNames() { return new String[] { "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", "KEEP", - "LIMIT", "META", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", - "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_MATCH", "DEV_METRICS", - "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "DIGIT", - "LETTER", "ESCAPE_SEQUENCE", "UNESCAPED_CHARS", "EXPONENT", "ASPERAND", - "BACKQUOTE", "BACKQUOTE_BLOCK", "UNDERSCORE", "UNQUOTED_ID_BODY", "QUOTED_STRING", + "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", "WHERE", + "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_MATCH", "DEV_METRICS", "UNKNOWN_CMD", + "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "DIGIT", "LETTER", + "ESCAPE_SEQUENCE", "UNESCAPED_CHARS", "EXPONENT", "ASPERAND", "BACKQUOTE", + "BACKQUOTE_BLOCK", "UNDERSCORE", "UNQUOTED_ID_BODY", "QUOTED_STRING", "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", "LP", "NOT", "NULL", "NULLS", "OR", "PARAM", "RLIKE", "RP", "TRUE", "EQ", @@ -98,8 +95,7 @@ private static String[] makeRuleNames() { "ENRICH_FIELD_WS", "MVEXPAND_PIPE", "MVEXPAND_DOT", "MVEXPAND_QUOTED_IDENTIFIER", "MVEXPAND_UNQUOTED_IDENTIFIER", "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", "MVEXPAND_WS", "SHOW_PIPE", "INFO", "SHOW_LINE_COMMENT", "SHOW_MULTILINE_COMMENT", - "SHOW_WS", "META_PIPE", "FUNCTIONS", "META_LINE_COMMENT", "META_MULTILINE_COMMENT", - "META_WS", "SETTING_CLOSING_BRACKET", "COLON", "SETTING", "SETTING_LINE_COMMENT", + "SHOW_WS", "SETTING_CLOSING_BRACKET", "COLON", "SETTING", "SETTING_LINE_COMMENT", "SETTTING_MULTILINE_COMMENT", "SETTING_WS", "LOOKUP_PIPE", "LOOKUP_COLON", "LOOKUP_COMMA", "LOOKUP_DOT", "LOOKUP_ON", "LOOKUP_UNQUOTED_SOURCE", "LOOKUP_QUOTED_SOURCE", "LOOKUP_LINE_COMMENT", "LOOKUP_MULTILINE_COMMENT", @@ -117,25 +113,25 @@ private static String[] makeRuleNames() { private static String[] makeLiteralNames() { return new String[] { null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", - "'grok'", "'keep'", "'limit'", "'meta'", "'mv_expand'", "'rename'", "'row'", - "'show'", "'sort'", "'stats'", "'where'", null, null, null, null, null, - null, null, null, "'|'", null, null, null, "'by'", "'and'", "'asc'", - "'='", "'::'", "','", "'desc'", "'.'", "'false'", "'first'", "'in'", - "'is'", "'last'", "'like'", "'('", "'not'", "'null'", "'nulls'", "'or'", - "'?'", "'rlike'", "')'", "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", - "'>'", "'>='", "'+'", "'-'", "'*'", "'/'", "'%'", null, null, "']'", - null, null, null, null, null, null, null, null, "'metadata'", null, null, - null, null, null, null, null, null, "'as'", null, null, null, "'on'", - "'with'", null, null, null, null, null, null, null, null, null, null, - "'info'", null, null, null, "'functions'", null, null, null, "':'" + "'grok'", "'keep'", "'limit'", "'mv_expand'", "'rename'", "'row'", "'show'", + "'sort'", "'stats'", "'where'", null, null, null, null, null, null, null, + null, "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", + "','", "'desc'", "'.'", "'false'", "'first'", "'in'", "'is'", "'last'", + "'like'", "'('", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", + "')'", "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", + "'+'", "'-'", "'*'", "'/'", "'%'", null, null, "']'", null, null, null, + null, null, null, null, null, "'metadata'", null, null, null, null, null, + null, null, null, "'as'", null, null, null, "'on'", "'with'", null, null, + null, null, null, null, null, null, null, null, "'info'", null, null, + null, "':'" }; } private static final String[] _LITERAL_NAMES = makeLiteralNames(); private static String[] makeSymbolicNames() { return new String[] { null, "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", - "KEEP", "LIMIT", "META", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", - "STATS", "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_MATCH", "DEV_METRICS", + "KEEP", "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", + "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_MATCH", "DEV_METRICS", "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", @@ -151,8 +147,7 @@ private static String[] makeSymbolicNames() { "ENRICH_MULTILINE_COMMENT", "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", "ENRICH_FIELD_MULTILINE_COMMENT", "ENRICH_FIELD_WS", "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", "MVEXPAND_WS", "INFO", "SHOW_LINE_COMMENT", - "SHOW_MULTILINE_COMMENT", "SHOW_WS", "FUNCTIONS", "META_LINE_COMMENT", - "META_MULTILINE_COMMENT", "META_WS", "COLON", "SETTING", "SETTING_LINE_COMMENT", + "SHOW_MULTILINE_COMMENT", "SHOW_WS", "COLON", "SETTING", "SETTING_LINE_COMMENT", "SETTTING_MULTILINE_COMMENT", "SETTING_WS", "LOOKUP_LINE_COMMENT", "LOOKUP_MULTILINE_COMMENT", "LOOKUP_WS", "LOOKUP_FIELD_LINE_COMMENT", "LOOKUP_FIELD_MULTILINE_COMMENT", "LOOKUP_FIELD_WS", "METRICS_LINE_COMMENT", "METRICS_MULTILINE_COMMENT", @@ -222,15 +217,15 @@ public EsqlBaseLexer(CharStream input) { @Override public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { switch (ruleIndex) { - case 17: + case 16: return DEV_INLINESTATS_sempred((RuleContext)_localctx, predIndex); - case 18: + case 17: return DEV_LOOKUP_sempred((RuleContext)_localctx, predIndex); - case 19: + case 18: return DEV_MATCH_sempred((RuleContext)_localctx, predIndex); - case 20: + case 19: return DEV_METRICS_sempred((RuleContext)_localctx, predIndex); - case 74: + case 73: return DEV_MATCH_OP_sempred((RuleContext)_localctx, predIndex); } return true; @@ -272,963 +267,931 @@ private boolean DEV_MATCH_OP_sempred(RuleContext _localctx, int predIndex) { } public static final String _serializedATN = - "\u0004\u0000}\u05c2\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ + "\u0004\u0000x\u0593\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ "\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff\u0006\uffff\uffff"+ - "\u0006\uffff\uffff\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+ - "\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002"+ - "\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007\u0002"+ - "\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b\u0002"+ - "\f\u0007\f\u0002\r\u0007\r\u0002\u000e\u0007\u000e\u0002\u000f\u0007\u000f"+ - "\u0002\u0010\u0007\u0010\u0002\u0011\u0007\u0011\u0002\u0012\u0007\u0012"+ - "\u0002\u0013\u0007\u0013\u0002\u0014\u0007\u0014\u0002\u0015\u0007\u0015"+ - "\u0002\u0016\u0007\u0016\u0002\u0017\u0007\u0017\u0002\u0018\u0007\u0018"+ - "\u0002\u0019\u0007\u0019\u0002\u001a\u0007\u001a\u0002\u001b\u0007\u001b"+ - "\u0002\u001c\u0007\u001c\u0002\u001d\u0007\u001d\u0002\u001e\u0007\u001e"+ - "\u0002\u001f\u0007\u001f\u0002 \u0007 \u0002!\u0007!\u0002\"\u0007\"\u0002"+ - "#\u0007#\u0002$\u0007$\u0002%\u0007%\u0002&\u0007&\u0002\'\u0007\'\u0002"+ - "(\u0007(\u0002)\u0007)\u0002*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002"+ - "-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u00021\u00071\u0002"+ - "2\u00072\u00023\u00073\u00024\u00074\u00025\u00075\u00026\u00076\u0002"+ - "7\u00077\u00028\u00078\u00029\u00079\u0002:\u0007:\u0002;\u0007;\u0002"+ - "<\u0007<\u0002=\u0007=\u0002>\u0007>\u0002?\u0007?\u0002@\u0007@\u0002"+ - "A\u0007A\u0002B\u0007B\u0002C\u0007C\u0002D\u0007D\u0002E\u0007E\u0002"+ - "F\u0007F\u0002G\u0007G\u0002H\u0007H\u0002I\u0007I\u0002J\u0007J\u0002"+ - "K\u0007K\u0002L\u0007L\u0002M\u0007M\u0002N\u0007N\u0002O\u0007O\u0002"+ - "P\u0007P\u0002Q\u0007Q\u0002R\u0007R\u0002S\u0007S\u0002T\u0007T\u0002"+ - "U\u0007U\u0002V\u0007V\u0002W\u0007W\u0002X\u0007X\u0002Y\u0007Y\u0002"+ - "Z\u0007Z\u0002[\u0007[\u0002\\\u0007\\\u0002]\u0007]\u0002^\u0007^\u0002"+ - "_\u0007_\u0002`\u0007`\u0002a\u0007a\u0002b\u0007b\u0002c\u0007c\u0002"+ - "d\u0007d\u0002e\u0007e\u0002f\u0007f\u0002g\u0007g\u0002h\u0007h\u0002"+ - "i\u0007i\u0002j\u0007j\u0002k\u0007k\u0002l\u0007l\u0002m\u0007m\u0002"+ - "n\u0007n\u0002o\u0007o\u0002p\u0007p\u0002q\u0007q\u0002r\u0007r\u0002"+ - "s\u0007s\u0002t\u0007t\u0002u\u0007u\u0002v\u0007v\u0002w\u0007w\u0002"+ - "x\u0007x\u0002y\u0007y\u0002z\u0007z\u0002{\u0007{\u0002|\u0007|\u0002"+ - "}\u0007}\u0002~\u0007~\u0002\u007f\u0007\u007f\u0002\u0080\u0007\u0080"+ - "\u0002\u0081\u0007\u0081\u0002\u0082\u0007\u0082\u0002\u0083\u0007\u0083"+ - "\u0002\u0084\u0007\u0084\u0002\u0085\u0007\u0085\u0002\u0086\u0007\u0086"+ - "\u0002\u0087\u0007\u0087\u0002\u0088\u0007\u0088\u0002\u0089\u0007\u0089"+ - "\u0002\u008a\u0007\u008a\u0002\u008b\u0007\u008b\u0002\u008c\u0007\u008c"+ - "\u0002\u008d\u0007\u008d\u0002\u008e\u0007\u008e\u0002\u008f\u0007\u008f"+ - "\u0002\u0090\u0007\u0090\u0002\u0091\u0007\u0091\u0002\u0092\u0007\u0092"+ - "\u0002\u0093\u0007\u0093\u0002\u0094\u0007\u0094\u0002\u0095\u0007\u0095"+ - "\u0002\u0096\u0007\u0096\u0002\u0097\u0007\u0097\u0002\u0098\u0007\u0098"+ - "\u0002\u0099\u0007\u0099\u0002\u009a\u0007\u009a\u0002\u009b\u0007\u009b"+ - "\u0002\u009c\u0007\u009c\u0002\u009d\u0007\u009d\u0002\u009e\u0007\u009e"+ - "\u0002\u009f\u0007\u009f\u0002\u00a0\u0007\u00a0\u0002\u00a1\u0007\u00a1"+ - "\u0002\u00a2\u0007\u00a2\u0002\u00a3\u0007\u00a3\u0002\u00a4\u0007\u00a4"+ - "\u0002\u00a5\u0007\u00a5\u0002\u00a6\u0007\u00a6\u0002\u00a7\u0007\u00a7"+ - "\u0002\u00a8\u0007\u00a8\u0002\u00a9\u0007\u00a9\u0002\u00aa\u0007\u00aa"+ - "\u0002\u00ab\u0007\u00ab\u0002\u00ac\u0007\u00ac\u0002\u00ad\u0007\u00ad"+ - "\u0002\u00ae\u0007\u00ae\u0002\u00af\u0007\u00af\u0002\u00b0\u0007\u00b0"+ - "\u0002\u00b1\u0007\u00b1\u0002\u00b2\u0007\u00b2\u0002\u00b3\u0007\u00b3"+ - "\u0002\u00b4\u0007\u00b4\u0002\u00b5\u0007\u00b5\u0002\u00b6\u0007\u00b6"+ - "\u0002\u00b7\u0007\u00b7\u0002\u00b8\u0007\u00b8\u0002\u00b9\u0007\u00b9"+ - "\u0002\u00ba\u0007\u00ba\u0002\u00bb\u0007\u00bb\u0002\u00bc\u0007\u00bc"+ - "\u0002\u00bd\u0007\u00bd\u0002\u00be\u0007\u00be\u0002\u00bf\u0007\u00bf"+ - "\u0002\u00c0\u0007\u00c0\u0002\u00c1\u0007\u00c1\u0002\u00c2\u0007\u00c2"+ - "\u0002\u00c3\u0007\u00c3\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000"+ - "\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000"+ + "\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002\u0002\u0007\u0002"+ + "\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002\u0005\u0007\u0005"+ + "\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007\u0002\b\u0007\b\u0002"+ + "\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b\u0002\f\u0007\f\u0002"+ + "\r\u0007\r\u0002\u000e\u0007\u000e\u0002\u000f\u0007\u000f\u0002\u0010"+ + "\u0007\u0010\u0002\u0011\u0007\u0011\u0002\u0012\u0007\u0012\u0002\u0013"+ + "\u0007\u0013\u0002\u0014\u0007\u0014\u0002\u0015\u0007\u0015\u0002\u0016"+ + "\u0007\u0016\u0002\u0017\u0007\u0017\u0002\u0018\u0007\u0018\u0002\u0019"+ + "\u0007\u0019\u0002\u001a\u0007\u001a\u0002\u001b\u0007\u001b\u0002\u001c"+ + "\u0007\u001c\u0002\u001d\u0007\u001d\u0002\u001e\u0007\u001e\u0002\u001f"+ + "\u0007\u001f\u0002 \u0007 \u0002!\u0007!\u0002\"\u0007\"\u0002#\u0007"+ + "#\u0002$\u0007$\u0002%\u0007%\u0002&\u0007&\u0002\'\u0007\'\u0002(\u0007"+ + "(\u0002)\u0007)\u0002*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002-\u0007"+ + "-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u00021\u00071\u00022\u0007"+ + "2\u00023\u00073\u00024\u00074\u00025\u00075\u00026\u00076\u00027\u0007"+ + "7\u00028\u00078\u00029\u00079\u0002:\u0007:\u0002;\u0007;\u0002<\u0007"+ + "<\u0002=\u0007=\u0002>\u0007>\u0002?\u0007?\u0002@\u0007@\u0002A\u0007"+ + "A\u0002B\u0007B\u0002C\u0007C\u0002D\u0007D\u0002E\u0007E\u0002F\u0007"+ + "F\u0002G\u0007G\u0002H\u0007H\u0002I\u0007I\u0002J\u0007J\u0002K\u0007"+ + "K\u0002L\u0007L\u0002M\u0007M\u0002N\u0007N\u0002O\u0007O\u0002P\u0007"+ + "P\u0002Q\u0007Q\u0002R\u0007R\u0002S\u0007S\u0002T\u0007T\u0002U\u0007"+ + "U\u0002V\u0007V\u0002W\u0007W\u0002X\u0007X\u0002Y\u0007Y\u0002Z\u0007"+ + "Z\u0002[\u0007[\u0002\\\u0007\\\u0002]\u0007]\u0002^\u0007^\u0002_\u0007"+ + "_\u0002`\u0007`\u0002a\u0007a\u0002b\u0007b\u0002c\u0007c\u0002d\u0007"+ + "d\u0002e\u0007e\u0002f\u0007f\u0002g\u0007g\u0002h\u0007h\u0002i\u0007"+ + "i\u0002j\u0007j\u0002k\u0007k\u0002l\u0007l\u0002m\u0007m\u0002n\u0007"+ + "n\u0002o\u0007o\u0002p\u0007p\u0002q\u0007q\u0002r\u0007r\u0002s\u0007"+ + "s\u0002t\u0007t\u0002u\u0007u\u0002v\u0007v\u0002w\u0007w\u0002x\u0007"+ + "x\u0002y\u0007y\u0002z\u0007z\u0002{\u0007{\u0002|\u0007|\u0002}\u0007"+ + "}\u0002~\u0007~\u0002\u007f\u0007\u007f\u0002\u0080\u0007\u0080\u0002"+ + "\u0081\u0007\u0081\u0002\u0082\u0007\u0082\u0002\u0083\u0007\u0083\u0002"+ + "\u0084\u0007\u0084\u0002\u0085\u0007\u0085\u0002\u0086\u0007\u0086\u0002"+ + "\u0087\u0007\u0087\u0002\u0088\u0007\u0088\u0002\u0089\u0007\u0089\u0002"+ + "\u008a\u0007\u008a\u0002\u008b\u0007\u008b\u0002\u008c\u0007\u008c\u0002"+ + "\u008d\u0007\u008d\u0002\u008e\u0007\u008e\u0002\u008f\u0007\u008f\u0002"+ + "\u0090\u0007\u0090\u0002\u0091\u0007\u0091\u0002\u0092\u0007\u0092\u0002"+ + "\u0093\u0007\u0093\u0002\u0094\u0007\u0094\u0002\u0095\u0007\u0095\u0002"+ + "\u0096\u0007\u0096\u0002\u0097\u0007\u0097\u0002\u0098\u0007\u0098\u0002"+ + "\u0099\u0007\u0099\u0002\u009a\u0007\u009a\u0002\u009b\u0007\u009b\u0002"+ + "\u009c\u0007\u009c\u0002\u009d\u0007\u009d\u0002\u009e\u0007\u009e\u0002"+ + "\u009f\u0007\u009f\u0002\u00a0\u0007\u00a0\u0002\u00a1\u0007\u00a1\u0002"+ + "\u00a2\u0007\u00a2\u0002\u00a3\u0007\u00a3\u0002\u00a4\u0007\u00a4\u0002"+ + "\u00a5\u0007\u00a5\u0002\u00a6\u0007\u00a6\u0002\u00a7\u0007\u00a7\u0002"+ + "\u00a8\u0007\u00a8\u0002\u00a9\u0007\u00a9\u0002\u00aa\u0007\u00aa\u0002"+ + "\u00ab\u0007\u00ab\u0002\u00ac\u0007\u00ac\u0002\u00ad\u0007\u00ad\u0002"+ + "\u00ae\u0007\u00ae\u0002\u00af\u0007\u00af\u0002\u00b0\u0007\u00b0\u0002"+ + "\u00b1\u0007\u00b1\u0002\u00b2\u0007\u00b2\u0002\u00b3\u0007\u00b3\u0002"+ + "\u00b4\u0007\u00b4\u0002\u00b5\u0007\u00b5\u0002\u00b6\u0007\u00b6\u0002"+ + "\u00b7\u0007\u00b7\u0002\u00b8\u0007\u00b8\u0002\u00b9\u0007\u00b9\u0002"+ + "\u00ba\u0007\u00ba\u0002\u00bb\u0007\u00bb\u0002\u00bc\u0007\u00bc\u0002"+ + "\u00bd\u0007\u00bd\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001"+ + "\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0000\u0001"+ "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ - "\u0001\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002"+ - "\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0003\u0001\u0003"+ - "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0004"+ - "\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004"+ - "\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005"+ - "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0006\u0001\u0006"+ - "\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0007"+ - "\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007"+ - "\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001"+ - "\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\n\u0001\n\u0001"+ - "\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+ - "\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"+ - "\u0001\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\f\u0001\f\u0001"+ - "\f\u0001\f\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001"+ - "\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001"+ - "\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001"+ - "\u000f\u0001\u000f\u0001\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0001"+ - "\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0011\u0001"+ - "\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001"+ - "\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001"+ - "\u0011\u0001\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001"+ - "\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001"+ - "\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001"+ - "\u0013\u0001\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001\u0014\u0001"+ - "\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001\u0014\u0001"+ - "\u0014\u0001\u0014\u0001\u0015\u0004\u0015\u024f\b\u0015\u000b\u0015\f"+ - "\u0015\u0250\u0001\u0015\u0001\u0015\u0001\u0016\u0001\u0016\u0001\u0016"+ - "\u0001\u0016\u0005\u0016\u0259\b\u0016\n\u0016\f\u0016\u025c\t\u0016\u0001"+ - "\u0016\u0003\u0016\u025f\b\u0016\u0001\u0016\u0003\u0016\u0262\b\u0016"+ - "\u0001\u0016\u0001\u0016\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017"+ - "\u0001\u0017\u0005\u0017\u026b\b\u0017\n\u0017\f\u0017\u026e\t\u0017\u0001"+ - "\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0018\u0004"+ - "\u0018\u0276\b\u0018\u000b\u0018\f\u0018\u0277\u0001\u0018\u0001\u0018"+ - "\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u001a\u0001\u001a"+ - "\u0001\u001b\u0001\u001b\u0001\u001c\u0001\u001c\u0001\u001c\u0001\u001d"+ - "\u0001\u001d\u0001\u001e\u0001\u001e\u0003\u001e\u028b\b\u001e\u0001\u001e"+ - "\u0004\u001e\u028e\b\u001e\u000b\u001e\f\u001e\u028f\u0001\u001f\u0001"+ - "\u001f\u0001 \u0001 \u0001!\u0001!\u0001!\u0003!\u0299\b!\u0001\"\u0001"+ - "\"\u0001#\u0001#\u0001#\u0003#\u02a0\b#\u0001$\u0001$\u0001$\u0005$\u02a5"+ - "\b$\n$\f$\u02a8\t$\u0001$\u0001$\u0001$\u0001$\u0001$\u0001$\u0005$\u02b0"+ - "\b$\n$\f$\u02b3\t$\u0001$\u0001$\u0001$\u0001$\u0001$\u0003$\u02ba\b$"+ - "\u0001$\u0003$\u02bd\b$\u0003$\u02bf\b$\u0001%\u0004%\u02c2\b%\u000b%"+ - "\f%\u02c3\u0001&\u0004&\u02c7\b&\u000b&\f&\u02c8\u0001&\u0001&\u0005&"+ - "\u02cd\b&\n&\f&\u02d0\t&\u0001&\u0001&\u0004&\u02d4\b&\u000b&\f&\u02d5"+ - "\u0001&\u0004&\u02d9\b&\u000b&\f&\u02da\u0001&\u0001&\u0005&\u02df\b&"+ - "\n&\f&\u02e2\t&\u0003&\u02e4\b&\u0001&\u0001&\u0001&\u0001&\u0004&\u02ea"+ - "\b&\u000b&\f&\u02eb\u0001&\u0001&\u0003&\u02f0\b&\u0001\'\u0001\'\u0001"+ - "\'\u0001(\u0001(\u0001(\u0001(\u0001)\u0001)\u0001)\u0001)\u0001*\u0001"+ - "*\u0001+\u0001+\u0001+\u0001,\u0001,\u0001-\u0001-\u0001-\u0001-\u0001"+ - "-\u0001.\u0001.\u0001/\u0001/\u0001/\u0001/\u0001/\u0001/\u00010\u0001"+ - "0\u00010\u00010\u00010\u00010\u00011\u00011\u00011\u00012\u00012\u0001"+ - "2\u00013\u00013\u00013\u00013\u00013\u00014\u00014\u00014\u00014\u0001"+ - "4\u00015\u00015\u00016\u00016\u00016\u00016\u00017\u00017\u00017\u0001"+ - "7\u00017\u00018\u00018\u00018\u00018\u00018\u00018\u00019\u00019\u0001"+ - "9\u0001:\u0001:\u0001;\u0001;\u0001;\u0001;\u0001;\u0001;\u0001<\u0001"+ - "<\u0001=\u0001=\u0001=\u0001=\u0001=\u0001>\u0001>\u0001>\u0001?\u0001"+ - "?\u0001?\u0001@\u0001@\u0001@\u0001A\u0001A\u0001B\u0001B\u0001B\u0001"+ - "C\u0001C\u0001D\u0001D\u0001D\u0001E\u0001E\u0001F\u0001F\u0001G\u0001"+ - "G\u0001H\u0001H\u0001I\u0001I\u0001J\u0001J\u0001J\u0001J\u0001J\u0001"+ - "K\u0001K\u0001K\u0003K\u036f\bK\u0001K\u0005K\u0372\bK\nK\fK\u0375\tK"+ - "\u0001K\u0001K\u0004K\u0379\bK\u000bK\fK\u037a\u0003K\u037d\bK\u0001L"+ - "\u0001L\u0001L\u0001L\u0001L\u0001M\u0001M\u0001M\u0001M\u0001M\u0001"+ - "N\u0001N\u0005N\u038b\bN\nN\fN\u038e\tN\u0001N\u0001N\u0003N\u0392\bN"+ - "\u0001N\u0004N\u0395\bN\u000bN\fN\u0396\u0003N\u0399\bN\u0001O\u0001O"+ - "\u0004O\u039d\bO\u000bO\fO\u039e\u0001O\u0001O\u0001P\u0001P\u0001Q\u0001"+ - "Q\u0001Q\u0001Q\u0001R\u0001R\u0001R\u0001R\u0001S\u0001S\u0001S\u0001"+ - "S\u0001T\u0001T\u0001T\u0001T\u0001T\u0001U\u0001U\u0001U\u0001U\u0001"+ - "U\u0001V\u0001V\u0001V\u0001V\u0001W\u0001W\u0001W\u0001W\u0001X\u0001"+ - "X\u0001X\u0001X\u0001Y\u0001Y\u0001Y\u0001Y\u0001Y\u0001Z\u0001Z\u0001"+ - "Z\u0001Z\u0001[\u0001[\u0001[\u0001[\u0001\\\u0001\\\u0001\\\u0001\\\u0001"+ - "]\u0001]\u0001]\u0001]\u0001^\u0001^\u0001^\u0001^\u0001_\u0001_\u0001"+ - "_\u0001_\u0001_\u0001_\u0001_\u0001_\u0001_\u0001`\u0001`\u0001`\u0003"+ - "`\u03ec\b`\u0001a\u0004a\u03ef\ba\u000ba\fa\u03f0\u0001b\u0001b\u0001"+ + "\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+ + "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0003\u0001\u0003\u0001"+ + "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0004\u0001"+ + "\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0004\u0001"+ + "\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0006\u0001\u0006\u0001"+ + "\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0007\u0001"+ + "\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+ + "\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0001\t\u0001"+ + "\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001"+ + "\t\u0001\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+ + "\n\u0001\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"+ + "\u0001\u000b\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001\f\u0001"+ + "\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\r\u0001\u000e\u0001\u000e"+ + "\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000e"+ + "\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f"+ + "\u0001\u000f\u0001\u000f\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010"+ + "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010"+ + "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0011"+ + "\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011"+ + "\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0012\u0001\u0012\u0001\u0012"+ + "\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012\u0001\u0012"+ + "\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013"+ + "\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0013\u0001\u0014"+ + "\u0004\u0014\u023b\b\u0014\u000b\u0014\f\u0014\u023c\u0001\u0014\u0001"+ + "\u0014\u0001\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0005\u0015\u0245"+ + "\b\u0015\n\u0015\f\u0015\u0248\t\u0015\u0001\u0015\u0003\u0015\u024b\b"+ + "\u0015\u0001\u0015\u0003\u0015\u024e\b\u0015\u0001\u0015\u0001\u0015\u0001"+ + "\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0005\u0016\u0257"+ + "\b\u0016\n\u0016\f\u0016\u025a\t\u0016\u0001\u0016\u0001\u0016\u0001\u0016"+ + "\u0001\u0016\u0001\u0016\u0001\u0017\u0004\u0017\u0262\b\u0017\u000b\u0017"+ + "\f\u0017\u0263\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018\u0001\u0018"+ + "\u0001\u0018\u0001\u0019\u0001\u0019\u0001\u001a\u0001\u001a\u0001\u001b"+ + "\u0001\u001b\u0001\u001b\u0001\u001c\u0001\u001c\u0001\u001d\u0001\u001d"+ + "\u0003\u001d\u0277\b\u001d\u0001\u001d\u0004\u001d\u027a\b\u001d\u000b"+ + "\u001d\f\u001d\u027b\u0001\u001e\u0001\u001e\u0001\u001f\u0001\u001f\u0001"+ + " \u0001 \u0001 \u0003 \u0285\b \u0001!\u0001!\u0001\"\u0001\"\u0001\""+ + "\u0003\"\u028c\b\"\u0001#\u0001#\u0001#\u0005#\u0291\b#\n#\f#\u0294\t"+ + "#\u0001#\u0001#\u0001#\u0001#\u0001#\u0001#\u0005#\u029c\b#\n#\f#\u029f"+ + "\t#\u0001#\u0001#\u0001#\u0001#\u0001#\u0003#\u02a6\b#\u0001#\u0003#\u02a9"+ + "\b#\u0003#\u02ab\b#\u0001$\u0004$\u02ae\b$\u000b$\f$\u02af\u0001%\u0004"+ + "%\u02b3\b%\u000b%\f%\u02b4\u0001%\u0001%\u0005%\u02b9\b%\n%\f%\u02bc\t"+ + "%\u0001%\u0001%\u0004%\u02c0\b%\u000b%\f%\u02c1\u0001%\u0004%\u02c5\b"+ + "%\u000b%\f%\u02c6\u0001%\u0001%\u0005%\u02cb\b%\n%\f%\u02ce\t%\u0003%"+ + "\u02d0\b%\u0001%\u0001%\u0001%\u0001%\u0004%\u02d6\b%\u000b%\f%\u02d7"+ + "\u0001%\u0001%\u0003%\u02dc\b%\u0001&\u0001&\u0001&\u0001\'\u0001\'\u0001"+ + "\'\u0001\'\u0001(\u0001(\u0001(\u0001(\u0001)\u0001)\u0001*\u0001*\u0001"+ + "*\u0001+\u0001+\u0001,\u0001,\u0001,\u0001,\u0001,\u0001-\u0001-\u0001"+ + ".\u0001.\u0001.\u0001.\u0001.\u0001.\u0001/\u0001/\u0001/\u0001/\u0001"+ + "/\u0001/\u00010\u00010\u00010\u00011\u00011\u00011\u00012\u00012\u0001"+ + "2\u00012\u00012\u00013\u00013\u00013\u00013\u00013\u00014\u00014\u0001"+ + "5\u00015\u00015\u00015\u00016\u00016\u00016\u00016\u00016\u00017\u0001"+ + "7\u00017\u00017\u00017\u00017\u00018\u00018\u00018\u00019\u00019\u0001"+ + ":\u0001:\u0001:\u0001:\u0001:\u0001:\u0001;\u0001;\u0001<\u0001<\u0001"+ + "<\u0001<\u0001<\u0001=\u0001=\u0001=\u0001>\u0001>\u0001>\u0001?\u0001"+ + "?\u0001?\u0001@\u0001@\u0001A\u0001A\u0001A\u0001B\u0001B\u0001C\u0001"+ + "C\u0001C\u0001D\u0001D\u0001E\u0001E\u0001F\u0001F\u0001G\u0001G\u0001"+ + "H\u0001H\u0001I\u0001I\u0001I\u0001I\u0001I\u0001J\u0001J\u0001J\u0003"+ + "J\u035b\bJ\u0001J\u0005J\u035e\bJ\nJ\fJ\u0361\tJ\u0001J\u0001J\u0004J"+ + "\u0365\bJ\u000bJ\fJ\u0366\u0003J\u0369\bJ\u0001K\u0001K\u0001K\u0001K"+ + "\u0001K\u0001L\u0001L\u0001L\u0001L\u0001L\u0001M\u0001M\u0005M\u0377"+ + "\bM\nM\fM\u037a\tM\u0001M\u0001M\u0003M\u037e\bM\u0001M\u0004M\u0381\b"+ + "M\u000bM\fM\u0382\u0003M\u0385\bM\u0001N\u0001N\u0004N\u0389\bN\u000b"+ + "N\fN\u038a\u0001N\u0001N\u0001O\u0001O\u0001P\u0001P\u0001P\u0001P\u0001"+ + "Q\u0001Q\u0001Q\u0001Q\u0001R\u0001R\u0001R\u0001R\u0001S\u0001S\u0001"+ + "S\u0001S\u0001S\u0001T\u0001T\u0001T\u0001T\u0001T\u0001U\u0001U\u0001"+ + "U\u0001U\u0001V\u0001V\u0001V\u0001V\u0001W\u0001W\u0001W\u0001W\u0001"+ + "X\u0001X\u0001X\u0001X\u0001X\u0001Y\u0001Y\u0001Y\u0001Y\u0001Z\u0001"+ + "Z\u0001Z\u0001Z\u0001[\u0001[\u0001[\u0001[\u0001\\\u0001\\\u0001\\\u0001"+ + "\\\u0001]\u0001]\u0001]\u0001]\u0001^\u0001^\u0001^\u0001^\u0001^\u0001"+ + "^\u0001^\u0001^\u0001^\u0001_\u0001_\u0001_\u0003_\u03d8\b_\u0001`\u0004"+ + "`\u03db\b`\u000b`\f`\u03dc\u0001a\u0001a\u0001a\u0001a\u0001b\u0001b\u0001"+ "b\u0001b\u0001c\u0001c\u0001c\u0001c\u0001d\u0001d\u0001d\u0001d\u0001"+ - "e\u0001e\u0001e\u0001e\u0001f\u0001f\u0001f\u0001f\u0001g\u0001g\u0001"+ + "e\u0001e\u0001e\u0001e\u0001f\u0001f\u0001f\u0001f\u0001f\u0001g\u0001"+ "g\u0001g\u0001g\u0001h\u0001h\u0001h\u0001h\u0001i\u0001i\u0001i\u0001"+ - "i\u0001j\u0001j\u0001j\u0001j\u0003j\u0418\bj\u0001k\u0001k\u0003k\u041c"+ - "\bk\u0001k\u0005k\u041f\bk\nk\fk\u0422\tk\u0001k\u0001k\u0003k\u0426\b"+ - "k\u0001k\u0004k\u0429\bk\u000bk\fk\u042a\u0003k\u042d\bk\u0001l\u0001"+ - "l\u0004l\u0431\bl\u000bl\fl\u0432\u0001m\u0001m\u0001m\u0001m\u0001n\u0001"+ - "n\u0001n\u0001n\u0001o\u0001o\u0001o\u0001o\u0001p\u0001p\u0001p\u0001"+ - "p\u0001p\u0001q\u0001q\u0001q\u0001q\u0001r\u0001r\u0001r\u0001r\u0001"+ - "s\u0001s\u0001s\u0001s\u0001t\u0001t\u0001t\u0001u\u0001u\u0001u\u0001"+ - "u\u0001v\u0001v\u0001v\u0001v\u0001w\u0001w\u0001w\u0001w\u0001x\u0001"+ - "x\u0001x\u0001x\u0001y\u0001y\u0001y\u0001y\u0001y\u0001z\u0001z\u0001"+ - "z\u0001z\u0001z\u0001{\u0001{\u0001{\u0001{\u0001{\u0001|\u0001|\u0001"+ - "|\u0001|\u0001|\u0001|\u0001|\u0001}\u0001}\u0001~\u0004~\u047e\b~\u000b"+ - "~\f~\u047f\u0001~\u0001~\u0003~\u0484\b~\u0001~\u0004~\u0487\b~\u000b"+ - "~\f~\u0488\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u0080"+ - "\u0001\u0080\u0001\u0080\u0001\u0080\u0001\u0081\u0001\u0081\u0001\u0081"+ - "\u0001\u0081\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0083"+ - "\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0084"+ - "\u0001\u0084\u0001\u0084\u0001\u0084\u0001\u0085\u0001\u0085\u0001\u0085"+ - "\u0001\u0085\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0087"+ - "\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0088\u0001\u0088\u0001\u0088"+ - "\u0001\u0088\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u008a"+ - "\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008b\u0001\u008b\u0001\u008b"+ - "\u0001\u008b\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008d"+ - "\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008e\u0001\u008e"+ - "\u0001\u008e\u0001\u008e\u0001\u008f\u0001\u008f\u0001\u008f\u0001\u008f"+ - "\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0091\u0001\u0091"+ - "\u0001\u0091\u0001\u0091\u0001\u0092\u0001\u0092\u0001\u0092\u0001\u0092"+ - "\u0001\u0093\u0001\u0093\u0001\u0093\u0001\u0093\u0001\u0094\u0001\u0094"+ - "\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0095\u0001\u0095\u0001\u0095"+ - "\u0001\u0095\u0001\u0095\u0001\u0096\u0001\u0096\u0001\u0096\u0001\u0096"+ - "\u0001\u0097\u0001\u0097\u0001\u0097\u0001\u0097\u0001\u0098\u0001\u0098"+ - "\u0001\u0098\u0001\u0098\u0001\u0099\u0001\u0099\u0001\u0099\u0001\u0099"+ - "\u0001\u0099\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009a"+ - "\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009b"+ - "\u0001\u009b\u0001\u009b\u0001\u009b\u0001\u009c\u0001\u009c\u0001\u009c"+ - "\u0001\u009c\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009e"+ - "\u0001\u009e\u0001\u009e\u0001\u009e\u0001\u009e\u0001\u009f\u0001\u009f"+ - "\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0004\u00a0"+ - "\u051f\b\u00a0\u000b\u00a0\f\u00a0\u0520\u0001\u00a1\u0001\u00a1\u0001"+ - "\u00a1\u0001\u00a1\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001"+ - "\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a4\u0001\u00a4\u0001"+ - "\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a5\u0001\u00a5\u0001\u00a5\u0001"+ - "\u00a5\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a7\u0001"+ - "\u00a7\u0001\u00a7\u0001\u00a7\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001"+ - "\u00a8\u0001\u00a8\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001\u00a9\u0001"+ - "\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00ab\u0001\u00ab\u0001"+ - "\u00ab\u0001\u00ab\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001\u00ac\u0001"+ - "\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ae\u0001\u00ae\u0001"+ - "\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00af\u0001\u00af\u0001"+ - "\u00af\u0001\u00af\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001"+ - "\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b2\u0001\u00b2\u0001"+ - "\u00b2\u0001\u00b2\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001"+ - "\u00b4\u0001\u00b4\u0001\u00b4\u0001\u00b4\u0001\u00b5\u0001\u00b5\u0001"+ - "\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001"+ - "\u00b6\u0001\u00b6\u0001\u00b6\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001"+ - "\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001"+ - "\u00b8\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00b9\u0001\u00ba\u0001"+ - "\u00ba\u0001\u00ba\u0001\u00ba\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001"+ - "\u00bb\u0001\u00bb\u0001\u00bb\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001"+ - "\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001"+ - "\u00bd\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00be\u0001\u00bf\u0001"+ - "\u00bf\u0001\u00bf\u0001\u00bf\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001"+ - "\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c1\u0001\u00c1\u0001\u00c1\u0001"+ - "\u00c1\u0001\u00c1\u0001\u00c1\u0001\u00c2\u0001\u00c2\u0001\u00c2\u0001"+ - "\u00c2\u0001\u00c2\u0001\u00c2\u0001\u00c3\u0001\u00c3\u0001\u00c3\u0001"+ - "\u00c3\u0001\u00c3\u0002\u026c\u02b1\u0000\u00c4\u0010\u0001\u0012\u0002"+ - "\u0014\u0003\u0016\u0004\u0018\u0005\u001a\u0006\u001c\u0007\u001e\b "+ - "\t\"\n$\u000b&\f(\r*\u000e,\u000f.\u00100\u00112\u00124\u00136\u00148"+ - "\u0015:\u0016<\u0017>\u0018@\u0019B\u001aD\u0000F\u0000H\u0000J\u0000"+ - "L\u0000N\u0000P\u0000R\u0000T\u0000V\u0000X\u001bZ\u001c\\\u001d^\u001e"+ - "`\u001fb d!f\"h#j$l%n&p\'r(t)v*x+z,|-~.\u0080/\u00820\u00841\u00862\u0088"+ - "3\u008a4\u008c5\u008e6\u00907\u00928\u00949\u0096:\u0098;\u009a<\u009c"+ - "=\u009e>\u00a0?\u00a2@\u00a4\u0000\u00a6A\u00a8B\u00aaC\u00acD\u00ae\u0000"+ - "\u00b0E\u00b2F\u00b4G\u00b6H\u00b8\u0000\u00ba\u0000\u00bcI\u00beJ\u00c0"+ - "K\u00c2\u0000\u00c4\u0000\u00c6\u0000\u00c8\u0000\u00ca\u0000\u00cc\u0000"+ - "\u00ceL\u00d0\u0000\u00d2M\u00d4\u0000\u00d6\u0000\u00d8N\u00daO\u00dc"+ - "P\u00de\u0000\u00e0\u0000\u00e2\u0000\u00e4\u0000\u00e6\u0000\u00e8Q\u00ea"+ - "R\u00ecS\u00eeT\u00f0\u0000\u00f2\u0000\u00f4\u0000\u00f6\u0000\u00f8"+ - "U\u00fa\u0000\u00fcV\u00feW\u0100X\u0102\u0000\u0104\u0000\u0106Y\u0108"+ - "Z\u010a\u0000\u010c[\u010e\u0000\u0110\\\u0112]\u0114^\u0116\u0000\u0118"+ - "\u0000\u011a\u0000\u011c\u0000\u011e\u0000\u0120\u0000\u0122\u0000\u0124"+ - "_\u0126`\u0128a\u012a\u0000\u012c\u0000\u012e\u0000\u0130\u0000\u0132"+ - "b\u0134c\u0136d\u0138\u0000\u013ae\u013cf\u013eg\u0140h\u0142\u0000\u0144"+ - "i\u0146j\u0148k\u014al\u014c\u0000\u014em\u0150n\u0152o\u0154p\u0156q"+ - "\u0158\u0000\u015a\u0000\u015c\u0000\u015e\u0000\u0160\u0000\u0162\u0000"+ - "\u0164\u0000\u0166r\u0168s\u016at\u016c\u0000\u016e\u0000\u0170\u0000"+ - "\u0172\u0000\u0174u\u0176v\u0178w\u017a\u0000\u017c\u0000\u017e\u0000"+ - "\u0180x\u0182y\u0184z\u0186\u0000\u0188\u0000\u018a{\u018c|\u018e}\u0190"+ - "\u0000\u0192\u0000\u0194\u0000\u0196\u0000\u0010\u0000\u0001\u0002\u0003"+ - "\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u000f#\u0002\u0000DDdd"+ - "\u0002\u0000IIii\u0002\u0000SSss\u0002\u0000EEee\u0002\u0000CCcc\u0002"+ - "\u0000TTtt\u0002\u0000RRrr\u0002\u0000OOoo\u0002\u0000PPpp\u0002\u0000"+ - "NNnn\u0002\u0000HHhh\u0002\u0000VVvv\u0002\u0000AAaa\u0002\u0000LLll\u0002"+ - "\u0000XXxx\u0002\u0000FFff\u0002\u0000MMmm\u0002\u0000GGgg\u0002\u0000"+ - "KKkk\u0002\u0000WWww\u0002\u0000UUuu\u0006\u0000\t\n\r\r //[[]]\u0002"+ - "\u0000\n\n\r\r\u0003\u0000\t\n\r\r \u0001\u000009\u0002\u0000AZaz\b\u0000"+ - "\"\"NNRRTT\\\\nnrrtt\u0004\u0000\n\n\r\r\"\"\\\\\u0002\u0000++--\u0001"+ - "\u0000``\u0002\u0000BBbb\u0002\u0000YYyy\u000b\u0000\t\n\r\r \"\",,/"+ - "/::==[[]]||\u0002\u0000**//\u000b\u0000\t\n\r\r \"#,,//::<<>?\\\\||\u05dd"+ - "\u0000\u0010\u0001\u0000\u0000\u0000\u0000\u0012\u0001\u0000\u0000\u0000"+ - "\u0000\u0014\u0001\u0000\u0000\u0000\u0000\u0016\u0001\u0000\u0000\u0000"+ - "\u0000\u0018\u0001\u0000\u0000\u0000\u0000\u001a\u0001\u0000\u0000\u0000"+ - "\u0000\u001c\u0001\u0000\u0000\u0000\u0000\u001e\u0001\u0000\u0000\u0000"+ - "\u0000 \u0001\u0000\u0000\u0000\u0000\"\u0001\u0000\u0000\u0000\u0000"+ - "$\u0001\u0000\u0000\u0000\u0000&\u0001\u0000\u0000\u0000\u0000(\u0001"+ - "\u0000\u0000\u0000\u0000*\u0001\u0000\u0000\u0000\u0000,\u0001\u0000\u0000"+ - "\u0000\u0000.\u0001\u0000\u0000\u0000\u00000\u0001\u0000\u0000\u0000\u0000"+ - "2\u0001\u0000\u0000\u0000\u00004\u0001\u0000\u0000\u0000\u00006\u0001"+ - "\u0000\u0000\u0000\u00008\u0001\u0000\u0000\u0000\u0000:\u0001\u0000\u0000"+ - "\u0000\u0000<\u0001\u0000\u0000\u0000\u0000>\u0001\u0000\u0000\u0000\u0000"+ - "@\u0001\u0000\u0000\u0000\u0001B\u0001\u0000\u0000\u0000\u0001X\u0001"+ - "\u0000\u0000\u0000\u0001Z\u0001\u0000\u0000\u0000\u0001\\\u0001\u0000"+ - "\u0000\u0000\u0001^\u0001\u0000\u0000\u0000\u0001`\u0001\u0000\u0000\u0000"+ - "\u0001b\u0001\u0000\u0000\u0000\u0001d\u0001\u0000\u0000\u0000\u0001f"+ - "\u0001\u0000\u0000\u0000\u0001h\u0001\u0000\u0000\u0000\u0001j\u0001\u0000"+ - "\u0000\u0000\u0001l\u0001\u0000\u0000\u0000\u0001n\u0001\u0000\u0000\u0000"+ - "\u0001p\u0001\u0000\u0000\u0000\u0001r\u0001\u0000\u0000\u0000\u0001t"+ - "\u0001\u0000\u0000\u0000\u0001v\u0001\u0000\u0000\u0000\u0001x\u0001\u0000"+ - "\u0000\u0000\u0001z\u0001\u0000\u0000\u0000\u0001|\u0001\u0000\u0000\u0000"+ - "\u0001~\u0001\u0000\u0000\u0000\u0001\u0080\u0001\u0000\u0000\u0000\u0001"+ - "\u0082\u0001\u0000\u0000\u0000\u0001\u0084\u0001\u0000\u0000\u0000\u0001"+ - "\u0086\u0001\u0000\u0000\u0000\u0001\u0088\u0001\u0000\u0000\u0000\u0001"+ - "\u008a\u0001\u0000\u0000\u0000\u0001\u008c\u0001\u0000\u0000\u0000\u0001"+ - "\u008e\u0001\u0000\u0000\u0000\u0001\u0090\u0001\u0000\u0000\u0000\u0001"+ - "\u0092\u0001\u0000\u0000\u0000\u0001\u0094\u0001\u0000\u0000\u0000\u0001"+ - "\u0096\u0001\u0000\u0000\u0000\u0001\u0098\u0001\u0000\u0000\u0000\u0001"+ - "\u009a\u0001\u0000\u0000\u0000\u0001\u009c\u0001\u0000\u0000\u0000\u0001"+ - "\u009e\u0001\u0000\u0000\u0000\u0001\u00a0\u0001\u0000\u0000\u0000\u0001"+ - "\u00a2\u0001\u0000\u0000\u0000\u0001\u00a4\u0001\u0000\u0000\u0000\u0001"+ - "\u00a6\u0001\u0000\u0000\u0000\u0001\u00a8\u0001\u0000\u0000\u0000\u0001"+ - "\u00aa\u0001\u0000\u0000\u0000\u0001\u00ac\u0001\u0000\u0000\u0000\u0001"+ - "\u00b0\u0001\u0000\u0000\u0000\u0001\u00b2\u0001\u0000\u0000\u0000\u0001"+ - "\u00b4\u0001\u0000\u0000\u0000\u0001\u00b6\u0001\u0000\u0000\u0000\u0002"+ - "\u00b8\u0001\u0000\u0000\u0000\u0002\u00ba\u0001\u0000\u0000\u0000\u0002"+ - "\u00bc\u0001\u0000\u0000\u0000\u0002\u00be\u0001\u0000\u0000\u0000\u0002"+ - "\u00c0\u0001\u0000\u0000\u0000\u0003\u00c2\u0001\u0000\u0000\u0000\u0003"+ - "\u00c4\u0001\u0000\u0000\u0000\u0003\u00c6\u0001\u0000\u0000\u0000\u0003"+ - "\u00c8\u0001\u0000\u0000\u0000\u0003\u00ca\u0001\u0000\u0000\u0000\u0003"+ - "\u00cc\u0001\u0000\u0000\u0000\u0003\u00ce\u0001\u0000\u0000\u0000\u0003"+ - "\u00d2\u0001\u0000\u0000\u0000\u0003\u00d4\u0001\u0000\u0000\u0000\u0003"+ - "\u00d6\u0001\u0000\u0000\u0000\u0003\u00d8\u0001\u0000\u0000\u0000\u0003"+ - "\u00da\u0001\u0000\u0000\u0000\u0003\u00dc\u0001\u0000\u0000\u0000\u0004"+ - "\u00de\u0001\u0000\u0000\u0000\u0004\u00e0\u0001\u0000\u0000\u0000\u0004"+ - "\u00e2\u0001\u0000\u0000\u0000\u0004\u00e8\u0001\u0000\u0000\u0000\u0004"+ - "\u00ea\u0001\u0000\u0000\u0000\u0004\u00ec\u0001\u0000\u0000\u0000\u0004"+ - "\u00ee\u0001\u0000\u0000\u0000\u0005\u00f0\u0001\u0000\u0000\u0000\u0005"+ - "\u00f2\u0001\u0000\u0000\u0000\u0005\u00f4\u0001\u0000\u0000\u0000\u0005"+ - "\u00f6\u0001\u0000\u0000\u0000\u0005\u00f8\u0001\u0000\u0000\u0000\u0005"+ - "\u00fa\u0001\u0000\u0000\u0000\u0005\u00fc\u0001\u0000\u0000\u0000\u0005"+ - "\u00fe\u0001\u0000\u0000\u0000\u0005\u0100\u0001\u0000\u0000\u0000\u0006"+ - "\u0102\u0001\u0000\u0000\u0000\u0006\u0104\u0001\u0000\u0000\u0000\u0006"+ - "\u0106\u0001\u0000\u0000\u0000\u0006\u0108\u0001\u0000\u0000\u0000\u0006"+ - "\u010c\u0001\u0000\u0000\u0000\u0006\u010e\u0001\u0000\u0000\u0000\u0006"+ - "\u0110\u0001\u0000\u0000\u0000\u0006\u0112\u0001\u0000\u0000\u0000\u0006"+ - "\u0114\u0001\u0000\u0000\u0000\u0007\u0116\u0001\u0000\u0000\u0000\u0007"+ - "\u0118\u0001\u0000\u0000\u0000\u0007\u011a\u0001\u0000\u0000\u0000\u0007"+ - "\u011c\u0001\u0000\u0000\u0000\u0007\u011e\u0001\u0000\u0000\u0000\u0007"+ - "\u0120\u0001\u0000\u0000\u0000\u0007\u0122\u0001\u0000\u0000\u0000\u0007"+ - "\u0124\u0001\u0000\u0000\u0000\u0007\u0126\u0001\u0000\u0000\u0000\u0007"+ - "\u0128\u0001\u0000\u0000\u0000\b\u012a\u0001\u0000\u0000\u0000\b\u012c"+ - "\u0001\u0000\u0000\u0000\b\u012e\u0001\u0000\u0000\u0000\b\u0130\u0001"+ - "\u0000\u0000\u0000\b\u0132\u0001\u0000\u0000\u0000\b\u0134\u0001\u0000"+ - "\u0000\u0000\b\u0136\u0001\u0000\u0000\u0000\t\u0138\u0001\u0000\u0000"+ - "\u0000\t\u013a\u0001\u0000\u0000\u0000\t\u013c\u0001\u0000\u0000\u0000"+ - "\t\u013e\u0001\u0000\u0000\u0000\t\u0140\u0001\u0000\u0000\u0000\n\u0142"+ - "\u0001\u0000\u0000\u0000\n\u0144\u0001\u0000\u0000\u0000\n\u0146\u0001"+ - "\u0000\u0000\u0000\n\u0148\u0001\u0000\u0000\u0000\n\u014a\u0001\u0000"+ - "\u0000\u0000\u000b\u014c\u0001\u0000\u0000\u0000\u000b\u014e\u0001\u0000"+ - "\u0000\u0000\u000b\u0150\u0001\u0000\u0000\u0000\u000b\u0152\u0001\u0000"+ - "\u0000\u0000\u000b\u0154\u0001\u0000\u0000\u0000\u000b\u0156\u0001\u0000"+ - "\u0000\u0000\f\u0158\u0001\u0000\u0000\u0000\f\u015a\u0001\u0000\u0000"+ - "\u0000\f\u015c\u0001\u0000\u0000\u0000\f\u015e\u0001\u0000\u0000\u0000"+ - "\f\u0160\u0001\u0000\u0000\u0000\f\u0162\u0001\u0000\u0000\u0000\f\u0164"+ - "\u0001\u0000\u0000\u0000\f\u0166\u0001\u0000\u0000\u0000\f\u0168\u0001"+ - "\u0000\u0000\u0000\f\u016a\u0001\u0000\u0000\u0000\r\u016c\u0001\u0000"+ - "\u0000\u0000\r\u016e\u0001\u0000\u0000\u0000\r\u0170\u0001\u0000\u0000"+ - "\u0000\r\u0172\u0001\u0000\u0000\u0000\r\u0174\u0001\u0000\u0000\u0000"+ - "\r\u0176\u0001\u0000\u0000\u0000\r\u0178\u0001\u0000\u0000\u0000\u000e"+ - "\u017a\u0001\u0000\u0000\u0000\u000e\u017c\u0001\u0000\u0000\u0000\u000e"+ - "\u017e\u0001\u0000\u0000\u0000\u000e\u0180\u0001\u0000\u0000\u0000\u000e"+ - "\u0182\u0001\u0000\u0000\u0000\u000e\u0184\u0001\u0000\u0000\u0000\u000f"+ - "\u0186\u0001\u0000\u0000\u0000\u000f\u0188\u0001\u0000\u0000\u0000\u000f"+ - "\u018a\u0001\u0000\u0000\u0000\u000f\u018c\u0001\u0000\u0000\u0000\u000f"+ - "\u018e\u0001\u0000\u0000\u0000\u000f\u0190\u0001\u0000\u0000\u0000\u000f"+ - "\u0192\u0001\u0000\u0000\u0000\u000f\u0194\u0001\u0000\u0000\u0000\u000f"+ - "\u0196\u0001\u0000\u0000\u0000\u0010\u0198\u0001\u0000\u0000\u0000\u0012"+ - "\u01a2\u0001\u0000\u0000\u0000\u0014\u01a9\u0001\u0000\u0000\u0000\u0016"+ - "\u01b2\u0001\u0000\u0000\u0000\u0018\u01b9\u0001\u0000\u0000\u0000\u001a"+ - "\u01c3\u0001\u0000\u0000\u0000\u001c\u01ca\u0001\u0000\u0000\u0000\u001e"+ - "\u01d1\u0001\u0000\u0000\u0000 \u01d8\u0001\u0000\u0000\u0000\"\u01e0"+ - "\u0001\u0000\u0000\u0000$\u01e7\u0001\u0000\u0000\u0000&\u01f3\u0001\u0000"+ - "\u0000\u0000(\u01fc\u0001\u0000\u0000\u0000*\u0202\u0001\u0000\u0000\u0000"+ - ",\u0209\u0001\u0000\u0000\u0000.\u0210\u0001\u0000\u0000\u00000\u0218"+ - "\u0001\u0000\u0000\u00002\u0220\u0001\u0000\u0000\u00004\u022f\u0001\u0000"+ - "\u0000\u00006\u0239\u0001\u0000\u0000\u00008\u0242\u0001\u0000\u0000\u0000"+ - ":\u024e\u0001\u0000\u0000\u0000<\u0254\u0001\u0000\u0000\u0000>\u0265"+ - "\u0001\u0000\u0000\u0000@\u0275\u0001\u0000\u0000\u0000B\u027b\u0001\u0000"+ - "\u0000\u0000D\u027f\u0001\u0000\u0000\u0000F\u0281\u0001\u0000\u0000\u0000"+ - "H\u0283\u0001\u0000\u0000\u0000J\u0286\u0001\u0000\u0000\u0000L\u0288"+ - "\u0001\u0000\u0000\u0000N\u0291\u0001\u0000\u0000\u0000P\u0293\u0001\u0000"+ - "\u0000\u0000R\u0298\u0001\u0000\u0000\u0000T\u029a\u0001\u0000\u0000\u0000"+ - "V\u029f\u0001\u0000\u0000\u0000X\u02be\u0001\u0000\u0000\u0000Z\u02c1"+ - "\u0001\u0000\u0000\u0000\\\u02ef\u0001\u0000\u0000\u0000^\u02f1\u0001"+ - "\u0000\u0000\u0000`\u02f4\u0001\u0000\u0000\u0000b\u02f8\u0001\u0000\u0000"+ - "\u0000d\u02fc\u0001\u0000\u0000\u0000f\u02fe\u0001\u0000\u0000\u0000h"+ - "\u0301\u0001\u0000\u0000\u0000j\u0303\u0001\u0000\u0000\u0000l\u0308\u0001"+ - "\u0000\u0000\u0000n\u030a\u0001\u0000\u0000\u0000p\u0310\u0001\u0000\u0000"+ - "\u0000r\u0316\u0001\u0000\u0000\u0000t\u0319\u0001\u0000\u0000\u0000v"+ - "\u031c\u0001\u0000\u0000\u0000x\u0321\u0001\u0000\u0000\u0000z\u0326\u0001"+ - "\u0000\u0000\u0000|\u0328\u0001\u0000\u0000\u0000~\u032c\u0001\u0000\u0000"+ - "\u0000\u0080\u0331\u0001\u0000\u0000\u0000\u0082\u0337\u0001\u0000\u0000"+ - "\u0000\u0084\u033a\u0001\u0000\u0000\u0000\u0086\u033c\u0001\u0000\u0000"+ - "\u0000\u0088\u0342\u0001\u0000\u0000\u0000\u008a\u0344\u0001\u0000\u0000"+ - "\u0000\u008c\u0349\u0001\u0000\u0000\u0000\u008e\u034c\u0001\u0000\u0000"+ - "\u0000\u0090\u034f\u0001\u0000\u0000\u0000\u0092\u0352\u0001\u0000\u0000"+ - "\u0000\u0094\u0354\u0001\u0000\u0000\u0000\u0096\u0357\u0001\u0000\u0000"+ - "\u0000\u0098\u0359\u0001\u0000\u0000\u0000\u009a\u035c\u0001\u0000\u0000"+ - "\u0000\u009c\u035e\u0001\u0000\u0000\u0000\u009e\u0360\u0001\u0000\u0000"+ - "\u0000\u00a0\u0362\u0001\u0000\u0000\u0000\u00a2\u0364\u0001\u0000\u0000"+ - "\u0000\u00a4\u0366\u0001\u0000\u0000\u0000\u00a6\u037c\u0001\u0000\u0000"+ - "\u0000\u00a8\u037e\u0001\u0000\u0000\u0000\u00aa\u0383\u0001\u0000\u0000"+ - "\u0000\u00ac\u0398\u0001\u0000\u0000\u0000\u00ae\u039a\u0001\u0000\u0000"+ - "\u0000\u00b0\u03a2\u0001\u0000\u0000\u0000\u00b2\u03a4\u0001\u0000\u0000"+ - "\u0000\u00b4\u03a8\u0001\u0000\u0000\u0000\u00b6\u03ac\u0001\u0000\u0000"+ - "\u0000\u00b8\u03b0\u0001\u0000\u0000\u0000\u00ba\u03b5\u0001\u0000\u0000"+ - "\u0000\u00bc\u03ba\u0001\u0000\u0000\u0000\u00be\u03be\u0001\u0000\u0000"+ - "\u0000\u00c0\u03c2\u0001\u0000\u0000\u0000\u00c2\u03c6\u0001\u0000\u0000"+ - "\u0000\u00c4\u03cb\u0001\u0000\u0000\u0000\u00c6\u03cf\u0001\u0000\u0000"+ - "\u0000\u00c8\u03d3\u0001\u0000\u0000\u0000\u00ca\u03d7\u0001\u0000\u0000"+ - "\u0000\u00cc\u03db\u0001\u0000\u0000\u0000\u00ce\u03df\u0001\u0000\u0000"+ - "\u0000\u00d0\u03eb\u0001\u0000\u0000\u0000\u00d2\u03ee\u0001\u0000\u0000"+ - "\u0000\u00d4\u03f2\u0001\u0000\u0000\u0000\u00d6\u03f6\u0001\u0000\u0000"+ - "\u0000\u00d8\u03fa\u0001\u0000\u0000\u0000\u00da\u03fe\u0001\u0000\u0000"+ - "\u0000\u00dc\u0402\u0001\u0000\u0000\u0000\u00de\u0406\u0001\u0000\u0000"+ - "\u0000\u00e0\u040b\u0001\u0000\u0000\u0000\u00e2\u040f\u0001\u0000\u0000"+ - "\u0000\u00e4\u0417\u0001\u0000\u0000\u0000\u00e6\u042c\u0001\u0000\u0000"+ - "\u0000\u00e8\u0430\u0001\u0000\u0000\u0000\u00ea\u0434\u0001\u0000\u0000"+ - "\u0000\u00ec\u0438\u0001\u0000\u0000\u0000\u00ee\u043c\u0001\u0000\u0000"+ - "\u0000\u00f0\u0440\u0001\u0000\u0000\u0000\u00f2\u0445\u0001\u0000\u0000"+ - "\u0000\u00f4\u0449\u0001\u0000\u0000\u0000\u00f6\u044d\u0001\u0000\u0000"+ - "\u0000\u00f8\u0451\u0001\u0000\u0000\u0000\u00fa\u0454\u0001\u0000\u0000"+ - "\u0000\u00fc\u0458\u0001\u0000\u0000\u0000\u00fe\u045c\u0001\u0000\u0000"+ - "\u0000\u0100\u0460\u0001\u0000\u0000\u0000\u0102\u0464\u0001\u0000\u0000"+ - "\u0000\u0104\u0469\u0001\u0000\u0000\u0000\u0106\u046e\u0001\u0000\u0000"+ - "\u0000\u0108\u0473\u0001\u0000\u0000\u0000\u010a\u047a\u0001\u0000\u0000"+ - "\u0000\u010c\u0483\u0001\u0000\u0000\u0000\u010e\u048a\u0001\u0000\u0000"+ - "\u0000\u0110\u048e\u0001\u0000\u0000\u0000\u0112\u0492\u0001\u0000\u0000"+ - "\u0000\u0114\u0496\u0001\u0000\u0000\u0000\u0116\u049a\u0001\u0000\u0000"+ - "\u0000\u0118\u04a0\u0001\u0000\u0000\u0000\u011a\u04a4\u0001\u0000\u0000"+ - "\u0000\u011c\u04a8\u0001\u0000\u0000\u0000\u011e\u04ac\u0001\u0000\u0000"+ - "\u0000\u0120\u04b0\u0001\u0000\u0000\u0000\u0122\u04b4\u0001\u0000\u0000"+ - "\u0000\u0124\u04b8\u0001\u0000\u0000\u0000\u0126\u04bc\u0001\u0000\u0000"+ - "\u0000\u0128\u04c0\u0001\u0000\u0000\u0000\u012a\u04c4\u0001\u0000\u0000"+ - "\u0000\u012c\u04c9\u0001\u0000\u0000\u0000\u012e\u04cd\u0001\u0000\u0000"+ - "\u0000\u0130\u04d1\u0001\u0000\u0000\u0000\u0132\u04d5\u0001\u0000\u0000"+ - "\u0000\u0134\u04d9\u0001\u0000\u0000\u0000\u0136\u04dd\u0001\u0000\u0000"+ - "\u0000\u0138\u04e1\u0001\u0000\u0000\u0000\u013a\u04e6\u0001\u0000\u0000"+ - "\u0000\u013c\u04eb\u0001\u0000\u0000\u0000\u013e\u04ef\u0001\u0000\u0000"+ - "\u0000\u0140\u04f3\u0001\u0000\u0000\u0000\u0142\u04f7\u0001\u0000\u0000"+ - "\u0000\u0144\u04fc\u0001\u0000\u0000\u0000\u0146\u0506\u0001\u0000\u0000"+ - "\u0000\u0148\u050a\u0001\u0000\u0000\u0000\u014a\u050e\u0001\u0000\u0000"+ - "\u0000\u014c\u0512\u0001\u0000\u0000\u0000\u014e\u0517\u0001\u0000\u0000"+ - "\u0000\u0150\u051e\u0001\u0000\u0000\u0000\u0152\u0522\u0001\u0000\u0000"+ - "\u0000\u0154\u0526\u0001\u0000\u0000\u0000\u0156\u052a\u0001\u0000\u0000"+ - "\u0000\u0158\u052e\u0001\u0000\u0000\u0000\u015a\u0533\u0001\u0000\u0000"+ - "\u0000\u015c\u0537\u0001\u0000\u0000\u0000\u015e\u053b\u0001\u0000\u0000"+ - "\u0000\u0160\u053f\u0001\u0000\u0000\u0000\u0162\u0544\u0001\u0000\u0000"+ - "\u0000\u0164\u0548\u0001\u0000\u0000\u0000\u0166\u054c\u0001\u0000\u0000"+ - "\u0000\u0168\u0550\u0001\u0000\u0000\u0000\u016a\u0554\u0001\u0000\u0000"+ - "\u0000\u016c\u0558\u0001\u0000\u0000\u0000\u016e\u055e\u0001\u0000\u0000"+ - "\u0000\u0170\u0562\u0001\u0000\u0000\u0000\u0172\u0566\u0001\u0000\u0000"+ - "\u0000\u0174\u056a\u0001\u0000\u0000\u0000\u0176\u056e\u0001\u0000\u0000"+ - "\u0000\u0178\u0572\u0001\u0000\u0000\u0000\u017a\u0576\u0001\u0000\u0000"+ - "\u0000\u017c\u057b\u0001\u0000\u0000\u0000\u017e\u0581\u0001\u0000\u0000"+ - "\u0000\u0180\u0587\u0001\u0000\u0000\u0000\u0182\u058b\u0001\u0000\u0000"+ - "\u0000\u0184\u058f\u0001\u0000\u0000\u0000\u0186\u0593\u0001\u0000\u0000"+ - "\u0000\u0188\u0599\u0001\u0000\u0000\u0000\u018a\u059f\u0001\u0000\u0000"+ - "\u0000\u018c\u05a3\u0001\u0000\u0000\u0000\u018e\u05a7\u0001\u0000\u0000"+ - "\u0000\u0190\u05ab\u0001\u0000\u0000\u0000\u0192\u05b1\u0001\u0000\u0000"+ - "\u0000\u0194\u05b7\u0001\u0000\u0000\u0000\u0196\u05bd\u0001\u0000\u0000"+ - "\u0000\u0198\u0199\u0007\u0000\u0000\u0000\u0199\u019a\u0007\u0001\u0000"+ - "\u0000\u019a\u019b\u0007\u0002\u0000\u0000\u019b\u019c\u0007\u0002\u0000"+ - "\u0000\u019c\u019d\u0007\u0003\u0000\u0000\u019d\u019e\u0007\u0004\u0000"+ - "\u0000\u019e\u019f\u0007\u0005\u0000\u0000\u019f\u01a0\u0001\u0000\u0000"+ - "\u0000\u01a0\u01a1\u0006\u0000\u0000\u0000\u01a1\u0011\u0001\u0000\u0000"+ - "\u0000\u01a2\u01a3\u0007\u0000\u0000\u0000\u01a3\u01a4\u0007\u0006\u0000"+ - "\u0000\u01a4\u01a5\u0007\u0007\u0000\u0000\u01a5\u01a6\u0007\b\u0000\u0000"+ - "\u01a6\u01a7\u0001\u0000\u0000\u0000\u01a7\u01a8\u0006\u0001\u0001\u0000"+ - "\u01a8\u0013\u0001\u0000\u0000\u0000\u01a9\u01aa\u0007\u0003\u0000\u0000"+ - "\u01aa\u01ab\u0007\t\u0000\u0000\u01ab\u01ac\u0007\u0006\u0000\u0000\u01ac"+ - "\u01ad\u0007\u0001\u0000\u0000\u01ad\u01ae\u0007\u0004\u0000\u0000\u01ae"+ - "\u01af\u0007\n\u0000\u0000\u01af\u01b0\u0001\u0000\u0000\u0000\u01b0\u01b1"+ - "\u0006\u0002\u0002\u0000\u01b1\u0015\u0001\u0000\u0000\u0000\u01b2\u01b3"+ - "\u0007\u0003\u0000\u0000\u01b3\u01b4\u0007\u000b\u0000\u0000\u01b4\u01b5"+ - "\u0007\f\u0000\u0000\u01b5\u01b6\u0007\r\u0000\u0000\u01b6\u01b7\u0001"+ - "\u0000\u0000\u0000\u01b7\u01b8\u0006\u0003\u0000\u0000\u01b8\u0017\u0001"+ - "\u0000\u0000\u0000\u01b9\u01ba\u0007\u0003\u0000\u0000\u01ba\u01bb\u0007"+ - "\u000e\u0000\u0000\u01bb\u01bc\u0007\b\u0000\u0000\u01bc\u01bd\u0007\r"+ - "\u0000\u0000\u01bd\u01be\u0007\f\u0000\u0000\u01be\u01bf\u0007\u0001\u0000"+ - "\u0000\u01bf\u01c0\u0007\t\u0000\u0000\u01c0\u01c1\u0001\u0000\u0000\u0000"+ - "\u01c1\u01c2\u0006\u0004\u0003\u0000\u01c2\u0019\u0001\u0000\u0000\u0000"+ - "\u01c3\u01c4\u0007\u000f\u0000\u0000\u01c4\u01c5\u0007\u0006\u0000\u0000"+ - "\u01c5\u01c6\u0007\u0007\u0000\u0000\u01c6\u01c7\u0007\u0010\u0000\u0000"+ - "\u01c7\u01c8\u0001\u0000\u0000\u0000\u01c8\u01c9\u0006\u0005\u0004\u0000"+ - "\u01c9\u001b\u0001\u0000\u0000\u0000\u01ca\u01cb\u0007\u0011\u0000\u0000"+ - "\u01cb\u01cc\u0007\u0006\u0000\u0000\u01cc\u01cd\u0007\u0007\u0000\u0000"+ - "\u01cd\u01ce\u0007\u0012\u0000\u0000\u01ce\u01cf\u0001\u0000\u0000\u0000"+ - "\u01cf\u01d0\u0006\u0006\u0000\u0000\u01d0\u001d\u0001\u0000\u0000\u0000"+ - "\u01d1\u01d2\u0007\u0012\u0000\u0000\u01d2\u01d3\u0007\u0003\u0000\u0000"+ - "\u01d3\u01d4\u0007\u0003\u0000\u0000\u01d4\u01d5\u0007\b\u0000\u0000\u01d5"+ - "\u01d6\u0001\u0000\u0000\u0000\u01d6\u01d7\u0006\u0007\u0001\u0000\u01d7"+ - "\u001f\u0001\u0000\u0000\u0000\u01d8\u01d9\u0007\r\u0000\u0000\u01d9\u01da"+ - "\u0007\u0001\u0000\u0000\u01da\u01db\u0007\u0010\u0000\u0000\u01db\u01dc"+ - "\u0007\u0001\u0000\u0000\u01dc\u01dd\u0007\u0005\u0000\u0000\u01dd\u01de"+ - "\u0001\u0000\u0000\u0000\u01de\u01df\u0006\b\u0000\u0000\u01df!\u0001"+ - "\u0000\u0000\u0000\u01e0\u01e1\u0007\u0010\u0000\u0000\u01e1\u01e2\u0007"+ - "\u0003\u0000\u0000\u01e2\u01e3\u0007\u0005\u0000\u0000\u01e3\u01e4\u0007"+ - "\f\u0000\u0000\u01e4\u01e5\u0001\u0000\u0000\u0000\u01e5\u01e6\u0006\t"+ - "\u0005\u0000\u01e6#\u0001\u0000\u0000\u0000\u01e7\u01e8\u0007\u0010\u0000"+ - "\u0000\u01e8\u01e9\u0007\u000b\u0000\u0000\u01e9\u01ea\u0005_\u0000\u0000"+ - "\u01ea\u01eb\u0007\u0003\u0000\u0000\u01eb\u01ec\u0007\u000e\u0000\u0000"+ - "\u01ec\u01ed\u0007\b\u0000\u0000\u01ed\u01ee\u0007\f\u0000\u0000\u01ee"+ - "\u01ef\u0007\t\u0000\u0000\u01ef\u01f0\u0007\u0000\u0000\u0000\u01f0\u01f1"+ - "\u0001\u0000\u0000\u0000\u01f1\u01f2\u0006\n\u0006\u0000\u01f2%\u0001"+ - "\u0000\u0000\u0000\u01f3\u01f4\u0007\u0006\u0000\u0000\u01f4\u01f5\u0007"+ - "\u0003\u0000\u0000\u01f5\u01f6\u0007\t\u0000\u0000\u01f6\u01f7\u0007\f"+ - "\u0000\u0000\u01f7\u01f8\u0007\u0010\u0000\u0000\u01f8\u01f9\u0007\u0003"+ - "\u0000\u0000\u01f9\u01fa\u0001\u0000\u0000\u0000\u01fa\u01fb\u0006\u000b"+ - "\u0007\u0000\u01fb\'\u0001\u0000\u0000\u0000\u01fc\u01fd\u0007\u0006\u0000"+ - "\u0000\u01fd\u01fe\u0007\u0007\u0000\u0000\u01fe\u01ff\u0007\u0013\u0000"+ - "\u0000\u01ff\u0200\u0001\u0000\u0000\u0000\u0200\u0201\u0006\f\u0000\u0000"+ - "\u0201)\u0001\u0000\u0000\u0000\u0202\u0203\u0007\u0002\u0000\u0000\u0203"+ - "\u0204\u0007\n\u0000\u0000\u0204\u0205\u0007\u0007\u0000\u0000\u0205\u0206"+ - "\u0007\u0013\u0000\u0000\u0206\u0207\u0001\u0000\u0000\u0000\u0207\u0208"+ - "\u0006\r\b\u0000\u0208+\u0001\u0000\u0000\u0000\u0209\u020a\u0007\u0002"+ - "\u0000\u0000\u020a\u020b\u0007\u0007\u0000\u0000\u020b\u020c\u0007\u0006"+ - "\u0000\u0000\u020c\u020d\u0007\u0005\u0000\u0000\u020d\u020e\u0001\u0000"+ - "\u0000\u0000\u020e\u020f\u0006\u000e\u0000\u0000\u020f-\u0001\u0000\u0000"+ - "\u0000\u0210\u0211\u0007\u0002\u0000\u0000\u0211\u0212\u0007\u0005\u0000"+ - "\u0000\u0212\u0213\u0007\f\u0000\u0000\u0213\u0214\u0007\u0005\u0000\u0000"+ - "\u0214\u0215\u0007\u0002\u0000\u0000\u0215\u0216\u0001\u0000\u0000\u0000"+ - "\u0216\u0217\u0006\u000f\u0000\u0000\u0217/\u0001\u0000\u0000\u0000\u0218"+ - "\u0219\u0007\u0013\u0000\u0000\u0219\u021a\u0007\n\u0000\u0000\u021a\u021b"+ - "\u0007\u0003\u0000\u0000\u021b\u021c\u0007\u0006\u0000\u0000\u021c\u021d"+ - "\u0007\u0003\u0000\u0000\u021d\u021e\u0001\u0000\u0000\u0000\u021e\u021f"+ - "\u0006\u0010\u0000\u0000\u021f1\u0001\u0000\u0000\u0000\u0220\u0221\u0004"+ - "\u0011\u0000\u0000\u0221\u0222\u0007\u0001\u0000\u0000\u0222\u0223\u0007"+ - "\t\u0000\u0000\u0223\u0224\u0007\r\u0000\u0000\u0224\u0225\u0007\u0001"+ - "\u0000\u0000\u0225\u0226\u0007\t\u0000\u0000\u0226\u0227\u0007\u0003\u0000"+ - "\u0000\u0227\u0228\u0007\u0002\u0000\u0000\u0228\u0229\u0007\u0005\u0000"+ - "\u0000\u0229\u022a\u0007\f\u0000\u0000\u022a\u022b\u0007\u0005\u0000\u0000"+ - "\u022b\u022c\u0007\u0002\u0000\u0000\u022c\u022d\u0001\u0000\u0000\u0000"+ - "\u022d\u022e\u0006\u0011\u0000\u0000\u022e3\u0001\u0000\u0000\u0000\u022f"+ - "\u0230\u0004\u0012\u0001\u0000\u0230\u0231\u0007\r\u0000\u0000\u0231\u0232"+ - "\u0007\u0007\u0000\u0000\u0232\u0233\u0007\u0007\u0000\u0000\u0233\u0234"+ - "\u0007\u0012\u0000\u0000\u0234\u0235\u0007\u0014\u0000\u0000\u0235\u0236"+ - "\u0007\b\u0000\u0000\u0236\u0237\u0001\u0000\u0000\u0000\u0237\u0238\u0006"+ - "\u0012\t\u0000\u02385\u0001\u0000\u0000\u0000\u0239\u023a\u0004\u0013"+ - "\u0002\u0000\u023a\u023b\u0007\u0010\u0000\u0000\u023b\u023c\u0007\f\u0000"+ - "\u0000\u023c\u023d\u0007\u0005\u0000\u0000\u023d\u023e\u0007\u0004\u0000"+ - "\u0000\u023e\u023f\u0007\n\u0000\u0000\u023f\u0240\u0001\u0000\u0000\u0000"+ - "\u0240\u0241\u0006\u0013\u0000\u0000\u02417\u0001\u0000\u0000\u0000\u0242"+ - "\u0243\u0004\u0014\u0003\u0000\u0243\u0244\u0007\u0010\u0000\u0000\u0244"+ - "\u0245\u0007\u0003\u0000\u0000\u0245\u0246\u0007\u0005\u0000\u0000\u0246"+ - "\u0247\u0007\u0006\u0000\u0000\u0247\u0248\u0007\u0001\u0000\u0000\u0248"+ - "\u0249\u0007\u0004\u0000\u0000\u0249\u024a\u0007\u0002\u0000\u0000\u024a"+ - "\u024b\u0001\u0000\u0000\u0000\u024b\u024c\u0006\u0014\n\u0000\u024c9"+ - "\u0001\u0000\u0000\u0000\u024d\u024f\b\u0015\u0000\u0000\u024e\u024d\u0001"+ - "\u0000\u0000\u0000\u024f\u0250\u0001\u0000\u0000\u0000\u0250\u024e\u0001"+ - "\u0000\u0000\u0000\u0250\u0251\u0001\u0000\u0000\u0000\u0251\u0252\u0001"+ - "\u0000\u0000\u0000\u0252\u0253\u0006\u0015\u0000\u0000\u0253;\u0001\u0000"+ - "\u0000\u0000\u0254\u0255\u0005/\u0000\u0000\u0255\u0256\u0005/\u0000\u0000"+ - "\u0256\u025a\u0001\u0000\u0000\u0000\u0257\u0259\b\u0016\u0000\u0000\u0258"+ - "\u0257\u0001\u0000\u0000\u0000\u0259\u025c\u0001\u0000\u0000\u0000\u025a"+ - "\u0258\u0001\u0000\u0000\u0000\u025a\u025b\u0001\u0000\u0000\u0000\u025b"+ - "\u025e\u0001\u0000\u0000\u0000\u025c\u025a\u0001\u0000\u0000\u0000\u025d"+ - "\u025f\u0005\r\u0000\u0000\u025e\u025d\u0001\u0000\u0000\u0000\u025e\u025f"+ - "\u0001\u0000\u0000\u0000\u025f\u0261\u0001\u0000\u0000\u0000\u0260\u0262"+ - "\u0005\n\u0000\u0000\u0261\u0260\u0001\u0000\u0000\u0000\u0261\u0262\u0001"+ - "\u0000\u0000\u0000\u0262\u0263\u0001\u0000\u0000\u0000\u0263\u0264\u0006"+ - "\u0016\u000b\u0000\u0264=\u0001\u0000\u0000\u0000\u0265\u0266\u0005/\u0000"+ - "\u0000\u0266\u0267\u0005*\u0000\u0000\u0267\u026c\u0001\u0000\u0000\u0000"+ - "\u0268\u026b\u0003>\u0017\u0000\u0269\u026b\t\u0000\u0000\u0000\u026a"+ - "\u0268\u0001\u0000\u0000\u0000\u026a\u0269\u0001\u0000\u0000\u0000\u026b"+ - "\u026e\u0001\u0000\u0000\u0000\u026c\u026d\u0001\u0000\u0000\u0000\u026c"+ - "\u026a\u0001\u0000\u0000\u0000\u026d\u026f\u0001\u0000\u0000\u0000\u026e"+ - "\u026c\u0001\u0000\u0000\u0000\u026f\u0270\u0005*\u0000\u0000\u0270\u0271"+ - "\u0005/\u0000\u0000\u0271\u0272\u0001\u0000\u0000\u0000\u0272\u0273\u0006"+ - "\u0017\u000b\u0000\u0273?\u0001\u0000\u0000\u0000\u0274\u0276\u0007\u0017"+ - "\u0000\u0000\u0275\u0274\u0001\u0000\u0000\u0000\u0276\u0277\u0001\u0000"+ - "\u0000\u0000\u0277\u0275\u0001\u0000\u0000\u0000\u0277\u0278\u0001\u0000"+ - "\u0000\u0000\u0278\u0279\u0001\u0000\u0000\u0000\u0279\u027a\u0006\u0018"+ - "\u000b\u0000\u027aA\u0001\u0000\u0000\u0000\u027b\u027c\u0005|\u0000\u0000"+ - "\u027c\u027d\u0001\u0000\u0000\u0000\u027d\u027e\u0006\u0019\f\u0000\u027e"+ - "C\u0001\u0000\u0000\u0000\u027f\u0280\u0007\u0018\u0000\u0000\u0280E\u0001"+ - "\u0000\u0000\u0000\u0281\u0282\u0007\u0019\u0000\u0000\u0282G\u0001\u0000"+ - "\u0000\u0000\u0283\u0284\u0005\\\u0000\u0000\u0284\u0285\u0007\u001a\u0000"+ - "\u0000\u0285I\u0001\u0000\u0000\u0000\u0286\u0287\b\u001b\u0000\u0000"+ - "\u0287K\u0001\u0000\u0000\u0000\u0288\u028a\u0007\u0003\u0000\u0000\u0289"+ - "\u028b\u0007\u001c\u0000\u0000\u028a\u0289\u0001\u0000\u0000\u0000\u028a"+ - "\u028b\u0001\u0000\u0000\u0000\u028b\u028d\u0001\u0000\u0000\u0000\u028c"+ - "\u028e\u0003D\u001a\u0000\u028d\u028c\u0001\u0000\u0000\u0000\u028e\u028f"+ - "\u0001\u0000\u0000\u0000\u028f\u028d\u0001\u0000\u0000\u0000\u028f\u0290"+ - "\u0001\u0000\u0000\u0000\u0290M\u0001\u0000\u0000\u0000\u0291\u0292\u0005"+ - "@\u0000\u0000\u0292O\u0001\u0000\u0000\u0000\u0293\u0294\u0005`\u0000"+ - "\u0000\u0294Q\u0001\u0000\u0000\u0000\u0295\u0299\b\u001d\u0000\u0000"+ - "\u0296\u0297\u0005`\u0000\u0000\u0297\u0299\u0005`\u0000\u0000\u0298\u0295"+ - "\u0001\u0000\u0000\u0000\u0298\u0296\u0001\u0000\u0000\u0000\u0299S\u0001"+ - "\u0000\u0000\u0000\u029a\u029b\u0005_\u0000\u0000\u029bU\u0001\u0000\u0000"+ - "\u0000\u029c\u02a0\u0003F\u001b\u0000\u029d\u02a0\u0003D\u001a\u0000\u029e"+ - "\u02a0\u0003T\"\u0000\u029f\u029c\u0001\u0000\u0000\u0000\u029f\u029d"+ - "\u0001\u0000\u0000\u0000\u029f\u029e\u0001\u0000\u0000\u0000\u02a0W\u0001"+ - "\u0000\u0000\u0000\u02a1\u02a6\u0005\"\u0000\u0000\u02a2\u02a5\u0003H"+ - "\u001c\u0000\u02a3\u02a5\u0003J\u001d\u0000\u02a4\u02a2\u0001\u0000\u0000"+ - "\u0000\u02a4\u02a3\u0001\u0000\u0000\u0000\u02a5\u02a8\u0001\u0000\u0000"+ - "\u0000\u02a6\u02a4\u0001\u0000\u0000\u0000\u02a6\u02a7\u0001\u0000\u0000"+ - "\u0000\u02a7\u02a9\u0001\u0000\u0000\u0000\u02a8\u02a6\u0001\u0000\u0000"+ - "\u0000\u02a9\u02bf\u0005\"\u0000\u0000\u02aa\u02ab\u0005\"\u0000\u0000"+ - "\u02ab\u02ac\u0005\"\u0000\u0000\u02ac\u02ad\u0005\"\u0000\u0000\u02ad"+ - "\u02b1\u0001\u0000\u0000\u0000\u02ae\u02b0\b\u0016\u0000\u0000\u02af\u02ae"+ - "\u0001\u0000\u0000\u0000\u02b0\u02b3\u0001\u0000\u0000\u0000\u02b1\u02b2"+ - "\u0001\u0000\u0000\u0000\u02b1\u02af\u0001\u0000\u0000\u0000\u02b2\u02b4"+ - "\u0001\u0000\u0000\u0000\u02b3\u02b1\u0001\u0000\u0000\u0000\u02b4\u02b5"+ - "\u0005\"\u0000\u0000\u02b5\u02b6\u0005\"\u0000\u0000\u02b6\u02b7\u0005"+ - "\"\u0000\u0000\u02b7\u02b9\u0001\u0000\u0000\u0000\u02b8\u02ba\u0005\""+ - "\u0000\u0000\u02b9\u02b8\u0001\u0000\u0000\u0000\u02b9\u02ba\u0001\u0000"+ - "\u0000\u0000\u02ba\u02bc\u0001\u0000\u0000\u0000\u02bb\u02bd\u0005\"\u0000"+ - "\u0000\u02bc\u02bb\u0001\u0000\u0000\u0000\u02bc\u02bd\u0001\u0000\u0000"+ - "\u0000\u02bd\u02bf\u0001\u0000\u0000\u0000\u02be\u02a1\u0001\u0000\u0000"+ - "\u0000\u02be\u02aa\u0001\u0000\u0000\u0000\u02bfY\u0001\u0000\u0000\u0000"+ - "\u02c0\u02c2\u0003D\u001a\u0000\u02c1\u02c0\u0001\u0000\u0000\u0000\u02c2"+ - "\u02c3\u0001\u0000\u0000\u0000\u02c3\u02c1\u0001\u0000\u0000\u0000\u02c3"+ - "\u02c4\u0001\u0000\u0000\u0000\u02c4[\u0001\u0000\u0000\u0000\u02c5\u02c7"+ - "\u0003D\u001a\u0000\u02c6\u02c5\u0001\u0000\u0000\u0000\u02c7\u02c8\u0001"+ - "\u0000\u0000\u0000\u02c8\u02c6\u0001\u0000\u0000\u0000\u02c8\u02c9\u0001"+ - "\u0000\u0000\u0000\u02c9\u02ca\u0001\u0000\u0000\u0000\u02ca\u02ce\u0003"+ - "l.\u0000\u02cb\u02cd\u0003D\u001a\u0000\u02cc\u02cb\u0001\u0000\u0000"+ - "\u0000\u02cd\u02d0\u0001\u0000\u0000\u0000\u02ce\u02cc\u0001\u0000\u0000"+ - "\u0000\u02ce\u02cf\u0001\u0000\u0000\u0000\u02cf\u02f0\u0001\u0000\u0000"+ - "\u0000\u02d0\u02ce\u0001\u0000\u0000\u0000\u02d1\u02d3\u0003l.\u0000\u02d2"+ - "\u02d4\u0003D\u001a\u0000\u02d3\u02d2\u0001\u0000\u0000\u0000\u02d4\u02d5"+ - "\u0001\u0000\u0000\u0000\u02d5\u02d3\u0001\u0000\u0000\u0000\u02d5\u02d6"+ - "\u0001\u0000\u0000\u0000\u02d6\u02f0\u0001\u0000\u0000\u0000\u02d7\u02d9"+ - "\u0003D\u001a\u0000\u02d8\u02d7\u0001\u0000\u0000\u0000\u02d9\u02da\u0001"+ - "\u0000\u0000\u0000\u02da\u02d8\u0001\u0000\u0000\u0000\u02da\u02db\u0001"+ - "\u0000\u0000\u0000\u02db\u02e3\u0001\u0000\u0000\u0000\u02dc\u02e0\u0003"+ - "l.\u0000\u02dd\u02df\u0003D\u001a\u0000\u02de\u02dd\u0001\u0000\u0000"+ - "\u0000\u02df\u02e2\u0001\u0000\u0000\u0000\u02e0\u02de\u0001\u0000\u0000"+ - "\u0000\u02e0\u02e1\u0001\u0000\u0000\u0000\u02e1\u02e4\u0001\u0000\u0000"+ - "\u0000\u02e2\u02e0\u0001\u0000\u0000\u0000\u02e3\u02dc\u0001\u0000\u0000"+ - "\u0000\u02e3\u02e4\u0001\u0000\u0000\u0000\u02e4\u02e5\u0001\u0000\u0000"+ - "\u0000\u02e5\u02e6\u0003L\u001e\u0000\u02e6\u02f0\u0001\u0000\u0000\u0000"+ - "\u02e7\u02e9\u0003l.\u0000\u02e8\u02ea\u0003D\u001a\u0000\u02e9\u02e8"+ - "\u0001\u0000\u0000\u0000\u02ea\u02eb\u0001\u0000\u0000\u0000\u02eb\u02e9"+ - "\u0001\u0000\u0000\u0000\u02eb\u02ec\u0001\u0000\u0000\u0000\u02ec\u02ed"+ - "\u0001\u0000\u0000\u0000\u02ed\u02ee\u0003L\u001e\u0000\u02ee\u02f0\u0001"+ - "\u0000\u0000\u0000\u02ef\u02c6\u0001\u0000\u0000\u0000\u02ef\u02d1\u0001"+ - "\u0000\u0000\u0000\u02ef\u02d8\u0001\u0000\u0000\u0000\u02ef\u02e7\u0001"+ - "\u0000\u0000\u0000\u02f0]\u0001\u0000\u0000\u0000\u02f1\u02f2\u0007\u001e"+ - "\u0000\u0000\u02f2\u02f3\u0007\u001f\u0000\u0000\u02f3_\u0001\u0000\u0000"+ - "\u0000\u02f4\u02f5\u0007\f\u0000\u0000\u02f5\u02f6\u0007\t\u0000\u0000"+ - "\u02f6\u02f7\u0007\u0000\u0000\u0000\u02f7a\u0001\u0000\u0000\u0000\u02f8"+ - "\u02f9\u0007\f\u0000\u0000\u02f9\u02fa\u0007\u0002\u0000\u0000\u02fa\u02fb"+ - "\u0007\u0004\u0000\u0000\u02fbc\u0001\u0000\u0000\u0000\u02fc\u02fd\u0005"+ - "=\u0000\u0000\u02fde\u0001\u0000\u0000\u0000\u02fe\u02ff\u0005:\u0000"+ - "\u0000\u02ff\u0300\u0005:\u0000\u0000\u0300g\u0001\u0000\u0000\u0000\u0301"+ - "\u0302\u0005,\u0000\u0000\u0302i\u0001\u0000\u0000\u0000\u0303\u0304\u0007"+ - "\u0000\u0000\u0000\u0304\u0305\u0007\u0003\u0000\u0000\u0305\u0306\u0007"+ - "\u0002\u0000\u0000\u0306\u0307\u0007\u0004\u0000\u0000\u0307k\u0001\u0000"+ - "\u0000\u0000\u0308\u0309\u0005.\u0000\u0000\u0309m\u0001\u0000\u0000\u0000"+ - "\u030a\u030b\u0007\u000f\u0000\u0000\u030b\u030c\u0007\f\u0000\u0000\u030c"+ - "\u030d\u0007\r\u0000\u0000\u030d\u030e\u0007\u0002\u0000\u0000\u030e\u030f"+ - "\u0007\u0003\u0000\u0000\u030fo\u0001\u0000\u0000\u0000\u0310\u0311\u0007"+ - "\u000f\u0000\u0000\u0311\u0312\u0007\u0001\u0000\u0000\u0312\u0313\u0007"+ - "\u0006\u0000\u0000\u0313\u0314\u0007\u0002\u0000\u0000\u0314\u0315\u0007"+ - "\u0005\u0000\u0000\u0315q\u0001\u0000\u0000\u0000\u0316\u0317\u0007\u0001"+ - "\u0000\u0000\u0317\u0318\u0007\t\u0000\u0000\u0318s\u0001\u0000\u0000"+ - "\u0000\u0319\u031a\u0007\u0001\u0000\u0000\u031a\u031b\u0007\u0002\u0000"+ - "\u0000\u031bu\u0001\u0000\u0000\u0000\u031c\u031d\u0007\r\u0000\u0000"+ - "\u031d\u031e\u0007\f\u0000\u0000\u031e\u031f\u0007\u0002\u0000\u0000\u031f"+ - "\u0320\u0007\u0005\u0000\u0000\u0320w\u0001\u0000\u0000\u0000\u0321\u0322"+ - "\u0007\r\u0000\u0000\u0322\u0323\u0007\u0001\u0000\u0000\u0323\u0324\u0007"+ - "\u0012\u0000\u0000\u0324\u0325\u0007\u0003\u0000\u0000\u0325y\u0001\u0000"+ - "\u0000\u0000\u0326\u0327\u0005(\u0000\u0000\u0327{\u0001\u0000\u0000\u0000"+ - "\u0328\u0329\u0007\t\u0000\u0000\u0329\u032a\u0007\u0007\u0000\u0000\u032a"+ - "\u032b\u0007\u0005\u0000\u0000\u032b}\u0001\u0000\u0000\u0000\u032c\u032d"+ - "\u0007\t\u0000\u0000\u032d\u032e\u0007\u0014\u0000\u0000\u032e\u032f\u0007"+ - "\r\u0000\u0000\u032f\u0330\u0007\r\u0000\u0000\u0330\u007f\u0001\u0000"+ - "\u0000\u0000\u0331\u0332\u0007\t\u0000\u0000\u0332\u0333\u0007\u0014\u0000"+ - "\u0000\u0333\u0334\u0007\r\u0000\u0000\u0334\u0335\u0007\r\u0000\u0000"+ - "\u0335\u0336\u0007\u0002\u0000\u0000\u0336\u0081\u0001\u0000\u0000\u0000"+ - "\u0337\u0338\u0007\u0007\u0000\u0000\u0338\u0339\u0007\u0006\u0000\u0000"+ - "\u0339\u0083\u0001\u0000\u0000\u0000\u033a\u033b\u0005?\u0000\u0000\u033b"+ - "\u0085\u0001\u0000\u0000\u0000\u033c\u033d\u0007\u0006\u0000\u0000\u033d"+ - "\u033e\u0007\r\u0000\u0000\u033e\u033f\u0007\u0001\u0000\u0000\u033f\u0340"+ - "\u0007\u0012\u0000\u0000\u0340\u0341\u0007\u0003\u0000\u0000\u0341\u0087"+ - "\u0001\u0000\u0000\u0000\u0342\u0343\u0005)\u0000\u0000\u0343\u0089\u0001"+ - "\u0000\u0000\u0000\u0344\u0345\u0007\u0005\u0000\u0000\u0345\u0346\u0007"+ - "\u0006\u0000\u0000\u0346\u0347\u0007\u0014\u0000\u0000\u0347\u0348\u0007"+ - "\u0003\u0000\u0000\u0348\u008b\u0001\u0000\u0000\u0000\u0349\u034a\u0005"+ - "=\u0000\u0000\u034a\u034b\u0005=\u0000\u0000\u034b\u008d\u0001\u0000\u0000"+ - "\u0000\u034c\u034d\u0005=\u0000\u0000\u034d\u034e\u0005~\u0000\u0000\u034e"+ - "\u008f\u0001\u0000\u0000\u0000\u034f\u0350\u0005!\u0000\u0000\u0350\u0351"+ - "\u0005=\u0000\u0000\u0351\u0091\u0001\u0000\u0000\u0000\u0352\u0353\u0005"+ - "<\u0000\u0000\u0353\u0093\u0001\u0000\u0000\u0000\u0354\u0355\u0005<\u0000"+ - "\u0000\u0355\u0356\u0005=\u0000\u0000\u0356\u0095\u0001\u0000\u0000\u0000"+ - "\u0357\u0358\u0005>\u0000\u0000\u0358\u0097\u0001\u0000\u0000\u0000\u0359"+ - "\u035a\u0005>\u0000\u0000\u035a\u035b\u0005=\u0000\u0000\u035b\u0099\u0001"+ - "\u0000\u0000\u0000\u035c\u035d\u0005+\u0000\u0000\u035d\u009b\u0001\u0000"+ - "\u0000\u0000\u035e\u035f\u0005-\u0000\u0000\u035f\u009d\u0001\u0000\u0000"+ - "\u0000\u0360\u0361\u0005*\u0000\u0000\u0361\u009f\u0001\u0000\u0000\u0000"+ - "\u0362\u0363\u0005/\u0000\u0000\u0363\u00a1\u0001\u0000\u0000\u0000\u0364"+ - "\u0365\u0005%\u0000\u0000\u0365\u00a3\u0001\u0000\u0000\u0000\u0366\u0367"+ - "\u0004J\u0004\u0000\u0367\u0368\u00036\u0013\u0000\u0368\u0369\u0001\u0000"+ - "\u0000\u0000\u0369\u036a\u0006J\r\u0000\u036a\u00a5\u0001\u0000\u0000"+ - "\u0000\u036b\u036e\u0003\u0084:\u0000\u036c\u036f\u0003F\u001b\u0000\u036d"+ - "\u036f\u0003T\"\u0000\u036e\u036c\u0001\u0000\u0000\u0000\u036e\u036d"+ - "\u0001\u0000\u0000\u0000\u036f\u0373\u0001\u0000\u0000\u0000\u0370\u0372"+ - "\u0003V#\u0000\u0371\u0370\u0001\u0000\u0000\u0000\u0372\u0375\u0001\u0000"+ - "\u0000\u0000\u0373\u0371\u0001\u0000\u0000\u0000\u0373\u0374\u0001\u0000"+ - "\u0000\u0000\u0374\u037d\u0001\u0000\u0000\u0000\u0375\u0373\u0001\u0000"+ - "\u0000\u0000\u0376\u0378\u0003\u0084:\u0000\u0377\u0379\u0003D\u001a\u0000"+ - "\u0378\u0377\u0001\u0000\u0000\u0000\u0379\u037a\u0001\u0000\u0000\u0000"+ - "\u037a\u0378\u0001\u0000\u0000\u0000\u037a\u037b\u0001\u0000\u0000\u0000"+ - "\u037b\u037d\u0001\u0000\u0000\u0000\u037c\u036b\u0001\u0000\u0000\u0000"+ - "\u037c\u0376\u0001\u0000\u0000\u0000\u037d\u00a7\u0001\u0000\u0000\u0000"+ - "\u037e\u037f\u0005[\u0000\u0000\u037f\u0380\u0001\u0000\u0000\u0000\u0380"+ - "\u0381\u0006L\u0000\u0000\u0381\u0382\u0006L\u0000\u0000\u0382\u00a9\u0001"+ - "\u0000\u0000\u0000\u0383\u0384\u0005]\u0000\u0000\u0384\u0385\u0001\u0000"+ - "\u0000\u0000\u0385\u0386\u0006M\f\u0000\u0386\u0387\u0006M\f\u0000\u0387"+ - "\u00ab\u0001\u0000\u0000\u0000\u0388\u038c\u0003F\u001b\u0000\u0389\u038b"+ - "\u0003V#\u0000\u038a\u0389\u0001\u0000\u0000\u0000\u038b\u038e\u0001\u0000"+ - "\u0000\u0000\u038c\u038a\u0001\u0000\u0000\u0000\u038c\u038d\u0001\u0000"+ - "\u0000\u0000\u038d\u0399\u0001\u0000\u0000\u0000\u038e\u038c\u0001\u0000"+ - "\u0000\u0000\u038f\u0392\u0003T\"\u0000\u0390\u0392\u0003N\u001f\u0000"+ - "\u0391\u038f\u0001\u0000\u0000\u0000\u0391\u0390\u0001\u0000\u0000\u0000"+ - "\u0392\u0394\u0001\u0000\u0000\u0000\u0393\u0395\u0003V#\u0000\u0394\u0393"+ - "\u0001\u0000\u0000\u0000\u0395\u0396\u0001\u0000\u0000\u0000\u0396\u0394"+ - "\u0001\u0000\u0000\u0000\u0396\u0397\u0001\u0000\u0000\u0000\u0397\u0399"+ - "\u0001\u0000\u0000\u0000\u0398\u0388\u0001\u0000\u0000\u0000\u0398\u0391"+ - "\u0001\u0000\u0000\u0000\u0399\u00ad\u0001\u0000\u0000\u0000\u039a\u039c"+ - "\u0003P \u0000\u039b\u039d\u0003R!\u0000\u039c\u039b\u0001\u0000\u0000"+ - "\u0000\u039d\u039e\u0001\u0000\u0000\u0000\u039e\u039c\u0001\u0000\u0000"+ - "\u0000\u039e\u039f\u0001\u0000\u0000\u0000\u039f\u03a0\u0001\u0000\u0000"+ - "\u0000\u03a0\u03a1\u0003P \u0000\u03a1\u00af\u0001\u0000\u0000\u0000\u03a2"+ - "\u03a3\u0003\u00aeO\u0000\u03a3\u00b1\u0001\u0000\u0000\u0000\u03a4\u03a5"+ - "\u0003<\u0016\u0000\u03a5\u03a6\u0001\u0000\u0000\u0000\u03a6\u03a7\u0006"+ - "Q\u000b\u0000\u03a7\u00b3\u0001\u0000\u0000\u0000\u03a8\u03a9\u0003>\u0017"+ - "\u0000\u03a9\u03aa\u0001\u0000\u0000\u0000\u03aa\u03ab\u0006R\u000b\u0000"+ - "\u03ab\u00b5\u0001\u0000\u0000\u0000\u03ac\u03ad\u0003@\u0018\u0000\u03ad"+ - "\u03ae\u0001\u0000\u0000\u0000\u03ae\u03af\u0006S\u000b\u0000\u03af\u00b7"+ - "\u0001\u0000\u0000\u0000\u03b0\u03b1\u0003\u00a8L\u0000\u03b1\u03b2\u0001"+ - "\u0000\u0000\u0000\u03b2\u03b3\u0006T\u000e\u0000\u03b3\u03b4\u0006T\u000f"+ - "\u0000\u03b4\u00b9\u0001\u0000\u0000\u0000\u03b5\u03b6\u0003B\u0019\u0000"+ - "\u03b6\u03b7\u0001\u0000\u0000\u0000\u03b7\u03b8\u0006U\u0010\u0000\u03b8"+ - "\u03b9\u0006U\f\u0000\u03b9\u00bb\u0001\u0000\u0000\u0000\u03ba\u03bb"+ - "\u0003@\u0018\u0000\u03bb\u03bc\u0001\u0000\u0000\u0000\u03bc\u03bd\u0006"+ - "V\u000b\u0000\u03bd\u00bd\u0001\u0000\u0000\u0000\u03be\u03bf\u0003<\u0016"+ - "\u0000\u03bf\u03c0\u0001\u0000\u0000\u0000\u03c0\u03c1\u0006W\u000b\u0000"+ - "\u03c1\u00bf\u0001\u0000\u0000\u0000\u03c2\u03c3\u0003>\u0017\u0000\u03c3"+ - "\u03c4\u0001\u0000\u0000\u0000\u03c4\u03c5\u0006X\u000b\u0000\u03c5\u00c1"+ - "\u0001\u0000\u0000\u0000\u03c6\u03c7\u0003B\u0019\u0000\u03c7\u03c8\u0001"+ - "\u0000\u0000\u0000\u03c8\u03c9\u0006Y\u0010\u0000\u03c9\u03ca\u0006Y\f"+ - "\u0000\u03ca\u00c3\u0001\u0000\u0000\u0000\u03cb\u03cc\u0003\u00a8L\u0000"+ - "\u03cc\u03cd\u0001\u0000\u0000\u0000\u03cd\u03ce\u0006Z\u000e\u0000\u03ce"+ - "\u00c5\u0001\u0000\u0000\u0000\u03cf\u03d0\u0003\u00aaM\u0000\u03d0\u03d1"+ - "\u0001\u0000\u0000\u0000\u03d1\u03d2\u0006[\u0011\u0000\u03d2\u00c7\u0001"+ - "\u0000\u0000\u0000\u03d3\u03d4\u0003\u014e\u009f\u0000\u03d4\u03d5\u0001"+ - "\u0000\u0000\u0000\u03d5\u03d6\u0006\\\u0012\u0000\u03d6\u00c9\u0001\u0000"+ - "\u0000\u0000\u03d7\u03d8\u0003h,\u0000\u03d8\u03d9\u0001\u0000\u0000\u0000"+ - "\u03d9\u03da\u0006]\u0013\u0000\u03da\u00cb\u0001\u0000\u0000\u0000\u03db"+ - "\u03dc\u0003d*\u0000\u03dc\u03dd\u0001\u0000\u0000\u0000\u03dd\u03de\u0006"+ - "^\u0014\u0000\u03de\u00cd\u0001\u0000\u0000\u0000\u03df\u03e0\u0007\u0010"+ - "\u0000\u0000\u03e0\u03e1\u0007\u0003\u0000\u0000\u03e1\u03e2\u0007\u0005"+ - "\u0000\u0000\u03e2\u03e3\u0007\f\u0000\u0000\u03e3\u03e4\u0007\u0000\u0000"+ - "\u0000\u03e4\u03e5\u0007\f\u0000\u0000\u03e5\u03e6\u0007\u0005\u0000\u0000"+ - "\u03e6\u03e7\u0007\f\u0000\u0000\u03e7\u00cf\u0001\u0000\u0000\u0000\u03e8"+ - "\u03ec\b \u0000\u0000\u03e9\u03ea\u0005/\u0000\u0000\u03ea\u03ec\b!\u0000"+ - "\u0000\u03eb\u03e8\u0001\u0000\u0000\u0000\u03eb\u03e9\u0001\u0000\u0000"+ - "\u0000\u03ec\u00d1\u0001\u0000\u0000\u0000\u03ed\u03ef\u0003\u00d0`\u0000"+ - "\u03ee\u03ed\u0001\u0000\u0000\u0000\u03ef\u03f0\u0001\u0000\u0000\u0000"+ - "\u03f0\u03ee\u0001\u0000\u0000\u0000\u03f0\u03f1\u0001\u0000\u0000\u0000"+ - "\u03f1\u00d3\u0001\u0000\u0000\u0000\u03f2\u03f3\u0003\u00d2a\u0000\u03f3"+ - "\u03f4\u0001\u0000\u0000\u0000\u03f4\u03f5\u0006b\u0015\u0000\u03f5\u00d5"+ - "\u0001\u0000\u0000\u0000\u03f6\u03f7\u0003X$\u0000\u03f7\u03f8\u0001\u0000"+ - "\u0000\u0000\u03f8\u03f9\u0006c\u0016\u0000\u03f9\u00d7\u0001\u0000\u0000"+ - "\u0000\u03fa\u03fb\u0003<\u0016\u0000\u03fb\u03fc\u0001\u0000\u0000\u0000"+ - "\u03fc\u03fd\u0006d\u000b\u0000\u03fd\u00d9\u0001\u0000\u0000\u0000\u03fe"+ - "\u03ff\u0003>\u0017\u0000\u03ff\u0400\u0001\u0000\u0000\u0000\u0400\u0401"+ - "\u0006e\u000b\u0000\u0401\u00db\u0001\u0000\u0000\u0000\u0402\u0403\u0003"+ - "@\u0018\u0000\u0403\u0404\u0001\u0000\u0000\u0000\u0404\u0405\u0006f\u000b"+ - "\u0000\u0405\u00dd\u0001\u0000\u0000\u0000\u0406\u0407\u0003B\u0019\u0000"+ - "\u0407\u0408\u0001\u0000\u0000\u0000\u0408\u0409\u0006g\u0010\u0000\u0409"+ - "\u040a\u0006g\f\u0000\u040a\u00df\u0001\u0000\u0000\u0000\u040b\u040c"+ - "\u0003l.\u0000\u040c\u040d\u0001\u0000\u0000\u0000\u040d\u040e\u0006h"+ - "\u0017\u0000\u040e\u00e1\u0001\u0000\u0000\u0000\u040f\u0410\u0003h,\u0000"+ - "\u0410\u0411\u0001\u0000\u0000\u0000\u0411\u0412\u0006i\u0013\u0000\u0412"+ - "\u00e3\u0001\u0000\u0000\u0000\u0413\u0418\u0003F\u001b\u0000\u0414\u0418"+ - "\u0003D\u001a\u0000\u0415\u0418\u0003T\"\u0000\u0416\u0418\u0003\u009e"+ - "G\u0000\u0417\u0413\u0001\u0000\u0000\u0000\u0417\u0414\u0001\u0000\u0000"+ - "\u0000\u0417\u0415\u0001\u0000\u0000\u0000\u0417\u0416\u0001\u0000\u0000"+ - "\u0000\u0418\u00e5\u0001\u0000\u0000\u0000\u0419\u041c\u0003F\u001b\u0000"+ - "\u041a\u041c\u0003\u009eG\u0000\u041b\u0419\u0001\u0000\u0000\u0000\u041b"+ - "\u041a\u0001\u0000\u0000\u0000\u041c\u0420\u0001\u0000\u0000\u0000\u041d"+ - "\u041f\u0003\u00e4j\u0000\u041e\u041d\u0001\u0000\u0000\u0000\u041f\u0422"+ - "\u0001\u0000\u0000\u0000\u0420\u041e\u0001\u0000\u0000\u0000\u0420\u0421"+ - "\u0001\u0000\u0000\u0000\u0421\u042d\u0001\u0000\u0000\u0000\u0422\u0420"+ - "\u0001\u0000\u0000\u0000\u0423\u0426\u0003T\"\u0000\u0424\u0426\u0003"+ - "N\u001f\u0000\u0425\u0423\u0001\u0000\u0000\u0000\u0425\u0424\u0001\u0000"+ - "\u0000\u0000\u0426\u0428\u0001\u0000\u0000\u0000\u0427\u0429\u0003\u00e4"+ - "j\u0000\u0428\u0427\u0001\u0000\u0000\u0000\u0429\u042a\u0001\u0000\u0000"+ - "\u0000\u042a\u0428\u0001\u0000\u0000\u0000\u042a\u042b\u0001\u0000\u0000"+ - "\u0000\u042b\u042d\u0001\u0000\u0000\u0000\u042c\u041b\u0001\u0000\u0000"+ - "\u0000\u042c\u0425\u0001\u0000\u0000\u0000\u042d\u00e7\u0001\u0000\u0000"+ - "\u0000\u042e\u0431\u0003\u00e6k\u0000\u042f\u0431\u0003\u00aeO\u0000\u0430"+ - "\u042e\u0001\u0000\u0000\u0000\u0430\u042f\u0001\u0000\u0000\u0000\u0431"+ - "\u0432\u0001\u0000\u0000\u0000\u0432\u0430\u0001\u0000\u0000\u0000\u0432"+ - "\u0433\u0001\u0000\u0000\u0000\u0433\u00e9\u0001\u0000\u0000\u0000\u0434"+ - "\u0435\u0003<\u0016\u0000\u0435\u0436\u0001\u0000\u0000\u0000\u0436\u0437"+ - "\u0006m\u000b\u0000\u0437\u00eb\u0001\u0000\u0000\u0000\u0438\u0439\u0003"+ - ">\u0017\u0000\u0439\u043a\u0001\u0000\u0000\u0000\u043a\u043b\u0006n\u000b"+ - "\u0000\u043b\u00ed\u0001\u0000\u0000\u0000\u043c\u043d\u0003@\u0018\u0000"+ - "\u043d\u043e\u0001\u0000\u0000\u0000\u043e\u043f\u0006o\u000b\u0000\u043f"+ - "\u00ef\u0001\u0000\u0000\u0000\u0440\u0441\u0003B\u0019\u0000\u0441\u0442"+ - "\u0001\u0000\u0000\u0000\u0442\u0443\u0006p\u0010\u0000\u0443\u0444\u0006"+ - "p\f\u0000\u0444\u00f1\u0001\u0000\u0000\u0000\u0445\u0446\u0003d*\u0000"+ - "\u0446\u0447\u0001\u0000\u0000\u0000\u0447\u0448\u0006q\u0014\u0000\u0448"+ - "\u00f3\u0001\u0000\u0000\u0000\u0449\u044a\u0003h,\u0000\u044a\u044b\u0001"+ - "\u0000\u0000\u0000\u044b\u044c\u0006r\u0013\u0000\u044c\u00f5\u0001\u0000"+ - "\u0000\u0000\u044d\u044e\u0003l.\u0000\u044e\u044f\u0001\u0000\u0000\u0000"+ - "\u044f\u0450\u0006s\u0017\u0000\u0450\u00f7\u0001\u0000\u0000\u0000\u0451"+ - "\u0452\u0007\f\u0000\u0000\u0452\u0453\u0007\u0002\u0000\u0000\u0453\u00f9"+ - "\u0001\u0000\u0000\u0000\u0454\u0455\u0003\u00e8l\u0000\u0455\u0456\u0001"+ - "\u0000\u0000\u0000\u0456\u0457\u0006u\u0018\u0000\u0457\u00fb\u0001\u0000"+ - "\u0000\u0000\u0458\u0459\u0003<\u0016\u0000\u0459\u045a\u0001\u0000\u0000"+ - "\u0000\u045a\u045b\u0006v\u000b\u0000\u045b\u00fd\u0001\u0000\u0000\u0000"+ - "\u045c\u045d\u0003>\u0017\u0000\u045d\u045e\u0001\u0000\u0000\u0000\u045e"+ - "\u045f\u0006w\u000b\u0000\u045f\u00ff\u0001\u0000\u0000\u0000\u0460\u0461"+ - "\u0003@\u0018\u0000\u0461\u0462\u0001\u0000\u0000\u0000\u0462\u0463\u0006"+ - "x\u000b\u0000\u0463\u0101\u0001\u0000\u0000\u0000\u0464\u0465\u0003B\u0019"+ - "\u0000\u0465\u0466\u0001\u0000\u0000\u0000\u0466\u0467\u0006y\u0010\u0000"+ - "\u0467\u0468\u0006y\f\u0000\u0468\u0103\u0001\u0000\u0000\u0000\u0469"+ - "\u046a\u0003\u00a8L\u0000\u046a\u046b\u0001\u0000\u0000\u0000\u046b\u046c"+ - "\u0006z\u000e\u0000\u046c\u046d\u0006z\u0019\u0000\u046d\u0105\u0001\u0000"+ - "\u0000\u0000\u046e\u046f\u0007\u0007\u0000\u0000\u046f\u0470\u0007\t\u0000"+ - "\u0000\u0470\u0471\u0001\u0000\u0000\u0000\u0471\u0472\u0006{\u001a\u0000"+ - "\u0472\u0107\u0001\u0000\u0000\u0000\u0473\u0474\u0007\u0013\u0000\u0000"+ - "\u0474\u0475\u0007\u0001\u0000\u0000\u0475\u0476\u0007\u0005\u0000\u0000"+ - "\u0476\u0477\u0007\n\u0000\u0000\u0477\u0478\u0001\u0000\u0000\u0000\u0478"+ - "\u0479\u0006|\u001a\u0000\u0479\u0109\u0001\u0000\u0000\u0000\u047a\u047b"+ - "\b\"\u0000\u0000\u047b\u010b\u0001\u0000\u0000\u0000\u047c\u047e\u0003"+ - "\u010a}\u0000\u047d\u047c\u0001\u0000\u0000\u0000\u047e\u047f\u0001\u0000"+ - "\u0000\u0000\u047f\u047d\u0001\u0000\u0000\u0000\u047f\u0480\u0001\u0000"+ - "\u0000\u0000\u0480\u0481\u0001\u0000\u0000\u0000\u0481\u0482\u0003\u014e"+ - "\u009f\u0000\u0482\u0484\u0001\u0000\u0000\u0000\u0483\u047d\u0001\u0000"+ - "\u0000\u0000\u0483\u0484\u0001\u0000\u0000\u0000\u0484\u0486\u0001\u0000"+ - "\u0000\u0000\u0485\u0487\u0003\u010a}\u0000\u0486\u0485\u0001\u0000\u0000"+ - "\u0000\u0487\u0488\u0001\u0000\u0000\u0000\u0488\u0486\u0001\u0000\u0000"+ - "\u0000\u0488\u0489\u0001\u0000\u0000\u0000\u0489\u010d\u0001\u0000\u0000"+ - "\u0000\u048a\u048b\u0003\u010c~\u0000\u048b\u048c\u0001\u0000\u0000\u0000"+ - "\u048c\u048d\u0006\u007f\u001b\u0000\u048d\u010f\u0001\u0000\u0000\u0000"+ - "\u048e\u048f\u0003<\u0016\u0000\u048f\u0490\u0001\u0000\u0000\u0000\u0490"+ - "\u0491\u0006\u0080\u000b\u0000\u0491\u0111\u0001\u0000\u0000\u0000\u0492"+ - "\u0493\u0003>\u0017\u0000\u0493\u0494\u0001\u0000\u0000\u0000\u0494\u0495"+ - "\u0006\u0081\u000b\u0000\u0495\u0113\u0001\u0000\u0000\u0000\u0496\u0497"+ - "\u0003@\u0018\u0000\u0497\u0498\u0001\u0000\u0000\u0000\u0498\u0499\u0006"+ - "\u0082\u000b\u0000\u0499\u0115\u0001\u0000\u0000\u0000\u049a\u049b\u0003"+ - "B\u0019\u0000\u049b\u049c\u0001\u0000\u0000\u0000\u049c\u049d\u0006\u0083"+ - "\u0010\u0000\u049d\u049e\u0006\u0083\f\u0000\u049e\u049f\u0006\u0083\f"+ - "\u0000\u049f\u0117\u0001\u0000\u0000\u0000\u04a0\u04a1\u0003d*\u0000\u04a1"+ - "\u04a2\u0001\u0000\u0000\u0000\u04a2\u04a3\u0006\u0084\u0014\u0000\u04a3"+ - "\u0119\u0001\u0000\u0000\u0000\u04a4\u04a5\u0003h,\u0000\u04a5\u04a6\u0001"+ - "\u0000\u0000\u0000\u04a6\u04a7\u0006\u0085\u0013\u0000\u04a7\u011b\u0001"+ - "\u0000\u0000\u0000\u04a8\u04a9\u0003l.\u0000\u04a9\u04aa\u0001\u0000\u0000"+ - "\u0000\u04aa\u04ab\u0006\u0086\u0017\u0000\u04ab\u011d\u0001\u0000\u0000"+ - "\u0000\u04ac\u04ad\u0003\u0108|\u0000\u04ad\u04ae\u0001\u0000\u0000\u0000"+ - "\u04ae\u04af\u0006\u0087\u001c\u0000\u04af\u011f\u0001\u0000\u0000\u0000"+ - "\u04b0\u04b1\u0003\u00e8l\u0000\u04b1\u04b2\u0001\u0000\u0000\u0000\u04b2"+ - "\u04b3\u0006\u0088\u0018\u0000\u04b3\u0121\u0001\u0000\u0000\u0000\u04b4"+ - "\u04b5\u0003\u00b0P\u0000\u04b5\u04b6\u0001\u0000\u0000\u0000\u04b6\u04b7"+ - "\u0006\u0089\u001d\u0000\u04b7\u0123\u0001\u0000\u0000\u0000\u04b8\u04b9"+ - "\u0003<\u0016\u0000\u04b9\u04ba\u0001\u0000\u0000\u0000\u04ba\u04bb\u0006"+ - "\u008a\u000b\u0000\u04bb\u0125\u0001\u0000\u0000\u0000\u04bc\u04bd\u0003"+ - ">\u0017\u0000\u04bd\u04be\u0001\u0000\u0000\u0000\u04be\u04bf\u0006\u008b"+ - "\u000b\u0000\u04bf\u0127\u0001\u0000\u0000\u0000\u04c0\u04c1\u0003@\u0018"+ - "\u0000\u04c1\u04c2\u0001\u0000\u0000\u0000\u04c2\u04c3\u0006\u008c\u000b"+ - "\u0000\u04c3\u0129\u0001\u0000\u0000\u0000\u04c4\u04c5\u0003B\u0019\u0000"+ - "\u04c5\u04c6\u0001\u0000\u0000\u0000\u04c6\u04c7\u0006\u008d\u0010\u0000"+ - "\u04c7\u04c8\u0006\u008d\f\u0000\u04c8\u012b\u0001\u0000\u0000\u0000\u04c9"+ - "\u04ca\u0003l.\u0000\u04ca\u04cb\u0001\u0000\u0000\u0000\u04cb\u04cc\u0006"+ - "\u008e\u0017\u0000\u04cc\u012d\u0001\u0000\u0000\u0000\u04cd\u04ce\u0003"+ - "\u00b0P\u0000\u04ce\u04cf\u0001\u0000\u0000\u0000\u04cf\u04d0\u0006\u008f"+ - "\u001d\u0000\u04d0\u012f\u0001\u0000\u0000\u0000\u04d1\u04d2\u0003\u00ac"+ - "N\u0000\u04d2\u04d3\u0001\u0000\u0000\u0000\u04d3\u04d4\u0006\u0090\u001e"+ - "\u0000\u04d4\u0131\u0001\u0000\u0000\u0000\u04d5\u04d6\u0003<\u0016\u0000"+ - "\u04d6\u04d7\u0001\u0000\u0000\u0000\u04d7\u04d8\u0006\u0091\u000b\u0000"+ - "\u04d8\u0133\u0001\u0000\u0000\u0000\u04d9\u04da\u0003>\u0017\u0000\u04da"+ - "\u04db\u0001\u0000\u0000\u0000\u04db\u04dc\u0006\u0092\u000b\u0000\u04dc"+ - "\u0135\u0001\u0000\u0000\u0000\u04dd\u04de\u0003@\u0018\u0000\u04de\u04df"+ - "\u0001\u0000\u0000\u0000\u04df\u04e0\u0006\u0093\u000b\u0000\u04e0\u0137"+ - "\u0001\u0000\u0000\u0000\u04e1\u04e2\u0003B\u0019\u0000\u04e2\u04e3\u0001"+ - "\u0000\u0000\u0000\u04e3\u04e4\u0006\u0094\u0010\u0000\u04e4\u04e5\u0006"+ - "\u0094\f\u0000\u04e5\u0139\u0001\u0000\u0000\u0000\u04e6\u04e7\u0007\u0001"+ - "\u0000\u0000\u04e7\u04e8\u0007\t\u0000\u0000\u04e8\u04e9\u0007\u000f\u0000"+ - "\u0000\u04e9\u04ea\u0007\u0007\u0000\u0000\u04ea\u013b\u0001\u0000\u0000"+ - "\u0000\u04eb\u04ec\u0003<\u0016\u0000\u04ec\u04ed\u0001\u0000\u0000\u0000"+ - "\u04ed\u04ee\u0006\u0096\u000b\u0000\u04ee\u013d\u0001\u0000\u0000\u0000"+ - "\u04ef\u04f0\u0003>\u0017\u0000\u04f0\u04f1\u0001\u0000\u0000\u0000\u04f1"+ - "\u04f2\u0006\u0097\u000b\u0000\u04f2\u013f\u0001\u0000\u0000\u0000\u04f3"+ - "\u04f4\u0003@\u0018\u0000\u04f4\u04f5\u0001\u0000\u0000\u0000\u04f5\u04f6"+ - "\u0006\u0098\u000b\u0000\u04f6\u0141\u0001\u0000\u0000\u0000\u04f7\u04f8"+ - "\u0003B\u0019\u0000\u04f8\u04f9\u0001\u0000\u0000\u0000\u04f9\u04fa\u0006"+ - "\u0099\u0010\u0000\u04fa\u04fb\u0006\u0099\f\u0000\u04fb\u0143\u0001\u0000"+ - "\u0000\u0000\u04fc\u04fd\u0007\u000f\u0000\u0000\u04fd\u04fe\u0007\u0014"+ - "\u0000\u0000\u04fe\u04ff\u0007\t\u0000\u0000\u04ff\u0500\u0007\u0004\u0000"+ - "\u0000\u0500\u0501\u0007\u0005\u0000\u0000\u0501\u0502\u0007\u0001\u0000"+ - "\u0000\u0502\u0503\u0007\u0007\u0000\u0000\u0503\u0504\u0007\t\u0000\u0000"+ - "\u0504\u0505\u0007\u0002\u0000\u0000\u0505\u0145\u0001\u0000\u0000\u0000"+ - "\u0506\u0507\u0003<\u0016\u0000\u0507\u0508\u0001\u0000\u0000\u0000\u0508"+ - "\u0509\u0006\u009b\u000b\u0000\u0509\u0147\u0001\u0000\u0000\u0000\u050a"+ - "\u050b\u0003>\u0017\u0000\u050b\u050c\u0001\u0000\u0000\u0000\u050c\u050d"+ - "\u0006\u009c\u000b\u0000\u050d\u0149\u0001\u0000\u0000\u0000\u050e\u050f"+ - "\u0003@\u0018\u0000\u050f\u0510\u0001\u0000\u0000\u0000\u0510\u0511\u0006"+ - "\u009d\u000b\u0000\u0511\u014b\u0001\u0000\u0000\u0000\u0512\u0513\u0003"+ - "\u00aaM\u0000\u0513\u0514\u0001\u0000\u0000\u0000\u0514\u0515\u0006\u009e"+ - "\u0011\u0000\u0515\u0516\u0006\u009e\f\u0000\u0516\u014d\u0001\u0000\u0000"+ - "\u0000\u0517\u0518\u0005:\u0000\u0000\u0518\u014f\u0001\u0000\u0000\u0000"+ - "\u0519\u051f\u0003N\u001f\u0000\u051a\u051f\u0003D\u001a\u0000\u051b\u051f"+ - "\u0003l.\u0000\u051c\u051f\u0003F\u001b\u0000\u051d\u051f\u0003T\"\u0000"+ - "\u051e\u0519\u0001\u0000\u0000\u0000\u051e\u051a\u0001\u0000\u0000\u0000"+ - "\u051e\u051b\u0001\u0000\u0000\u0000\u051e\u051c\u0001\u0000\u0000\u0000"+ - "\u051e\u051d\u0001\u0000\u0000\u0000\u051f\u0520\u0001\u0000\u0000\u0000"+ - "\u0520\u051e\u0001\u0000\u0000\u0000\u0520\u0521\u0001\u0000\u0000\u0000"+ - "\u0521\u0151\u0001\u0000\u0000\u0000\u0522\u0523\u0003<\u0016\u0000\u0523"+ - "\u0524\u0001\u0000\u0000\u0000\u0524\u0525\u0006\u00a1\u000b\u0000\u0525"+ - "\u0153\u0001\u0000\u0000\u0000\u0526\u0527\u0003>\u0017\u0000\u0527\u0528"+ - "\u0001\u0000\u0000\u0000\u0528\u0529\u0006\u00a2\u000b\u0000\u0529\u0155"+ - "\u0001\u0000\u0000\u0000\u052a\u052b\u0003@\u0018\u0000\u052b\u052c\u0001"+ - "\u0000\u0000\u0000\u052c\u052d\u0006\u00a3\u000b\u0000\u052d\u0157\u0001"+ - "\u0000\u0000\u0000\u052e\u052f\u0003B\u0019\u0000\u052f\u0530\u0001\u0000"+ - "\u0000\u0000\u0530\u0531\u0006\u00a4\u0010\u0000\u0531\u0532\u0006\u00a4"+ - "\f\u0000\u0532\u0159\u0001\u0000\u0000\u0000\u0533\u0534\u0003\u014e\u009f"+ - "\u0000\u0534\u0535\u0001\u0000\u0000\u0000\u0535\u0536\u0006\u00a5\u0012"+ - "\u0000\u0536\u015b\u0001\u0000\u0000\u0000\u0537\u0538\u0003h,\u0000\u0538"+ - "\u0539\u0001\u0000\u0000\u0000\u0539\u053a\u0006\u00a6\u0013\u0000\u053a"+ - "\u015d\u0001\u0000\u0000\u0000\u053b\u053c\u0003l.\u0000\u053c\u053d\u0001"+ - "\u0000\u0000\u0000\u053d\u053e\u0006\u00a7\u0017\u0000\u053e\u015f\u0001"+ - "\u0000\u0000\u0000\u053f\u0540\u0003\u0106{\u0000\u0540\u0541\u0001\u0000"+ - "\u0000\u0000\u0541\u0542\u0006\u00a8\u001f\u0000\u0542\u0543\u0006\u00a8"+ - " \u0000\u0543\u0161\u0001\u0000\u0000\u0000\u0544\u0545\u0003\u00d2a\u0000"+ - "\u0545\u0546\u0001\u0000\u0000\u0000\u0546\u0547\u0006\u00a9\u0015\u0000"+ - "\u0547\u0163\u0001\u0000\u0000\u0000\u0548\u0549\u0003X$\u0000\u0549\u054a"+ - "\u0001\u0000\u0000\u0000\u054a\u054b\u0006\u00aa\u0016\u0000\u054b\u0165"+ - "\u0001\u0000\u0000\u0000\u054c\u054d\u0003<\u0016\u0000\u054d\u054e\u0001"+ - "\u0000\u0000\u0000\u054e\u054f\u0006\u00ab\u000b\u0000\u054f\u0167\u0001"+ - "\u0000\u0000\u0000\u0550\u0551\u0003>\u0017\u0000\u0551\u0552\u0001\u0000"+ - "\u0000\u0000\u0552\u0553\u0006\u00ac\u000b\u0000\u0553\u0169\u0001\u0000"+ - "\u0000\u0000\u0554\u0555\u0003@\u0018\u0000\u0555\u0556\u0001\u0000\u0000"+ - "\u0000\u0556\u0557\u0006\u00ad\u000b\u0000\u0557\u016b\u0001\u0000\u0000"+ - "\u0000\u0558\u0559\u0003B\u0019\u0000\u0559\u055a\u0001\u0000\u0000\u0000"+ - "\u055a\u055b\u0006\u00ae\u0010\u0000\u055b\u055c\u0006\u00ae\f\u0000\u055c"+ - "\u055d\u0006\u00ae\f\u0000\u055d\u016d\u0001\u0000\u0000\u0000\u055e\u055f"+ - "\u0003h,\u0000\u055f\u0560\u0001\u0000\u0000\u0000\u0560\u0561\u0006\u00af"+ - "\u0013\u0000\u0561\u016f\u0001\u0000\u0000\u0000\u0562\u0563\u0003l.\u0000"+ - "\u0563\u0564\u0001\u0000\u0000\u0000\u0564\u0565\u0006\u00b0\u0017\u0000"+ - "\u0565\u0171\u0001\u0000\u0000\u0000\u0566\u0567\u0003\u00e8l\u0000\u0567"+ - "\u0568\u0001\u0000\u0000\u0000\u0568\u0569\u0006\u00b1\u0018\u0000\u0569"+ - "\u0173\u0001\u0000\u0000\u0000\u056a\u056b\u0003<\u0016\u0000\u056b\u056c"+ - "\u0001\u0000\u0000\u0000\u056c\u056d\u0006\u00b2\u000b\u0000\u056d\u0175"+ - "\u0001\u0000\u0000\u0000\u056e\u056f\u0003>\u0017\u0000\u056f\u0570\u0001"+ - "\u0000\u0000\u0000\u0570\u0571\u0006\u00b3\u000b\u0000\u0571\u0177\u0001"+ - "\u0000\u0000\u0000\u0572\u0573\u0003@\u0018\u0000\u0573\u0574\u0001\u0000"+ - "\u0000\u0000\u0574\u0575\u0006\u00b4\u000b\u0000\u0575\u0179\u0001\u0000"+ - "\u0000\u0000\u0576\u0577\u0003B\u0019\u0000\u0577\u0578\u0001\u0000\u0000"+ - "\u0000\u0578\u0579\u0006\u00b5\u0010\u0000\u0579\u057a\u0006\u00b5\f\u0000"+ - "\u057a\u017b\u0001\u0000\u0000\u0000\u057b\u057c\u0003\u00d2a\u0000\u057c"+ - "\u057d\u0001\u0000\u0000\u0000\u057d\u057e\u0006\u00b6\u0015\u0000\u057e"+ - "\u057f\u0006\u00b6\f\u0000\u057f\u0580\u0006\u00b6!\u0000\u0580\u017d"+ - "\u0001\u0000\u0000\u0000\u0581\u0582\u0003X$\u0000\u0582\u0583\u0001\u0000"+ - "\u0000\u0000\u0583\u0584\u0006\u00b7\u0016\u0000\u0584\u0585\u0006\u00b7"+ - "\f\u0000\u0585\u0586\u0006\u00b7!\u0000\u0586\u017f\u0001\u0000\u0000"+ - "\u0000\u0587\u0588\u0003<\u0016\u0000\u0588\u0589\u0001\u0000\u0000\u0000"+ - "\u0589\u058a\u0006\u00b8\u000b\u0000\u058a\u0181\u0001\u0000\u0000\u0000"+ - "\u058b\u058c\u0003>\u0017\u0000\u058c\u058d\u0001\u0000\u0000\u0000\u058d"+ - "\u058e\u0006\u00b9\u000b\u0000\u058e\u0183\u0001\u0000\u0000\u0000\u058f"+ - "\u0590\u0003@\u0018\u0000\u0590\u0591\u0001\u0000\u0000\u0000\u0591\u0592"+ - "\u0006\u00ba\u000b\u0000\u0592\u0185\u0001\u0000\u0000\u0000\u0593\u0594"+ - "\u0003\u014e\u009f\u0000\u0594\u0595\u0001\u0000\u0000\u0000\u0595\u0596"+ - "\u0006\u00bb\u0012\u0000\u0596\u0597\u0006\u00bb\f\u0000\u0597\u0598\u0006"+ - "\u00bb\n\u0000\u0598\u0187\u0001\u0000\u0000\u0000\u0599\u059a\u0003h"+ - ",\u0000\u059a\u059b\u0001\u0000\u0000\u0000\u059b\u059c\u0006\u00bc\u0013"+ - "\u0000\u059c\u059d\u0006\u00bc\f\u0000\u059d\u059e\u0006\u00bc\n\u0000"+ - "\u059e\u0189\u0001\u0000\u0000\u0000\u059f\u05a0\u0003<\u0016\u0000\u05a0"+ - "\u05a1\u0001\u0000\u0000\u0000\u05a1\u05a2\u0006\u00bd\u000b\u0000\u05a2"+ - "\u018b\u0001\u0000\u0000\u0000\u05a3\u05a4\u0003>\u0017\u0000\u05a4\u05a5"+ - "\u0001\u0000\u0000\u0000\u05a5\u05a6\u0006\u00be\u000b\u0000\u05a6\u018d"+ - "\u0001\u0000\u0000\u0000\u05a7\u05a8\u0003@\u0018\u0000\u05a8\u05a9\u0001"+ - "\u0000\u0000\u0000\u05a9\u05aa\u0006\u00bf\u000b\u0000\u05aa\u018f\u0001"+ - "\u0000\u0000\u0000\u05ab\u05ac\u0003\u00b0P\u0000\u05ac\u05ad\u0001\u0000"+ - "\u0000\u0000\u05ad\u05ae\u0006\u00c0\f\u0000\u05ae\u05af\u0006\u00c0\u0000"+ - "\u0000\u05af\u05b0\u0006\u00c0\u001d\u0000\u05b0\u0191\u0001\u0000\u0000"+ - "\u0000\u05b1\u05b2\u0003\u00acN\u0000\u05b2\u05b3\u0001\u0000\u0000\u0000"+ - "\u05b3\u05b4\u0006\u00c1\f\u0000\u05b4\u05b5\u0006\u00c1\u0000\u0000\u05b5"+ - "\u05b6\u0006\u00c1\u001e\u0000\u05b6\u0193\u0001\u0000\u0000\u0000\u05b7"+ - "\u05b8\u0003^\'\u0000\u05b8\u05b9\u0001\u0000\u0000\u0000\u05b9\u05ba"+ - "\u0006\u00c2\f\u0000\u05ba\u05bb\u0006\u00c2\u0000\u0000\u05bb\u05bc\u0006"+ - "\u00c2\"\u0000\u05bc\u0195\u0001\u0000\u0000\u0000\u05bd\u05be\u0003B"+ - "\u0019\u0000\u05be\u05bf\u0001\u0000\u0000\u0000\u05bf\u05c0\u0006\u00c3"+ - "\u0010\u0000\u05c0\u05c1\u0006\u00c3\f\u0000\u05c1\u0197\u0001\u0000\u0000"+ - "\u0000B\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f"+ - "\r\u000e\u000f\u0250\u025a\u025e\u0261\u026a\u026c\u0277\u028a\u028f\u0298"+ - "\u029f\u02a4\u02a6\u02b1\u02b9\u02bc\u02be\u02c3\u02c8\u02ce\u02d5\u02da"+ - "\u02e0\u02e3\u02eb\u02ef\u036e\u0373\u037a\u037c\u038c\u0391\u0396\u0398"+ - "\u039e\u03eb\u03f0\u0417\u041b\u0420\u0425\u042a\u042c\u0430\u0432\u047f"+ - "\u0483\u0488\u051e\u0520#\u0005\u0001\u0000\u0005\u0004\u0000\u0005\u0006"+ - "\u0000\u0005\u0002\u0000\u0005\u0003\u0000\u0005\n\u0000\u0005\b\u0000"+ - "\u0005\u0005\u0000\u0005\t\u0000\u0005\f\u0000\u0005\u000e\u0000\u0000"+ - "\u0001\u0000\u0004\u0000\u0000\u0007\u0014\u0000\u0007B\u0000\u0005\u0000"+ - "\u0000\u0007\u001a\u0000\u0007C\u0000\u0007m\u0000\u0007#\u0000\u0007"+ - "!\u0000\u0007M\u0000\u0007\u001b\u0000\u0007%\u0000\u0007Q\u0000\u0005"+ - "\u000b\u0000\u0005\u0007\u0000\u0007[\u0000\u0007Z\u0000\u0007E\u0000"+ - "\u0007D\u0000\u0007Y\u0000\u0005\r\u0000\u0005\u000f\u0000\u0007\u001e"+ - "\u0000"; + "i\u0003i\u0404\bi\u0001j\u0001j\u0003j\u0408\bj\u0001j\u0005j\u040b\b"+ + "j\nj\fj\u040e\tj\u0001j\u0001j\u0003j\u0412\bj\u0001j\u0004j\u0415\bj"+ + "\u000bj\fj\u0416\u0003j\u0419\bj\u0001k\u0001k\u0004k\u041d\bk\u000bk"+ + "\fk\u041e\u0001l\u0001l\u0001l\u0001l\u0001m\u0001m\u0001m\u0001m\u0001"+ + "n\u0001n\u0001n\u0001n\u0001o\u0001o\u0001o\u0001o\u0001o\u0001p\u0001"+ + "p\u0001p\u0001p\u0001q\u0001q\u0001q\u0001q\u0001r\u0001r\u0001r\u0001"+ + "r\u0001s\u0001s\u0001s\u0001t\u0001t\u0001t\u0001t\u0001u\u0001u\u0001"+ + "u\u0001u\u0001v\u0001v\u0001v\u0001v\u0001w\u0001w\u0001w\u0001w\u0001"+ + "x\u0001x\u0001x\u0001x\u0001x\u0001y\u0001y\u0001y\u0001y\u0001y\u0001"+ + "z\u0001z\u0001z\u0001z\u0001z\u0001{\u0001{\u0001{\u0001{\u0001{\u0001"+ + "{\u0001{\u0001|\u0001|\u0001}\u0004}\u046a\b}\u000b}\f}\u046b\u0001}\u0001"+ + "}\u0003}\u0470\b}\u0001}\u0004}\u0473\b}\u000b}\f}\u0474\u0001~\u0001"+ + "~\u0001~\u0001~\u0001\u007f\u0001\u007f\u0001\u007f\u0001\u007f\u0001"+ + "\u0080\u0001\u0080\u0001\u0080\u0001\u0080\u0001\u0081\u0001\u0081\u0001"+ + "\u0081\u0001\u0081\u0001\u0082\u0001\u0082\u0001\u0082\u0001\u0082\u0001"+ + "\u0082\u0001\u0082\u0001\u0083\u0001\u0083\u0001\u0083\u0001\u0083\u0001"+ + "\u0084\u0001\u0084\u0001\u0084\u0001\u0084\u0001\u0085\u0001\u0085\u0001"+ + "\u0085\u0001\u0085\u0001\u0086\u0001\u0086\u0001\u0086\u0001\u0086\u0001"+ + "\u0087\u0001\u0087\u0001\u0087\u0001\u0087\u0001\u0088\u0001\u0088\u0001"+ + "\u0088\u0001\u0088\u0001\u0089\u0001\u0089\u0001\u0089\u0001\u0089\u0001"+ + "\u008a\u0001\u008a\u0001\u008a\u0001\u008a\u0001\u008b\u0001\u008b\u0001"+ + "\u008b\u0001\u008b\u0001\u008c\u0001\u008c\u0001\u008c\u0001\u008c\u0001"+ + "\u008c\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008d\u0001\u008e\u0001"+ + "\u008e\u0001\u008e\u0001\u008e\u0001\u008f\u0001\u008f\u0001\u008f\u0001"+ + "\u008f\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0090\u0001\u0091\u0001"+ + "\u0091\u0001\u0091\u0001\u0091\u0001\u0092\u0001\u0092\u0001\u0092\u0001"+ + "\u0092\u0001\u0093\u0001\u0093\u0001\u0093\u0001\u0093\u0001\u0093\u0001"+ + "\u0094\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0094\u0001\u0095\u0001"+ + "\u0095\u0001\u0095\u0001\u0095\u0001\u0096\u0001\u0096\u0001\u0096\u0001"+ + "\u0096\u0001\u0097\u0001\u0097\u0001\u0097\u0001\u0097\u0001\u0098\u0001"+ + "\u0098\u0001\u0098\u0001\u0098\u0001\u0098\u0001\u0099\u0001\u0099\u0001"+ + "\u009a\u0001\u009a\u0001\u009a\u0001\u009a\u0001\u009a\u0004\u009a\u04f0"+ + "\b\u009a\u000b\u009a\f\u009a\u04f1\u0001\u009b\u0001\u009b\u0001\u009b"+ + "\u0001\u009b\u0001\u009c\u0001\u009c\u0001\u009c\u0001\u009c\u0001\u009d"+ + "\u0001\u009d\u0001\u009d\u0001\u009d\u0001\u009e\u0001\u009e\u0001\u009e"+ + "\u0001\u009e\u0001\u009e\u0001\u009f\u0001\u009f\u0001\u009f\u0001\u009f"+ + "\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a0\u0001\u00a1\u0001\u00a1"+ + "\u0001\u00a1\u0001\u00a1\u0001\u00a2\u0001\u00a2\u0001\u00a2\u0001\u00a2"+ + "\u0001\u00a2\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a3\u0001\u00a4"+ + "\u0001\u00a4\u0001\u00a4\u0001\u00a4\u0001\u00a5\u0001\u00a5\u0001\u00a5"+ + "\u0001\u00a5\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a6\u0001\u00a7"+ + "\u0001\u00a7\u0001\u00a7\u0001\u00a7\u0001\u00a8\u0001\u00a8\u0001\u00a8"+ + "\u0001\u00a8\u0001\u00a8\u0001\u00a8\u0001\u00a9\u0001\u00a9\u0001\u00a9"+ + "\u0001\u00a9\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00aa\u0001\u00ab"+ + "\u0001\u00ab\u0001\u00ab\u0001\u00ab\u0001\u00ac\u0001\u00ac\u0001\u00ac"+ + "\u0001\u00ac\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ad\u0001\u00ae"+ + "\u0001\u00ae\u0001\u00ae\u0001\u00ae\u0001\u00af\u0001\u00af\u0001\u00af"+ + "\u0001\u00af\u0001\u00af\u0001\u00b0\u0001\u00b0\u0001\u00b0\u0001\u00b0"+ + "\u0001\u00b0\u0001\u00b0\u0001\u00b1\u0001\u00b1\u0001\u00b1\u0001\u00b1"+ + "\u0001\u00b1\u0001\u00b1\u0001\u00b2\u0001\u00b2\u0001\u00b2\u0001\u00b2"+ + "\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001\u00b3\u0001\u00b4\u0001\u00b4"+ + "\u0001\u00b4\u0001\u00b4\u0001\u00b5\u0001\u00b5\u0001\u00b5\u0001\u00b5"+ + "\u0001\u00b5\u0001\u00b5\u0001\u00b6\u0001\u00b6\u0001\u00b6\u0001\u00b6"+ + "\u0001\u00b6\u0001\u00b6\u0001\u00b7\u0001\u00b7\u0001\u00b7\u0001\u00b7"+ + "\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b8\u0001\u00b9\u0001\u00b9"+ + "\u0001\u00b9\u0001\u00b9\u0001\u00ba\u0001\u00ba\u0001\u00ba\u0001\u00ba"+ + "\u0001\u00ba\u0001\u00ba\u0001\u00bb\u0001\u00bb\u0001\u00bb\u0001\u00bb"+ + "\u0001\u00bb\u0001\u00bb\u0001\u00bc\u0001\u00bc\u0001\u00bc\u0001\u00bc"+ + "\u0001\u00bc\u0001\u00bc\u0001\u00bd\u0001\u00bd\u0001\u00bd\u0001\u00bd"+ + "\u0001\u00bd\u0002\u0258\u029d\u0000\u00be\u000f\u0001\u0011\u0002\u0013"+ + "\u0003\u0015\u0004\u0017\u0005\u0019\u0006\u001b\u0007\u001d\b\u001f\t"+ + "!\n#\u000b%\f\'\r)\u000e+\u000f-\u0010/\u00111\u00123\u00135\u00147\u0015"+ + "9\u0016;\u0017=\u0018?\u0019A\u0000C\u0000E\u0000G\u0000I\u0000K\u0000"+ + "M\u0000O\u0000Q\u0000S\u0000U\u001aW\u001bY\u001c[\u001d]\u001e_\u001f"+ + "a c!e\"g#i$k%m&o\'q(s)u*w+y,{-}.\u007f/\u00810\u00831\u00852\u00873\u0089"+ + "4\u008b5\u008d6\u008f7\u00918\u00939\u0095:\u0097;\u0099<\u009b=\u009d"+ + ">\u009f?\u00a1\u0000\u00a3@\u00a5A\u00a7B\u00a9C\u00ab\u0000\u00adD\u00af"+ + "E\u00b1F\u00b3G\u00b5\u0000\u00b7\u0000\u00b9H\u00bbI\u00bdJ\u00bf\u0000"+ + "\u00c1\u0000\u00c3\u0000\u00c5\u0000\u00c7\u0000\u00c9\u0000\u00cbK\u00cd"+ + "\u0000\u00cfL\u00d1\u0000\u00d3\u0000\u00d5M\u00d7N\u00d9O\u00db\u0000"+ + "\u00dd\u0000\u00df\u0000\u00e1\u0000\u00e3\u0000\u00e5P\u00e7Q\u00e9R"+ + "\u00ebS\u00ed\u0000\u00ef\u0000\u00f1\u0000\u00f3\u0000\u00f5T\u00f7\u0000"+ + "\u00f9U\u00fbV\u00fdW\u00ff\u0000\u0101\u0000\u0103X\u0105Y\u0107\u0000"+ + "\u0109Z\u010b\u0000\u010d[\u010f\\\u0111]\u0113\u0000\u0115\u0000\u0117"+ + "\u0000\u0119\u0000\u011b\u0000\u011d\u0000\u011f\u0000\u0121^\u0123_\u0125"+ + "`\u0127\u0000\u0129\u0000\u012b\u0000\u012d\u0000\u012fa\u0131b\u0133"+ + "c\u0135\u0000\u0137d\u0139e\u013bf\u013dg\u013f\u0000\u0141h\u0143i\u0145"+ + "j\u0147k\u0149l\u014b\u0000\u014d\u0000\u014f\u0000\u0151\u0000\u0153"+ + "\u0000\u0155\u0000\u0157\u0000\u0159m\u015bn\u015do\u015f\u0000\u0161"+ + "\u0000\u0163\u0000\u0165\u0000\u0167p\u0169q\u016br\u016d\u0000\u016f"+ + "\u0000\u0171\u0000\u0173s\u0175t\u0177u\u0179\u0000\u017b\u0000\u017d"+ + "v\u017fw\u0181x\u0183\u0000\u0185\u0000\u0187\u0000\u0189\u0000\u000f"+ + "\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e"+ + "#\u0002\u0000DDdd\u0002\u0000IIii\u0002\u0000SSss\u0002\u0000EEee\u0002"+ + "\u0000CCcc\u0002\u0000TTtt\u0002\u0000RRrr\u0002\u0000OOoo\u0002\u0000"+ + "PPpp\u0002\u0000NNnn\u0002\u0000HHhh\u0002\u0000VVvv\u0002\u0000AAaa\u0002"+ + "\u0000LLll\u0002\u0000XXxx\u0002\u0000FFff\u0002\u0000MMmm\u0002\u0000"+ + "GGgg\u0002\u0000KKkk\u0002\u0000WWww\u0002\u0000UUuu\u0006\u0000\t\n\r"+ + "\r //[[]]\u0002\u0000\n\n\r\r\u0003\u0000\t\n\r\r \u0001\u000009\u0002"+ + "\u0000AZaz\b\u0000\"\"NNRRTT\\\\nnrrtt\u0004\u0000\n\n\r\r\"\"\\\\\u0002"+ + "\u0000++--\u0001\u0000``\u0002\u0000BBbb\u0002\u0000YYyy\u000b\u0000\t"+ + "\n\r\r \"\",,//::==[[]]||\u0002\u0000**//\u000b\u0000\t\n\r\r \"#,,"+ + "//::<<>?\\\\||\u05af\u0000\u000f\u0001\u0000\u0000\u0000\u0000\u0011\u0001"+ + "\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000\u0000\u0000\u0015\u0001"+ + "\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000\u0000\u0000\u0019\u0001"+ + "\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000\u0000\u0000\u001d\u0001"+ + "\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000\u0000\u0000!\u0001\u0000"+ + "\u0000\u0000\u0000#\u0001\u0000\u0000\u0000\u0000%\u0001\u0000\u0000\u0000"+ + "\u0000\'\u0001\u0000\u0000\u0000\u0000)\u0001\u0000\u0000\u0000\u0000"+ + "+\u0001\u0000\u0000\u0000\u0000-\u0001\u0000\u0000\u0000\u0000/\u0001"+ + "\u0000\u0000\u0000\u00001\u0001\u0000\u0000\u0000\u00003\u0001\u0000\u0000"+ + "\u0000\u00005\u0001\u0000\u0000\u0000\u00007\u0001\u0000\u0000\u0000\u0000"+ + "9\u0001\u0000\u0000\u0000\u0000;\u0001\u0000\u0000\u0000\u0000=\u0001"+ + "\u0000\u0000\u0000\u0001?\u0001\u0000\u0000\u0000\u0001U\u0001\u0000\u0000"+ + "\u0000\u0001W\u0001\u0000\u0000\u0000\u0001Y\u0001\u0000\u0000\u0000\u0001"+ + "[\u0001\u0000\u0000\u0000\u0001]\u0001\u0000\u0000\u0000\u0001_\u0001"+ + "\u0000\u0000\u0000\u0001a\u0001\u0000\u0000\u0000\u0001c\u0001\u0000\u0000"+ + "\u0000\u0001e\u0001\u0000\u0000\u0000\u0001g\u0001\u0000\u0000\u0000\u0001"+ + "i\u0001\u0000\u0000\u0000\u0001k\u0001\u0000\u0000\u0000\u0001m\u0001"+ + "\u0000\u0000\u0000\u0001o\u0001\u0000\u0000\u0000\u0001q\u0001\u0000\u0000"+ + "\u0000\u0001s\u0001\u0000\u0000\u0000\u0001u\u0001\u0000\u0000\u0000\u0001"+ + "w\u0001\u0000\u0000\u0000\u0001y\u0001\u0000\u0000\u0000\u0001{\u0001"+ + "\u0000\u0000\u0000\u0001}\u0001\u0000\u0000\u0000\u0001\u007f\u0001\u0000"+ + "\u0000\u0000\u0001\u0081\u0001\u0000\u0000\u0000\u0001\u0083\u0001\u0000"+ + "\u0000\u0000\u0001\u0085\u0001\u0000\u0000\u0000\u0001\u0087\u0001\u0000"+ + "\u0000\u0000\u0001\u0089\u0001\u0000\u0000\u0000\u0001\u008b\u0001\u0000"+ + "\u0000\u0000\u0001\u008d\u0001\u0000\u0000\u0000\u0001\u008f\u0001\u0000"+ + "\u0000\u0000\u0001\u0091\u0001\u0000\u0000\u0000\u0001\u0093\u0001\u0000"+ + "\u0000\u0000\u0001\u0095\u0001\u0000\u0000\u0000\u0001\u0097\u0001\u0000"+ + "\u0000\u0000\u0001\u0099\u0001\u0000\u0000\u0000\u0001\u009b\u0001\u0000"+ + "\u0000\u0000\u0001\u009d\u0001\u0000\u0000\u0000\u0001\u009f\u0001\u0000"+ + "\u0000\u0000\u0001\u00a1\u0001\u0000\u0000\u0000\u0001\u00a3\u0001\u0000"+ + "\u0000\u0000\u0001\u00a5\u0001\u0000\u0000\u0000\u0001\u00a7\u0001\u0000"+ + "\u0000\u0000\u0001\u00a9\u0001\u0000\u0000\u0000\u0001\u00ad\u0001\u0000"+ + "\u0000\u0000\u0001\u00af\u0001\u0000\u0000\u0000\u0001\u00b1\u0001\u0000"+ + "\u0000\u0000\u0001\u00b3\u0001\u0000\u0000\u0000\u0002\u00b5\u0001\u0000"+ + "\u0000\u0000\u0002\u00b7\u0001\u0000\u0000\u0000\u0002\u00b9\u0001\u0000"+ + "\u0000\u0000\u0002\u00bb\u0001\u0000\u0000\u0000\u0002\u00bd\u0001\u0000"+ + "\u0000\u0000\u0003\u00bf\u0001\u0000\u0000\u0000\u0003\u00c1\u0001\u0000"+ + "\u0000\u0000\u0003\u00c3\u0001\u0000\u0000\u0000\u0003\u00c5\u0001\u0000"+ + "\u0000\u0000\u0003\u00c7\u0001\u0000\u0000\u0000\u0003\u00c9\u0001\u0000"+ + "\u0000\u0000\u0003\u00cb\u0001\u0000\u0000\u0000\u0003\u00cf\u0001\u0000"+ + "\u0000\u0000\u0003\u00d1\u0001\u0000\u0000\u0000\u0003\u00d3\u0001\u0000"+ + "\u0000\u0000\u0003\u00d5\u0001\u0000\u0000\u0000\u0003\u00d7\u0001\u0000"+ + "\u0000\u0000\u0003\u00d9\u0001\u0000\u0000\u0000\u0004\u00db\u0001\u0000"+ + "\u0000\u0000\u0004\u00dd\u0001\u0000\u0000\u0000\u0004\u00df\u0001\u0000"+ + "\u0000\u0000\u0004\u00e5\u0001\u0000\u0000\u0000\u0004\u00e7\u0001\u0000"+ + "\u0000\u0000\u0004\u00e9\u0001\u0000\u0000\u0000\u0004\u00eb\u0001\u0000"+ + "\u0000\u0000\u0005\u00ed\u0001\u0000\u0000\u0000\u0005\u00ef\u0001\u0000"+ + "\u0000\u0000\u0005\u00f1\u0001\u0000\u0000\u0000\u0005\u00f3\u0001\u0000"+ + "\u0000\u0000\u0005\u00f5\u0001\u0000\u0000\u0000\u0005\u00f7\u0001\u0000"+ + "\u0000\u0000\u0005\u00f9\u0001\u0000\u0000\u0000\u0005\u00fb\u0001\u0000"+ + "\u0000\u0000\u0005\u00fd\u0001\u0000\u0000\u0000\u0006\u00ff\u0001\u0000"+ + "\u0000\u0000\u0006\u0101\u0001\u0000\u0000\u0000\u0006\u0103\u0001\u0000"+ + "\u0000\u0000\u0006\u0105\u0001\u0000\u0000\u0000\u0006\u0109\u0001\u0000"+ + "\u0000\u0000\u0006\u010b\u0001\u0000\u0000\u0000\u0006\u010d\u0001\u0000"+ + "\u0000\u0000\u0006\u010f\u0001\u0000\u0000\u0000\u0006\u0111\u0001\u0000"+ + "\u0000\u0000\u0007\u0113\u0001\u0000\u0000\u0000\u0007\u0115\u0001\u0000"+ + "\u0000\u0000\u0007\u0117\u0001\u0000\u0000\u0000\u0007\u0119\u0001\u0000"+ + "\u0000\u0000\u0007\u011b\u0001\u0000\u0000\u0000\u0007\u011d\u0001\u0000"+ + "\u0000\u0000\u0007\u011f\u0001\u0000\u0000\u0000\u0007\u0121\u0001\u0000"+ + "\u0000\u0000\u0007\u0123\u0001\u0000\u0000\u0000\u0007\u0125\u0001\u0000"+ + "\u0000\u0000\b\u0127\u0001\u0000\u0000\u0000\b\u0129\u0001\u0000\u0000"+ + "\u0000\b\u012b\u0001\u0000\u0000\u0000\b\u012d\u0001\u0000\u0000\u0000"+ + "\b\u012f\u0001\u0000\u0000\u0000\b\u0131\u0001\u0000\u0000\u0000\b\u0133"+ + "\u0001\u0000\u0000\u0000\t\u0135\u0001\u0000\u0000\u0000\t\u0137\u0001"+ + "\u0000\u0000\u0000\t\u0139\u0001\u0000\u0000\u0000\t\u013b\u0001\u0000"+ + "\u0000\u0000\t\u013d\u0001\u0000\u0000\u0000\n\u013f\u0001\u0000\u0000"+ + "\u0000\n\u0141\u0001\u0000\u0000\u0000\n\u0143\u0001\u0000\u0000\u0000"+ + "\n\u0145\u0001\u0000\u0000\u0000\n\u0147\u0001\u0000\u0000\u0000\n\u0149"+ + "\u0001\u0000\u0000\u0000\u000b\u014b\u0001\u0000\u0000\u0000\u000b\u014d"+ + "\u0001\u0000\u0000\u0000\u000b\u014f\u0001\u0000\u0000\u0000\u000b\u0151"+ + "\u0001\u0000\u0000\u0000\u000b\u0153\u0001\u0000\u0000\u0000\u000b\u0155"+ + "\u0001\u0000\u0000\u0000\u000b\u0157\u0001\u0000\u0000\u0000\u000b\u0159"+ + "\u0001\u0000\u0000\u0000\u000b\u015b\u0001\u0000\u0000\u0000\u000b\u015d"+ + "\u0001\u0000\u0000\u0000\f\u015f\u0001\u0000\u0000\u0000\f\u0161\u0001"+ + "\u0000\u0000\u0000\f\u0163\u0001\u0000\u0000\u0000\f\u0165\u0001\u0000"+ + "\u0000\u0000\f\u0167\u0001\u0000\u0000\u0000\f\u0169\u0001\u0000\u0000"+ + "\u0000\f\u016b\u0001\u0000\u0000\u0000\r\u016d\u0001\u0000\u0000\u0000"+ + "\r\u016f\u0001\u0000\u0000\u0000\r\u0171\u0001\u0000\u0000\u0000\r\u0173"+ + "\u0001\u0000\u0000\u0000\r\u0175\u0001\u0000\u0000\u0000\r\u0177\u0001"+ + "\u0000\u0000\u0000\u000e\u0179\u0001\u0000\u0000\u0000\u000e\u017b\u0001"+ + "\u0000\u0000\u0000\u000e\u017d\u0001\u0000\u0000\u0000\u000e\u017f\u0001"+ + "\u0000\u0000\u0000\u000e\u0181\u0001\u0000\u0000\u0000\u000e\u0183\u0001"+ + "\u0000\u0000\u0000\u000e\u0185\u0001\u0000\u0000\u0000\u000e\u0187\u0001"+ + "\u0000\u0000\u0000\u000e\u0189\u0001\u0000\u0000\u0000\u000f\u018b\u0001"+ + "\u0000\u0000\u0000\u0011\u0195\u0001\u0000\u0000\u0000\u0013\u019c\u0001"+ + "\u0000\u0000\u0000\u0015\u01a5\u0001\u0000\u0000\u0000\u0017\u01ac\u0001"+ + "\u0000\u0000\u0000\u0019\u01b6\u0001\u0000\u0000\u0000\u001b\u01bd\u0001"+ + "\u0000\u0000\u0000\u001d\u01c4\u0001\u0000\u0000\u0000\u001f\u01cb\u0001"+ + "\u0000\u0000\u0000!\u01d3\u0001\u0000\u0000\u0000#\u01df\u0001\u0000\u0000"+ + "\u0000%\u01e8\u0001\u0000\u0000\u0000\'\u01ee\u0001\u0000\u0000\u0000"+ + ")\u01f5\u0001\u0000\u0000\u0000+\u01fc\u0001\u0000\u0000\u0000-\u0204"+ + "\u0001\u0000\u0000\u0000/\u020c\u0001\u0000\u0000\u00001\u021b\u0001\u0000"+ + "\u0000\u00003\u0225\u0001\u0000\u0000\u00005\u022e\u0001\u0000\u0000\u0000"+ + "7\u023a\u0001\u0000\u0000\u00009\u0240\u0001\u0000\u0000\u0000;\u0251"+ + "\u0001\u0000\u0000\u0000=\u0261\u0001\u0000\u0000\u0000?\u0267\u0001\u0000"+ + "\u0000\u0000A\u026b\u0001\u0000\u0000\u0000C\u026d\u0001\u0000\u0000\u0000"+ + "E\u026f\u0001\u0000\u0000\u0000G\u0272\u0001\u0000\u0000\u0000I\u0274"+ + "\u0001\u0000\u0000\u0000K\u027d\u0001\u0000\u0000\u0000M\u027f\u0001\u0000"+ + "\u0000\u0000O\u0284\u0001\u0000\u0000\u0000Q\u0286\u0001\u0000\u0000\u0000"+ + "S\u028b\u0001\u0000\u0000\u0000U\u02aa\u0001\u0000\u0000\u0000W\u02ad"+ + "\u0001\u0000\u0000\u0000Y\u02db\u0001\u0000\u0000\u0000[\u02dd\u0001\u0000"+ + "\u0000\u0000]\u02e0\u0001\u0000\u0000\u0000_\u02e4\u0001\u0000\u0000\u0000"+ + "a\u02e8\u0001\u0000\u0000\u0000c\u02ea\u0001\u0000\u0000\u0000e\u02ed"+ + "\u0001\u0000\u0000\u0000g\u02ef\u0001\u0000\u0000\u0000i\u02f4\u0001\u0000"+ + "\u0000\u0000k\u02f6\u0001\u0000\u0000\u0000m\u02fc\u0001\u0000\u0000\u0000"+ + "o\u0302\u0001\u0000\u0000\u0000q\u0305\u0001\u0000\u0000\u0000s\u0308"+ + "\u0001\u0000\u0000\u0000u\u030d\u0001\u0000\u0000\u0000w\u0312\u0001\u0000"+ + "\u0000\u0000y\u0314\u0001\u0000\u0000\u0000{\u0318\u0001\u0000\u0000\u0000"+ + "}\u031d\u0001\u0000\u0000\u0000\u007f\u0323\u0001\u0000\u0000\u0000\u0081"+ + "\u0326\u0001\u0000\u0000\u0000\u0083\u0328\u0001\u0000\u0000\u0000\u0085"+ + "\u032e\u0001\u0000\u0000\u0000\u0087\u0330\u0001\u0000\u0000\u0000\u0089"+ + "\u0335\u0001\u0000\u0000\u0000\u008b\u0338\u0001\u0000\u0000\u0000\u008d"+ + "\u033b\u0001\u0000\u0000\u0000\u008f\u033e\u0001\u0000\u0000\u0000\u0091"+ + "\u0340\u0001\u0000\u0000\u0000\u0093\u0343\u0001\u0000\u0000\u0000\u0095"+ + "\u0345\u0001\u0000\u0000\u0000\u0097\u0348\u0001\u0000\u0000\u0000\u0099"+ + "\u034a\u0001\u0000\u0000\u0000\u009b\u034c\u0001\u0000\u0000\u0000\u009d"+ + "\u034e\u0001\u0000\u0000\u0000\u009f\u0350\u0001\u0000\u0000\u0000\u00a1"+ + "\u0352\u0001\u0000\u0000\u0000\u00a3\u0368\u0001\u0000\u0000\u0000\u00a5"+ + "\u036a\u0001\u0000\u0000\u0000\u00a7\u036f\u0001\u0000\u0000\u0000\u00a9"+ + "\u0384\u0001\u0000\u0000\u0000\u00ab\u0386\u0001\u0000\u0000\u0000\u00ad"+ + "\u038e\u0001\u0000\u0000\u0000\u00af\u0390\u0001\u0000\u0000\u0000\u00b1"+ + "\u0394\u0001\u0000\u0000\u0000\u00b3\u0398\u0001\u0000\u0000\u0000\u00b5"+ + "\u039c\u0001\u0000\u0000\u0000\u00b7\u03a1\u0001\u0000\u0000\u0000\u00b9"+ + "\u03a6\u0001\u0000\u0000\u0000\u00bb\u03aa\u0001\u0000\u0000\u0000\u00bd"+ + "\u03ae\u0001\u0000\u0000\u0000\u00bf\u03b2\u0001\u0000\u0000\u0000\u00c1"+ + "\u03b7\u0001\u0000\u0000\u0000\u00c3\u03bb\u0001\u0000\u0000\u0000\u00c5"+ + "\u03bf\u0001\u0000\u0000\u0000\u00c7\u03c3\u0001\u0000\u0000\u0000\u00c9"+ + "\u03c7\u0001\u0000\u0000\u0000\u00cb\u03cb\u0001\u0000\u0000\u0000\u00cd"+ + "\u03d7\u0001\u0000\u0000\u0000\u00cf\u03da\u0001\u0000\u0000\u0000\u00d1"+ + "\u03de\u0001\u0000\u0000\u0000\u00d3\u03e2\u0001\u0000\u0000\u0000\u00d5"+ + "\u03e6\u0001\u0000\u0000\u0000\u00d7\u03ea\u0001\u0000\u0000\u0000\u00d9"+ + "\u03ee\u0001\u0000\u0000\u0000\u00db\u03f2\u0001\u0000\u0000\u0000\u00dd"+ + "\u03f7\u0001\u0000\u0000\u0000\u00df\u03fb\u0001\u0000\u0000\u0000\u00e1"+ + "\u0403\u0001\u0000\u0000\u0000\u00e3\u0418\u0001\u0000\u0000\u0000\u00e5"+ + "\u041c\u0001\u0000\u0000\u0000\u00e7\u0420\u0001\u0000\u0000\u0000\u00e9"+ + "\u0424\u0001\u0000\u0000\u0000\u00eb\u0428\u0001\u0000\u0000\u0000\u00ed"+ + "\u042c\u0001\u0000\u0000\u0000\u00ef\u0431\u0001\u0000\u0000\u0000\u00f1"+ + "\u0435\u0001\u0000\u0000\u0000\u00f3\u0439\u0001\u0000\u0000\u0000\u00f5"+ + "\u043d\u0001\u0000\u0000\u0000\u00f7\u0440\u0001\u0000\u0000\u0000\u00f9"+ + "\u0444\u0001\u0000\u0000\u0000\u00fb\u0448\u0001\u0000\u0000\u0000\u00fd"+ + "\u044c\u0001\u0000\u0000\u0000\u00ff\u0450\u0001\u0000\u0000\u0000\u0101"+ + "\u0455\u0001\u0000\u0000\u0000\u0103\u045a\u0001\u0000\u0000\u0000\u0105"+ + "\u045f\u0001\u0000\u0000\u0000\u0107\u0466\u0001\u0000\u0000\u0000\u0109"+ + "\u046f\u0001\u0000\u0000\u0000\u010b\u0476\u0001\u0000\u0000\u0000\u010d"+ + "\u047a\u0001\u0000\u0000\u0000\u010f\u047e\u0001\u0000\u0000\u0000\u0111"+ + "\u0482\u0001\u0000\u0000\u0000\u0113\u0486\u0001\u0000\u0000\u0000\u0115"+ + "\u048c\u0001\u0000\u0000\u0000\u0117\u0490\u0001\u0000\u0000\u0000\u0119"+ + "\u0494\u0001\u0000\u0000\u0000\u011b\u0498\u0001\u0000\u0000\u0000\u011d"+ + "\u049c\u0001\u0000\u0000\u0000\u011f\u04a0\u0001\u0000\u0000\u0000\u0121"+ + "\u04a4\u0001\u0000\u0000\u0000\u0123\u04a8\u0001\u0000\u0000\u0000\u0125"+ + "\u04ac\u0001\u0000\u0000\u0000\u0127\u04b0\u0001\u0000\u0000\u0000\u0129"+ + "\u04b5\u0001\u0000\u0000\u0000\u012b\u04b9\u0001\u0000\u0000\u0000\u012d"+ + "\u04bd\u0001\u0000\u0000\u0000\u012f\u04c1\u0001\u0000\u0000\u0000\u0131"+ + "\u04c5\u0001\u0000\u0000\u0000\u0133\u04c9\u0001\u0000\u0000\u0000\u0135"+ + "\u04cd\u0001\u0000\u0000\u0000\u0137\u04d2\u0001\u0000\u0000\u0000\u0139"+ + "\u04d7\u0001\u0000\u0000\u0000\u013b\u04db\u0001\u0000\u0000\u0000\u013d"+ + "\u04df\u0001\u0000\u0000\u0000\u013f\u04e3\u0001\u0000\u0000\u0000\u0141"+ + "\u04e8\u0001\u0000\u0000\u0000\u0143\u04ef\u0001\u0000\u0000\u0000\u0145"+ + "\u04f3\u0001\u0000\u0000\u0000\u0147\u04f7\u0001\u0000\u0000\u0000\u0149"+ + "\u04fb\u0001\u0000\u0000\u0000\u014b\u04ff\u0001\u0000\u0000\u0000\u014d"+ + "\u0504\u0001\u0000\u0000\u0000\u014f\u0508\u0001\u0000\u0000\u0000\u0151"+ + "\u050c\u0001\u0000\u0000\u0000\u0153\u0510\u0001\u0000\u0000\u0000\u0155"+ + "\u0515\u0001\u0000\u0000\u0000\u0157\u0519\u0001\u0000\u0000\u0000\u0159"+ + "\u051d\u0001\u0000\u0000\u0000\u015b\u0521\u0001\u0000\u0000\u0000\u015d"+ + "\u0525\u0001\u0000\u0000\u0000\u015f\u0529\u0001\u0000\u0000\u0000\u0161"+ + "\u052f\u0001\u0000\u0000\u0000\u0163\u0533\u0001\u0000\u0000\u0000\u0165"+ + "\u0537\u0001\u0000\u0000\u0000\u0167\u053b\u0001\u0000\u0000\u0000\u0169"+ + "\u053f\u0001\u0000\u0000\u0000\u016b\u0543\u0001\u0000\u0000\u0000\u016d"+ + "\u0547\u0001\u0000\u0000\u0000\u016f\u054c\u0001\u0000\u0000\u0000\u0171"+ + "\u0552\u0001\u0000\u0000\u0000\u0173\u0558\u0001\u0000\u0000\u0000\u0175"+ + "\u055c\u0001\u0000\u0000\u0000\u0177\u0560\u0001\u0000\u0000\u0000\u0179"+ + "\u0564\u0001\u0000\u0000\u0000\u017b\u056a\u0001\u0000\u0000\u0000\u017d"+ + "\u0570\u0001\u0000\u0000\u0000\u017f\u0574\u0001\u0000\u0000\u0000\u0181"+ + "\u0578\u0001\u0000\u0000\u0000\u0183\u057c\u0001\u0000\u0000\u0000\u0185"+ + "\u0582\u0001\u0000\u0000\u0000\u0187\u0588\u0001\u0000\u0000\u0000\u0189"+ + "\u058e\u0001\u0000\u0000\u0000\u018b\u018c\u0007\u0000\u0000\u0000\u018c"+ + "\u018d\u0007\u0001\u0000\u0000\u018d\u018e\u0007\u0002\u0000\u0000\u018e"+ + "\u018f\u0007\u0002\u0000\u0000\u018f\u0190\u0007\u0003\u0000\u0000\u0190"+ + "\u0191\u0007\u0004\u0000\u0000\u0191\u0192\u0007\u0005\u0000\u0000\u0192"+ + "\u0193\u0001\u0000\u0000\u0000\u0193\u0194\u0006\u0000\u0000\u0000\u0194"+ + "\u0010\u0001\u0000\u0000\u0000\u0195\u0196\u0007\u0000\u0000\u0000\u0196"+ + "\u0197\u0007\u0006\u0000\u0000\u0197\u0198\u0007\u0007\u0000\u0000\u0198"+ + "\u0199\u0007\b\u0000\u0000\u0199\u019a\u0001\u0000\u0000\u0000\u019a\u019b"+ + "\u0006\u0001\u0001\u0000\u019b\u0012\u0001\u0000\u0000\u0000\u019c\u019d"+ + "\u0007\u0003\u0000\u0000\u019d\u019e\u0007\t\u0000\u0000\u019e\u019f\u0007"+ + "\u0006\u0000\u0000\u019f\u01a0\u0007\u0001\u0000\u0000\u01a0\u01a1\u0007"+ + "\u0004\u0000\u0000\u01a1\u01a2\u0007\n\u0000\u0000\u01a2\u01a3\u0001\u0000"+ + "\u0000\u0000\u01a3\u01a4\u0006\u0002\u0002\u0000\u01a4\u0014\u0001\u0000"+ + "\u0000\u0000\u01a5\u01a6\u0007\u0003\u0000\u0000\u01a6\u01a7\u0007\u000b"+ + "\u0000\u0000\u01a7\u01a8\u0007\f\u0000\u0000\u01a8\u01a9\u0007\r\u0000"+ + "\u0000\u01a9\u01aa\u0001\u0000\u0000\u0000\u01aa\u01ab\u0006\u0003\u0000"+ + "\u0000\u01ab\u0016\u0001\u0000\u0000\u0000\u01ac\u01ad\u0007\u0003\u0000"+ + "\u0000\u01ad\u01ae\u0007\u000e\u0000\u0000\u01ae\u01af\u0007\b\u0000\u0000"+ + "\u01af\u01b0\u0007\r\u0000\u0000\u01b0\u01b1\u0007\f\u0000\u0000\u01b1"+ + "\u01b2\u0007\u0001\u0000\u0000\u01b2\u01b3\u0007\t\u0000\u0000\u01b3\u01b4"+ + "\u0001\u0000\u0000\u0000\u01b4\u01b5\u0006\u0004\u0003\u0000\u01b5\u0018"+ + "\u0001\u0000\u0000\u0000\u01b6\u01b7\u0007\u000f\u0000\u0000\u01b7\u01b8"+ + "\u0007\u0006\u0000\u0000\u01b8\u01b9\u0007\u0007\u0000\u0000\u01b9\u01ba"+ + "\u0007\u0010\u0000\u0000\u01ba\u01bb\u0001\u0000\u0000\u0000\u01bb\u01bc"+ + "\u0006\u0005\u0004\u0000\u01bc\u001a\u0001\u0000\u0000\u0000\u01bd\u01be"+ + "\u0007\u0011\u0000\u0000\u01be\u01bf\u0007\u0006\u0000\u0000\u01bf\u01c0"+ + "\u0007\u0007\u0000\u0000\u01c0\u01c1\u0007\u0012\u0000\u0000\u01c1\u01c2"+ + "\u0001\u0000\u0000\u0000\u01c2\u01c3\u0006\u0006\u0000\u0000\u01c3\u001c"+ + "\u0001\u0000\u0000\u0000\u01c4\u01c5\u0007\u0012\u0000\u0000\u01c5\u01c6"+ + "\u0007\u0003\u0000\u0000\u01c6\u01c7\u0007\u0003\u0000\u0000\u01c7\u01c8"+ + "\u0007\b\u0000\u0000\u01c8\u01c9\u0001\u0000\u0000\u0000\u01c9\u01ca\u0006"+ + "\u0007\u0001\u0000\u01ca\u001e\u0001\u0000\u0000\u0000\u01cb\u01cc\u0007"+ + "\r\u0000\u0000\u01cc\u01cd\u0007\u0001\u0000\u0000\u01cd\u01ce\u0007\u0010"+ + "\u0000\u0000\u01ce\u01cf\u0007\u0001\u0000\u0000\u01cf\u01d0\u0007\u0005"+ + "\u0000\u0000\u01d0\u01d1\u0001\u0000\u0000\u0000\u01d1\u01d2\u0006\b\u0000"+ + "\u0000\u01d2 \u0001\u0000\u0000\u0000\u01d3\u01d4\u0007\u0010\u0000\u0000"+ + "\u01d4\u01d5\u0007\u000b\u0000\u0000\u01d5\u01d6\u0005_\u0000\u0000\u01d6"+ + "\u01d7\u0007\u0003\u0000\u0000\u01d7\u01d8\u0007\u000e\u0000\u0000\u01d8"+ + "\u01d9\u0007\b\u0000\u0000\u01d9\u01da\u0007\f\u0000\u0000\u01da\u01db"+ + "\u0007\t\u0000\u0000\u01db\u01dc\u0007\u0000\u0000\u0000\u01dc\u01dd\u0001"+ + "\u0000\u0000\u0000\u01dd\u01de\u0006\t\u0005\u0000\u01de\"\u0001\u0000"+ + "\u0000\u0000\u01df\u01e0\u0007\u0006\u0000\u0000\u01e0\u01e1\u0007\u0003"+ + "\u0000\u0000\u01e1\u01e2\u0007\t\u0000\u0000\u01e2\u01e3\u0007\f\u0000"+ + "\u0000\u01e3\u01e4\u0007\u0010\u0000\u0000\u01e4\u01e5\u0007\u0003\u0000"+ + "\u0000\u01e5\u01e6\u0001\u0000\u0000\u0000\u01e6\u01e7\u0006\n\u0006\u0000"+ + "\u01e7$\u0001\u0000\u0000\u0000\u01e8\u01e9\u0007\u0006\u0000\u0000\u01e9"+ + "\u01ea\u0007\u0007\u0000\u0000\u01ea\u01eb\u0007\u0013\u0000\u0000\u01eb"+ + "\u01ec\u0001\u0000\u0000\u0000\u01ec\u01ed\u0006\u000b\u0000\u0000\u01ed"+ + "&\u0001\u0000\u0000\u0000\u01ee\u01ef\u0007\u0002\u0000\u0000\u01ef\u01f0"+ + "\u0007\n\u0000\u0000\u01f0\u01f1\u0007\u0007\u0000\u0000\u01f1\u01f2\u0007"+ + "\u0013\u0000\u0000\u01f2\u01f3\u0001\u0000\u0000\u0000\u01f3\u01f4\u0006"+ + "\f\u0007\u0000\u01f4(\u0001\u0000\u0000\u0000\u01f5\u01f6\u0007\u0002"+ + "\u0000\u0000\u01f6\u01f7\u0007\u0007\u0000\u0000\u01f7\u01f8\u0007\u0006"+ + "\u0000\u0000\u01f8\u01f9\u0007\u0005\u0000\u0000\u01f9\u01fa\u0001\u0000"+ + "\u0000\u0000\u01fa\u01fb\u0006\r\u0000\u0000\u01fb*\u0001\u0000\u0000"+ + "\u0000\u01fc\u01fd\u0007\u0002\u0000\u0000\u01fd\u01fe\u0007\u0005\u0000"+ + "\u0000\u01fe\u01ff\u0007\f\u0000\u0000\u01ff\u0200\u0007\u0005\u0000\u0000"+ + "\u0200\u0201\u0007\u0002\u0000\u0000\u0201\u0202\u0001\u0000\u0000\u0000"+ + "\u0202\u0203\u0006\u000e\u0000\u0000\u0203,\u0001\u0000\u0000\u0000\u0204"+ + "\u0205\u0007\u0013\u0000\u0000\u0205\u0206\u0007\n\u0000\u0000\u0206\u0207"+ + "\u0007\u0003\u0000\u0000\u0207\u0208\u0007\u0006\u0000\u0000\u0208\u0209"+ + "\u0007\u0003\u0000\u0000\u0209\u020a\u0001\u0000\u0000\u0000\u020a\u020b"+ + "\u0006\u000f\u0000\u0000\u020b.\u0001\u0000\u0000\u0000\u020c\u020d\u0004"+ + "\u0010\u0000\u0000\u020d\u020e\u0007\u0001\u0000\u0000\u020e\u020f\u0007"+ + "\t\u0000\u0000\u020f\u0210\u0007\r\u0000\u0000\u0210\u0211\u0007\u0001"+ + "\u0000\u0000\u0211\u0212\u0007\t\u0000\u0000\u0212\u0213\u0007\u0003\u0000"+ + "\u0000\u0213\u0214\u0007\u0002\u0000\u0000\u0214\u0215\u0007\u0005\u0000"+ + "\u0000\u0215\u0216\u0007\f\u0000\u0000\u0216\u0217\u0007\u0005\u0000\u0000"+ + "\u0217\u0218\u0007\u0002\u0000\u0000\u0218\u0219\u0001\u0000\u0000\u0000"+ + "\u0219\u021a\u0006\u0010\u0000\u0000\u021a0\u0001\u0000\u0000\u0000\u021b"+ + "\u021c\u0004\u0011\u0001\u0000\u021c\u021d\u0007\r\u0000\u0000\u021d\u021e"+ + "\u0007\u0007\u0000\u0000\u021e\u021f\u0007\u0007\u0000\u0000\u021f\u0220"+ + "\u0007\u0012\u0000\u0000\u0220\u0221\u0007\u0014\u0000\u0000\u0221\u0222"+ + "\u0007\b\u0000\u0000\u0222\u0223\u0001\u0000\u0000\u0000\u0223\u0224\u0006"+ + "\u0011\b\u0000\u02242\u0001\u0000\u0000\u0000\u0225\u0226\u0004\u0012"+ + "\u0002\u0000\u0226\u0227\u0007\u0010\u0000\u0000\u0227\u0228\u0007\f\u0000"+ + "\u0000\u0228\u0229\u0007\u0005\u0000\u0000\u0229\u022a\u0007\u0004\u0000"+ + "\u0000\u022a\u022b\u0007\n\u0000\u0000\u022b\u022c\u0001\u0000\u0000\u0000"+ + "\u022c\u022d\u0006\u0012\u0000\u0000\u022d4\u0001\u0000\u0000\u0000\u022e"+ + "\u022f\u0004\u0013\u0003\u0000\u022f\u0230\u0007\u0010\u0000\u0000\u0230"+ + "\u0231\u0007\u0003\u0000\u0000\u0231\u0232\u0007\u0005\u0000\u0000\u0232"+ + "\u0233\u0007\u0006\u0000\u0000\u0233\u0234\u0007\u0001\u0000\u0000\u0234"+ + "\u0235\u0007\u0004\u0000\u0000\u0235\u0236\u0007\u0002\u0000\u0000\u0236"+ + "\u0237\u0001\u0000\u0000\u0000\u0237\u0238\u0006\u0013\t\u0000\u02386"+ + "\u0001\u0000\u0000\u0000\u0239\u023b\b\u0015\u0000\u0000\u023a\u0239\u0001"+ + "\u0000\u0000\u0000\u023b\u023c\u0001\u0000\u0000\u0000\u023c\u023a\u0001"+ + "\u0000\u0000\u0000\u023c\u023d\u0001\u0000\u0000\u0000\u023d\u023e\u0001"+ + "\u0000\u0000\u0000\u023e\u023f\u0006\u0014\u0000\u0000\u023f8\u0001\u0000"+ + "\u0000\u0000\u0240\u0241\u0005/\u0000\u0000\u0241\u0242\u0005/\u0000\u0000"+ + "\u0242\u0246\u0001\u0000\u0000\u0000\u0243\u0245\b\u0016\u0000\u0000\u0244"+ + "\u0243\u0001\u0000\u0000\u0000\u0245\u0248\u0001\u0000\u0000\u0000\u0246"+ + "\u0244\u0001\u0000\u0000\u0000\u0246\u0247\u0001\u0000\u0000\u0000\u0247"+ + "\u024a\u0001\u0000\u0000\u0000\u0248\u0246\u0001\u0000\u0000\u0000\u0249"+ + "\u024b\u0005\r\u0000\u0000\u024a\u0249\u0001\u0000\u0000\u0000\u024a\u024b"+ + "\u0001\u0000\u0000\u0000\u024b\u024d\u0001\u0000\u0000\u0000\u024c\u024e"+ + "\u0005\n\u0000\u0000\u024d\u024c\u0001\u0000\u0000\u0000\u024d\u024e\u0001"+ + "\u0000\u0000\u0000\u024e\u024f\u0001\u0000\u0000\u0000\u024f\u0250\u0006"+ + "\u0015\n\u0000\u0250:\u0001\u0000\u0000\u0000\u0251\u0252\u0005/\u0000"+ + "\u0000\u0252\u0253\u0005*\u0000\u0000\u0253\u0258\u0001\u0000\u0000\u0000"+ + "\u0254\u0257\u0003;\u0016\u0000\u0255\u0257\t\u0000\u0000\u0000\u0256"+ + "\u0254\u0001\u0000\u0000\u0000\u0256\u0255\u0001\u0000\u0000\u0000\u0257"+ + "\u025a\u0001\u0000\u0000\u0000\u0258\u0259\u0001\u0000\u0000\u0000\u0258"+ + "\u0256\u0001\u0000\u0000\u0000\u0259\u025b\u0001\u0000\u0000\u0000\u025a"+ + "\u0258\u0001\u0000\u0000\u0000\u025b\u025c\u0005*\u0000\u0000\u025c\u025d"+ + "\u0005/\u0000\u0000\u025d\u025e\u0001\u0000\u0000\u0000\u025e\u025f\u0006"+ + "\u0016\n\u0000\u025f<\u0001\u0000\u0000\u0000\u0260\u0262\u0007\u0017"+ + "\u0000\u0000\u0261\u0260\u0001\u0000\u0000\u0000\u0262\u0263\u0001\u0000"+ + "\u0000\u0000\u0263\u0261\u0001\u0000\u0000\u0000\u0263\u0264\u0001\u0000"+ + "\u0000\u0000\u0264\u0265\u0001\u0000\u0000\u0000\u0265\u0266\u0006\u0017"+ + "\n\u0000\u0266>\u0001\u0000\u0000\u0000\u0267\u0268\u0005|\u0000\u0000"+ + "\u0268\u0269\u0001\u0000\u0000\u0000\u0269\u026a\u0006\u0018\u000b\u0000"+ + "\u026a@\u0001\u0000\u0000\u0000\u026b\u026c\u0007\u0018\u0000\u0000\u026c"+ + "B\u0001\u0000\u0000\u0000\u026d\u026e\u0007\u0019\u0000\u0000\u026eD\u0001"+ + "\u0000\u0000\u0000\u026f\u0270\u0005\\\u0000\u0000\u0270\u0271\u0007\u001a"+ + "\u0000\u0000\u0271F\u0001\u0000\u0000\u0000\u0272\u0273\b\u001b\u0000"+ + "\u0000\u0273H\u0001\u0000\u0000\u0000\u0274\u0276\u0007\u0003\u0000\u0000"+ + "\u0275\u0277\u0007\u001c\u0000\u0000\u0276\u0275\u0001\u0000\u0000\u0000"+ + "\u0276\u0277\u0001\u0000\u0000\u0000\u0277\u0279\u0001\u0000\u0000\u0000"+ + "\u0278\u027a\u0003A\u0019\u0000\u0279\u0278\u0001\u0000\u0000\u0000\u027a"+ + "\u027b\u0001\u0000\u0000\u0000\u027b\u0279\u0001\u0000\u0000\u0000\u027b"+ + "\u027c\u0001\u0000\u0000\u0000\u027cJ\u0001\u0000\u0000\u0000\u027d\u027e"+ + "\u0005@\u0000\u0000\u027eL\u0001\u0000\u0000\u0000\u027f\u0280\u0005`"+ + "\u0000\u0000\u0280N\u0001\u0000\u0000\u0000\u0281\u0285\b\u001d\u0000"+ + "\u0000\u0282\u0283\u0005`\u0000\u0000\u0283\u0285\u0005`\u0000\u0000\u0284"+ + "\u0281\u0001\u0000\u0000\u0000\u0284\u0282\u0001\u0000\u0000\u0000\u0285"+ + "P\u0001\u0000\u0000\u0000\u0286\u0287\u0005_\u0000\u0000\u0287R\u0001"+ + "\u0000\u0000\u0000\u0288\u028c\u0003C\u001a\u0000\u0289\u028c\u0003A\u0019"+ + "\u0000\u028a\u028c\u0003Q!\u0000\u028b\u0288\u0001\u0000\u0000\u0000\u028b"+ + "\u0289\u0001\u0000\u0000\u0000\u028b\u028a\u0001\u0000\u0000\u0000\u028c"+ + "T\u0001\u0000\u0000\u0000\u028d\u0292\u0005\"\u0000\u0000\u028e\u0291"+ + "\u0003E\u001b\u0000\u028f\u0291\u0003G\u001c\u0000\u0290\u028e\u0001\u0000"+ + "\u0000\u0000\u0290\u028f\u0001\u0000\u0000\u0000\u0291\u0294\u0001\u0000"+ + "\u0000\u0000\u0292\u0290\u0001\u0000\u0000\u0000\u0292\u0293\u0001\u0000"+ + "\u0000\u0000\u0293\u0295\u0001\u0000\u0000\u0000\u0294\u0292\u0001\u0000"+ + "\u0000\u0000\u0295\u02ab\u0005\"\u0000\u0000\u0296\u0297\u0005\"\u0000"+ + "\u0000\u0297\u0298\u0005\"\u0000\u0000\u0298\u0299\u0005\"\u0000\u0000"+ + "\u0299\u029d\u0001\u0000\u0000\u0000\u029a\u029c\b\u0016\u0000\u0000\u029b"+ + "\u029a\u0001\u0000\u0000\u0000\u029c\u029f\u0001\u0000\u0000\u0000\u029d"+ + "\u029e\u0001\u0000\u0000\u0000\u029d\u029b\u0001\u0000\u0000\u0000\u029e"+ + "\u02a0\u0001\u0000\u0000\u0000\u029f\u029d\u0001\u0000\u0000\u0000\u02a0"+ + "\u02a1\u0005\"\u0000\u0000\u02a1\u02a2\u0005\"\u0000\u0000\u02a2\u02a3"+ + "\u0005\"\u0000\u0000\u02a3\u02a5\u0001\u0000\u0000\u0000\u02a4\u02a6\u0005"+ + "\"\u0000\u0000\u02a5\u02a4\u0001\u0000\u0000\u0000\u02a5\u02a6\u0001\u0000"+ + "\u0000\u0000\u02a6\u02a8\u0001\u0000\u0000\u0000\u02a7\u02a9\u0005\"\u0000"+ + "\u0000\u02a8\u02a7\u0001\u0000\u0000\u0000\u02a8\u02a9\u0001\u0000\u0000"+ + "\u0000\u02a9\u02ab\u0001\u0000\u0000\u0000\u02aa\u028d\u0001\u0000\u0000"+ + "\u0000\u02aa\u0296\u0001\u0000\u0000\u0000\u02abV\u0001\u0000\u0000\u0000"+ + "\u02ac\u02ae\u0003A\u0019\u0000\u02ad\u02ac\u0001\u0000\u0000\u0000\u02ae"+ + "\u02af\u0001\u0000\u0000\u0000\u02af\u02ad\u0001\u0000\u0000\u0000\u02af"+ + "\u02b0\u0001\u0000\u0000\u0000\u02b0X\u0001\u0000\u0000\u0000\u02b1\u02b3"+ + "\u0003A\u0019\u0000\u02b2\u02b1\u0001\u0000\u0000\u0000\u02b3\u02b4\u0001"+ + "\u0000\u0000\u0000\u02b4\u02b2\u0001\u0000\u0000\u0000\u02b4\u02b5\u0001"+ + "\u0000\u0000\u0000\u02b5\u02b6\u0001\u0000\u0000\u0000\u02b6\u02ba\u0003"+ + "i-\u0000\u02b7\u02b9\u0003A\u0019\u0000\u02b8\u02b7\u0001\u0000\u0000"+ + "\u0000\u02b9\u02bc\u0001\u0000\u0000\u0000\u02ba\u02b8\u0001\u0000\u0000"+ + "\u0000\u02ba\u02bb\u0001\u0000\u0000\u0000\u02bb\u02dc\u0001\u0000\u0000"+ + "\u0000\u02bc\u02ba\u0001\u0000\u0000\u0000\u02bd\u02bf\u0003i-\u0000\u02be"+ + "\u02c0\u0003A\u0019\u0000\u02bf\u02be\u0001\u0000\u0000\u0000\u02c0\u02c1"+ + "\u0001\u0000\u0000\u0000\u02c1\u02bf\u0001\u0000\u0000\u0000\u02c1\u02c2"+ + "\u0001\u0000\u0000\u0000\u02c2\u02dc\u0001\u0000\u0000\u0000\u02c3\u02c5"+ + "\u0003A\u0019\u0000\u02c4\u02c3\u0001\u0000\u0000\u0000\u02c5\u02c6\u0001"+ + "\u0000\u0000\u0000\u02c6\u02c4\u0001\u0000\u0000\u0000\u02c6\u02c7\u0001"+ + "\u0000\u0000\u0000\u02c7\u02cf\u0001\u0000\u0000\u0000\u02c8\u02cc\u0003"+ + "i-\u0000\u02c9\u02cb\u0003A\u0019\u0000\u02ca\u02c9\u0001\u0000\u0000"+ + "\u0000\u02cb\u02ce\u0001\u0000\u0000\u0000\u02cc\u02ca\u0001\u0000\u0000"+ + "\u0000\u02cc\u02cd\u0001\u0000\u0000\u0000\u02cd\u02d0\u0001\u0000\u0000"+ + "\u0000\u02ce\u02cc\u0001\u0000\u0000\u0000\u02cf\u02c8\u0001\u0000\u0000"+ + "\u0000\u02cf\u02d0\u0001\u0000\u0000\u0000\u02d0\u02d1\u0001\u0000\u0000"+ + "\u0000\u02d1\u02d2\u0003I\u001d\u0000\u02d2\u02dc\u0001\u0000\u0000\u0000"+ + "\u02d3\u02d5\u0003i-\u0000\u02d4\u02d6\u0003A\u0019\u0000\u02d5\u02d4"+ + "\u0001\u0000\u0000\u0000\u02d6\u02d7\u0001\u0000\u0000\u0000\u02d7\u02d5"+ + "\u0001\u0000\u0000\u0000\u02d7\u02d8\u0001\u0000\u0000\u0000\u02d8\u02d9"+ + "\u0001\u0000\u0000\u0000\u02d9\u02da\u0003I\u001d\u0000\u02da\u02dc\u0001"+ + "\u0000\u0000\u0000\u02db\u02b2\u0001\u0000\u0000\u0000\u02db\u02bd\u0001"+ + "\u0000\u0000\u0000\u02db\u02c4\u0001\u0000\u0000\u0000\u02db\u02d3\u0001"+ + "\u0000\u0000\u0000\u02dcZ\u0001\u0000\u0000\u0000\u02dd\u02de\u0007\u001e"+ + "\u0000\u0000\u02de\u02df\u0007\u001f\u0000\u0000\u02df\\\u0001\u0000\u0000"+ + "\u0000\u02e0\u02e1\u0007\f\u0000\u0000\u02e1\u02e2\u0007\t\u0000\u0000"+ + "\u02e2\u02e3\u0007\u0000\u0000\u0000\u02e3^\u0001\u0000\u0000\u0000\u02e4"+ + "\u02e5\u0007\f\u0000\u0000\u02e5\u02e6\u0007\u0002\u0000\u0000\u02e6\u02e7"+ + "\u0007\u0004\u0000\u0000\u02e7`\u0001\u0000\u0000\u0000\u02e8\u02e9\u0005"+ + "=\u0000\u0000\u02e9b\u0001\u0000\u0000\u0000\u02ea\u02eb\u0005:\u0000"+ + "\u0000\u02eb\u02ec\u0005:\u0000\u0000\u02ecd\u0001\u0000\u0000\u0000\u02ed"+ + "\u02ee\u0005,\u0000\u0000\u02eef\u0001\u0000\u0000\u0000\u02ef\u02f0\u0007"+ + "\u0000\u0000\u0000\u02f0\u02f1\u0007\u0003\u0000\u0000\u02f1\u02f2\u0007"+ + "\u0002\u0000\u0000\u02f2\u02f3\u0007\u0004\u0000\u0000\u02f3h\u0001\u0000"+ + "\u0000\u0000\u02f4\u02f5\u0005.\u0000\u0000\u02f5j\u0001\u0000\u0000\u0000"+ + "\u02f6\u02f7\u0007\u000f\u0000\u0000\u02f7\u02f8\u0007\f\u0000\u0000\u02f8"+ + "\u02f9\u0007\r\u0000\u0000\u02f9\u02fa\u0007\u0002\u0000\u0000\u02fa\u02fb"+ + "\u0007\u0003\u0000\u0000\u02fbl\u0001\u0000\u0000\u0000\u02fc\u02fd\u0007"+ + "\u000f\u0000\u0000\u02fd\u02fe\u0007\u0001\u0000\u0000\u02fe\u02ff\u0007"+ + "\u0006\u0000\u0000\u02ff\u0300\u0007\u0002\u0000\u0000\u0300\u0301\u0007"+ + "\u0005\u0000\u0000\u0301n\u0001\u0000\u0000\u0000\u0302\u0303\u0007\u0001"+ + "\u0000\u0000\u0303\u0304\u0007\t\u0000\u0000\u0304p\u0001\u0000\u0000"+ + "\u0000\u0305\u0306\u0007\u0001\u0000\u0000\u0306\u0307\u0007\u0002\u0000"+ + "\u0000\u0307r\u0001\u0000\u0000\u0000\u0308\u0309\u0007\r\u0000\u0000"+ + "\u0309\u030a\u0007\f\u0000\u0000\u030a\u030b\u0007\u0002\u0000\u0000\u030b"+ + "\u030c\u0007\u0005\u0000\u0000\u030ct\u0001\u0000\u0000\u0000\u030d\u030e"+ + "\u0007\r\u0000\u0000\u030e\u030f\u0007\u0001\u0000\u0000\u030f\u0310\u0007"+ + "\u0012\u0000\u0000\u0310\u0311\u0007\u0003\u0000\u0000\u0311v\u0001\u0000"+ + "\u0000\u0000\u0312\u0313\u0005(\u0000\u0000\u0313x\u0001\u0000\u0000\u0000"+ + "\u0314\u0315\u0007\t\u0000\u0000\u0315\u0316\u0007\u0007\u0000\u0000\u0316"+ + "\u0317\u0007\u0005\u0000\u0000\u0317z\u0001\u0000\u0000\u0000\u0318\u0319"+ + "\u0007\t\u0000\u0000\u0319\u031a\u0007\u0014\u0000\u0000\u031a\u031b\u0007"+ + "\r\u0000\u0000\u031b\u031c\u0007\r\u0000\u0000\u031c|\u0001\u0000\u0000"+ + "\u0000\u031d\u031e\u0007\t\u0000\u0000\u031e\u031f\u0007\u0014\u0000\u0000"+ + "\u031f\u0320\u0007\r\u0000\u0000\u0320\u0321\u0007\r\u0000\u0000\u0321"+ + "\u0322\u0007\u0002\u0000\u0000\u0322~\u0001\u0000\u0000\u0000\u0323\u0324"+ + "\u0007\u0007\u0000\u0000\u0324\u0325\u0007\u0006\u0000\u0000\u0325\u0080"+ + "\u0001\u0000\u0000\u0000\u0326\u0327\u0005?\u0000\u0000\u0327\u0082\u0001"+ + "\u0000\u0000\u0000\u0328\u0329\u0007\u0006\u0000\u0000\u0329\u032a\u0007"+ + "\r\u0000\u0000\u032a\u032b\u0007\u0001\u0000\u0000\u032b\u032c\u0007\u0012"+ + "\u0000\u0000\u032c\u032d\u0007\u0003\u0000\u0000\u032d\u0084\u0001\u0000"+ + "\u0000\u0000\u032e\u032f\u0005)\u0000\u0000\u032f\u0086\u0001\u0000\u0000"+ + "\u0000\u0330\u0331\u0007\u0005\u0000\u0000\u0331\u0332\u0007\u0006\u0000"+ + "\u0000\u0332\u0333\u0007\u0014\u0000\u0000\u0333\u0334\u0007\u0003\u0000"+ + "\u0000\u0334\u0088\u0001\u0000\u0000\u0000\u0335\u0336\u0005=\u0000\u0000"+ + "\u0336\u0337\u0005=\u0000\u0000\u0337\u008a\u0001\u0000\u0000\u0000\u0338"+ + "\u0339\u0005=\u0000\u0000\u0339\u033a\u0005~\u0000\u0000\u033a\u008c\u0001"+ + "\u0000\u0000\u0000\u033b\u033c\u0005!\u0000\u0000\u033c\u033d\u0005=\u0000"+ + "\u0000\u033d\u008e\u0001\u0000\u0000\u0000\u033e\u033f\u0005<\u0000\u0000"+ + "\u033f\u0090\u0001\u0000\u0000\u0000\u0340\u0341\u0005<\u0000\u0000\u0341"+ + "\u0342\u0005=\u0000\u0000\u0342\u0092\u0001\u0000\u0000\u0000\u0343\u0344"+ + "\u0005>\u0000\u0000\u0344\u0094\u0001\u0000\u0000\u0000\u0345\u0346\u0005"+ + ">\u0000\u0000\u0346\u0347\u0005=\u0000\u0000\u0347\u0096\u0001\u0000\u0000"+ + "\u0000\u0348\u0349\u0005+\u0000\u0000\u0349\u0098\u0001\u0000\u0000\u0000"+ + "\u034a\u034b\u0005-\u0000\u0000\u034b\u009a\u0001\u0000\u0000\u0000\u034c"+ + "\u034d\u0005*\u0000\u0000\u034d\u009c\u0001\u0000\u0000\u0000\u034e\u034f"+ + "\u0005/\u0000\u0000\u034f\u009e\u0001\u0000\u0000\u0000\u0350\u0351\u0005"+ + "%\u0000\u0000\u0351\u00a0\u0001\u0000\u0000\u0000\u0352\u0353\u0004I\u0004"+ + "\u0000\u0353\u0354\u00033\u0012\u0000\u0354\u0355\u0001\u0000\u0000\u0000"+ + "\u0355\u0356\u0006I\f\u0000\u0356\u00a2\u0001\u0000\u0000\u0000\u0357"+ + "\u035a\u0003\u00819\u0000\u0358\u035b\u0003C\u001a\u0000\u0359\u035b\u0003"+ + "Q!\u0000\u035a\u0358\u0001\u0000\u0000\u0000\u035a\u0359\u0001\u0000\u0000"+ + "\u0000\u035b\u035f\u0001\u0000\u0000\u0000\u035c\u035e\u0003S\"\u0000"+ + "\u035d\u035c\u0001\u0000\u0000\u0000\u035e\u0361\u0001\u0000\u0000\u0000"+ + "\u035f\u035d\u0001\u0000\u0000\u0000\u035f\u0360\u0001\u0000\u0000\u0000"+ + "\u0360\u0369\u0001\u0000\u0000\u0000\u0361\u035f\u0001\u0000\u0000\u0000"+ + "\u0362\u0364\u0003\u00819\u0000\u0363\u0365\u0003A\u0019\u0000\u0364\u0363"+ + "\u0001\u0000\u0000\u0000\u0365\u0366\u0001\u0000\u0000\u0000\u0366\u0364"+ + "\u0001\u0000\u0000\u0000\u0366\u0367\u0001\u0000\u0000\u0000\u0367\u0369"+ + "\u0001\u0000\u0000\u0000\u0368\u0357\u0001\u0000\u0000\u0000\u0368\u0362"+ + "\u0001\u0000\u0000\u0000\u0369\u00a4\u0001\u0000\u0000\u0000\u036a\u036b"+ + "\u0005[\u0000\u0000\u036b\u036c\u0001\u0000\u0000\u0000\u036c\u036d\u0006"+ + "K\u0000\u0000\u036d\u036e\u0006K\u0000\u0000\u036e\u00a6\u0001\u0000\u0000"+ + "\u0000\u036f\u0370\u0005]\u0000\u0000\u0370\u0371\u0001\u0000\u0000\u0000"+ + "\u0371\u0372\u0006L\u000b\u0000\u0372\u0373\u0006L\u000b\u0000\u0373\u00a8"+ + "\u0001\u0000\u0000\u0000\u0374\u0378\u0003C\u001a\u0000\u0375\u0377\u0003"+ + "S\"\u0000\u0376\u0375\u0001\u0000\u0000\u0000\u0377\u037a\u0001\u0000"+ + "\u0000\u0000\u0378\u0376\u0001\u0000\u0000\u0000\u0378\u0379\u0001\u0000"+ + "\u0000\u0000\u0379\u0385\u0001\u0000\u0000\u0000\u037a\u0378\u0001\u0000"+ + "\u0000\u0000\u037b\u037e\u0003Q!\u0000\u037c\u037e\u0003K\u001e\u0000"+ + "\u037d\u037b\u0001\u0000\u0000\u0000\u037d\u037c\u0001\u0000\u0000\u0000"+ + "\u037e\u0380\u0001\u0000\u0000\u0000\u037f\u0381\u0003S\"\u0000\u0380"+ + "\u037f\u0001\u0000\u0000\u0000\u0381\u0382\u0001\u0000\u0000\u0000\u0382"+ + "\u0380\u0001\u0000\u0000\u0000\u0382\u0383\u0001\u0000\u0000\u0000\u0383"+ + "\u0385\u0001\u0000\u0000\u0000\u0384\u0374\u0001\u0000\u0000\u0000\u0384"+ + "\u037d\u0001\u0000\u0000\u0000\u0385\u00aa\u0001\u0000\u0000\u0000\u0386"+ + "\u0388\u0003M\u001f\u0000\u0387\u0389\u0003O \u0000\u0388\u0387\u0001"+ + "\u0000\u0000\u0000\u0389\u038a\u0001\u0000\u0000\u0000\u038a\u0388\u0001"+ + "\u0000\u0000\u0000\u038a\u038b\u0001\u0000\u0000\u0000\u038b\u038c\u0001"+ + "\u0000\u0000\u0000\u038c\u038d\u0003M\u001f\u0000\u038d\u00ac\u0001\u0000"+ + "\u0000\u0000\u038e\u038f\u0003\u00abN\u0000\u038f\u00ae\u0001\u0000\u0000"+ + "\u0000\u0390\u0391\u00039\u0015\u0000\u0391\u0392\u0001\u0000\u0000\u0000"+ + "\u0392\u0393\u0006P\n\u0000\u0393\u00b0\u0001\u0000\u0000\u0000\u0394"+ + "\u0395\u0003;\u0016\u0000\u0395\u0396\u0001\u0000\u0000\u0000\u0396\u0397"+ + "\u0006Q\n\u0000\u0397\u00b2\u0001\u0000\u0000\u0000\u0398\u0399\u0003"+ + "=\u0017\u0000\u0399\u039a\u0001\u0000\u0000\u0000\u039a\u039b\u0006R\n"+ + "\u0000\u039b\u00b4\u0001\u0000\u0000\u0000\u039c\u039d\u0003\u00a5K\u0000"+ + "\u039d\u039e\u0001\u0000\u0000\u0000\u039e\u039f\u0006S\r\u0000\u039f"+ + "\u03a0\u0006S\u000e\u0000\u03a0\u00b6\u0001\u0000\u0000\u0000\u03a1\u03a2"+ + "\u0003?\u0018\u0000\u03a2\u03a3\u0001\u0000\u0000\u0000\u03a3\u03a4\u0006"+ + "T\u000f\u0000\u03a4\u03a5\u0006T\u000b\u0000\u03a5\u00b8\u0001\u0000\u0000"+ + "\u0000\u03a6\u03a7\u0003=\u0017\u0000\u03a7\u03a8\u0001\u0000\u0000\u0000"+ + "\u03a8\u03a9\u0006U\n\u0000\u03a9\u00ba\u0001\u0000\u0000\u0000\u03aa"+ + "\u03ab\u00039\u0015\u0000\u03ab\u03ac\u0001\u0000\u0000\u0000\u03ac\u03ad"+ + "\u0006V\n\u0000\u03ad\u00bc\u0001\u0000\u0000\u0000\u03ae\u03af\u0003"+ + ";\u0016\u0000\u03af\u03b0\u0001\u0000\u0000\u0000\u03b0\u03b1\u0006W\n"+ + "\u0000\u03b1\u00be\u0001\u0000\u0000\u0000\u03b2\u03b3\u0003?\u0018\u0000"+ + "\u03b3\u03b4\u0001\u0000\u0000\u0000\u03b4\u03b5\u0006X\u000f\u0000\u03b5"+ + "\u03b6\u0006X\u000b\u0000\u03b6\u00c0\u0001\u0000\u0000\u0000\u03b7\u03b8"+ + "\u0003\u00a5K\u0000\u03b8\u03b9\u0001\u0000\u0000\u0000\u03b9\u03ba\u0006"+ + "Y\r\u0000\u03ba\u00c2\u0001\u0000\u0000\u0000\u03bb\u03bc\u0003\u00a7"+ + "L\u0000\u03bc\u03bd\u0001\u0000\u0000\u0000\u03bd\u03be\u0006Z\u0010\u0000"+ + "\u03be\u00c4\u0001\u0000\u0000\u0000\u03bf\u03c0\u0003\u0141\u0099\u0000"+ + "\u03c0\u03c1\u0001\u0000\u0000\u0000\u03c1\u03c2\u0006[\u0011\u0000\u03c2"+ + "\u00c6\u0001\u0000\u0000\u0000\u03c3\u03c4\u0003e+\u0000\u03c4\u03c5\u0001"+ + "\u0000\u0000\u0000\u03c5\u03c6\u0006\\\u0012\u0000\u03c6\u00c8\u0001\u0000"+ + "\u0000\u0000\u03c7\u03c8\u0003a)\u0000\u03c8\u03c9\u0001\u0000\u0000\u0000"+ + "\u03c9\u03ca\u0006]\u0013\u0000\u03ca\u00ca\u0001\u0000\u0000\u0000\u03cb"+ + "\u03cc\u0007\u0010\u0000\u0000\u03cc\u03cd\u0007\u0003\u0000\u0000\u03cd"+ + "\u03ce\u0007\u0005\u0000\u0000\u03ce\u03cf\u0007\f\u0000\u0000\u03cf\u03d0"+ + "\u0007\u0000\u0000\u0000\u03d0\u03d1\u0007\f\u0000\u0000\u03d1\u03d2\u0007"+ + "\u0005\u0000\u0000\u03d2\u03d3\u0007\f\u0000\u0000\u03d3\u00cc\u0001\u0000"+ + "\u0000\u0000\u03d4\u03d8\b \u0000\u0000\u03d5\u03d6\u0005/\u0000\u0000"+ + "\u03d6\u03d8\b!\u0000\u0000\u03d7\u03d4\u0001\u0000\u0000\u0000\u03d7"+ + "\u03d5\u0001\u0000\u0000\u0000\u03d8\u00ce\u0001\u0000\u0000\u0000\u03d9"+ + "\u03db\u0003\u00cd_\u0000\u03da\u03d9\u0001\u0000\u0000\u0000\u03db\u03dc"+ + "\u0001\u0000\u0000\u0000\u03dc\u03da\u0001\u0000\u0000\u0000\u03dc\u03dd"+ + "\u0001\u0000\u0000\u0000\u03dd\u00d0\u0001\u0000\u0000\u0000\u03de\u03df"+ + "\u0003\u00cf`\u0000\u03df\u03e0\u0001\u0000\u0000\u0000\u03e0\u03e1\u0006"+ + "a\u0014\u0000\u03e1\u00d2\u0001\u0000\u0000\u0000\u03e2\u03e3\u0003U#"+ + "\u0000\u03e3\u03e4\u0001\u0000\u0000\u0000\u03e4\u03e5\u0006b\u0015\u0000"+ + "\u03e5\u00d4\u0001\u0000\u0000\u0000\u03e6\u03e7\u00039\u0015\u0000\u03e7"+ + "\u03e8\u0001\u0000\u0000\u0000\u03e8\u03e9\u0006c\n\u0000\u03e9\u00d6"+ + "\u0001\u0000\u0000\u0000\u03ea\u03eb\u0003;\u0016\u0000\u03eb\u03ec\u0001"+ + "\u0000\u0000\u0000\u03ec\u03ed\u0006d\n\u0000\u03ed\u00d8\u0001\u0000"+ + "\u0000\u0000\u03ee\u03ef\u0003=\u0017\u0000\u03ef\u03f0\u0001\u0000\u0000"+ + "\u0000\u03f0\u03f1\u0006e\n\u0000\u03f1\u00da\u0001\u0000\u0000\u0000"+ + "\u03f2\u03f3\u0003?\u0018\u0000\u03f3\u03f4\u0001\u0000\u0000\u0000\u03f4"+ + "\u03f5\u0006f\u000f\u0000\u03f5\u03f6\u0006f\u000b\u0000\u03f6\u00dc\u0001"+ + "\u0000\u0000\u0000\u03f7\u03f8\u0003i-\u0000\u03f8\u03f9\u0001\u0000\u0000"+ + "\u0000\u03f9\u03fa\u0006g\u0016\u0000\u03fa\u00de\u0001\u0000\u0000\u0000"+ + "\u03fb\u03fc\u0003e+\u0000\u03fc\u03fd\u0001\u0000\u0000\u0000\u03fd\u03fe"+ + "\u0006h\u0012\u0000\u03fe\u00e0\u0001\u0000\u0000\u0000\u03ff\u0404\u0003"+ + "C\u001a\u0000\u0400\u0404\u0003A\u0019\u0000\u0401\u0404\u0003Q!\u0000"+ + "\u0402\u0404\u0003\u009bF\u0000\u0403\u03ff\u0001\u0000\u0000\u0000\u0403"+ + "\u0400\u0001\u0000\u0000\u0000\u0403\u0401\u0001\u0000\u0000\u0000\u0403"+ + "\u0402\u0001\u0000\u0000\u0000\u0404\u00e2\u0001\u0000\u0000\u0000\u0405"+ + "\u0408\u0003C\u001a\u0000\u0406\u0408\u0003\u009bF\u0000\u0407\u0405\u0001"+ + "\u0000\u0000\u0000\u0407\u0406\u0001\u0000\u0000\u0000\u0408\u040c\u0001"+ + "\u0000\u0000\u0000\u0409\u040b\u0003\u00e1i\u0000\u040a\u0409\u0001\u0000"+ + "\u0000\u0000\u040b\u040e\u0001\u0000\u0000\u0000\u040c\u040a\u0001\u0000"+ + "\u0000\u0000\u040c\u040d\u0001\u0000\u0000\u0000\u040d\u0419\u0001\u0000"+ + "\u0000\u0000\u040e\u040c\u0001\u0000\u0000\u0000\u040f\u0412\u0003Q!\u0000"+ + "\u0410\u0412\u0003K\u001e\u0000\u0411\u040f\u0001\u0000\u0000\u0000\u0411"+ + "\u0410\u0001\u0000\u0000\u0000\u0412\u0414\u0001\u0000\u0000\u0000\u0413"+ + "\u0415\u0003\u00e1i\u0000\u0414\u0413\u0001\u0000\u0000\u0000\u0415\u0416"+ + "\u0001\u0000\u0000\u0000\u0416\u0414\u0001\u0000\u0000\u0000\u0416\u0417"+ + "\u0001\u0000\u0000\u0000\u0417\u0419\u0001\u0000\u0000\u0000\u0418\u0407"+ + "\u0001\u0000\u0000\u0000\u0418\u0411\u0001\u0000\u0000\u0000\u0419\u00e4"+ + "\u0001\u0000\u0000\u0000\u041a\u041d\u0003\u00e3j\u0000\u041b\u041d\u0003"+ + "\u00abN\u0000\u041c\u041a\u0001\u0000\u0000\u0000\u041c\u041b\u0001\u0000"+ + "\u0000\u0000\u041d\u041e\u0001\u0000\u0000\u0000\u041e\u041c\u0001\u0000"+ + "\u0000\u0000\u041e\u041f\u0001\u0000\u0000\u0000\u041f\u00e6\u0001\u0000"+ + "\u0000\u0000\u0420\u0421\u00039\u0015\u0000\u0421\u0422\u0001\u0000\u0000"+ + "\u0000\u0422\u0423\u0006l\n\u0000\u0423\u00e8\u0001\u0000\u0000\u0000"+ + "\u0424\u0425\u0003;\u0016\u0000\u0425\u0426\u0001\u0000\u0000\u0000\u0426"+ + "\u0427\u0006m\n\u0000\u0427\u00ea\u0001\u0000\u0000\u0000\u0428\u0429"+ + "\u0003=\u0017\u0000\u0429\u042a\u0001\u0000\u0000\u0000\u042a\u042b\u0006"+ + "n\n\u0000\u042b\u00ec\u0001\u0000\u0000\u0000\u042c\u042d\u0003?\u0018"+ + "\u0000\u042d\u042e\u0001\u0000\u0000\u0000\u042e\u042f\u0006o\u000f\u0000"+ + "\u042f\u0430\u0006o\u000b\u0000\u0430\u00ee\u0001\u0000\u0000\u0000\u0431"+ + "\u0432\u0003a)\u0000\u0432\u0433\u0001\u0000\u0000\u0000\u0433\u0434\u0006"+ + "p\u0013\u0000\u0434\u00f0\u0001\u0000\u0000\u0000\u0435\u0436\u0003e+"+ + "\u0000\u0436\u0437\u0001\u0000\u0000\u0000\u0437\u0438\u0006q\u0012\u0000"+ + "\u0438\u00f2\u0001\u0000\u0000\u0000\u0439\u043a\u0003i-\u0000\u043a\u043b"+ + "\u0001\u0000\u0000\u0000\u043b\u043c\u0006r\u0016\u0000\u043c\u00f4\u0001"+ + "\u0000\u0000\u0000\u043d\u043e\u0007\f\u0000\u0000\u043e\u043f\u0007\u0002"+ + "\u0000\u0000\u043f\u00f6\u0001\u0000\u0000\u0000\u0440\u0441\u0003\u00e5"+ + "k\u0000\u0441\u0442\u0001\u0000\u0000\u0000\u0442\u0443\u0006t\u0017\u0000"+ + "\u0443\u00f8\u0001\u0000\u0000\u0000\u0444\u0445\u00039\u0015\u0000\u0445"+ + "\u0446\u0001\u0000\u0000\u0000\u0446\u0447\u0006u\n\u0000\u0447\u00fa"+ + "\u0001\u0000\u0000\u0000\u0448\u0449\u0003;\u0016\u0000\u0449\u044a\u0001"+ + "\u0000\u0000\u0000\u044a\u044b\u0006v\n\u0000\u044b\u00fc\u0001\u0000"+ + "\u0000\u0000\u044c\u044d\u0003=\u0017\u0000\u044d\u044e\u0001\u0000\u0000"+ + "\u0000\u044e\u044f\u0006w\n\u0000\u044f\u00fe\u0001\u0000\u0000\u0000"+ + "\u0450\u0451\u0003?\u0018\u0000\u0451\u0452\u0001\u0000\u0000\u0000\u0452"+ + "\u0453\u0006x\u000f\u0000\u0453\u0454\u0006x\u000b\u0000\u0454\u0100\u0001"+ + "\u0000\u0000\u0000\u0455\u0456\u0003\u00a5K\u0000\u0456\u0457\u0001\u0000"+ + "\u0000\u0000\u0457\u0458\u0006y\r\u0000\u0458\u0459\u0006y\u0018\u0000"+ + "\u0459\u0102\u0001\u0000\u0000\u0000\u045a\u045b\u0007\u0007\u0000\u0000"+ + "\u045b\u045c\u0007\t\u0000\u0000\u045c\u045d\u0001\u0000\u0000\u0000\u045d"+ + "\u045e\u0006z\u0019\u0000\u045e\u0104\u0001\u0000\u0000\u0000\u045f\u0460"+ + "\u0007\u0013\u0000\u0000\u0460\u0461\u0007\u0001\u0000\u0000\u0461\u0462"+ + "\u0007\u0005\u0000\u0000\u0462\u0463\u0007\n\u0000\u0000\u0463\u0464\u0001"+ + "\u0000\u0000\u0000\u0464\u0465\u0006{\u0019\u0000\u0465\u0106\u0001\u0000"+ + "\u0000\u0000\u0466\u0467\b\"\u0000\u0000\u0467\u0108\u0001\u0000\u0000"+ + "\u0000\u0468\u046a\u0003\u0107|\u0000\u0469\u0468\u0001\u0000\u0000\u0000"+ + "\u046a\u046b\u0001\u0000\u0000\u0000\u046b\u0469\u0001\u0000\u0000\u0000"+ + "\u046b\u046c\u0001\u0000\u0000\u0000\u046c\u046d\u0001\u0000\u0000\u0000"+ + "\u046d\u046e\u0003\u0141\u0099\u0000\u046e\u0470\u0001\u0000\u0000\u0000"+ + "\u046f\u0469\u0001\u0000\u0000\u0000\u046f\u0470\u0001\u0000\u0000\u0000"+ + "\u0470\u0472\u0001\u0000\u0000\u0000\u0471\u0473\u0003\u0107|\u0000\u0472"+ + "\u0471\u0001\u0000\u0000\u0000\u0473\u0474\u0001\u0000\u0000\u0000\u0474"+ + "\u0472\u0001\u0000\u0000\u0000\u0474\u0475\u0001\u0000\u0000\u0000\u0475"+ + "\u010a\u0001\u0000\u0000\u0000\u0476\u0477\u0003\u0109}\u0000\u0477\u0478"+ + "\u0001\u0000\u0000\u0000\u0478\u0479\u0006~\u001a\u0000\u0479\u010c\u0001"+ + "\u0000\u0000\u0000\u047a\u047b\u00039\u0015\u0000\u047b\u047c\u0001\u0000"+ + "\u0000\u0000\u047c\u047d\u0006\u007f\n\u0000\u047d\u010e\u0001\u0000\u0000"+ + "\u0000\u047e\u047f\u0003;\u0016\u0000\u047f\u0480\u0001\u0000\u0000\u0000"+ + "\u0480\u0481\u0006\u0080\n\u0000\u0481\u0110\u0001\u0000\u0000\u0000\u0482"+ + "\u0483\u0003=\u0017\u0000\u0483\u0484\u0001\u0000\u0000\u0000\u0484\u0485"+ + "\u0006\u0081\n\u0000\u0485\u0112\u0001\u0000\u0000\u0000\u0486\u0487\u0003"+ + "?\u0018\u0000\u0487\u0488\u0001\u0000\u0000\u0000\u0488\u0489\u0006\u0082"+ + "\u000f\u0000\u0489\u048a\u0006\u0082\u000b\u0000\u048a\u048b\u0006\u0082"+ + "\u000b\u0000\u048b\u0114\u0001\u0000\u0000\u0000\u048c\u048d\u0003a)\u0000"+ + "\u048d\u048e\u0001\u0000\u0000\u0000\u048e\u048f\u0006\u0083\u0013\u0000"+ + "\u048f\u0116\u0001\u0000\u0000\u0000\u0490\u0491\u0003e+\u0000\u0491\u0492"+ + "\u0001\u0000\u0000\u0000\u0492\u0493\u0006\u0084\u0012\u0000\u0493\u0118"+ + "\u0001\u0000\u0000\u0000\u0494\u0495\u0003i-\u0000\u0495\u0496\u0001\u0000"+ + "\u0000\u0000\u0496\u0497\u0006\u0085\u0016\u0000\u0497\u011a\u0001\u0000"+ + "\u0000\u0000\u0498\u0499\u0003\u0105{\u0000\u0499\u049a\u0001\u0000\u0000"+ + "\u0000\u049a\u049b\u0006\u0086\u001b\u0000\u049b\u011c\u0001\u0000\u0000"+ + "\u0000\u049c\u049d\u0003\u00e5k\u0000\u049d\u049e\u0001\u0000\u0000\u0000"+ + "\u049e\u049f\u0006\u0087\u0017\u0000\u049f\u011e\u0001\u0000\u0000\u0000"+ + "\u04a0\u04a1\u0003\u00adO\u0000\u04a1\u04a2\u0001\u0000\u0000\u0000\u04a2"+ + "\u04a3\u0006\u0088\u001c\u0000\u04a3\u0120\u0001\u0000\u0000\u0000\u04a4"+ + "\u04a5\u00039\u0015\u0000\u04a5\u04a6\u0001\u0000\u0000\u0000\u04a6\u04a7"+ + "\u0006\u0089\n\u0000\u04a7\u0122\u0001\u0000\u0000\u0000\u04a8\u04a9\u0003"+ + ";\u0016\u0000\u04a9\u04aa\u0001\u0000\u0000\u0000\u04aa\u04ab\u0006\u008a"+ + "\n\u0000\u04ab\u0124\u0001\u0000\u0000\u0000\u04ac\u04ad\u0003=\u0017"+ + "\u0000\u04ad\u04ae\u0001\u0000\u0000\u0000\u04ae\u04af\u0006\u008b\n\u0000"+ + "\u04af\u0126\u0001\u0000\u0000\u0000\u04b0\u04b1\u0003?\u0018\u0000\u04b1"+ + "\u04b2\u0001\u0000\u0000\u0000\u04b2\u04b3\u0006\u008c\u000f\u0000\u04b3"+ + "\u04b4\u0006\u008c\u000b\u0000\u04b4\u0128\u0001\u0000\u0000\u0000\u04b5"+ + "\u04b6\u0003i-\u0000\u04b6\u04b7\u0001\u0000\u0000\u0000\u04b7\u04b8\u0006"+ + "\u008d\u0016\u0000\u04b8\u012a\u0001\u0000\u0000\u0000\u04b9\u04ba\u0003"+ + "\u00adO\u0000\u04ba\u04bb\u0001\u0000\u0000\u0000\u04bb\u04bc\u0006\u008e"+ + "\u001c\u0000\u04bc\u012c\u0001\u0000\u0000\u0000\u04bd\u04be\u0003\u00a9"+ + "M\u0000\u04be\u04bf\u0001\u0000\u0000\u0000\u04bf\u04c0\u0006\u008f\u001d"+ + "\u0000\u04c0\u012e\u0001\u0000\u0000\u0000\u04c1\u04c2\u00039\u0015\u0000"+ + "\u04c2\u04c3\u0001\u0000\u0000\u0000\u04c3\u04c4\u0006\u0090\n\u0000\u04c4"+ + "\u0130\u0001\u0000\u0000\u0000\u04c5\u04c6\u0003;\u0016\u0000\u04c6\u04c7"+ + "\u0001\u0000\u0000\u0000\u04c7\u04c8\u0006\u0091\n\u0000\u04c8\u0132\u0001"+ + "\u0000\u0000\u0000\u04c9\u04ca\u0003=\u0017\u0000\u04ca\u04cb\u0001\u0000"+ + "\u0000\u0000\u04cb\u04cc\u0006\u0092\n\u0000\u04cc\u0134\u0001\u0000\u0000"+ + "\u0000\u04cd\u04ce\u0003?\u0018\u0000\u04ce\u04cf\u0001\u0000\u0000\u0000"+ + "\u04cf\u04d0\u0006\u0093\u000f\u0000\u04d0\u04d1\u0006\u0093\u000b\u0000"+ + "\u04d1\u0136\u0001\u0000\u0000\u0000\u04d2\u04d3\u0007\u0001\u0000\u0000"+ + "\u04d3\u04d4\u0007\t\u0000\u0000\u04d4\u04d5\u0007\u000f\u0000\u0000\u04d5"+ + "\u04d6\u0007\u0007\u0000\u0000\u04d6\u0138\u0001\u0000\u0000\u0000\u04d7"+ + "\u04d8\u00039\u0015\u0000\u04d8\u04d9\u0001\u0000\u0000\u0000\u04d9\u04da"+ + "\u0006\u0095\n\u0000\u04da\u013a\u0001\u0000\u0000\u0000\u04db\u04dc\u0003"+ + ";\u0016\u0000\u04dc\u04dd\u0001\u0000\u0000\u0000\u04dd\u04de\u0006\u0096"+ + "\n\u0000\u04de\u013c\u0001\u0000\u0000\u0000\u04df\u04e0\u0003=\u0017"+ + "\u0000\u04e0\u04e1\u0001\u0000\u0000\u0000\u04e1\u04e2\u0006\u0097\n\u0000"+ + "\u04e2\u013e\u0001\u0000\u0000\u0000\u04e3\u04e4\u0003\u00a7L\u0000\u04e4"+ + "\u04e5\u0001\u0000\u0000\u0000\u04e5\u04e6\u0006\u0098\u0010\u0000\u04e6"+ + "\u04e7\u0006\u0098\u000b\u0000\u04e7\u0140\u0001\u0000\u0000\u0000\u04e8"+ + "\u04e9\u0005:\u0000\u0000\u04e9\u0142\u0001\u0000\u0000\u0000\u04ea\u04f0"+ + "\u0003K\u001e\u0000\u04eb\u04f0\u0003A\u0019\u0000\u04ec\u04f0\u0003i"+ + "-\u0000\u04ed\u04f0\u0003C\u001a\u0000\u04ee\u04f0\u0003Q!\u0000\u04ef"+ + "\u04ea\u0001\u0000\u0000\u0000\u04ef\u04eb\u0001\u0000\u0000\u0000\u04ef"+ + "\u04ec\u0001\u0000\u0000\u0000\u04ef\u04ed\u0001\u0000\u0000\u0000\u04ef"+ + "\u04ee\u0001\u0000\u0000\u0000\u04f0\u04f1\u0001\u0000\u0000\u0000\u04f1"+ + "\u04ef\u0001\u0000\u0000\u0000\u04f1\u04f2\u0001\u0000\u0000\u0000\u04f2"+ + "\u0144\u0001\u0000\u0000\u0000\u04f3\u04f4\u00039\u0015\u0000\u04f4\u04f5"+ + "\u0001\u0000\u0000\u0000\u04f5\u04f6\u0006\u009b\n\u0000\u04f6\u0146\u0001"+ + "\u0000\u0000\u0000\u04f7\u04f8\u0003;\u0016\u0000\u04f8\u04f9\u0001\u0000"+ + "\u0000\u0000\u04f9\u04fa\u0006\u009c\n\u0000\u04fa\u0148\u0001\u0000\u0000"+ + "\u0000\u04fb\u04fc\u0003=\u0017\u0000\u04fc\u04fd\u0001\u0000\u0000\u0000"+ + "\u04fd\u04fe\u0006\u009d\n\u0000\u04fe\u014a\u0001\u0000\u0000\u0000\u04ff"+ + "\u0500\u0003?\u0018\u0000\u0500\u0501\u0001\u0000\u0000\u0000\u0501\u0502"+ + "\u0006\u009e\u000f\u0000\u0502\u0503\u0006\u009e\u000b\u0000\u0503\u014c"+ + "\u0001\u0000\u0000\u0000\u0504\u0505\u0003\u0141\u0099\u0000\u0505\u0506"+ + "\u0001\u0000\u0000\u0000\u0506\u0507\u0006\u009f\u0011\u0000\u0507\u014e"+ + "\u0001\u0000\u0000\u0000\u0508\u0509\u0003e+\u0000\u0509\u050a\u0001\u0000"+ + "\u0000\u0000\u050a\u050b\u0006\u00a0\u0012\u0000\u050b\u0150\u0001\u0000"+ + "\u0000\u0000\u050c\u050d\u0003i-\u0000\u050d\u050e\u0001\u0000\u0000\u0000"+ + "\u050e\u050f\u0006\u00a1\u0016\u0000\u050f\u0152\u0001\u0000\u0000\u0000"+ + "\u0510\u0511\u0003\u0103z\u0000\u0511\u0512\u0001\u0000\u0000\u0000\u0512"+ + "\u0513\u0006\u00a2\u001e\u0000\u0513\u0514\u0006\u00a2\u001f\u0000\u0514"+ + "\u0154\u0001\u0000\u0000\u0000\u0515\u0516\u0003\u00cf`\u0000\u0516\u0517"+ + "\u0001\u0000\u0000\u0000\u0517\u0518\u0006\u00a3\u0014\u0000\u0518\u0156"+ + "\u0001\u0000\u0000\u0000\u0519\u051a\u0003U#\u0000\u051a\u051b\u0001\u0000"+ + "\u0000\u0000\u051b\u051c\u0006\u00a4\u0015\u0000\u051c\u0158\u0001\u0000"+ + "\u0000\u0000\u051d\u051e\u00039\u0015\u0000\u051e\u051f\u0001\u0000\u0000"+ + "\u0000\u051f\u0520\u0006\u00a5\n\u0000\u0520\u015a\u0001\u0000\u0000\u0000"+ + "\u0521\u0522\u0003;\u0016\u0000\u0522\u0523\u0001\u0000\u0000\u0000\u0523"+ + "\u0524\u0006\u00a6\n\u0000\u0524\u015c\u0001\u0000\u0000\u0000\u0525\u0526"+ + "\u0003=\u0017\u0000\u0526\u0527\u0001\u0000\u0000\u0000\u0527\u0528\u0006"+ + "\u00a7\n\u0000\u0528\u015e\u0001\u0000\u0000\u0000\u0529\u052a\u0003?"+ + "\u0018\u0000\u052a\u052b\u0001\u0000\u0000\u0000\u052b\u052c\u0006\u00a8"+ + "\u000f\u0000\u052c\u052d\u0006\u00a8\u000b\u0000\u052d\u052e\u0006\u00a8"+ + "\u000b\u0000\u052e\u0160\u0001\u0000\u0000\u0000\u052f\u0530\u0003e+\u0000"+ + "\u0530\u0531\u0001\u0000\u0000\u0000\u0531\u0532\u0006\u00a9\u0012\u0000"+ + "\u0532\u0162\u0001\u0000\u0000\u0000\u0533\u0534\u0003i-\u0000\u0534\u0535"+ + "\u0001\u0000\u0000\u0000\u0535\u0536\u0006\u00aa\u0016\u0000\u0536\u0164"+ + "\u0001\u0000\u0000\u0000\u0537\u0538\u0003\u00e5k\u0000\u0538\u0539\u0001"+ + "\u0000\u0000\u0000\u0539\u053a\u0006\u00ab\u0017\u0000\u053a\u0166\u0001"+ + "\u0000\u0000\u0000\u053b\u053c\u00039\u0015\u0000\u053c\u053d\u0001\u0000"+ + "\u0000\u0000\u053d\u053e\u0006\u00ac\n\u0000\u053e\u0168\u0001\u0000\u0000"+ + "\u0000\u053f\u0540\u0003;\u0016\u0000\u0540\u0541\u0001\u0000\u0000\u0000"+ + "\u0541\u0542\u0006\u00ad\n\u0000\u0542\u016a\u0001\u0000\u0000\u0000\u0543"+ + "\u0544\u0003=\u0017\u0000\u0544\u0545\u0001\u0000\u0000\u0000\u0545\u0546"+ + "\u0006\u00ae\n\u0000\u0546\u016c\u0001\u0000\u0000\u0000\u0547\u0548\u0003"+ + "?\u0018\u0000\u0548\u0549\u0001\u0000\u0000\u0000\u0549\u054a\u0006\u00af"+ + "\u000f\u0000\u054a\u054b\u0006\u00af\u000b\u0000\u054b\u016e\u0001\u0000"+ + "\u0000\u0000\u054c\u054d\u0003\u00cf`\u0000\u054d\u054e\u0001\u0000\u0000"+ + "\u0000\u054e\u054f\u0006\u00b0\u0014\u0000\u054f\u0550\u0006\u00b0\u000b"+ + "\u0000\u0550\u0551\u0006\u00b0 \u0000\u0551\u0170\u0001\u0000\u0000\u0000"+ + "\u0552\u0553\u0003U#\u0000\u0553\u0554\u0001\u0000\u0000\u0000\u0554\u0555"+ + "\u0006\u00b1\u0015\u0000\u0555\u0556\u0006\u00b1\u000b\u0000\u0556\u0557"+ + "\u0006\u00b1 \u0000\u0557\u0172\u0001\u0000\u0000\u0000\u0558\u0559\u0003"+ + "9\u0015\u0000\u0559\u055a\u0001\u0000\u0000\u0000\u055a\u055b\u0006\u00b2"+ + "\n\u0000\u055b\u0174\u0001\u0000\u0000\u0000\u055c\u055d\u0003;\u0016"+ + "\u0000\u055d\u055e\u0001\u0000\u0000\u0000\u055e\u055f\u0006\u00b3\n\u0000"+ + "\u055f\u0176\u0001\u0000\u0000\u0000\u0560\u0561\u0003=\u0017\u0000\u0561"+ + "\u0562\u0001\u0000\u0000\u0000\u0562\u0563\u0006\u00b4\n\u0000\u0563\u0178"+ + "\u0001\u0000\u0000\u0000\u0564\u0565\u0003\u0141\u0099\u0000\u0565\u0566"+ + "\u0001\u0000\u0000\u0000\u0566\u0567\u0006\u00b5\u0011\u0000\u0567\u0568"+ + "\u0006\u00b5\u000b\u0000\u0568\u0569\u0006\u00b5\t\u0000\u0569\u017a\u0001"+ + "\u0000\u0000\u0000\u056a\u056b\u0003e+\u0000\u056b\u056c\u0001\u0000\u0000"+ + "\u0000\u056c\u056d\u0006\u00b6\u0012\u0000\u056d\u056e\u0006\u00b6\u000b"+ + "\u0000\u056e\u056f\u0006\u00b6\t\u0000\u056f\u017c\u0001\u0000\u0000\u0000"+ + "\u0570\u0571\u00039\u0015\u0000\u0571\u0572\u0001\u0000\u0000\u0000\u0572"+ + "\u0573\u0006\u00b7\n\u0000\u0573\u017e\u0001\u0000\u0000\u0000\u0574\u0575"+ + "\u0003;\u0016\u0000\u0575\u0576\u0001\u0000\u0000\u0000\u0576\u0577\u0006"+ + "\u00b8\n\u0000\u0577\u0180\u0001\u0000\u0000\u0000\u0578\u0579\u0003="+ + "\u0017\u0000\u0579\u057a\u0001\u0000\u0000\u0000\u057a\u057b\u0006\u00b9"+ + "\n\u0000\u057b\u0182\u0001\u0000\u0000\u0000\u057c\u057d\u0003\u00adO"+ + "\u0000\u057d\u057e\u0001\u0000\u0000\u0000\u057e\u057f\u0006\u00ba\u000b"+ + "\u0000\u057f\u0580\u0006\u00ba\u0000\u0000\u0580\u0581\u0006\u00ba\u001c"+ + "\u0000\u0581\u0184\u0001\u0000\u0000\u0000\u0582\u0583\u0003\u00a9M\u0000"+ + "\u0583\u0584\u0001\u0000\u0000\u0000\u0584\u0585\u0006\u00bb\u000b\u0000"+ + "\u0585\u0586\u0006\u00bb\u0000\u0000\u0586\u0587\u0006\u00bb\u001d\u0000"+ + "\u0587\u0186\u0001\u0000\u0000\u0000\u0588\u0589\u0003[&\u0000\u0589\u058a"+ + "\u0001\u0000\u0000\u0000\u058a\u058b\u0006\u00bc\u000b\u0000\u058b\u058c"+ + "\u0006\u00bc\u0000\u0000\u058c\u058d\u0006\u00bc!\u0000\u058d\u0188\u0001"+ + "\u0000\u0000\u0000\u058e\u058f\u0003?\u0018\u0000\u058f\u0590\u0001\u0000"+ + "\u0000\u0000\u0590\u0591\u0006\u00bd\u000f\u0000\u0591\u0592\u0006\u00bd"+ + "\u000b\u0000\u0592\u018a\u0001\u0000\u0000\u0000A\u0000\u0001\u0002\u0003"+ + "\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u023c\u0246\u024a\u024d"+ + "\u0256\u0258\u0263\u0276\u027b\u0284\u028b\u0290\u0292\u029d\u02a5\u02a8"+ + "\u02aa\u02af\u02b4\u02ba\u02c1\u02c6\u02cc\u02cf\u02d7\u02db\u035a\u035f"+ + "\u0366\u0368\u0378\u037d\u0382\u0384\u038a\u03d7\u03dc\u0403\u0407\u040c"+ + "\u0411\u0416\u0418\u041c\u041e\u046b\u046f\u0474\u04ef\u04f1\"\u0005\u0001"+ + "\u0000\u0005\u0004\u0000\u0005\u0006\u0000\u0005\u0002\u0000\u0005\u0003"+ + "\u0000\u0005\b\u0000\u0005\u0005\u0000\u0005\t\u0000\u0005\u000b\u0000"+ + "\u0005\r\u0000\u0000\u0001\u0000\u0004\u0000\u0000\u0007\u0013\u0000\u0007"+ + "A\u0000\u0005\u0000\u0000\u0007\u0019\u0000\u0007B\u0000\u0007h\u0000"+ + "\u0007\"\u0000\u0007 \u0000\u0007L\u0000\u0007\u001a\u0000\u0007$\u0000"+ + "\u0007P\u0000\u0005\n\u0000\u0005\u0007\u0000\u0007Z\u0000\u0007Y\u0000"+ + "\u0007D\u0000\u0007C\u0000\u0007X\u0000\u0005\f\u0000\u0005\u000e\u0000"+ + "\u0007\u001d\u0000"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp index f7eed3e9be796..ae34d683403fb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp @@ -9,7 +9,6 @@ null 'grok' 'keep' 'limit' -'meta' 'mv_expand' 'rename' 'row' @@ -104,10 +103,6 @@ null null null null -'functions' -null -null -null ':' null null @@ -137,7 +132,6 @@ FROM GROK KEEP LIMIT -META MV_EXPAND RENAME ROW @@ -232,10 +226,6 @@ INFO SHOW_LINE_COMMENT SHOW_MULTILINE_COMMENT SHOW_WS -FUNCTIONS -META_LINE_COMMENT -META_MULTILINE_COMMENT -META_WS COLON SETTING SETTING_LINE_COMMENT @@ -309,7 +299,6 @@ comparisonOperator explainCommand subqueryExpression showCommand -metaCommand enrichCommand enrichWithClause lookupCommand @@ -317,4 +306,4 @@ inlinestatsCommand atn: -[4, 1, 125, 578, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 128, 8, 1, 10, 1, 12, 1, 131, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 140, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 158, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 170, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 177, 8, 5, 10, 5, 12, 5, 180, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 187, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 193, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 201, 8, 5, 10, 5, 12, 5, 204, 9, 5, 1, 6, 1, 6, 3, 6, 208, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 215, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 220, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 231, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 237, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 245, 8, 9, 10, 9, 12, 9, 248, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 258, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 263, 8, 10, 10, 10, 12, 10, 266, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 274, 8, 11, 10, 11, 12, 11, 277, 9, 11, 3, 11, 279, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 5, 14, 291, 8, 14, 10, 14, 12, 14, 294, 9, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 3, 15, 301, 8, 15, 1, 16, 1, 16, 1, 16, 1, 16, 5, 16, 307, 8, 16, 10, 16, 12, 16, 310, 9, 16, 1, 16, 3, 16, 313, 8, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 3, 17, 320, 8, 17, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 3, 20, 328, 8, 20, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 334, 8, 21, 10, 21, 12, 21, 337, 9, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 5, 23, 347, 8, 23, 10, 23, 12, 23, 350, 9, 23, 1, 23, 3, 23, 353, 8, 23, 1, 23, 1, 23, 3, 23, 357, 8, 23, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 3, 25, 364, 8, 25, 1, 25, 1, 25, 3, 25, 368, 8, 25, 1, 26, 1, 26, 1, 26, 5, 26, 373, 8, 26, 10, 26, 12, 26, 376, 9, 26, 1, 27, 1, 27, 1, 27, 5, 27, 381, 8, 27, 10, 27, 12, 27, 384, 9, 27, 1, 28, 1, 28, 1, 28, 5, 28, 389, 8, 28, 10, 28, 12, 28, 392, 9, 28, 1, 29, 1, 29, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 411, 8, 31, 10, 31, 12, 31, 414, 9, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 422, 8, 31, 10, 31, 12, 31, 425, 9, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 433, 8, 31, 10, 31, 12, 31, 436, 9, 31, 1, 31, 1, 31, 3, 31, 440, 8, 31, 1, 32, 1, 32, 3, 32, 444, 8, 32, 1, 33, 1, 33, 1, 33, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 453, 8, 34, 10, 34, 12, 34, 456, 9, 34, 1, 35, 1, 35, 3, 35, 460, 8, 35, 1, 35, 1, 35, 3, 35, 464, 8, 35, 1, 36, 1, 36, 1, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 476, 8, 38, 10, 38, 12, 38, 479, 9, 38, 1, 39, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 1, 40, 3, 40, 489, 8, 40, 1, 41, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 5, 43, 501, 8, 43, 10, 43, 12, 43, 504, 9, 43, 1, 44, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 46, 1, 46, 3, 46, 514, 8, 46, 1, 47, 3, 47, 517, 8, 47, 1, 47, 1, 47, 1, 48, 3, 48, 522, 8, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 55, 3, 55, 547, 8, 55, 1, 55, 1, 55, 1, 55, 1, 55, 5, 55, 553, 8, 55, 10, 55, 12, 55, 556, 9, 55, 3, 55, 558, 8, 55, 1, 56, 1, 56, 1, 56, 3, 56, 563, 8, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 576, 8, 58, 1, 58, 0, 4, 2, 10, 18, 20, 59, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 0, 8, 1, 0, 60, 61, 1, 0, 62, 64, 2, 0, 27, 27, 77, 77, 1, 0, 68, 69, 2, 0, 32, 32, 36, 36, 2, 0, 39, 39, 42, 42, 2, 0, 38, 38, 52, 52, 2, 0, 53, 53, 55, 59, 603, 0, 118, 1, 0, 0, 0, 2, 121, 1, 0, 0, 0, 4, 139, 1, 0, 0, 0, 6, 157, 1, 0, 0, 0, 8, 159, 1, 0, 0, 0, 10, 192, 1, 0, 0, 0, 12, 219, 1, 0, 0, 0, 14, 221, 1, 0, 0, 0, 16, 230, 1, 0, 0, 0, 18, 236, 1, 0, 0, 0, 20, 257, 1, 0, 0, 0, 22, 267, 1, 0, 0, 0, 24, 282, 1, 0, 0, 0, 26, 284, 1, 0, 0, 0, 28, 287, 1, 0, 0, 0, 30, 300, 1, 0, 0, 0, 32, 302, 1, 0, 0, 0, 34, 319, 1, 0, 0, 0, 36, 321, 1, 0, 0, 0, 38, 323, 1, 0, 0, 0, 40, 327, 1, 0, 0, 0, 42, 329, 1, 0, 0, 0, 44, 338, 1, 0, 0, 0, 46, 342, 1, 0, 0, 0, 48, 358, 1, 0, 0, 0, 50, 361, 1, 0, 0, 0, 52, 369, 1, 0, 0, 0, 54, 377, 1, 0, 0, 0, 56, 385, 1, 0, 0, 0, 58, 393, 1, 0, 0, 0, 60, 395, 1, 0, 0, 0, 62, 439, 1, 0, 0, 0, 64, 443, 1, 0, 0, 0, 66, 445, 1, 0, 0, 0, 68, 448, 1, 0, 0, 0, 70, 457, 1, 0, 0, 0, 72, 465, 1, 0, 0, 0, 74, 468, 1, 0, 0, 0, 76, 471, 1, 0, 0, 0, 78, 480, 1, 0, 0, 0, 80, 484, 1, 0, 0, 0, 82, 490, 1, 0, 0, 0, 84, 494, 1, 0, 0, 0, 86, 497, 1, 0, 0, 0, 88, 505, 1, 0, 0, 0, 90, 509, 1, 0, 0, 0, 92, 513, 1, 0, 0, 0, 94, 516, 1, 0, 0, 0, 96, 521, 1, 0, 0, 0, 98, 525, 1, 0, 0, 0, 100, 527, 1, 0, 0, 0, 102, 529, 1, 0, 0, 0, 104, 532, 1, 0, 0, 0, 106, 536, 1, 0, 0, 0, 108, 539, 1, 0, 0, 0, 110, 542, 1, 0, 0, 0, 112, 562, 1, 0, 0, 0, 114, 566, 1, 0, 0, 0, 116, 571, 1, 0, 0, 0, 118, 119, 3, 2, 1, 0, 119, 120, 5, 0, 0, 1, 120, 1, 1, 0, 0, 0, 121, 122, 6, 1, -1, 0, 122, 123, 3, 4, 2, 0, 123, 129, 1, 0, 0, 0, 124, 125, 10, 1, 0, 0, 125, 126, 5, 26, 0, 0, 126, 128, 3, 6, 3, 0, 127, 124, 1, 0, 0, 0, 128, 131, 1, 0, 0, 0, 129, 127, 1, 0, 0, 0, 129, 130, 1, 0, 0, 0, 130, 3, 1, 0, 0, 0, 131, 129, 1, 0, 0, 0, 132, 140, 3, 102, 51, 0, 133, 140, 3, 32, 16, 0, 134, 140, 3, 108, 54, 0, 135, 140, 3, 26, 13, 0, 136, 140, 3, 106, 53, 0, 137, 138, 4, 2, 1, 0, 138, 140, 3, 46, 23, 0, 139, 132, 1, 0, 0, 0, 139, 133, 1, 0, 0, 0, 139, 134, 1, 0, 0, 0, 139, 135, 1, 0, 0, 0, 139, 136, 1, 0, 0, 0, 139, 137, 1, 0, 0, 0, 140, 5, 1, 0, 0, 0, 141, 158, 3, 48, 24, 0, 142, 158, 3, 8, 4, 0, 143, 158, 3, 72, 36, 0, 144, 158, 3, 66, 33, 0, 145, 158, 3, 50, 25, 0, 146, 158, 3, 68, 34, 0, 147, 158, 3, 74, 37, 0, 148, 158, 3, 76, 38, 0, 149, 158, 3, 80, 40, 0, 150, 158, 3, 82, 41, 0, 151, 158, 3, 110, 55, 0, 152, 158, 3, 84, 42, 0, 153, 154, 4, 3, 2, 0, 154, 158, 3, 116, 58, 0, 155, 156, 4, 3, 3, 0, 156, 158, 3, 114, 57, 0, 157, 141, 1, 0, 0, 0, 157, 142, 1, 0, 0, 0, 157, 143, 1, 0, 0, 0, 157, 144, 1, 0, 0, 0, 157, 145, 1, 0, 0, 0, 157, 146, 1, 0, 0, 0, 157, 147, 1, 0, 0, 0, 157, 148, 1, 0, 0, 0, 157, 149, 1, 0, 0, 0, 157, 150, 1, 0, 0, 0, 157, 151, 1, 0, 0, 0, 157, 152, 1, 0, 0, 0, 157, 153, 1, 0, 0, 0, 157, 155, 1, 0, 0, 0, 158, 7, 1, 0, 0, 0, 159, 160, 5, 17, 0, 0, 160, 161, 3, 10, 5, 0, 161, 9, 1, 0, 0, 0, 162, 163, 6, 5, -1, 0, 163, 164, 5, 45, 0, 0, 164, 193, 3, 10, 5, 8, 165, 193, 3, 16, 8, 0, 166, 193, 3, 12, 6, 0, 167, 169, 3, 16, 8, 0, 168, 170, 5, 45, 0, 0, 169, 168, 1, 0, 0, 0, 169, 170, 1, 0, 0, 0, 170, 171, 1, 0, 0, 0, 171, 172, 5, 40, 0, 0, 172, 173, 5, 44, 0, 0, 173, 178, 3, 16, 8, 0, 174, 175, 5, 35, 0, 0, 175, 177, 3, 16, 8, 0, 176, 174, 1, 0, 0, 0, 177, 180, 1, 0, 0, 0, 178, 176, 1, 0, 0, 0, 178, 179, 1, 0, 0, 0, 179, 181, 1, 0, 0, 0, 180, 178, 1, 0, 0, 0, 181, 182, 5, 51, 0, 0, 182, 193, 1, 0, 0, 0, 183, 184, 3, 16, 8, 0, 184, 186, 5, 41, 0, 0, 185, 187, 5, 45, 0, 0, 186, 185, 1, 0, 0, 0, 186, 187, 1, 0, 0, 0, 187, 188, 1, 0, 0, 0, 188, 189, 5, 46, 0, 0, 189, 193, 1, 0, 0, 0, 190, 191, 4, 5, 4, 0, 191, 193, 3, 14, 7, 0, 192, 162, 1, 0, 0, 0, 192, 165, 1, 0, 0, 0, 192, 166, 1, 0, 0, 0, 192, 167, 1, 0, 0, 0, 192, 183, 1, 0, 0, 0, 192, 190, 1, 0, 0, 0, 193, 202, 1, 0, 0, 0, 194, 195, 10, 5, 0, 0, 195, 196, 5, 31, 0, 0, 196, 201, 3, 10, 5, 6, 197, 198, 10, 4, 0, 0, 198, 199, 5, 48, 0, 0, 199, 201, 3, 10, 5, 5, 200, 194, 1, 0, 0, 0, 200, 197, 1, 0, 0, 0, 201, 204, 1, 0, 0, 0, 202, 200, 1, 0, 0, 0, 202, 203, 1, 0, 0, 0, 203, 11, 1, 0, 0, 0, 204, 202, 1, 0, 0, 0, 205, 207, 3, 16, 8, 0, 206, 208, 5, 45, 0, 0, 207, 206, 1, 0, 0, 0, 207, 208, 1, 0, 0, 0, 208, 209, 1, 0, 0, 0, 209, 210, 5, 43, 0, 0, 210, 211, 3, 98, 49, 0, 211, 220, 1, 0, 0, 0, 212, 214, 3, 16, 8, 0, 213, 215, 5, 45, 0, 0, 214, 213, 1, 0, 0, 0, 214, 215, 1, 0, 0, 0, 215, 216, 1, 0, 0, 0, 216, 217, 5, 50, 0, 0, 217, 218, 3, 98, 49, 0, 218, 220, 1, 0, 0, 0, 219, 205, 1, 0, 0, 0, 219, 212, 1, 0, 0, 0, 220, 13, 1, 0, 0, 0, 221, 222, 3, 16, 8, 0, 222, 223, 5, 20, 0, 0, 223, 224, 3, 98, 49, 0, 224, 15, 1, 0, 0, 0, 225, 231, 3, 18, 9, 0, 226, 227, 3, 18, 9, 0, 227, 228, 3, 100, 50, 0, 228, 229, 3, 18, 9, 0, 229, 231, 1, 0, 0, 0, 230, 225, 1, 0, 0, 0, 230, 226, 1, 0, 0, 0, 231, 17, 1, 0, 0, 0, 232, 233, 6, 9, -1, 0, 233, 237, 3, 20, 10, 0, 234, 235, 7, 0, 0, 0, 235, 237, 3, 18, 9, 3, 236, 232, 1, 0, 0, 0, 236, 234, 1, 0, 0, 0, 237, 246, 1, 0, 0, 0, 238, 239, 10, 2, 0, 0, 239, 240, 7, 1, 0, 0, 240, 245, 3, 18, 9, 3, 241, 242, 10, 1, 0, 0, 242, 243, 7, 0, 0, 0, 243, 245, 3, 18, 9, 2, 244, 238, 1, 0, 0, 0, 244, 241, 1, 0, 0, 0, 245, 248, 1, 0, 0, 0, 246, 244, 1, 0, 0, 0, 246, 247, 1, 0, 0, 0, 247, 19, 1, 0, 0, 0, 248, 246, 1, 0, 0, 0, 249, 250, 6, 10, -1, 0, 250, 258, 3, 62, 31, 0, 251, 258, 3, 52, 26, 0, 252, 258, 3, 22, 11, 0, 253, 254, 5, 44, 0, 0, 254, 255, 3, 10, 5, 0, 255, 256, 5, 51, 0, 0, 256, 258, 1, 0, 0, 0, 257, 249, 1, 0, 0, 0, 257, 251, 1, 0, 0, 0, 257, 252, 1, 0, 0, 0, 257, 253, 1, 0, 0, 0, 258, 264, 1, 0, 0, 0, 259, 260, 10, 1, 0, 0, 260, 261, 5, 34, 0, 0, 261, 263, 3, 24, 12, 0, 262, 259, 1, 0, 0, 0, 263, 266, 1, 0, 0, 0, 264, 262, 1, 0, 0, 0, 264, 265, 1, 0, 0, 0, 265, 21, 1, 0, 0, 0, 266, 264, 1, 0, 0, 0, 267, 268, 3, 58, 29, 0, 268, 278, 5, 44, 0, 0, 269, 279, 5, 62, 0, 0, 270, 275, 3, 10, 5, 0, 271, 272, 5, 35, 0, 0, 272, 274, 3, 10, 5, 0, 273, 271, 1, 0, 0, 0, 274, 277, 1, 0, 0, 0, 275, 273, 1, 0, 0, 0, 275, 276, 1, 0, 0, 0, 276, 279, 1, 0, 0, 0, 277, 275, 1, 0, 0, 0, 278, 269, 1, 0, 0, 0, 278, 270, 1, 0, 0, 0, 278, 279, 1, 0, 0, 0, 279, 280, 1, 0, 0, 0, 280, 281, 5, 51, 0, 0, 281, 23, 1, 0, 0, 0, 282, 283, 3, 58, 29, 0, 283, 25, 1, 0, 0, 0, 284, 285, 5, 13, 0, 0, 285, 286, 3, 28, 14, 0, 286, 27, 1, 0, 0, 0, 287, 292, 3, 30, 15, 0, 288, 289, 5, 35, 0, 0, 289, 291, 3, 30, 15, 0, 290, 288, 1, 0, 0, 0, 291, 294, 1, 0, 0, 0, 292, 290, 1, 0, 0, 0, 292, 293, 1, 0, 0, 0, 293, 29, 1, 0, 0, 0, 294, 292, 1, 0, 0, 0, 295, 301, 3, 10, 5, 0, 296, 297, 3, 52, 26, 0, 297, 298, 5, 33, 0, 0, 298, 299, 3, 10, 5, 0, 299, 301, 1, 0, 0, 0, 300, 295, 1, 0, 0, 0, 300, 296, 1, 0, 0, 0, 301, 31, 1, 0, 0, 0, 302, 303, 5, 6, 0, 0, 303, 308, 3, 34, 17, 0, 304, 305, 5, 35, 0, 0, 305, 307, 3, 34, 17, 0, 306, 304, 1, 0, 0, 0, 307, 310, 1, 0, 0, 0, 308, 306, 1, 0, 0, 0, 308, 309, 1, 0, 0, 0, 309, 312, 1, 0, 0, 0, 310, 308, 1, 0, 0, 0, 311, 313, 3, 40, 20, 0, 312, 311, 1, 0, 0, 0, 312, 313, 1, 0, 0, 0, 313, 33, 1, 0, 0, 0, 314, 315, 3, 36, 18, 0, 315, 316, 5, 109, 0, 0, 316, 317, 3, 38, 19, 0, 317, 320, 1, 0, 0, 0, 318, 320, 3, 38, 19, 0, 319, 314, 1, 0, 0, 0, 319, 318, 1, 0, 0, 0, 320, 35, 1, 0, 0, 0, 321, 322, 5, 77, 0, 0, 322, 37, 1, 0, 0, 0, 323, 324, 7, 2, 0, 0, 324, 39, 1, 0, 0, 0, 325, 328, 3, 42, 21, 0, 326, 328, 3, 44, 22, 0, 327, 325, 1, 0, 0, 0, 327, 326, 1, 0, 0, 0, 328, 41, 1, 0, 0, 0, 329, 330, 5, 76, 0, 0, 330, 335, 5, 77, 0, 0, 331, 332, 5, 35, 0, 0, 332, 334, 5, 77, 0, 0, 333, 331, 1, 0, 0, 0, 334, 337, 1, 0, 0, 0, 335, 333, 1, 0, 0, 0, 335, 336, 1, 0, 0, 0, 336, 43, 1, 0, 0, 0, 337, 335, 1, 0, 0, 0, 338, 339, 5, 66, 0, 0, 339, 340, 3, 42, 21, 0, 340, 341, 5, 67, 0, 0, 341, 45, 1, 0, 0, 0, 342, 343, 5, 21, 0, 0, 343, 348, 3, 34, 17, 0, 344, 345, 5, 35, 0, 0, 345, 347, 3, 34, 17, 0, 346, 344, 1, 0, 0, 0, 347, 350, 1, 0, 0, 0, 348, 346, 1, 0, 0, 0, 348, 349, 1, 0, 0, 0, 349, 352, 1, 0, 0, 0, 350, 348, 1, 0, 0, 0, 351, 353, 3, 28, 14, 0, 352, 351, 1, 0, 0, 0, 352, 353, 1, 0, 0, 0, 353, 356, 1, 0, 0, 0, 354, 355, 5, 30, 0, 0, 355, 357, 3, 28, 14, 0, 356, 354, 1, 0, 0, 0, 356, 357, 1, 0, 0, 0, 357, 47, 1, 0, 0, 0, 358, 359, 5, 4, 0, 0, 359, 360, 3, 28, 14, 0, 360, 49, 1, 0, 0, 0, 361, 363, 5, 16, 0, 0, 362, 364, 3, 28, 14, 0, 363, 362, 1, 0, 0, 0, 363, 364, 1, 0, 0, 0, 364, 367, 1, 0, 0, 0, 365, 366, 5, 30, 0, 0, 366, 368, 3, 28, 14, 0, 367, 365, 1, 0, 0, 0, 367, 368, 1, 0, 0, 0, 368, 51, 1, 0, 0, 0, 369, 374, 3, 58, 29, 0, 370, 371, 5, 37, 0, 0, 371, 373, 3, 58, 29, 0, 372, 370, 1, 0, 0, 0, 373, 376, 1, 0, 0, 0, 374, 372, 1, 0, 0, 0, 374, 375, 1, 0, 0, 0, 375, 53, 1, 0, 0, 0, 376, 374, 1, 0, 0, 0, 377, 382, 3, 60, 30, 0, 378, 379, 5, 37, 0, 0, 379, 381, 3, 60, 30, 0, 380, 378, 1, 0, 0, 0, 381, 384, 1, 0, 0, 0, 382, 380, 1, 0, 0, 0, 382, 383, 1, 0, 0, 0, 383, 55, 1, 0, 0, 0, 384, 382, 1, 0, 0, 0, 385, 390, 3, 54, 27, 0, 386, 387, 5, 35, 0, 0, 387, 389, 3, 54, 27, 0, 388, 386, 1, 0, 0, 0, 389, 392, 1, 0, 0, 0, 390, 388, 1, 0, 0, 0, 390, 391, 1, 0, 0, 0, 391, 57, 1, 0, 0, 0, 392, 390, 1, 0, 0, 0, 393, 394, 7, 3, 0, 0, 394, 59, 1, 0, 0, 0, 395, 396, 5, 81, 0, 0, 396, 61, 1, 0, 0, 0, 397, 440, 5, 46, 0, 0, 398, 399, 3, 96, 48, 0, 399, 400, 5, 68, 0, 0, 400, 440, 1, 0, 0, 0, 401, 440, 3, 94, 47, 0, 402, 440, 3, 96, 48, 0, 403, 440, 3, 90, 45, 0, 404, 440, 3, 64, 32, 0, 405, 440, 3, 98, 49, 0, 406, 407, 5, 66, 0, 0, 407, 412, 3, 92, 46, 0, 408, 409, 5, 35, 0, 0, 409, 411, 3, 92, 46, 0, 410, 408, 1, 0, 0, 0, 411, 414, 1, 0, 0, 0, 412, 410, 1, 0, 0, 0, 412, 413, 1, 0, 0, 0, 413, 415, 1, 0, 0, 0, 414, 412, 1, 0, 0, 0, 415, 416, 5, 67, 0, 0, 416, 440, 1, 0, 0, 0, 417, 418, 5, 66, 0, 0, 418, 423, 3, 90, 45, 0, 419, 420, 5, 35, 0, 0, 420, 422, 3, 90, 45, 0, 421, 419, 1, 0, 0, 0, 422, 425, 1, 0, 0, 0, 423, 421, 1, 0, 0, 0, 423, 424, 1, 0, 0, 0, 424, 426, 1, 0, 0, 0, 425, 423, 1, 0, 0, 0, 426, 427, 5, 67, 0, 0, 427, 440, 1, 0, 0, 0, 428, 429, 5, 66, 0, 0, 429, 434, 3, 98, 49, 0, 430, 431, 5, 35, 0, 0, 431, 433, 3, 98, 49, 0, 432, 430, 1, 0, 0, 0, 433, 436, 1, 0, 0, 0, 434, 432, 1, 0, 0, 0, 434, 435, 1, 0, 0, 0, 435, 437, 1, 0, 0, 0, 436, 434, 1, 0, 0, 0, 437, 438, 5, 67, 0, 0, 438, 440, 1, 0, 0, 0, 439, 397, 1, 0, 0, 0, 439, 398, 1, 0, 0, 0, 439, 401, 1, 0, 0, 0, 439, 402, 1, 0, 0, 0, 439, 403, 1, 0, 0, 0, 439, 404, 1, 0, 0, 0, 439, 405, 1, 0, 0, 0, 439, 406, 1, 0, 0, 0, 439, 417, 1, 0, 0, 0, 439, 428, 1, 0, 0, 0, 440, 63, 1, 0, 0, 0, 441, 444, 5, 49, 0, 0, 442, 444, 5, 65, 0, 0, 443, 441, 1, 0, 0, 0, 443, 442, 1, 0, 0, 0, 444, 65, 1, 0, 0, 0, 445, 446, 5, 9, 0, 0, 446, 447, 5, 28, 0, 0, 447, 67, 1, 0, 0, 0, 448, 449, 5, 15, 0, 0, 449, 454, 3, 70, 35, 0, 450, 451, 5, 35, 0, 0, 451, 453, 3, 70, 35, 0, 452, 450, 1, 0, 0, 0, 453, 456, 1, 0, 0, 0, 454, 452, 1, 0, 0, 0, 454, 455, 1, 0, 0, 0, 455, 69, 1, 0, 0, 0, 456, 454, 1, 0, 0, 0, 457, 459, 3, 10, 5, 0, 458, 460, 7, 4, 0, 0, 459, 458, 1, 0, 0, 0, 459, 460, 1, 0, 0, 0, 460, 463, 1, 0, 0, 0, 461, 462, 5, 47, 0, 0, 462, 464, 7, 5, 0, 0, 463, 461, 1, 0, 0, 0, 463, 464, 1, 0, 0, 0, 464, 71, 1, 0, 0, 0, 465, 466, 5, 8, 0, 0, 466, 467, 3, 56, 28, 0, 467, 73, 1, 0, 0, 0, 468, 469, 5, 2, 0, 0, 469, 470, 3, 56, 28, 0, 470, 75, 1, 0, 0, 0, 471, 472, 5, 12, 0, 0, 472, 477, 3, 78, 39, 0, 473, 474, 5, 35, 0, 0, 474, 476, 3, 78, 39, 0, 475, 473, 1, 0, 0, 0, 476, 479, 1, 0, 0, 0, 477, 475, 1, 0, 0, 0, 477, 478, 1, 0, 0, 0, 478, 77, 1, 0, 0, 0, 479, 477, 1, 0, 0, 0, 480, 481, 3, 54, 27, 0, 481, 482, 5, 85, 0, 0, 482, 483, 3, 54, 27, 0, 483, 79, 1, 0, 0, 0, 484, 485, 5, 1, 0, 0, 485, 486, 3, 20, 10, 0, 486, 488, 3, 98, 49, 0, 487, 489, 3, 86, 43, 0, 488, 487, 1, 0, 0, 0, 488, 489, 1, 0, 0, 0, 489, 81, 1, 0, 0, 0, 490, 491, 5, 7, 0, 0, 491, 492, 3, 20, 10, 0, 492, 493, 3, 98, 49, 0, 493, 83, 1, 0, 0, 0, 494, 495, 5, 11, 0, 0, 495, 496, 3, 52, 26, 0, 496, 85, 1, 0, 0, 0, 497, 502, 3, 88, 44, 0, 498, 499, 5, 35, 0, 0, 499, 501, 3, 88, 44, 0, 500, 498, 1, 0, 0, 0, 501, 504, 1, 0, 0, 0, 502, 500, 1, 0, 0, 0, 502, 503, 1, 0, 0, 0, 503, 87, 1, 0, 0, 0, 504, 502, 1, 0, 0, 0, 505, 506, 3, 58, 29, 0, 506, 507, 5, 33, 0, 0, 507, 508, 3, 62, 31, 0, 508, 89, 1, 0, 0, 0, 509, 510, 7, 6, 0, 0, 510, 91, 1, 0, 0, 0, 511, 514, 3, 94, 47, 0, 512, 514, 3, 96, 48, 0, 513, 511, 1, 0, 0, 0, 513, 512, 1, 0, 0, 0, 514, 93, 1, 0, 0, 0, 515, 517, 7, 0, 0, 0, 516, 515, 1, 0, 0, 0, 516, 517, 1, 0, 0, 0, 517, 518, 1, 0, 0, 0, 518, 519, 5, 29, 0, 0, 519, 95, 1, 0, 0, 0, 520, 522, 7, 0, 0, 0, 521, 520, 1, 0, 0, 0, 521, 522, 1, 0, 0, 0, 522, 523, 1, 0, 0, 0, 523, 524, 5, 28, 0, 0, 524, 97, 1, 0, 0, 0, 525, 526, 5, 27, 0, 0, 526, 99, 1, 0, 0, 0, 527, 528, 7, 7, 0, 0, 528, 101, 1, 0, 0, 0, 529, 530, 5, 5, 0, 0, 530, 531, 3, 104, 52, 0, 531, 103, 1, 0, 0, 0, 532, 533, 5, 66, 0, 0, 533, 534, 3, 2, 1, 0, 534, 535, 5, 67, 0, 0, 535, 105, 1, 0, 0, 0, 536, 537, 5, 14, 0, 0, 537, 538, 5, 101, 0, 0, 538, 107, 1, 0, 0, 0, 539, 540, 5, 10, 0, 0, 540, 541, 5, 105, 0, 0, 541, 109, 1, 0, 0, 0, 542, 543, 5, 3, 0, 0, 543, 546, 5, 91, 0, 0, 544, 545, 5, 89, 0, 0, 545, 547, 3, 54, 27, 0, 546, 544, 1, 0, 0, 0, 546, 547, 1, 0, 0, 0, 547, 557, 1, 0, 0, 0, 548, 549, 5, 90, 0, 0, 549, 554, 3, 112, 56, 0, 550, 551, 5, 35, 0, 0, 551, 553, 3, 112, 56, 0, 552, 550, 1, 0, 0, 0, 553, 556, 1, 0, 0, 0, 554, 552, 1, 0, 0, 0, 554, 555, 1, 0, 0, 0, 555, 558, 1, 0, 0, 0, 556, 554, 1, 0, 0, 0, 557, 548, 1, 0, 0, 0, 557, 558, 1, 0, 0, 0, 558, 111, 1, 0, 0, 0, 559, 560, 3, 54, 27, 0, 560, 561, 5, 33, 0, 0, 561, 563, 1, 0, 0, 0, 562, 559, 1, 0, 0, 0, 562, 563, 1, 0, 0, 0, 563, 564, 1, 0, 0, 0, 564, 565, 3, 54, 27, 0, 565, 113, 1, 0, 0, 0, 566, 567, 5, 19, 0, 0, 567, 568, 3, 34, 17, 0, 568, 569, 5, 89, 0, 0, 569, 570, 3, 56, 28, 0, 570, 115, 1, 0, 0, 0, 571, 572, 5, 18, 0, 0, 572, 575, 3, 28, 14, 0, 573, 574, 5, 30, 0, 0, 574, 576, 3, 28, 14, 0, 575, 573, 1, 0, 0, 0, 575, 576, 1, 0, 0, 0, 576, 117, 1, 0, 0, 0, 54, 129, 139, 157, 169, 178, 186, 192, 200, 202, 207, 214, 219, 230, 236, 244, 246, 257, 264, 275, 278, 292, 300, 308, 312, 319, 327, 335, 348, 352, 356, 363, 367, 374, 382, 390, 412, 423, 434, 439, 443, 454, 459, 463, 477, 488, 502, 513, 516, 521, 546, 554, 557, 562, 575] \ No newline at end of file +[4, 1, 120, 572, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 126, 8, 1, 10, 1, 12, 1, 129, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 137, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 155, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 167, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 174, 8, 5, 10, 5, 12, 5, 177, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 184, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 190, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 198, 8, 5, 10, 5, 12, 5, 201, 9, 5, 1, 6, 1, 6, 3, 6, 205, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 212, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 217, 8, 6, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 228, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 234, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 242, 8, 9, 10, 9, 12, 9, 245, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 255, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 260, 8, 10, 10, 10, 12, 10, 263, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 271, 8, 11, 10, 11, 12, 11, 274, 9, 11, 3, 11, 276, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 5, 14, 288, 8, 14, 10, 14, 12, 14, 291, 9, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 3, 15, 298, 8, 15, 1, 16, 1, 16, 1, 16, 1, 16, 5, 16, 304, 8, 16, 10, 16, 12, 16, 307, 9, 16, 1, 16, 3, 16, 310, 8, 16, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 3, 17, 317, 8, 17, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 3, 20, 325, 8, 20, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 331, 8, 21, 10, 21, 12, 21, 334, 9, 21, 1, 22, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 23, 5, 23, 344, 8, 23, 10, 23, 12, 23, 347, 9, 23, 1, 23, 3, 23, 350, 8, 23, 1, 23, 1, 23, 3, 23, 354, 8, 23, 1, 24, 1, 24, 1, 24, 1, 25, 1, 25, 3, 25, 361, 8, 25, 1, 25, 1, 25, 3, 25, 365, 8, 25, 1, 26, 1, 26, 1, 26, 5, 26, 370, 8, 26, 10, 26, 12, 26, 373, 9, 26, 1, 27, 1, 27, 1, 27, 5, 27, 378, 8, 27, 10, 27, 12, 27, 381, 9, 27, 1, 28, 1, 28, 1, 28, 5, 28, 386, 8, 28, 10, 28, 12, 28, 389, 9, 28, 1, 29, 1, 29, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 408, 8, 31, 10, 31, 12, 31, 411, 9, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 419, 8, 31, 10, 31, 12, 31, 422, 9, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 1, 31, 5, 31, 430, 8, 31, 10, 31, 12, 31, 433, 9, 31, 1, 31, 1, 31, 3, 31, 437, 8, 31, 1, 32, 1, 32, 3, 32, 441, 8, 32, 1, 33, 1, 33, 1, 33, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 450, 8, 34, 10, 34, 12, 34, 453, 9, 34, 1, 35, 1, 35, 3, 35, 457, 8, 35, 1, 35, 1, 35, 3, 35, 461, 8, 35, 1, 36, 1, 36, 1, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 473, 8, 38, 10, 38, 12, 38, 476, 9, 38, 1, 39, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 1, 40, 3, 40, 486, 8, 40, 1, 41, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 5, 43, 498, 8, 43, 10, 43, 12, 43, 501, 9, 43, 1, 44, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 46, 1, 46, 3, 46, 511, 8, 46, 1, 47, 3, 47, 514, 8, 47, 1, 47, 1, 47, 1, 48, 3, 48, 519, 8, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 52, 1, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 3, 54, 541, 8, 54, 1, 54, 1, 54, 1, 54, 1, 54, 5, 54, 547, 8, 54, 10, 54, 12, 54, 550, 9, 54, 3, 54, 552, 8, 54, 1, 55, 1, 55, 1, 55, 3, 55, 557, 8, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 57, 3, 57, 570, 8, 57, 1, 57, 0, 4, 2, 10, 18, 20, 58, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 0, 8, 1, 0, 59, 60, 1, 0, 61, 63, 2, 0, 26, 26, 76, 76, 1, 0, 67, 68, 2, 0, 31, 31, 35, 35, 2, 0, 38, 38, 41, 41, 2, 0, 37, 37, 51, 51, 2, 0, 52, 52, 54, 58, 597, 0, 116, 1, 0, 0, 0, 2, 119, 1, 0, 0, 0, 4, 136, 1, 0, 0, 0, 6, 154, 1, 0, 0, 0, 8, 156, 1, 0, 0, 0, 10, 189, 1, 0, 0, 0, 12, 216, 1, 0, 0, 0, 14, 218, 1, 0, 0, 0, 16, 227, 1, 0, 0, 0, 18, 233, 1, 0, 0, 0, 20, 254, 1, 0, 0, 0, 22, 264, 1, 0, 0, 0, 24, 279, 1, 0, 0, 0, 26, 281, 1, 0, 0, 0, 28, 284, 1, 0, 0, 0, 30, 297, 1, 0, 0, 0, 32, 299, 1, 0, 0, 0, 34, 316, 1, 0, 0, 0, 36, 318, 1, 0, 0, 0, 38, 320, 1, 0, 0, 0, 40, 324, 1, 0, 0, 0, 42, 326, 1, 0, 0, 0, 44, 335, 1, 0, 0, 0, 46, 339, 1, 0, 0, 0, 48, 355, 1, 0, 0, 0, 50, 358, 1, 0, 0, 0, 52, 366, 1, 0, 0, 0, 54, 374, 1, 0, 0, 0, 56, 382, 1, 0, 0, 0, 58, 390, 1, 0, 0, 0, 60, 392, 1, 0, 0, 0, 62, 436, 1, 0, 0, 0, 64, 440, 1, 0, 0, 0, 66, 442, 1, 0, 0, 0, 68, 445, 1, 0, 0, 0, 70, 454, 1, 0, 0, 0, 72, 462, 1, 0, 0, 0, 74, 465, 1, 0, 0, 0, 76, 468, 1, 0, 0, 0, 78, 477, 1, 0, 0, 0, 80, 481, 1, 0, 0, 0, 82, 487, 1, 0, 0, 0, 84, 491, 1, 0, 0, 0, 86, 494, 1, 0, 0, 0, 88, 502, 1, 0, 0, 0, 90, 506, 1, 0, 0, 0, 92, 510, 1, 0, 0, 0, 94, 513, 1, 0, 0, 0, 96, 518, 1, 0, 0, 0, 98, 522, 1, 0, 0, 0, 100, 524, 1, 0, 0, 0, 102, 526, 1, 0, 0, 0, 104, 529, 1, 0, 0, 0, 106, 533, 1, 0, 0, 0, 108, 536, 1, 0, 0, 0, 110, 556, 1, 0, 0, 0, 112, 560, 1, 0, 0, 0, 114, 565, 1, 0, 0, 0, 116, 117, 3, 2, 1, 0, 117, 118, 5, 0, 0, 1, 118, 1, 1, 0, 0, 0, 119, 120, 6, 1, -1, 0, 120, 121, 3, 4, 2, 0, 121, 127, 1, 0, 0, 0, 122, 123, 10, 1, 0, 0, 123, 124, 5, 25, 0, 0, 124, 126, 3, 6, 3, 0, 125, 122, 1, 0, 0, 0, 126, 129, 1, 0, 0, 0, 127, 125, 1, 0, 0, 0, 127, 128, 1, 0, 0, 0, 128, 3, 1, 0, 0, 0, 129, 127, 1, 0, 0, 0, 130, 137, 3, 102, 51, 0, 131, 137, 3, 32, 16, 0, 132, 137, 3, 26, 13, 0, 133, 137, 3, 106, 53, 0, 134, 135, 4, 2, 1, 0, 135, 137, 3, 46, 23, 0, 136, 130, 1, 0, 0, 0, 136, 131, 1, 0, 0, 0, 136, 132, 1, 0, 0, 0, 136, 133, 1, 0, 0, 0, 136, 134, 1, 0, 0, 0, 137, 5, 1, 0, 0, 0, 138, 155, 3, 48, 24, 0, 139, 155, 3, 8, 4, 0, 140, 155, 3, 72, 36, 0, 141, 155, 3, 66, 33, 0, 142, 155, 3, 50, 25, 0, 143, 155, 3, 68, 34, 0, 144, 155, 3, 74, 37, 0, 145, 155, 3, 76, 38, 0, 146, 155, 3, 80, 40, 0, 147, 155, 3, 82, 41, 0, 148, 155, 3, 108, 54, 0, 149, 155, 3, 84, 42, 0, 150, 151, 4, 3, 2, 0, 151, 155, 3, 114, 57, 0, 152, 153, 4, 3, 3, 0, 153, 155, 3, 112, 56, 0, 154, 138, 1, 0, 0, 0, 154, 139, 1, 0, 0, 0, 154, 140, 1, 0, 0, 0, 154, 141, 1, 0, 0, 0, 154, 142, 1, 0, 0, 0, 154, 143, 1, 0, 0, 0, 154, 144, 1, 0, 0, 0, 154, 145, 1, 0, 0, 0, 154, 146, 1, 0, 0, 0, 154, 147, 1, 0, 0, 0, 154, 148, 1, 0, 0, 0, 154, 149, 1, 0, 0, 0, 154, 150, 1, 0, 0, 0, 154, 152, 1, 0, 0, 0, 155, 7, 1, 0, 0, 0, 156, 157, 5, 16, 0, 0, 157, 158, 3, 10, 5, 0, 158, 9, 1, 0, 0, 0, 159, 160, 6, 5, -1, 0, 160, 161, 5, 44, 0, 0, 161, 190, 3, 10, 5, 8, 162, 190, 3, 16, 8, 0, 163, 190, 3, 12, 6, 0, 164, 166, 3, 16, 8, 0, 165, 167, 5, 44, 0, 0, 166, 165, 1, 0, 0, 0, 166, 167, 1, 0, 0, 0, 167, 168, 1, 0, 0, 0, 168, 169, 5, 39, 0, 0, 169, 170, 5, 43, 0, 0, 170, 175, 3, 16, 8, 0, 171, 172, 5, 34, 0, 0, 172, 174, 3, 16, 8, 0, 173, 171, 1, 0, 0, 0, 174, 177, 1, 0, 0, 0, 175, 173, 1, 0, 0, 0, 175, 176, 1, 0, 0, 0, 176, 178, 1, 0, 0, 0, 177, 175, 1, 0, 0, 0, 178, 179, 5, 50, 0, 0, 179, 190, 1, 0, 0, 0, 180, 181, 3, 16, 8, 0, 181, 183, 5, 40, 0, 0, 182, 184, 5, 44, 0, 0, 183, 182, 1, 0, 0, 0, 183, 184, 1, 0, 0, 0, 184, 185, 1, 0, 0, 0, 185, 186, 5, 45, 0, 0, 186, 190, 1, 0, 0, 0, 187, 188, 4, 5, 4, 0, 188, 190, 3, 14, 7, 0, 189, 159, 1, 0, 0, 0, 189, 162, 1, 0, 0, 0, 189, 163, 1, 0, 0, 0, 189, 164, 1, 0, 0, 0, 189, 180, 1, 0, 0, 0, 189, 187, 1, 0, 0, 0, 190, 199, 1, 0, 0, 0, 191, 192, 10, 5, 0, 0, 192, 193, 5, 30, 0, 0, 193, 198, 3, 10, 5, 6, 194, 195, 10, 4, 0, 0, 195, 196, 5, 47, 0, 0, 196, 198, 3, 10, 5, 5, 197, 191, 1, 0, 0, 0, 197, 194, 1, 0, 0, 0, 198, 201, 1, 0, 0, 0, 199, 197, 1, 0, 0, 0, 199, 200, 1, 0, 0, 0, 200, 11, 1, 0, 0, 0, 201, 199, 1, 0, 0, 0, 202, 204, 3, 16, 8, 0, 203, 205, 5, 44, 0, 0, 204, 203, 1, 0, 0, 0, 204, 205, 1, 0, 0, 0, 205, 206, 1, 0, 0, 0, 206, 207, 5, 42, 0, 0, 207, 208, 3, 98, 49, 0, 208, 217, 1, 0, 0, 0, 209, 211, 3, 16, 8, 0, 210, 212, 5, 44, 0, 0, 211, 210, 1, 0, 0, 0, 211, 212, 1, 0, 0, 0, 212, 213, 1, 0, 0, 0, 213, 214, 5, 49, 0, 0, 214, 215, 3, 98, 49, 0, 215, 217, 1, 0, 0, 0, 216, 202, 1, 0, 0, 0, 216, 209, 1, 0, 0, 0, 217, 13, 1, 0, 0, 0, 218, 219, 3, 16, 8, 0, 219, 220, 5, 19, 0, 0, 220, 221, 3, 98, 49, 0, 221, 15, 1, 0, 0, 0, 222, 228, 3, 18, 9, 0, 223, 224, 3, 18, 9, 0, 224, 225, 3, 100, 50, 0, 225, 226, 3, 18, 9, 0, 226, 228, 1, 0, 0, 0, 227, 222, 1, 0, 0, 0, 227, 223, 1, 0, 0, 0, 228, 17, 1, 0, 0, 0, 229, 230, 6, 9, -1, 0, 230, 234, 3, 20, 10, 0, 231, 232, 7, 0, 0, 0, 232, 234, 3, 18, 9, 3, 233, 229, 1, 0, 0, 0, 233, 231, 1, 0, 0, 0, 234, 243, 1, 0, 0, 0, 235, 236, 10, 2, 0, 0, 236, 237, 7, 1, 0, 0, 237, 242, 3, 18, 9, 3, 238, 239, 10, 1, 0, 0, 239, 240, 7, 0, 0, 0, 240, 242, 3, 18, 9, 2, 241, 235, 1, 0, 0, 0, 241, 238, 1, 0, 0, 0, 242, 245, 1, 0, 0, 0, 243, 241, 1, 0, 0, 0, 243, 244, 1, 0, 0, 0, 244, 19, 1, 0, 0, 0, 245, 243, 1, 0, 0, 0, 246, 247, 6, 10, -1, 0, 247, 255, 3, 62, 31, 0, 248, 255, 3, 52, 26, 0, 249, 255, 3, 22, 11, 0, 250, 251, 5, 43, 0, 0, 251, 252, 3, 10, 5, 0, 252, 253, 5, 50, 0, 0, 253, 255, 1, 0, 0, 0, 254, 246, 1, 0, 0, 0, 254, 248, 1, 0, 0, 0, 254, 249, 1, 0, 0, 0, 254, 250, 1, 0, 0, 0, 255, 261, 1, 0, 0, 0, 256, 257, 10, 1, 0, 0, 257, 258, 5, 33, 0, 0, 258, 260, 3, 24, 12, 0, 259, 256, 1, 0, 0, 0, 260, 263, 1, 0, 0, 0, 261, 259, 1, 0, 0, 0, 261, 262, 1, 0, 0, 0, 262, 21, 1, 0, 0, 0, 263, 261, 1, 0, 0, 0, 264, 265, 3, 58, 29, 0, 265, 275, 5, 43, 0, 0, 266, 276, 5, 61, 0, 0, 267, 272, 3, 10, 5, 0, 268, 269, 5, 34, 0, 0, 269, 271, 3, 10, 5, 0, 270, 268, 1, 0, 0, 0, 271, 274, 1, 0, 0, 0, 272, 270, 1, 0, 0, 0, 272, 273, 1, 0, 0, 0, 273, 276, 1, 0, 0, 0, 274, 272, 1, 0, 0, 0, 275, 266, 1, 0, 0, 0, 275, 267, 1, 0, 0, 0, 275, 276, 1, 0, 0, 0, 276, 277, 1, 0, 0, 0, 277, 278, 5, 50, 0, 0, 278, 23, 1, 0, 0, 0, 279, 280, 3, 58, 29, 0, 280, 25, 1, 0, 0, 0, 281, 282, 5, 12, 0, 0, 282, 283, 3, 28, 14, 0, 283, 27, 1, 0, 0, 0, 284, 289, 3, 30, 15, 0, 285, 286, 5, 34, 0, 0, 286, 288, 3, 30, 15, 0, 287, 285, 1, 0, 0, 0, 288, 291, 1, 0, 0, 0, 289, 287, 1, 0, 0, 0, 289, 290, 1, 0, 0, 0, 290, 29, 1, 0, 0, 0, 291, 289, 1, 0, 0, 0, 292, 298, 3, 10, 5, 0, 293, 294, 3, 52, 26, 0, 294, 295, 5, 32, 0, 0, 295, 296, 3, 10, 5, 0, 296, 298, 1, 0, 0, 0, 297, 292, 1, 0, 0, 0, 297, 293, 1, 0, 0, 0, 298, 31, 1, 0, 0, 0, 299, 300, 5, 6, 0, 0, 300, 305, 3, 34, 17, 0, 301, 302, 5, 34, 0, 0, 302, 304, 3, 34, 17, 0, 303, 301, 1, 0, 0, 0, 304, 307, 1, 0, 0, 0, 305, 303, 1, 0, 0, 0, 305, 306, 1, 0, 0, 0, 306, 309, 1, 0, 0, 0, 307, 305, 1, 0, 0, 0, 308, 310, 3, 40, 20, 0, 309, 308, 1, 0, 0, 0, 309, 310, 1, 0, 0, 0, 310, 33, 1, 0, 0, 0, 311, 312, 3, 36, 18, 0, 312, 313, 5, 104, 0, 0, 313, 314, 3, 38, 19, 0, 314, 317, 1, 0, 0, 0, 315, 317, 3, 38, 19, 0, 316, 311, 1, 0, 0, 0, 316, 315, 1, 0, 0, 0, 317, 35, 1, 0, 0, 0, 318, 319, 5, 76, 0, 0, 319, 37, 1, 0, 0, 0, 320, 321, 7, 2, 0, 0, 321, 39, 1, 0, 0, 0, 322, 325, 3, 42, 21, 0, 323, 325, 3, 44, 22, 0, 324, 322, 1, 0, 0, 0, 324, 323, 1, 0, 0, 0, 325, 41, 1, 0, 0, 0, 326, 327, 5, 75, 0, 0, 327, 332, 5, 76, 0, 0, 328, 329, 5, 34, 0, 0, 329, 331, 5, 76, 0, 0, 330, 328, 1, 0, 0, 0, 331, 334, 1, 0, 0, 0, 332, 330, 1, 0, 0, 0, 332, 333, 1, 0, 0, 0, 333, 43, 1, 0, 0, 0, 334, 332, 1, 0, 0, 0, 335, 336, 5, 65, 0, 0, 336, 337, 3, 42, 21, 0, 337, 338, 5, 66, 0, 0, 338, 45, 1, 0, 0, 0, 339, 340, 5, 20, 0, 0, 340, 345, 3, 34, 17, 0, 341, 342, 5, 34, 0, 0, 342, 344, 3, 34, 17, 0, 343, 341, 1, 0, 0, 0, 344, 347, 1, 0, 0, 0, 345, 343, 1, 0, 0, 0, 345, 346, 1, 0, 0, 0, 346, 349, 1, 0, 0, 0, 347, 345, 1, 0, 0, 0, 348, 350, 3, 28, 14, 0, 349, 348, 1, 0, 0, 0, 349, 350, 1, 0, 0, 0, 350, 353, 1, 0, 0, 0, 351, 352, 5, 29, 0, 0, 352, 354, 3, 28, 14, 0, 353, 351, 1, 0, 0, 0, 353, 354, 1, 0, 0, 0, 354, 47, 1, 0, 0, 0, 355, 356, 5, 4, 0, 0, 356, 357, 3, 28, 14, 0, 357, 49, 1, 0, 0, 0, 358, 360, 5, 15, 0, 0, 359, 361, 3, 28, 14, 0, 360, 359, 1, 0, 0, 0, 360, 361, 1, 0, 0, 0, 361, 364, 1, 0, 0, 0, 362, 363, 5, 29, 0, 0, 363, 365, 3, 28, 14, 0, 364, 362, 1, 0, 0, 0, 364, 365, 1, 0, 0, 0, 365, 51, 1, 0, 0, 0, 366, 371, 3, 58, 29, 0, 367, 368, 5, 36, 0, 0, 368, 370, 3, 58, 29, 0, 369, 367, 1, 0, 0, 0, 370, 373, 1, 0, 0, 0, 371, 369, 1, 0, 0, 0, 371, 372, 1, 0, 0, 0, 372, 53, 1, 0, 0, 0, 373, 371, 1, 0, 0, 0, 374, 379, 3, 60, 30, 0, 375, 376, 5, 36, 0, 0, 376, 378, 3, 60, 30, 0, 377, 375, 1, 0, 0, 0, 378, 381, 1, 0, 0, 0, 379, 377, 1, 0, 0, 0, 379, 380, 1, 0, 0, 0, 380, 55, 1, 0, 0, 0, 381, 379, 1, 0, 0, 0, 382, 387, 3, 54, 27, 0, 383, 384, 5, 34, 0, 0, 384, 386, 3, 54, 27, 0, 385, 383, 1, 0, 0, 0, 386, 389, 1, 0, 0, 0, 387, 385, 1, 0, 0, 0, 387, 388, 1, 0, 0, 0, 388, 57, 1, 0, 0, 0, 389, 387, 1, 0, 0, 0, 390, 391, 7, 3, 0, 0, 391, 59, 1, 0, 0, 0, 392, 393, 5, 80, 0, 0, 393, 61, 1, 0, 0, 0, 394, 437, 5, 45, 0, 0, 395, 396, 3, 96, 48, 0, 396, 397, 5, 67, 0, 0, 397, 437, 1, 0, 0, 0, 398, 437, 3, 94, 47, 0, 399, 437, 3, 96, 48, 0, 400, 437, 3, 90, 45, 0, 401, 437, 3, 64, 32, 0, 402, 437, 3, 98, 49, 0, 403, 404, 5, 65, 0, 0, 404, 409, 3, 92, 46, 0, 405, 406, 5, 34, 0, 0, 406, 408, 3, 92, 46, 0, 407, 405, 1, 0, 0, 0, 408, 411, 1, 0, 0, 0, 409, 407, 1, 0, 0, 0, 409, 410, 1, 0, 0, 0, 410, 412, 1, 0, 0, 0, 411, 409, 1, 0, 0, 0, 412, 413, 5, 66, 0, 0, 413, 437, 1, 0, 0, 0, 414, 415, 5, 65, 0, 0, 415, 420, 3, 90, 45, 0, 416, 417, 5, 34, 0, 0, 417, 419, 3, 90, 45, 0, 418, 416, 1, 0, 0, 0, 419, 422, 1, 0, 0, 0, 420, 418, 1, 0, 0, 0, 420, 421, 1, 0, 0, 0, 421, 423, 1, 0, 0, 0, 422, 420, 1, 0, 0, 0, 423, 424, 5, 66, 0, 0, 424, 437, 1, 0, 0, 0, 425, 426, 5, 65, 0, 0, 426, 431, 3, 98, 49, 0, 427, 428, 5, 34, 0, 0, 428, 430, 3, 98, 49, 0, 429, 427, 1, 0, 0, 0, 430, 433, 1, 0, 0, 0, 431, 429, 1, 0, 0, 0, 431, 432, 1, 0, 0, 0, 432, 434, 1, 0, 0, 0, 433, 431, 1, 0, 0, 0, 434, 435, 5, 66, 0, 0, 435, 437, 1, 0, 0, 0, 436, 394, 1, 0, 0, 0, 436, 395, 1, 0, 0, 0, 436, 398, 1, 0, 0, 0, 436, 399, 1, 0, 0, 0, 436, 400, 1, 0, 0, 0, 436, 401, 1, 0, 0, 0, 436, 402, 1, 0, 0, 0, 436, 403, 1, 0, 0, 0, 436, 414, 1, 0, 0, 0, 436, 425, 1, 0, 0, 0, 437, 63, 1, 0, 0, 0, 438, 441, 5, 48, 0, 0, 439, 441, 5, 64, 0, 0, 440, 438, 1, 0, 0, 0, 440, 439, 1, 0, 0, 0, 441, 65, 1, 0, 0, 0, 442, 443, 5, 9, 0, 0, 443, 444, 5, 27, 0, 0, 444, 67, 1, 0, 0, 0, 445, 446, 5, 14, 0, 0, 446, 451, 3, 70, 35, 0, 447, 448, 5, 34, 0, 0, 448, 450, 3, 70, 35, 0, 449, 447, 1, 0, 0, 0, 450, 453, 1, 0, 0, 0, 451, 449, 1, 0, 0, 0, 451, 452, 1, 0, 0, 0, 452, 69, 1, 0, 0, 0, 453, 451, 1, 0, 0, 0, 454, 456, 3, 10, 5, 0, 455, 457, 7, 4, 0, 0, 456, 455, 1, 0, 0, 0, 456, 457, 1, 0, 0, 0, 457, 460, 1, 0, 0, 0, 458, 459, 5, 46, 0, 0, 459, 461, 7, 5, 0, 0, 460, 458, 1, 0, 0, 0, 460, 461, 1, 0, 0, 0, 461, 71, 1, 0, 0, 0, 462, 463, 5, 8, 0, 0, 463, 464, 3, 56, 28, 0, 464, 73, 1, 0, 0, 0, 465, 466, 5, 2, 0, 0, 466, 467, 3, 56, 28, 0, 467, 75, 1, 0, 0, 0, 468, 469, 5, 11, 0, 0, 469, 474, 3, 78, 39, 0, 470, 471, 5, 34, 0, 0, 471, 473, 3, 78, 39, 0, 472, 470, 1, 0, 0, 0, 473, 476, 1, 0, 0, 0, 474, 472, 1, 0, 0, 0, 474, 475, 1, 0, 0, 0, 475, 77, 1, 0, 0, 0, 476, 474, 1, 0, 0, 0, 477, 478, 3, 54, 27, 0, 478, 479, 5, 84, 0, 0, 479, 480, 3, 54, 27, 0, 480, 79, 1, 0, 0, 0, 481, 482, 5, 1, 0, 0, 482, 483, 3, 20, 10, 0, 483, 485, 3, 98, 49, 0, 484, 486, 3, 86, 43, 0, 485, 484, 1, 0, 0, 0, 485, 486, 1, 0, 0, 0, 486, 81, 1, 0, 0, 0, 487, 488, 5, 7, 0, 0, 488, 489, 3, 20, 10, 0, 489, 490, 3, 98, 49, 0, 490, 83, 1, 0, 0, 0, 491, 492, 5, 10, 0, 0, 492, 493, 3, 52, 26, 0, 493, 85, 1, 0, 0, 0, 494, 499, 3, 88, 44, 0, 495, 496, 5, 34, 0, 0, 496, 498, 3, 88, 44, 0, 497, 495, 1, 0, 0, 0, 498, 501, 1, 0, 0, 0, 499, 497, 1, 0, 0, 0, 499, 500, 1, 0, 0, 0, 500, 87, 1, 0, 0, 0, 501, 499, 1, 0, 0, 0, 502, 503, 3, 58, 29, 0, 503, 504, 5, 32, 0, 0, 504, 505, 3, 62, 31, 0, 505, 89, 1, 0, 0, 0, 506, 507, 7, 6, 0, 0, 507, 91, 1, 0, 0, 0, 508, 511, 3, 94, 47, 0, 509, 511, 3, 96, 48, 0, 510, 508, 1, 0, 0, 0, 510, 509, 1, 0, 0, 0, 511, 93, 1, 0, 0, 0, 512, 514, 7, 0, 0, 0, 513, 512, 1, 0, 0, 0, 513, 514, 1, 0, 0, 0, 514, 515, 1, 0, 0, 0, 515, 516, 5, 28, 0, 0, 516, 95, 1, 0, 0, 0, 517, 519, 7, 0, 0, 0, 518, 517, 1, 0, 0, 0, 518, 519, 1, 0, 0, 0, 519, 520, 1, 0, 0, 0, 520, 521, 5, 27, 0, 0, 521, 97, 1, 0, 0, 0, 522, 523, 5, 26, 0, 0, 523, 99, 1, 0, 0, 0, 524, 525, 7, 7, 0, 0, 525, 101, 1, 0, 0, 0, 526, 527, 5, 5, 0, 0, 527, 528, 3, 104, 52, 0, 528, 103, 1, 0, 0, 0, 529, 530, 5, 65, 0, 0, 530, 531, 3, 2, 1, 0, 531, 532, 5, 66, 0, 0, 532, 105, 1, 0, 0, 0, 533, 534, 5, 13, 0, 0, 534, 535, 5, 100, 0, 0, 535, 107, 1, 0, 0, 0, 536, 537, 5, 3, 0, 0, 537, 540, 5, 90, 0, 0, 538, 539, 5, 88, 0, 0, 539, 541, 3, 54, 27, 0, 540, 538, 1, 0, 0, 0, 540, 541, 1, 0, 0, 0, 541, 551, 1, 0, 0, 0, 542, 543, 5, 89, 0, 0, 543, 548, 3, 110, 55, 0, 544, 545, 5, 34, 0, 0, 545, 547, 3, 110, 55, 0, 546, 544, 1, 0, 0, 0, 547, 550, 1, 0, 0, 0, 548, 546, 1, 0, 0, 0, 548, 549, 1, 0, 0, 0, 549, 552, 1, 0, 0, 0, 550, 548, 1, 0, 0, 0, 551, 542, 1, 0, 0, 0, 551, 552, 1, 0, 0, 0, 552, 109, 1, 0, 0, 0, 553, 554, 3, 54, 27, 0, 554, 555, 5, 32, 0, 0, 555, 557, 1, 0, 0, 0, 556, 553, 1, 0, 0, 0, 556, 557, 1, 0, 0, 0, 557, 558, 1, 0, 0, 0, 558, 559, 3, 54, 27, 0, 559, 111, 1, 0, 0, 0, 560, 561, 5, 18, 0, 0, 561, 562, 3, 34, 17, 0, 562, 563, 5, 88, 0, 0, 563, 564, 3, 56, 28, 0, 564, 113, 1, 0, 0, 0, 565, 566, 5, 17, 0, 0, 566, 569, 3, 28, 14, 0, 567, 568, 5, 29, 0, 0, 568, 570, 3, 28, 14, 0, 569, 567, 1, 0, 0, 0, 569, 570, 1, 0, 0, 0, 570, 115, 1, 0, 0, 0, 54, 127, 136, 154, 166, 175, 183, 189, 197, 199, 204, 211, 216, 227, 233, 241, 243, 254, 261, 272, 275, 289, 297, 305, 309, 316, 324, 332, 345, 349, 353, 360, 364, 371, 379, 387, 409, 420, 431, 436, 440, 451, 456, 460, 474, 485, 499, 510, 513, 518, 540, 548, 551, 556, 569] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java index 578da6fe786ac..cf1c5fa691d1e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java @@ -26,31 +26,30 @@ public class EsqlBaseParser extends ParserConfig { new PredictionContextCache(); public static final int DISSECT=1, DROP=2, ENRICH=3, EVAL=4, EXPLAIN=5, FROM=6, GROK=7, KEEP=8, - LIMIT=9, META=10, MV_EXPAND=11, RENAME=12, ROW=13, SHOW=14, SORT=15, STATS=16, - WHERE=17, DEV_INLINESTATS=18, DEV_LOOKUP=19, DEV_MATCH=20, DEV_METRICS=21, - UNKNOWN_CMD=22, LINE_COMMENT=23, MULTILINE_COMMENT=24, WS=25, PIPE=26, - QUOTED_STRING=27, INTEGER_LITERAL=28, DECIMAL_LITERAL=29, BY=30, AND=31, - ASC=32, ASSIGN=33, CAST_OP=34, COMMA=35, DESC=36, DOT=37, FALSE=38, FIRST=39, - IN=40, IS=41, LAST=42, LIKE=43, LP=44, NOT=45, NULL=46, NULLS=47, OR=48, - PARAM=49, RLIKE=50, RP=51, TRUE=52, EQ=53, CIEQ=54, NEQ=55, LT=56, LTE=57, - GT=58, GTE=59, PLUS=60, MINUS=61, ASTERISK=62, SLASH=63, PERCENT=64, NAMED_OR_POSITIONAL_PARAM=65, - OPENING_BRACKET=66, CLOSING_BRACKET=67, UNQUOTED_IDENTIFIER=68, QUOTED_IDENTIFIER=69, - EXPR_LINE_COMMENT=70, EXPR_MULTILINE_COMMENT=71, EXPR_WS=72, EXPLAIN_WS=73, - EXPLAIN_LINE_COMMENT=74, EXPLAIN_MULTILINE_COMMENT=75, METADATA=76, UNQUOTED_SOURCE=77, - FROM_LINE_COMMENT=78, FROM_MULTILINE_COMMENT=79, FROM_WS=80, ID_PATTERN=81, - PROJECT_LINE_COMMENT=82, PROJECT_MULTILINE_COMMENT=83, PROJECT_WS=84, - AS=85, RENAME_LINE_COMMENT=86, RENAME_MULTILINE_COMMENT=87, RENAME_WS=88, - ON=89, WITH=90, ENRICH_POLICY_NAME=91, ENRICH_LINE_COMMENT=92, ENRICH_MULTILINE_COMMENT=93, - ENRICH_WS=94, ENRICH_FIELD_LINE_COMMENT=95, ENRICH_FIELD_MULTILINE_COMMENT=96, - ENRICH_FIELD_WS=97, MVEXPAND_LINE_COMMENT=98, MVEXPAND_MULTILINE_COMMENT=99, - MVEXPAND_WS=100, INFO=101, SHOW_LINE_COMMENT=102, SHOW_MULTILINE_COMMENT=103, - SHOW_WS=104, FUNCTIONS=105, META_LINE_COMMENT=106, META_MULTILINE_COMMENT=107, - META_WS=108, COLON=109, SETTING=110, SETTING_LINE_COMMENT=111, SETTTING_MULTILINE_COMMENT=112, - SETTING_WS=113, LOOKUP_LINE_COMMENT=114, LOOKUP_MULTILINE_COMMENT=115, - LOOKUP_WS=116, LOOKUP_FIELD_LINE_COMMENT=117, LOOKUP_FIELD_MULTILINE_COMMENT=118, - LOOKUP_FIELD_WS=119, METRICS_LINE_COMMENT=120, METRICS_MULTILINE_COMMENT=121, - METRICS_WS=122, CLOSING_METRICS_LINE_COMMENT=123, CLOSING_METRICS_MULTILINE_COMMENT=124, - CLOSING_METRICS_WS=125; + LIMIT=9, MV_EXPAND=10, RENAME=11, ROW=12, SHOW=13, SORT=14, STATS=15, + WHERE=16, DEV_INLINESTATS=17, DEV_LOOKUP=18, DEV_MATCH=19, DEV_METRICS=20, + UNKNOWN_CMD=21, LINE_COMMENT=22, MULTILINE_COMMENT=23, WS=24, PIPE=25, + QUOTED_STRING=26, INTEGER_LITERAL=27, DECIMAL_LITERAL=28, BY=29, AND=30, + ASC=31, ASSIGN=32, CAST_OP=33, COMMA=34, DESC=35, DOT=36, FALSE=37, FIRST=38, + IN=39, IS=40, LAST=41, LIKE=42, LP=43, NOT=44, NULL=45, NULLS=46, OR=47, + PARAM=48, RLIKE=49, RP=50, TRUE=51, EQ=52, CIEQ=53, NEQ=54, LT=55, LTE=56, + GT=57, GTE=58, PLUS=59, MINUS=60, ASTERISK=61, SLASH=62, PERCENT=63, NAMED_OR_POSITIONAL_PARAM=64, + OPENING_BRACKET=65, CLOSING_BRACKET=66, UNQUOTED_IDENTIFIER=67, QUOTED_IDENTIFIER=68, + EXPR_LINE_COMMENT=69, EXPR_MULTILINE_COMMENT=70, EXPR_WS=71, EXPLAIN_WS=72, + EXPLAIN_LINE_COMMENT=73, EXPLAIN_MULTILINE_COMMENT=74, METADATA=75, UNQUOTED_SOURCE=76, + FROM_LINE_COMMENT=77, FROM_MULTILINE_COMMENT=78, FROM_WS=79, ID_PATTERN=80, + PROJECT_LINE_COMMENT=81, PROJECT_MULTILINE_COMMENT=82, PROJECT_WS=83, + AS=84, RENAME_LINE_COMMENT=85, RENAME_MULTILINE_COMMENT=86, RENAME_WS=87, + ON=88, WITH=89, ENRICH_POLICY_NAME=90, ENRICH_LINE_COMMENT=91, ENRICH_MULTILINE_COMMENT=92, + ENRICH_WS=93, ENRICH_FIELD_LINE_COMMENT=94, ENRICH_FIELD_MULTILINE_COMMENT=95, + ENRICH_FIELD_WS=96, MVEXPAND_LINE_COMMENT=97, MVEXPAND_MULTILINE_COMMENT=98, + MVEXPAND_WS=99, INFO=100, SHOW_LINE_COMMENT=101, SHOW_MULTILINE_COMMENT=102, + SHOW_WS=103, COLON=104, SETTING=105, SETTING_LINE_COMMENT=106, SETTTING_MULTILINE_COMMENT=107, + SETTING_WS=108, LOOKUP_LINE_COMMENT=109, LOOKUP_MULTILINE_COMMENT=110, + LOOKUP_WS=111, LOOKUP_FIELD_LINE_COMMENT=112, LOOKUP_FIELD_MULTILINE_COMMENT=113, + LOOKUP_FIELD_WS=114, METRICS_LINE_COMMENT=115, METRICS_MULTILINE_COMMENT=116, + METRICS_WS=117, CLOSING_METRICS_LINE_COMMENT=118, CLOSING_METRICS_MULTILINE_COMMENT=119, + CLOSING_METRICS_WS=120; public static final int RULE_singleStatement = 0, RULE_query = 1, RULE_sourceCommand = 2, RULE_processingCommand = 3, RULE_whereCommand = 4, RULE_booleanExpression = 5, RULE_regexBooleanExpression = 6, @@ -69,8 +68,8 @@ public class EsqlBaseParser extends ParserConfig { RULE_booleanValue = 45, RULE_numericValue = 46, RULE_decimalValue = 47, RULE_integerValue = 48, RULE_string = 49, RULE_comparisonOperator = 50, RULE_explainCommand = 51, RULE_subqueryExpression = 52, RULE_showCommand = 53, - RULE_metaCommand = 54, RULE_enrichCommand = 55, RULE_enrichWithClause = 56, - RULE_lookupCommand = 57, RULE_inlinestatsCommand = 58; + RULE_enrichCommand = 54, RULE_enrichWithClause = 55, RULE_lookupCommand = 56, + RULE_inlinestatsCommand = 57; private static String[] makeRuleNames() { return new String[] { "singleStatement", "query", "sourceCommand", "processingCommand", "whereCommand", @@ -84,8 +83,8 @@ private static String[] makeRuleNames() { "dropCommand", "renameCommand", "renameClause", "dissectCommand", "grokCommand", "mvExpandCommand", "commandOptions", "commandOption", "booleanValue", "numericValue", "decimalValue", "integerValue", "string", "comparisonOperator", - "explainCommand", "subqueryExpression", "showCommand", "metaCommand", - "enrichCommand", "enrichWithClause", "lookupCommand", "inlinestatsCommand" + "explainCommand", "subqueryExpression", "showCommand", "enrichCommand", + "enrichWithClause", "lookupCommand", "inlinestatsCommand" }; } public static final String[] ruleNames = makeRuleNames(); @@ -93,25 +92,25 @@ private static String[] makeRuleNames() { private static String[] makeLiteralNames() { return new String[] { null, "'dissect'", "'drop'", "'enrich'", "'eval'", "'explain'", "'from'", - "'grok'", "'keep'", "'limit'", "'meta'", "'mv_expand'", "'rename'", "'row'", - "'show'", "'sort'", "'stats'", "'where'", null, null, null, null, null, - null, null, null, "'|'", null, null, null, "'by'", "'and'", "'asc'", - "'='", "'::'", "','", "'desc'", "'.'", "'false'", "'first'", "'in'", - "'is'", "'last'", "'like'", "'('", "'not'", "'null'", "'nulls'", "'or'", - "'?'", "'rlike'", "')'", "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", - "'>'", "'>='", "'+'", "'-'", "'*'", "'/'", "'%'", null, null, "']'", - null, null, null, null, null, null, null, null, "'metadata'", null, null, - null, null, null, null, null, null, "'as'", null, null, null, "'on'", - "'with'", null, null, null, null, null, null, null, null, null, null, - "'info'", null, null, null, "'functions'", null, null, null, "':'" + "'grok'", "'keep'", "'limit'", "'mv_expand'", "'rename'", "'row'", "'show'", + "'sort'", "'stats'", "'where'", null, null, null, null, null, null, null, + null, "'|'", null, null, null, "'by'", "'and'", "'asc'", "'='", "'::'", + "','", "'desc'", "'.'", "'false'", "'first'", "'in'", "'is'", "'last'", + "'like'", "'('", "'not'", "'null'", "'nulls'", "'or'", "'?'", "'rlike'", + "')'", "'true'", "'=='", "'=~'", "'!='", "'<'", "'<='", "'>'", "'>='", + "'+'", "'-'", "'*'", "'/'", "'%'", null, null, "']'", null, null, null, + null, null, null, null, null, "'metadata'", null, null, null, null, null, + null, null, null, "'as'", null, null, null, "'on'", "'with'", null, null, + null, null, null, null, null, null, null, null, "'info'", null, null, + null, "':'" }; } private static final String[] _LITERAL_NAMES = makeLiteralNames(); private static String[] makeSymbolicNames() { return new String[] { null, "DISSECT", "DROP", "ENRICH", "EVAL", "EXPLAIN", "FROM", "GROK", - "KEEP", "LIMIT", "META", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", - "STATS", "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_MATCH", "DEV_METRICS", + "KEEP", "LIMIT", "MV_EXPAND", "RENAME", "ROW", "SHOW", "SORT", "STATS", + "WHERE", "DEV_INLINESTATS", "DEV_LOOKUP", "DEV_MATCH", "DEV_METRICS", "UNKNOWN_CMD", "LINE_COMMENT", "MULTILINE_COMMENT", "WS", "PIPE", "QUOTED_STRING", "INTEGER_LITERAL", "DECIMAL_LITERAL", "BY", "AND", "ASC", "ASSIGN", "CAST_OP", "COMMA", "DESC", "DOT", "FALSE", "FIRST", "IN", "IS", "LAST", "LIKE", @@ -127,8 +126,7 @@ private static String[] makeSymbolicNames() { "ENRICH_MULTILINE_COMMENT", "ENRICH_WS", "ENRICH_FIELD_LINE_COMMENT", "ENRICH_FIELD_MULTILINE_COMMENT", "ENRICH_FIELD_WS", "MVEXPAND_LINE_COMMENT", "MVEXPAND_MULTILINE_COMMENT", "MVEXPAND_WS", "INFO", "SHOW_LINE_COMMENT", - "SHOW_MULTILINE_COMMENT", "SHOW_WS", "FUNCTIONS", "META_LINE_COMMENT", - "META_MULTILINE_COMMENT", "META_WS", "COLON", "SETTING", "SETTING_LINE_COMMENT", + "SHOW_MULTILINE_COMMENT", "SHOW_WS", "COLON", "SETTING", "SETTING_LINE_COMMENT", "SETTTING_MULTILINE_COMMENT", "SETTING_WS", "LOOKUP_LINE_COMMENT", "LOOKUP_MULTILINE_COMMENT", "LOOKUP_WS", "LOOKUP_FIELD_LINE_COMMENT", "LOOKUP_FIELD_MULTILINE_COMMENT", "LOOKUP_FIELD_WS", "METRICS_LINE_COMMENT", "METRICS_MULTILINE_COMMENT", @@ -220,9 +218,9 @@ public final SingleStatementContext singleStatement() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(118); + setState(116); query(0); - setState(119); + setState(117); match(EOF); } } @@ -318,11 +316,11 @@ private QueryContext query(int _p) throws RecognitionException { _ctx = _localctx; _prevctx = _localctx; - setState(122); + setState(120); sourceCommand(); } _ctx.stop = _input.LT(-1); - setState(129); + setState(127); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -333,16 +331,16 @@ private QueryContext query(int _p) throws RecognitionException { { _localctx = new CompositeQueryContext(new QueryContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_query); - setState(124); + setState(122); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(125); + setState(123); match(PIPE); - setState(126); + setState(124); processingCommand(); } } } - setState(131); + setState(129); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); } @@ -367,9 +365,6 @@ public ExplainCommandContext explainCommand() { public FromCommandContext fromCommand() { return getRuleContext(FromCommandContext.class,0); } - public MetaCommandContext metaCommand() { - return getRuleContext(MetaCommandContext.class,0); - } public RowCommandContext rowCommand() { return getRuleContext(RowCommandContext.class,0); } @@ -403,50 +398,43 @@ public final SourceCommandContext sourceCommand() throws RecognitionException { SourceCommandContext _localctx = new SourceCommandContext(_ctx, getState()); enterRule(_localctx, 4, RULE_sourceCommand); try { - setState(139); + setState(136); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,1,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(132); + setState(130); explainCommand(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(133); + setState(131); fromCommand(); } break; case 3: enterOuterAlt(_localctx, 3); { - setState(134); - metaCommand(); + setState(132); + rowCommand(); } break; case 4: enterOuterAlt(_localctx, 4); { - setState(135); - rowCommand(); + setState(133); + showCommand(); } break; case 5: enterOuterAlt(_localctx, 5); { - setState(136); - showCommand(); - } - break; - case 6: - enterOuterAlt(_localctx, 6); - { - setState(137); + setState(134); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(138); + setState(135); metricsCommand(); } break; @@ -531,108 +519,108 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce ProcessingCommandContext _localctx = new ProcessingCommandContext(_ctx, getState()); enterRule(_localctx, 6, RULE_processingCommand); try { - setState(157); + setState(154); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(141); + setState(138); evalCommand(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(142); + setState(139); whereCommand(); } break; case 3: enterOuterAlt(_localctx, 3); { - setState(143); + setState(140); keepCommand(); } break; case 4: enterOuterAlt(_localctx, 4); { - setState(144); + setState(141); limitCommand(); } break; case 5: enterOuterAlt(_localctx, 5); { - setState(145); + setState(142); statsCommand(); } break; case 6: enterOuterAlt(_localctx, 6); { - setState(146); + setState(143); sortCommand(); } break; case 7: enterOuterAlt(_localctx, 7); { - setState(147); + setState(144); dropCommand(); } break; case 8: enterOuterAlt(_localctx, 8); { - setState(148); + setState(145); renameCommand(); } break; case 9: enterOuterAlt(_localctx, 9); { - setState(149); + setState(146); dissectCommand(); } break; case 10: enterOuterAlt(_localctx, 10); { - setState(150); + setState(147); grokCommand(); } break; case 11: enterOuterAlt(_localctx, 11); { - setState(151); + setState(148); enrichCommand(); } break; case 12: enterOuterAlt(_localctx, 12); { - setState(152); + setState(149); mvExpandCommand(); } break; case 13: enterOuterAlt(_localctx, 13); { - setState(153); + setState(150); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(154); + setState(151); inlinestatsCommand(); } break; case 14: enterOuterAlt(_localctx, 14); { - setState(155); + setState(152); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(156); + setState(153); lookupCommand(); } break; @@ -681,9 +669,9 @@ public final WhereCommandContext whereCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(159); + setState(156); match(WHERE); - setState(160); + setState(157); booleanExpression(0); } } @@ -899,7 +887,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc int _alt; enterOuterAlt(_localctx, 1); { - setState(192); + setState(189); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) { case 1: @@ -908,9 +896,9 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _ctx = _localctx; _prevctx = _localctx; - setState(163); + setState(160); match(NOT); - setState(164); + setState(161); booleanExpression(8); } break; @@ -919,7 +907,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new BooleanDefaultContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(165); + setState(162); valueExpression(); } break; @@ -928,7 +916,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new RegexExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(166); + setState(163); regexBooleanExpression(); } break; @@ -937,41 +925,41 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalInContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(167); + setState(164); valueExpression(); - setState(169); + setState(166); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(168); + setState(165); match(NOT); } } - setState(171); + setState(168); match(IN); - setState(172); + setState(169); match(LP); - setState(173); + setState(170); valueExpression(); - setState(178); + setState(175); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(174); + setState(171); match(COMMA); - setState(175); + setState(172); valueExpression(); } } - setState(180); + setState(177); _errHandler.sync(this); _la = _input.LA(1); } - setState(181); + setState(178); match(RP); } break; @@ -980,21 +968,21 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new IsNullContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(183); + setState(180); valueExpression(); - setState(184); + setState(181); match(IS); - setState(186); + setState(183); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(185); + setState(182); match(NOT); } } - setState(188); + setState(185); match(NULL); } break; @@ -1003,15 +991,15 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new MatchExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(190); + setState(187); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(191); + setState(188); matchBooleanExpression(); } break; } _ctx.stop = _input.LT(-1); - setState(202); + setState(199); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1019,7 +1007,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(200); + setState(197); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) { case 1: @@ -1027,11 +1015,11 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(194); + setState(191); if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)"); - setState(195); + setState(192); ((LogicalBinaryContext)_localctx).operator = match(AND); - setState(196); + setState(193); ((LogicalBinaryContext)_localctx).right = booleanExpression(6); } break; @@ -1040,18 +1028,18 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(197); + setState(194); if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)"); - setState(198); + setState(195); ((LogicalBinaryContext)_localctx).operator = match(OR); - setState(199); + setState(196); ((LogicalBinaryContext)_localctx).right = booleanExpression(5); } break; } } } - setState(204); + setState(201); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); } @@ -1106,48 +1094,48 @@ public final RegexBooleanExpressionContext regexBooleanExpression() throws Recog enterRule(_localctx, 12, RULE_regexBooleanExpression); int _la; try { - setState(219); + setState(216); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(205); + setState(202); valueExpression(); - setState(207); + setState(204); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(206); + setState(203); match(NOT); } } - setState(209); + setState(206); ((RegexBooleanExpressionContext)_localctx).kind = match(LIKE); - setState(210); + setState(207); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(212); + setState(209); valueExpression(); - setState(214); + setState(211); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(213); + setState(210); match(NOT); } } - setState(216); + setState(213); ((RegexBooleanExpressionContext)_localctx).kind = match(RLIKE); - setState(217); + setState(214); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; @@ -1200,11 +1188,11 @@ public final MatchBooleanExpressionContext matchBooleanExpression() throws Recog try { enterOuterAlt(_localctx, 1); { - setState(221); + setState(218); valueExpression(); - setState(222); + setState(219); match(DEV_MATCH); - setState(223); + setState(220); ((MatchBooleanExpressionContext)_localctx).queryString = string(); } } @@ -1288,14 +1276,14 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio ValueExpressionContext _localctx = new ValueExpressionContext(_ctx, getState()); enterRule(_localctx, 16, RULE_valueExpression); try { - setState(230); + setState(227); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) { case 1: _localctx = new ValueExpressionDefaultContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(225); + setState(222); operatorExpression(0); } break; @@ -1303,11 +1291,11 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio _localctx = new ComparisonContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(226); + setState(223); ((ComparisonContext)_localctx).left = operatorExpression(0); - setState(227); + setState(224); comparisonOperator(); - setState(228); + setState(225); ((ComparisonContext)_localctx).right = operatorExpression(0); } break; @@ -1432,7 +1420,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE int _alt; enterOuterAlt(_localctx, 1); { - setState(236); + setState(233); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) { case 1: @@ -1441,7 +1429,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _ctx = _localctx; _prevctx = _localctx; - setState(233); + setState(230); primaryExpression(0); } break; @@ -1450,7 +1438,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticUnaryContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(234); + setState(231); ((ArithmeticUnaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1461,13 +1449,13 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(235); + setState(232); operatorExpression(3); } break; } _ctx.stop = _input.LT(-1); - setState(246); + setState(243); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,15,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1475,7 +1463,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(244); + setState(241); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) { case 1: @@ -1483,12 +1471,12 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(238); + setState(235); if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); - setState(239); + setState(236); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); - if ( !(((((_la - 62)) & ~0x3f) == 0 && ((1L << (_la - 62)) & 7L) != 0)) ) { + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & -2305843009213693952L) != 0)) ) { ((ArithmeticBinaryContext)_localctx).operator = (Token)_errHandler.recoverInline(this); } else { @@ -1496,7 +1484,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(240); + setState(237); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(3); } break; @@ -1505,9 +1493,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(241); + setState(238); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(242); + setState(239); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1518,14 +1506,14 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(243); + setState(240); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(2); } break; } } } - setState(248); + setState(245); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,15,_ctx); } @@ -1683,7 +1671,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc int _alt; enterOuterAlt(_localctx, 1); { - setState(257); + setState(254); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) { case 1: @@ -1692,7 +1680,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _ctx = _localctx; _prevctx = _localctx; - setState(250); + setState(247); constant(); } break; @@ -1701,7 +1689,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new DereferenceContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(251); + setState(248); qualifiedName(); } break; @@ -1710,7 +1698,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new FunctionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(252); + setState(249); functionExpression(); } break; @@ -1719,17 +1707,17 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new ParenthesizedExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(253); + setState(250); match(LP); - setState(254); + setState(251); booleanExpression(0); - setState(255); + setState(252); match(RP); } break; } _ctx.stop = _input.LT(-1); - setState(264); + setState(261); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,17,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1740,16 +1728,16 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc { _localctx = new InlineCastContext(new PrimaryExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_primaryExpression); - setState(259); + setState(256); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(260); + setState(257); match(CAST_OP); - setState(261); + setState(258); dataType(); } } } - setState(266); + setState(263); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,17,_ctx); } @@ -1811,37 +1799,37 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(267); + setState(264); identifier(); - setState(268); + setState(265); match(LP); - setState(278); + setState(275); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) { case 1: { - setState(269); + setState(266); match(ASTERISK); } break; case 2: { { - setState(270); + setState(267); booleanExpression(0); - setState(275); + setState(272); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(271); + setState(268); match(COMMA); - setState(272); + setState(269); booleanExpression(0); } } - setState(277); + setState(274); _errHandler.sync(this); _la = _input.LA(1); } @@ -1849,7 +1837,7 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx } break; } - setState(280); + setState(277); match(RP); } } @@ -1907,7 +1895,7 @@ public final DataTypeContext dataType() throws RecognitionException { _localctx = new ToDataTypeContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(282); + setState(279); identifier(); } } @@ -1954,9 +1942,9 @@ public final RowCommandContext rowCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(284); + setState(281); match(ROW); - setState(285); + setState(282); fields(); } } @@ -2010,23 +1998,23 @@ public final FieldsContext fields() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(287); + setState(284); field(); - setState(292); + setState(289); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,20,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(288); + setState(285); match(COMMA); - setState(289); + setState(286); field(); } } } - setState(294); + setState(291); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,20,_ctx); } @@ -2076,24 +2064,24 @@ public final FieldContext field() throws RecognitionException { FieldContext _localctx = new FieldContext(_ctx, getState()); enterRule(_localctx, 30, RULE_field); try { - setState(300); + setState(297); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,21,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(295); + setState(292); booleanExpression(0); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(296); + setState(293); qualifiedName(); - setState(297); + setState(294); match(ASSIGN); - setState(298); + setState(295); booleanExpression(0); } break; @@ -2153,34 +2141,34 @@ public final FromCommandContext fromCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(302); + setState(299); match(FROM); - setState(303); + setState(300); indexPattern(); - setState(308); + setState(305); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,22,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(304); + setState(301); match(COMMA); - setState(305); + setState(302); indexPattern(); } } } - setState(310); + setState(307); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,22,_ctx); } - setState(312); + setState(309); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) { case 1: { - setState(311); + setState(308); metadata(); } break; @@ -2231,24 +2219,24 @@ public final IndexPatternContext indexPattern() throws RecognitionException { IndexPatternContext _localctx = new IndexPatternContext(_ctx, getState()); enterRule(_localctx, 34, RULE_indexPattern); try { - setState(319); + setState(316); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(314); + setState(311); clusterString(); - setState(315); + setState(312); match(COLON); - setState(316); + setState(313); indexString(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(318); + setState(315); indexString(); } break; @@ -2294,7 +2282,7 @@ public final ClusterStringContext clusterString() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(321); + setState(318); match(UNQUOTED_SOURCE); } } @@ -2340,7 +2328,7 @@ public final IndexStringContext indexString() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(323); + setState(320); _la = _input.LA(1); if ( !(_la==QUOTED_STRING || _la==UNQUOTED_SOURCE) ) { _errHandler.recoverInline(this); @@ -2395,20 +2383,20 @@ public final MetadataContext metadata() throws RecognitionException { MetadataContext _localctx = new MetadataContext(_ctx, getState()); enterRule(_localctx, 40, RULE_metadata); try { - setState(327); + setState(324); _errHandler.sync(this); switch (_input.LA(1)) { case METADATA: enterOuterAlt(_localctx, 1); { - setState(325); + setState(322); metadataOption(); } break; case OPENING_BRACKET: enterOuterAlt(_localctx, 2); { - setState(326); + setState(323); deprecated_metadata(); } break; @@ -2465,25 +2453,25 @@ public final MetadataOptionContext metadataOption() throws RecognitionException int _alt; enterOuterAlt(_localctx, 1); { - setState(329); + setState(326); match(METADATA); - setState(330); + setState(327); match(UNQUOTED_SOURCE); - setState(335); + setState(332); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,26,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(331); + setState(328); match(COMMA); - setState(332); + setState(329); match(UNQUOTED_SOURCE); } } } - setState(337); + setState(334); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,26,_ctx); } @@ -2532,11 +2520,11 @@ public final Deprecated_metadataContext deprecated_metadata() throws Recognition try { enterOuterAlt(_localctx, 1); { - setState(338); + setState(335); match(OPENING_BRACKET); - setState(339); + setState(336); metadataOption(); - setState(340); + setState(337); match(CLOSING_BRACKET); } } @@ -2600,46 +2588,46 @@ public final MetricsCommandContext metricsCommand() throws RecognitionException int _alt; enterOuterAlt(_localctx, 1); { - setState(342); + setState(339); match(DEV_METRICS); - setState(343); + setState(340); indexPattern(); - setState(348); + setState(345); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,27,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(344); + setState(341); match(COMMA); - setState(345); + setState(342); indexPattern(); } } } - setState(350); + setState(347); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,27,_ctx); } - setState(352); + setState(349); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) { case 1: { - setState(351); + setState(348); ((MetricsCommandContext)_localctx).aggregates = fields(); } break; } - setState(356); + setState(353); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) { case 1: { - setState(354); + setState(351); match(BY); - setState(355); + setState(352); ((MetricsCommandContext)_localctx).grouping = fields(); } break; @@ -2689,9 +2677,9 @@ public final EvalCommandContext evalCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(358); + setState(355); match(EVAL); - setState(359); + setState(356); fields(); } } @@ -2744,26 +2732,26 @@ public final StatsCommandContext statsCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(361); + setState(358); match(STATS); - setState(363); + setState(360); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) { case 1: { - setState(362); + setState(359); ((StatsCommandContext)_localctx).stats = fields(); } break; } - setState(367); + setState(364); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) { case 1: { - setState(365); + setState(362); match(BY); - setState(366); + setState(363); ((StatsCommandContext)_localctx).grouping = fields(); } break; @@ -2820,23 +2808,23 @@ public final QualifiedNameContext qualifiedName() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(369); + setState(366); identifier(); - setState(374); + setState(371); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,32,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(370); + setState(367); match(DOT); - setState(371); + setState(368); identifier(); } } } - setState(376); + setState(373); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,32,_ctx); } @@ -2892,23 +2880,23 @@ public final QualifiedNamePatternContext qualifiedNamePattern() throws Recogniti int _alt; enterOuterAlt(_localctx, 1); { - setState(377); + setState(374); identifierPattern(); - setState(382); + setState(379); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,33,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(378); + setState(375); match(DOT); - setState(379); + setState(376); identifierPattern(); } } } - setState(384); + setState(381); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,33,_ctx); } @@ -2964,23 +2952,23 @@ public final QualifiedNamePatternsContext qualifiedNamePatterns() throws Recogni int _alt; enterOuterAlt(_localctx, 1); { - setState(385); + setState(382); qualifiedNamePattern(); - setState(390); + setState(387); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,34,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(386); + setState(383); match(COMMA); - setState(387); + setState(384); qualifiedNamePattern(); } } } - setState(392); + setState(389); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,34,_ctx); } @@ -3028,7 +3016,7 @@ public final IdentifierContext identifier() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(393); + setState(390); _la = _input.LA(1); if ( !(_la==UNQUOTED_IDENTIFIER || _la==QUOTED_IDENTIFIER) ) { _errHandler.recoverInline(this); @@ -3080,7 +3068,7 @@ public final IdentifierPatternContext identifierPattern() throws RecognitionExce try { enterOuterAlt(_localctx, 1); { - setState(395); + setState(392); match(ID_PATTERN); } } @@ -3351,14 +3339,14 @@ public final ConstantContext constant() throws RecognitionException { enterRule(_localctx, 62, RULE_constant); int _la; try { - setState(439); + setState(436); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,38,_ctx) ) { case 1: _localctx = new NullLiteralContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(397); + setState(394); match(NULL); } break; @@ -3366,9 +3354,9 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new QualifiedIntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(398); + setState(395); integerValue(); - setState(399); + setState(396); match(UNQUOTED_IDENTIFIER); } break; @@ -3376,7 +3364,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new DecimalLiteralContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(401); + setState(398); decimalValue(); } break; @@ -3384,7 +3372,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new IntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(402); + setState(399); integerValue(); } break; @@ -3392,7 +3380,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanLiteralContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(403); + setState(400); booleanValue(); } break; @@ -3400,7 +3388,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new InputParamsContext(_localctx); enterOuterAlt(_localctx, 6); { - setState(404); + setState(401); params(); } break; @@ -3408,7 +3396,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringLiteralContext(_localctx); enterOuterAlt(_localctx, 7); { - setState(405); + setState(402); string(); } break; @@ -3416,27 +3404,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new NumericArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 8); { - setState(406); + setState(403); match(OPENING_BRACKET); - setState(407); + setState(404); numericValue(); - setState(412); + setState(409); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(408); + setState(405); match(COMMA); - setState(409); + setState(406); numericValue(); } } - setState(414); + setState(411); _errHandler.sync(this); _la = _input.LA(1); } - setState(415); + setState(412); match(CLOSING_BRACKET); } break; @@ -3444,27 +3432,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 9); { - setState(417); + setState(414); match(OPENING_BRACKET); - setState(418); + setState(415); booleanValue(); - setState(423); + setState(420); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(419); + setState(416); match(COMMA); - setState(420); + setState(417); booleanValue(); } } - setState(425); + setState(422); _errHandler.sync(this); _la = _input.LA(1); } - setState(426); + setState(423); match(CLOSING_BRACKET); } break; @@ -3472,27 +3460,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 10); { - setState(428); + setState(425); match(OPENING_BRACKET); - setState(429); + setState(426); string(); - setState(434); + setState(431); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(430); + setState(427); match(COMMA); - setState(431); + setState(428); string(); } } - setState(436); + setState(433); _errHandler.sync(this); _la = _input.LA(1); } - setState(437); + setState(434); match(CLOSING_BRACKET); } break; @@ -3566,14 +3554,14 @@ public final ParamsContext params() throws RecognitionException { ParamsContext _localctx = new ParamsContext(_ctx, getState()); enterRule(_localctx, 64, RULE_params); try { - setState(443); + setState(440); _errHandler.sync(this); switch (_input.LA(1)) { case PARAM: _localctx = new InputParamContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(441); + setState(438); match(PARAM); } break; @@ -3581,7 +3569,7 @@ public final ParamsContext params() throws RecognitionException { _localctx = new InputNamedOrPositionalParamContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(442); + setState(439); match(NAMED_OR_POSITIONAL_PARAM); } break; @@ -3630,9 +3618,9 @@ public final LimitCommandContext limitCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(445); + setState(442); match(LIMIT); - setState(446); + setState(443); match(INTEGER_LITERAL); } } @@ -3687,25 +3675,25 @@ public final SortCommandContext sortCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(448); + setState(445); match(SORT); - setState(449); + setState(446); orderExpression(); - setState(454); + setState(451); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,40,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(450); + setState(447); match(COMMA); - setState(451); + setState(448); orderExpression(); } } } - setState(456); + setState(453); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,40,_ctx); } @@ -3761,14 +3749,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(457); + setState(454); booleanExpression(0); - setState(459); + setState(456); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,41,_ctx) ) { case 1: { - setState(458); + setState(455); ((OrderExpressionContext)_localctx).ordering = _input.LT(1); _la = _input.LA(1); if ( !(_la==ASC || _la==DESC) ) { @@ -3782,14 +3770,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio } break; } - setState(463); + setState(460); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,42,_ctx) ) { case 1: { - setState(461); + setState(458); match(NULLS); - setState(462); + setState(459); ((OrderExpressionContext)_localctx).nullOrdering = _input.LT(1); _la = _input.LA(1); if ( !(_la==FIRST || _la==LAST) ) { @@ -3848,9 +3836,9 @@ public final KeepCommandContext keepCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(465); + setState(462); match(KEEP); - setState(466); + setState(463); qualifiedNamePatterns(); } } @@ -3897,9 +3885,9 @@ public final DropCommandContext dropCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(468); + setState(465); match(DROP); - setState(469); + setState(466); qualifiedNamePatterns(); } } @@ -3954,25 +3942,25 @@ public final RenameCommandContext renameCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(471); + setState(468); match(RENAME); - setState(472); + setState(469); renameClause(); - setState(477); + setState(474); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,43,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(473); + setState(470); match(COMMA); - setState(474); + setState(471); renameClause(); } } } - setState(479); + setState(476); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,43,_ctx); } @@ -4026,11 +4014,11 @@ public final RenameClauseContext renameClause() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(480); + setState(477); ((RenameClauseContext)_localctx).oldName = qualifiedNamePattern(); - setState(481); + setState(478); match(AS); - setState(482); + setState(479); ((RenameClauseContext)_localctx).newName = qualifiedNamePattern(); } } @@ -4083,18 +4071,18 @@ public final DissectCommandContext dissectCommand() throws RecognitionException try { enterOuterAlt(_localctx, 1); { - setState(484); + setState(481); match(DISSECT); - setState(485); + setState(482); primaryExpression(0); - setState(486); + setState(483); string(); - setState(488); + setState(485); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,44,_ctx) ) { case 1: { - setState(487); + setState(484); commandOptions(); } break; @@ -4147,11 +4135,11 @@ public final GrokCommandContext grokCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(490); + setState(487); match(GROK); - setState(491); + setState(488); primaryExpression(0); - setState(492); + setState(489); string(); } } @@ -4198,9 +4186,9 @@ public final MvExpandCommandContext mvExpandCommand() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(494); + setState(491); match(MV_EXPAND); - setState(495); + setState(492); qualifiedName(); } } @@ -4254,23 +4242,23 @@ public final CommandOptionsContext commandOptions() throws RecognitionException int _alt; enterOuterAlt(_localctx, 1); { - setState(497); + setState(494); commandOption(); - setState(502); + setState(499); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,45,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(498); + setState(495); match(COMMA); - setState(499); + setState(496); commandOption(); } } } - setState(504); + setState(501); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,45,_ctx); } @@ -4322,11 +4310,11 @@ public final CommandOptionContext commandOption() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(505); + setState(502); identifier(); - setState(506); + setState(503); match(ASSIGN); - setState(507); + setState(504); constant(); } } @@ -4372,7 +4360,7 @@ public final BooleanValueContext booleanValue() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(509); + setState(506); _la = _input.LA(1); if ( !(_la==FALSE || _la==TRUE) ) { _errHandler.recoverInline(this); @@ -4427,20 +4415,20 @@ public final NumericValueContext numericValue() throws RecognitionException { NumericValueContext _localctx = new NumericValueContext(_ctx, getState()); enterRule(_localctx, 92, RULE_numericValue); try { - setState(513); + setState(510); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(511); + setState(508); decimalValue(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(512); + setState(509); integerValue(); } break; @@ -4489,12 +4477,12 @@ public final DecimalValueContext decimalValue() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(516); + setState(513); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(515); + setState(512); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4507,7 +4495,7 @@ public final DecimalValueContext decimalValue() throws RecognitionException { } } - setState(518); + setState(515); match(DECIMAL_LITERAL); } } @@ -4554,12 +4542,12 @@ public final IntegerValueContext integerValue() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(521); + setState(518); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(520); + setState(517); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4572,7 +4560,7 @@ public final IntegerValueContext integerValue() throws RecognitionException { } } - setState(523); + setState(520); match(INTEGER_LITERAL); } } @@ -4616,7 +4604,7 @@ public final StringContext string() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(525); + setState(522); match(QUOTED_STRING); } } @@ -4666,9 +4654,9 @@ public final ComparisonOperatorContext comparisonOperator() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(527); + setState(524); _la = _input.LA(1); - if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 1125899906842624000L) != 0)) ) { + if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 562949953421312000L) != 0)) ) { _errHandler.recoverInline(this); } else { @@ -4721,9 +4709,9 @@ public final ExplainCommandContext explainCommand() throws RecognitionException try { enterOuterAlt(_localctx, 1); { - setState(529); + setState(526); match(EXPLAIN); - setState(530); + setState(527); subqueryExpression(); } } @@ -4771,11 +4759,11 @@ public final SubqueryExpressionContext subqueryExpression() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(532); + setState(529); match(OPENING_BRACKET); - setState(533); + setState(530); query(0); - setState(534); + setState(531); match(CLOSING_BRACKET); } } @@ -4832,9 +4820,9 @@ public final ShowCommandContext showCommand() throws RecognitionException { _localctx = new ShowInfoContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(536); + setState(533); match(SHOW); - setState(537); + setState(534); match(INFO); } } @@ -4849,65 +4837,6 @@ public final ShowCommandContext showCommand() throws RecognitionException { return _localctx; } - @SuppressWarnings("CheckReturnValue") - public static class MetaCommandContext extends ParserRuleContext { - @SuppressWarnings("this-escape") - public MetaCommandContext(ParserRuleContext parent, int invokingState) { - super(parent, invokingState); - } - @Override public int getRuleIndex() { return RULE_metaCommand; } - - @SuppressWarnings("this-escape") - public MetaCommandContext() { } - public void copyFrom(MetaCommandContext ctx) { - super.copyFrom(ctx); - } - } - @SuppressWarnings("CheckReturnValue") - public static class MetaFunctionsContext extends MetaCommandContext { - public TerminalNode META() { return getToken(EsqlBaseParser.META, 0); } - public TerminalNode FUNCTIONS() { return getToken(EsqlBaseParser.FUNCTIONS, 0); } - @SuppressWarnings("this-escape") - public MetaFunctionsContext(MetaCommandContext ctx) { copyFrom(ctx); } - @Override - public void enterRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterMetaFunctions(this); - } - @Override - public void exitRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitMetaFunctions(this); - } - @Override - public T accept(ParseTreeVisitor visitor) { - if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitMetaFunctions(this); - else return visitor.visitChildren(this); - } - } - - public final MetaCommandContext metaCommand() throws RecognitionException { - MetaCommandContext _localctx = new MetaCommandContext(_ctx, getState()); - enterRule(_localctx, 108, RULE_metaCommand); - try { - _localctx = new MetaFunctionsContext(_localctx); - enterOuterAlt(_localctx, 1); - { - setState(539); - match(META); - setState(540); - match(FUNCTIONS); - } - } - catch (RecognitionException re) { - _localctx.exception = re; - _errHandler.reportError(this, re); - _errHandler.recover(this, re); - } - finally { - exitRule(); - } - return _localctx; - } - @SuppressWarnings("CheckReturnValue") public static class EnrichCommandContext extends ParserRuleContext { public Token policyName; @@ -4951,51 +4880,51 @@ public T accept(ParseTreeVisitor visitor) { public final EnrichCommandContext enrichCommand() throws RecognitionException { EnrichCommandContext _localctx = new EnrichCommandContext(_ctx, getState()); - enterRule(_localctx, 110, RULE_enrichCommand); + enterRule(_localctx, 108, RULE_enrichCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(542); + setState(536); match(ENRICH); - setState(543); + setState(537); ((EnrichCommandContext)_localctx).policyName = match(ENRICH_POLICY_NAME); - setState(546); + setState(540); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,49,_ctx) ) { case 1: { - setState(544); + setState(538); match(ON); - setState(545); + setState(539); ((EnrichCommandContext)_localctx).matchField = qualifiedNamePattern(); } break; } - setState(557); + setState(551); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,51,_ctx) ) { case 1: { - setState(548); + setState(542); match(WITH); - setState(549); + setState(543); enrichWithClause(); - setState(554); + setState(548); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,50,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(550); + setState(544); match(COMMA); - setState(551); + setState(545); enrichWithClause(); } } } - setState(556); + setState(550); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,50,_ctx); } @@ -5048,23 +4977,23 @@ public T accept(ParseTreeVisitor visitor) { public final EnrichWithClauseContext enrichWithClause() throws RecognitionException { EnrichWithClauseContext _localctx = new EnrichWithClauseContext(_ctx, getState()); - enterRule(_localctx, 112, RULE_enrichWithClause); + enterRule(_localctx, 110, RULE_enrichWithClause); try { enterOuterAlt(_localctx, 1); { - setState(562); + setState(556); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,52,_ctx) ) { case 1: { - setState(559); + setState(553); ((EnrichWithClauseContext)_localctx).newName = qualifiedNamePattern(); - setState(560); + setState(554); match(ASSIGN); } break; } - setState(564); + setState(558); ((EnrichWithClauseContext)_localctx).enrichField = qualifiedNamePattern(); } } @@ -5113,17 +5042,17 @@ public T accept(ParseTreeVisitor visitor) { public final LookupCommandContext lookupCommand() throws RecognitionException { LookupCommandContext _localctx = new LookupCommandContext(_ctx, getState()); - enterRule(_localctx, 114, RULE_lookupCommand); + enterRule(_localctx, 112, RULE_lookupCommand); try { enterOuterAlt(_localctx, 1); { - setState(566); + setState(560); match(DEV_LOOKUP); - setState(567); + setState(561); ((LookupCommandContext)_localctx).tableName = indexPattern(); - setState(568); + setState(562); match(ON); - setState(569); + setState(563); ((LookupCommandContext)_localctx).matchFields = qualifiedNamePatterns(); } } @@ -5172,22 +5101,22 @@ public T accept(ParseTreeVisitor visitor) { public final InlinestatsCommandContext inlinestatsCommand() throws RecognitionException { InlinestatsCommandContext _localctx = new InlinestatsCommandContext(_ctx, getState()); - enterRule(_localctx, 116, RULE_inlinestatsCommand); + enterRule(_localctx, 114, RULE_inlinestatsCommand); try { enterOuterAlt(_localctx, 1); { - setState(571); + setState(565); match(DEV_INLINESTATS); - setState(572); + setState(566); ((InlinestatsCommandContext)_localctx).stats = fields(); - setState(575); + setState(569); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,53,_ctx) ) { case 1: { - setState(573); + setState(567); match(BY); - setState(574); + setState(568); ((InlinestatsCommandContext)_localctx).grouping = fields(); } break; @@ -5274,7 +5203,7 @@ private boolean primaryExpression_sempred(PrimaryExpressionContext _localctx, in } public static final String _serializedATN = - "\u0004\u0001}\u0242\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+ + "\u0004\u0001x\u023c\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001\u0002"+ "\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004\u0002"+ "\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007\u0002"+ "\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b\u0002"+ @@ -5289,359 +5218,355 @@ private boolean primaryExpression_sempred(PrimaryExpressionContext _localctx, in "(\u0007(\u0002)\u0007)\u0002*\u0007*\u0002+\u0007+\u0002,\u0007,\u0002"+ "-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u00021\u00071\u0002"+ "2\u00072\u00023\u00073\u00024\u00074\u00025\u00075\u00026\u00076\u0002"+ - "7\u00077\u00028\u00078\u00029\u00079\u0002:\u0007:\u0001\u0000\u0001\u0000"+ - "\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ - "\u0001\u0001\u0005\u0001\u0080\b\u0001\n\u0001\f\u0001\u0083\t\u0001\u0001"+ - "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+ - "\u0002\u0003\u0002\u008c\b\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ - "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ + "7\u00077\u00028\u00078\u00029\u00079\u0001\u0000\u0001\u0000\u0001\u0000"+ + "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ + "\u0005\u0001~\b\u0001\n\u0001\f\u0001\u0081\t\u0001\u0001\u0002\u0001"+ + "\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002\u0089"+ + "\b\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ - "\u0003\u0003\u0003\u009e\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ - "\u0005\u0003\u0005\u00aa\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0005\u0005\u00b1\b\u0005\n\u0005\f\u0005\u00b4\t\u0005"+ - "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005"+ - "\u00bb\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005"+ - "\u00c1\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ - "\u0001\u0005\u0005\u0005\u00c9\b\u0005\n\u0005\f\u0005\u00cc\t\u0005\u0001"+ - "\u0006\u0001\u0006\u0003\u0006\u00d0\b\u0006\u0001\u0006\u0001\u0006\u0001"+ - "\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00d7\b\u0006\u0001\u0006\u0001"+ - "\u0006\u0001\u0006\u0003\u0006\u00dc\b\u0006\u0001\u0007\u0001\u0007\u0001"+ - "\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0003\b\u00e7"+ - "\b\b\u0001\t\u0001\t\u0001\t\u0001\t\u0003\t\u00ed\b\t\u0001\t\u0001\t"+ - "\u0001\t\u0001\t\u0001\t\u0001\t\u0005\t\u00f5\b\t\n\t\f\t\u00f8\t\t\u0001"+ - "\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0003\n\u0102"+ - "\b\n\u0001\n\u0001\n\u0001\n\u0005\n\u0107\b\n\n\n\f\n\u010a\t\n\u0001"+ - "\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0005"+ - "\u000b\u0112\b\u000b\n\u000b\f\u000b\u0115\t\u000b\u0003\u000b\u0117\b"+ - "\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\r\u0001\r\u0001\r"+ - "\u0001\u000e\u0001\u000e\u0001\u000e\u0005\u000e\u0123\b\u000e\n\u000e"+ - "\f\u000e\u0126\t\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f"+ - "\u0001\u000f\u0003\u000f\u012d\b\u000f\u0001\u0010\u0001\u0010\u0001\u0010"+ - "\u0001\u0010\u0005\u0010\u0133\b\u0010\n\u0010\f\u0010\u0136\t\u0010\u0001"+ - "\u0010\u0003\u0010\u0139\b\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001"+ - "\u0011\u0001\u0011\u0003\u0011\u0140\b\u0011\u0001\u0012\u0001\u0012\u0001"+ - "\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0003\u0014\u0148\b\u0014\u0001"+ - "\u0015\u0001\u0015\u0001\u0015\u0001\u0015\u0005\u0015\u014e\b\u0015\n"+ - "\u0015\f\u0015\u0151\t\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0001"+ - "\u0016\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0005\u0017\u015b"+ - "\b\u0017\n\u0017\f\u0017\u015e\t\u0017\u0001\u0017\u0003\u0017\u0161\b"+ - "\u0017\u0001\u0017\u0001\u0017\u0003\u0017\u0165\b\u0017\u0001\u0018\u0001"+ - "\u0018\u0001\u0018\u0001\u0019\u0001\u0019\u0003\u0019\u016c\b\u0019\u0001"+ - "\u0019\u0001\u0019\u0003\u0019\u0170\b\u0019\u0001\u001a\u0001\u001a\u0001"+ - "\u001a\u0005\u001a\u0175\b\u001a\n\u001a\f\u001a\u0178\t\u001a\u0001\u001b"+ - "\u0001\u001b\u0001\u001b\u0005\u001b\u017d\b\u001b\n\u001b\f\u001b\u0180"+ - "\t\u001b\u0001\u001c\u0001\u001c\u0001\u001c\u0005\u001c\u0185\b\u001c"+ - "\n\u001c\f\u001c\u0188\t\u001c\u0001\u001d\u0001\u001d\u0001\u001e\u0001"+ - "\u001e\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001"+ - "\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001"+ - "\u001f\u0001\u001f\u0005\u001f\u019b\b\u001f\n\u001f\f\u001f\u019e\t\u001f"+ + "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0003\u0003\u009b"+ + "\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00a7"+ + "\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005"+ + "\u0005\u00ae\b\u0005\n\u0005\f\u0005\u00b1\t\u0005\u0001\u0005\u0001\u0005"+ + "\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00b8\b\u0005\u0001\u0005"+ + "\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00be\b\u0005\u0001\u0005"+ + "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005"+ + "\u00c6\b\u0005\n\u0005\f\u0005\u00c9\t\u0005\u0001\u0006\u0001\u0006\u0003"+ + "\u0006\u00cd\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001"+ + "\u0006\u0003\u0006\u00d4\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003"+ + "\u0006\u00d9\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001"+ + "\b\u0001\b\u0001\b\u0001\b\u0001\b\u0003\b\u00e4\b\b\u0001\t\u0001\t\u0001"+ + "\t\u0001\t\u0003\t\u00ea\b\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001"+ + "\t\u0005\t\u00f2\b\t\n\t\f\t\u00f5\t\t\u0001\n\u0001\n\u0001\n\u0001\n"+ + "\u0001\n\u0001\n\u0001\n\u0001\n\u0003\n\u00ff\b\n\u0001\n\u0001\n\u0001"+ + "\n\u0005\n\u0104\b\n\n\n\f\n\u0107\t\n\u0001\u000b\u0001\u000b\u0001\u000b"+ + "\u0001\u000b\u0001\u000b\u0001\u000b\u0005\u000b\u010f\b\u000b\n\u000b"+ + "\f\u000b\u0112\t\u000b\u0003\u000b\u0114\b\u000b\u0001\u000b\u0001\u000b"+ + "\u0001\f\u0001\f\u0001\r\u0001\r\u0001\r\u0001\u000e\u0001\u000e\u0001"+ + "\u000e\u0005\u000e\u0120\b\u000e\n\u000e\f\u000e\u0123\t\u000e\u0001\u000f"+ + "\u0001\u000f\u0001\u000f\u0001\u000f\u0001\u000f\u0003\u000f\u012a\b\u000f"+ + "\u0001\u0010\u0001\u0010\u0001\u0010\u0001\u0010\u0005\u0010\u0130\b\u0010"+ + "\n\u0010\f\u0010\u0133\t\u0010\u0001\u0010\u0003\u0010\u0136\b\u0010\u0001"+ + "\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0003\u0011\u013d"+ + "\b\u0011\u0001\u0012\u0001\u0012\u0001\u0013\u0001\u0013\u0001\u0014\u0001"+ + "\u0014\u0003\u0014\u0145\b\u0014\u0001\u0015\u0001\u0015\u0001\u0015\u0001"+ + "\u0015\u0005\u0015\u014b\b\u0015\n\u0015\f\u0015\u014e\t\u0015\u0001\u0016"+ + "\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0017\u0001\u0017\u0001\u0017"+ + "\u0001\u0017\u0005\u0017\u0158\b\u0017\n\u0017\f\u0017\u015b\t\u0017\u0001"+ + "\u0017\u0003\u0017\u015e\b\u0017\u0001\u0017\u0001\u0017\u0003\u0017\u0162"+ + "\b\u0017\u0001\u0018\u0001\u0018\u0001\u0018\u0001\u0019\u0001\u0019\u0003"+ + "\u0019\u0169\b\u0019\u0001\u0019\u0001\u0019\u0003\u0019\u016d\b\u0019"+ + "\u0001\u001a\u0001\u001a\u0001\u001a\u0005\u001a\u0172\b\u001a\n\u001a"+ + "\f\u001a\u0175\t\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0005\u001b"+ + "\u017a\b\u001b\n\u001b\f\u001b\u017d\t\u001b\u0001\u001c\u0001\u001c\u0001"+ + "\u001c\u0005\u001c\u0182\b\u001c\n\u001c\f\u001c\u0185\t\u001c\u0001\u001d"+ + "\u0001\u001d\u0001\u001e\u0001\u001e\u0001\u001f\u0001\u001f\u0001\u001f"+ "\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f"+ - "\u0005\u001f\u01a6\b\u001f\n\u001f\f\u001f\u01a9\t\u001f\u0001\u001f\u0001"+ - "\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u01b1"+ - "\b\u001f\n\u001f\f\u001f\u01b4\t\u001f\u0001\u001f\u0001\u001f\u0003\u001f"+ - "\u01b8\b\u001f\u0001 \u0001 \u0003 \u01bc\b \u0001!\u0001!\u0001!\u0001"+ - "\"\u0001\"\u0001\"\u0001\"\u0005\"\u01c5\b\"\n\"\f\"\u01c8\t\"\u0001#"+ - "\u0001#\u0003#\u01cc\b#\u0001#\u0001#\u0003#\u01d0\b#\u0001$\u0001$\u0001"+ - "$\u0001%\u0001%\u0001%\u0001&\u0001&\u0001&\u0001&\u0005&\u01dc\b&\n&"+ - "\f&\u01df\t&\u0001\'\u0001\'\u0001\'\u0001\'\u0001(\u0001(\u0001(\u0001"+ - "(\u0003(\u01e9\b(\u0001)\u0001)\u0001)\u0001)\u0001*\u0001*\u0001*\u0001"+ - "+\u0001+\u0001+\u0005+\u01f5\b+\n+\f+\u01f8\t+\u0001,\u0001,\u0001,\u0001"+ - ",\u0001-\u0001-\u0001.\u0001.\u0003.\u0202\b.\u0001/\u0003/\u0205\b/\u0001"+ - "/\u0001/\u00010\u00030\u020a\b0\u00010\u00010\u00011\u00011\u00012\u0001"+ - "2\u00013\u00013\u00013\u00014\u00014\u00014\u00014\u00015\u00015\u0001"+ - "5\u00016\u00016\u00016\u00017\u00017\u00017\u00017\u00037\u0223\b7\u0001"+ - "7\u00017\u00017\u00017\u00057\u0229\b7\n7\f7\u022c\t7\u00037\u022e\b7"+ - "\u00018\u00018\u00018\u00038\u0233\b8\u00018\u00018\u00019\u00019\u0001"+ - "9\u00019\u00019\u0001:\u0001:\u0001:\u0001:\u0003:\u0240\b:\u0001:\u0000"+ - "\u0004\u0002\n\u0012\u0014;\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010"+ - "\u0012\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPR"+ - "TVXZ\\^`bdfhjlnprt\u0000\b\u0001\u0000<=\u0001\u0000>@\u0002\u0000\u001b"+ - "\u001bMM\u0001\u0000DE\u0002\u0000 $$\u0002\u0000\'\'**\u0002\u0000&"+ - "&44\u0002\u0000557;\u025b\u0000v\u0001\u0000\u0000\u0000\u0002y\u0001"+ - "\u0000\u0000\u0000\u0004\u008b\u0001\u0000\u0000\u0000\u0006\u009d\u0001"+ - "\u0000\u0000\u0000\b\u009f\u0001\u0000\u0000\u0000\n\u00c0\u0001\u0000"+ - "\u0000\u0000\f\u00db\u0001\u0000\u0000\u0000\u000e\u00dd\u0001\u0000\u0000"+ - "\u0000\u0010\u00e6\u0001\u0000\u0000\u0000\u0012\u00ec\u0001\u0000\u0000"+ - "\u0000\u0014\u0101\u0001\u0000\u0000\u0000\u0016\u010b\u0001\u0000\u0000"+ - "\u0000\u0018\u011a\u0001\u0000\u0000\u0000\u001a\u011c\u0001\u0000\u0000"+ - "\u0000\u001c\u011f\u0001\u0000\u0000\u0000\u001e\u012c\u0001\u0000\u0000"+ - "\u0000 \u012e\u0001\u0000\u0000\u0000\"\u013f\u0001\u0000\u0000\u0000"+ - "$\u0141\u0001\u0000\u0000\u0000&\u0143\u0001\u0000\u0000\u0000(\u0147"+ - "\u0001\u0000\u0000\u0000*\u0149\u0001\u0000\u0000\u0000,\u0152\u0001\u0000"+ - "\u0000\u0000.\u0156\u0001\u0000\u0000\u00000\u0166\u0001\u0000\u0000\u0000"+ - "2\u0169\u0001\u0000\u0000\u00004\u0171\u0001\u0000\u0000\u00006\u0179"+ - "\u0001\u0000\u0000\u00008\u0181\u0001\u0000\u0000\u0000:\u0189\u0001\u0000"+ - "\u0000\u0000<\u018b\u0001\u0000\u0000\u0000>\u01b7\u0001\u0000\u0000\u0000"+ - "@\u01bb\u0001\u0000\u0000\u0000B\u01bd\u0001\u0000\u0000\u0000D\u01c0"+ - "\u0001\u0000\u0000\u0000F\u01c9\u0001\u0000\u0000\u0000H\u01d1\u0001\u0000"+ - "\u0000\u0000J\u01d4\u0001\u0000\u0000\u0000L\u01d7\u0001\u0000\u0000\u0000"+ - "N\u01e0\u0001\u0000\u0000\u0000P\u01e4\u0001\u0000\u0000\u0000R\u01ea"+ - "\u0001\u0000\u0000\u0000T\u01ee\u0001\u0000\u0000\u0000V\u01f1\u0001\u0000"+ - "\u0000\u0000X\u01f9\u0001\u0000\u0000\u0000Z\u01fd\u0001\u0000\u0000\u0000"+ - "\\\u0201\u0001\u0000\u0000\u0000^\u0204\u0001\u0000\u0000\u0000`\u0209"+ - "\u0001\u0000\u0000\u0000b\u020d\u0001\u0000\u0000\u0000d\u020f\u0001\u0000"+ - "\u0000\u0000f\u0211\u0001\u0000\u0000\u0000h\u0214\u0001\u0000\u0000\u0000"+ - "j\u0218\u0001\u0000\u0000\u0000l\u021b\u0001\u0000\u0000\u0000n\u021e"+ - "\u0001\u0000\u0000\u0000p\u0232\u0001\u0000\u0000\u0000r\u0236\u0001\u0000"+ - "\u0000\u0000t\u023b\u0001\u0000\u0000\u0000vw\u0003\u0002\u0001\u0000"+ - "wx\u0005\u0000\u0000\u0001x\u0001\u0001\u0000\u0000\u0000yz\u0006\u0001"+ - "\uffff\uffff\u0000z{\u0003\u0004\u0002\u0000{\u0081\u0001\u0000\u0000"+ - "\u0000|}\n\u0001\u0000\u0000}~\u0005\u001a\u0000\u0000~\u0080\u0003\u0006"+ - "\u0003\u0000\u007f|\u0001\u0000\u0000\u0000\u0080\u0083\u0001\u0000\u0000"+ - "\u0000\u0081\u007f\u0001\u0000\u0000\u0000\u0081\u0082\u0001\u0000\u0000"+ - "\u0000\u0082\u0003\u0001\u0000\u0000\u0000\u0083\u0081\u0001\u0000\u0000"+ - "\u0000\u0084\u008c\u0003f3\u0000\u0085\u008c\u0003 \u0010\u0000\u0086"+ - "\u008c\u0003l6\u0000\u0087\u008c\u0003\u001a\r\u0000\u0088\u008c\u0003"+ - "j5\u0000\u0089\u008a\u0004\u0002\u0001\u0000\u008a\u008c\u0003.\u0017"+ - "\u0000\u008b\u0084\u0001\u0000\u0000\u0000\u008b\u0085\u0001\u0000\u0000"+ - "\u0000\u008b\u0086\u0001\u0000\u0000\u0000\u008b\u0087\u0001\u0000\u0000"+ - "\u0000\u008b\u0088\u0001\u0000\u0000\u0000\u008b\u0089\u0001\u0000\u0000"+ - "\u0000\u008c\u0005\u0001\u0000\u0000\u0000\u008d\u009e\u00030\u0018\u0000"+ - "\u008e\u009e\u0003\b\u0004\u0000\u008f\u009e\u0003H$\u0000\u0090\u009e"+ - "\u0003B!\u0000\u0091\u009e\u00032\u0019\u0000\u0092\u009e\u0003D\"\u0000"+ - "\u0093\u009e\u0003J%\u0000\u0094\u009e\u0003L&\u0000\u0095\u009e\u0003"+ - "P(\u0000\u0096\u009e\u0003R)\u0000\u0097\u009e\u0003n7\u0000\u0098\u009e"+ - "\u0003T*\u0000\u0099\u009a\u0004\u0003\u0002\u0000\u009a\u009e\u0003t"+ - ":\u0000\u009b\u009c\u0004\u0003\u0003\u0000\u009c\u009e\u0003r9\u0000"+ - "\u009d\u008d\u0001\u0000\u0000\u0000\u009d\u008e\u0001\u0000\u0000\u0000"+ - "\u009d\u008f\u0001\u0000\u0000\u0000\u009d\u0090\u0001\u0000\u0000\u0000"+ - "\u009d\u0091\u0001\u0000\u0000\u0000\u009d\u0092\u0001\u0000\u0000\u0000"+ - "\u009d\u0093\u0001\u0000\u0000\u0000\u009d\u0094\u0001\u0000\u0000\u0000"+ - "\u009d\u0095\u0001\u0000\u0000\u0000\u009d\u0096\u0001\u0000\u0000\u0000"+ - "\u009d\u0097\u0001\u0000\u0000\u0000\u009d\u0098\u0001\u0000\u0000\u0000"+ - "\u009d\u0099\u0001\u0000\u0000\u0000\u009d\u009b\u0001\u0000\u0000\u0000"+ - "\u009e\u0007\u0001\u0000\u0000\u0000\u009f\u00a0\u0005\u0011\u0000\u0000"+ - "\u00a0\u00a1\u0003\n\u0005\u0000\u00a1\t\u0001\u0000\u0000\u0000\u00a2"+ - "\u00a3\u0006\u0005\uffff\uffff\u0000\u00a3\u00a4\u0005-\u0000\u0000\u00a4"+ - "\u00c1\u0003\n\u0005\b\u00a5\u00c1\u0003\u0010\b\u0000\u00a6\u00c1\u0003"+ - "\f\u0006\u0000\u00a7\u00a9\u0003\u0010\b\u0000\u00a8\u00aa\u0005-\u0000"+ - "\u0000\u00a9\u00a8\u0001\u0000\u0000\u0000\u00a9\u00aa\u0001\u0000\u0000"+ - "\u0000\u00aa\u00ab\u0001\u0000\u0000\u0000\u00ab\u00ac\u0005(\u0000\u0000"+ - "\u00ac\u00ad\u0005,\u0000\u0000\u00ad\u00b2\u0003\u0010\b\u0000\u00ae"+ - "\u00af\u0005#\u0000\u0000\u00af\u00b1\u0003\u0010\b\u0000\u00b0\u00ae"+ - "\u0001\u0000\u0000\u0000\u00b1\u00b4\u0001\u0000\u0000\u0000\u00b2\u00b0"+ - "\u0001\u0000\u0000\u0000\u00b2\u00b3\u0001\u0000\u0000\u0000\u00b3\u00b5"+ - "\u0001\u0000\u0000\u0000\u00b4\u00b2\u0001\u0000\u0000\u0000\u00b5\u00b6"+ - "\u00053\u0000\u0000\u00b6\u00c1\u0001\u0000\u0000\u0000\u00b7\u00b8\u0003"+ - "\u0010\b\u0000\u00b8\u00ba\u0005)\u0000\u0000\u00b9\u00bb\u0005-\u0000"+ - "\u0000\u00ba\u00b9\u0001\u0000\u0000\u0000\u00ba\u00bb\u0001\u0000\u0000"+ - "\u0000\u00bb\u00bc\u0001\u0000\u0000\u0000\u00bc\u00bd\u0005.\u0000\u0000"+ - "\u00bd\u00c1\u0001\u0000\u0000\u0000\u00be\u00bf\u0004\u0005\u0004\u0000"+ - "\u00bf\u00c1\u0003\u000e\u0007\u0000\u00c0\u00a2\u0001\u0000\u0000\u0000"+ - "\u00c0\u00a5\u0001\u0000\u0000\u0000\u00c0\u00a6\u0001\u0000\u0000\u0000"+ - "\u00c0\u00a7\u0001\u0000\u0000\u0000\u00c0\u00b7\u0001\u0000\u0000\u0000"+ - "\u00c0\u00be\u0001\u0000\u0000\u0000\u00c1\u00ca\u0001\u0000\u0000\u0000"+ - "\u00c2\u00c3\n\u0005\u0000\u0000\u00c3\u00c4\u0005\u001f\u0000\u0000\u00c4"+ - "\u00c9\u0003\n\u0005\u0006\u00c5\u00c6\n\u0004\u0000\u0000\u00c6\u00c7"+ - "\u00050\u0000\u0000\u00c7\u00c9\u0003\n\u0005\u0005\u00c8\u00c2\u0001"+ - "\u0000\u0000\u0000\u00c8\u00c5\u0001\u0000\u0000\u0000\u00c9\u00cc\u0001"+ - "\u0000\u0000\u0000\u00ca\u00c8\u0001\u0000\u0000\u0000\u00ca\u00cb\u0001"+ - "\u0000\u0000\u0000\u00cb\u000b\u0001\u0000\u0000\u0000\u00cc\u00ca\u0001"+ - "\u0000\u0000\u0000\u00cd\u00cf\u0003\u0010\b\u0000\u00ce\u00d0\u0005-"+ - "\u0000\u0000\u00cf\u00ce\u0001\u0000\u0000\u0000\u00cf\u00d0\u0001\u0000"+ - "\u0000\u0000\u00d0\u00d1\u0001\u0000\u0000\u0000\u00d1\u00d2\u0005+\u0000"+ - "\u0000\u00d2\u00d3\u0003b1\u0000\u00d3\u00dc\u0001\u0000\u0000\u0000\u00d4"+ - "\u00d6\u0003\u0010\b\u0000\u00d5\u00d7\u0005-\u0000\u0000\u00d6\u00d5"+ - "\u0001\u0000\u0000\u0000\u00d6\u00d7\u0001\u0000\u0000\u0000\u00d7\u00d8"+ - "\u0001\u0000\u0000\u0000\u00d8\u00d9\u00052\u0000\u0000\u00d9\u00da\u0003"+ - "b1\u0000\u00da\u00dc\u0001\u0000\u0000\u0000\u00db\u00cd\u0001\u0000\u0000"+ - "\u0000\u00db\u00d4\u0001\u0000\u0000\u0000\u00dc\r\u0001\u0000\u0000\u0000"+ - "\u00dd\u00de\u0003\u0010\b\u0000\u00de\u00df\u0005\u0014\u0000\u0000\u00df"+ - "\u00e0\u0003b1\u0000\u00e0\u000f\u0001\u0000\u0000\u0000\u00e1\u00e7\u0003"+ - "\u0012\t\u0000\u00e2\u00e3\u0003\u0012\t\u0000\u00e3\u00e4\u0003d2\u0000"+ - "\u00e4\u00e5\u0003\u0012\t\u0000\u00e5\u00e7\u0001\u0000\u0000\u0000\u00e6"+ - "\u00e1\u0001\u0000\u0000\u0000\u00e6\u00e2\u0001\u0000\u0000\u0000\u00e7"+ - "\u0011\u0001\u0000\u0000\u0000\u00e8\u00e9\u0006\t\uffff\uffff\u0000\u00e9"+ - "\u00ed\u0003\u0014\n\u0000\u00ea\u00eb\u0007\u0000\u0000\u0000\u00eb\u00ed"+ - "\u0003\u0012\t\u0003\u00ec\u00e8\u0001\u0000\u0000\u0000\u00ec\u00ea\u0001"+ - "\u0000\u0000\u0000\u00ed\u00f6\u0001\u0000\u0000\u0000\u00ee\u00ef\n\u0002"+ - "\u0000\u0000\u00ef\u00f0\u0007\u0001\u0000\u0000\u00f0\u00f5\u0003\u0012"+ - "\t\u0003\u00f1\u00f2\n\u0001\u0000\u0000\u00f2\u00f3\u0007\u0000\u0000"+ - "\u0000\u00f3\u00f5\u0003\u0012\t\u0002\u00f4\u00ee\u0001\u0000\u0000\u0000"+ - "\u00f4\u00f1\u0001\u0000\u0000\u0000\u00f5\u00f8\u0001\u0000\u0000\u0000"+ - "\u00f6\u00f4\u0001\u0000\u0000\u0000\u00f6\u00f7\u0001\u0000\u0000\u0000"+ - "\u00f7\u0013\u0001\u0000\u0000\u0000\u00f8\u00f6\u0001\u0000\u0000\u0000"+ - "\u00f9\u00fa\u0006\n\uffff\uffff\u0000\u00fa\u0102\u0003>\u001f\u0000"+ - "\u00fb\u0102\u00034\u001a\u0000\u00fc\u0102\u0003\u0016\u000b\u0000\u00fd"+ - "\u00fe\u0005,\u0000\u0000\u00fe\u00ff\u0003\n\u0005\u0000\u00ff\u0100"+ - "\u00053\u0000\u0000\u0100\u0102\u0001\u0000\u0000\u0000\u0101\u00f9\u0001"+ - "\u0000\u0000\u0000\u0101\u00fb\u0001\u0000\u0000\u0000\u0101\u00fc\u0001"+ - "\u0000\u0000\u0000\u0101\u00fd\u0001\u0000\u0000\u0000\u0102\u0108\u0001"+ - "\u0000\u0000\u0000\u0103\u0104\n\u0001\u0000\u0000\u0104\u0105\u0005\""+ - "\u0000\u0000\u0105\u0107\u0003\u0018\f\u0000\u0106\u0103\u0001\u0000\u0000"+ - "\u0000\u0107\u010a\u0001\u0000\u0000\u0000\u0108\u0106\u0001\u0000\u0000"+ - "\u0000\u0108\u0109\u0001\u0000\u0000\u0000\u0109\u0015\u0001\u0000\u0000"+ - "\u0000\u010a\u0108\u0001\u0000\u0000\u0000\u010b\u010c\u0003:\u001d\u0000"+ - "\u010c\u0116\u0005,\u0000\u0000\u010d\u0117\u0005>\u0000\u0000\u010e\u0113"+ - "\u0003\n\u0005\u0000\u010f\u0110\u0005#\u0000\u0000\u0110\u0112\u0003"+ - "\n\u0005\u0000\u0111\u010f\u0001\u0000\u0000\u0000\u0112\u0115\u0001\u0000"+ - "\u0000\u0000\u0113\u0111\u0001\u0000\u0000\u0000\u0113\u0114\u0001\u0000"+ - "\u0000\u0000\u0114\u0117\u0001\u0000\u0000\u0000\u0115\u0113\u0001\u0000"+ - "\u0000\u0000\u0116\u010d\u0001\u0000\u0000\u0000\u0116\u010e\u0001\u0000"+ - "\u0000\u0000\u0116\u0117\u0001\u0000\u0000\u0000\u0117\u0118\u0001\u0000"+ - "\u0000\u0000\u0118\u0119\u00053\u0000\u0000\u0119\u0017\u0001\u0000\u0000"+ - "\u0000\u011a\u011b\u0003:\u001d\u0000\u011b\u0019\u0001\u0000\u0000\u0000"+ - "\u011c\u011d\u0005\r\u0000\u0000\u011d\u011e\u0003\u001c\u000e\u0000\u011e"+ - "\u001b\u0001\u0000\u0000\u0000\u011f\u0124\u0003\u001e\u000f\u0000\u0120"+ - "\u0121\u0005#\u0000\u0000\u0121\u0123\u0003\u001e\u000f\u0000\u0122\u0120"+ - "\u0001\u0000\u0000\u0000\u0123\u0126\u0001\u0000\u0000\u0000\u0124\u0122"+ - "\u0001\u0000\u0000\u0000\u0124\u0125\u0001\u0000\u0000\u0000\u0125\u001d"+ - "\u0001\u0000\u0000\u0000\u0126\u0124\u0001\u0000\u0000\u0000\u0127\u012d"+ - "\u0003\n\u0005\u0000\u0128\u0129\u00034\u001a\u0000\u0129\u012a\u0005"+ - "!\u0000\u0000\u012a\u012b\u0003\n\u0005\u0000\u012b\u012d\u0001\u0000"+ - "\u0000\u0000\u012c\u0127\u0001\u0000\u0000\u0000\u012c\u0128\u0001\u0000"+ - "\u0000\u0000\u012d\u001f\u0001\u0000\u0000\u0000\u012e\u012f\u0005\u0006"+ - "\u0000\u0000\u012f\u0134\u0003\"\u0011\u0000\u0130\u0131\u0005#\u0000"+ - "\u0000\u0131\u0133\u0003\"\u0011\u0000\u0132\u0130\u0001\u0000\u0000\u0000"+ - "\u0133\u0136\u0001\u0000\u0000\u0000\u0134\u0132\u0001\u0000\u0000\u0000"+ - "\u0134\u0135\u0001\u0000\u0000\u0000\u0135\u0138\u0001\u0000\u0000\u0000"+ - "\u0136\u0134\u0001\u0000\u0000\u0000\u0137\u0139\u0003(\u0014\u0000\u0138"+ - "\u0137\u0001\u0000\u0000\u0000\u0138\u0139\u0001\u0000\u0000\u0000\u0139"+ - "!\u0001\u0000\u0000\u0000\u013a\u013b\u0003$\u0012\u0000\u013b\u013c\u0005"+ - "m\u0000\u0000\u013c\u013d\u0003&\u0013\u0000\u013d\u0140\u0001\u0000\u0000"+ - "\u0000\u013e\u0140\u0003&\u0013\u0000\u013f\u013a\u0001\u0000\u0000\u0000"+ - "\u013f\u013e\u0001\u0000\u0000\u0000\u0140#\u0001\u0000\u0000\u0000\u0141"+ - "\u0142\u0005M\u0000\u0000\u0142%\u0001\u0000\u0000\u0000\u0143\u0144\u0007"+ - "\u0002\u0000\u0000\u0144\'\u0001\u0000\u0000\u0000\u0145\u0148\u0003*"+ - "\u0015\u0000\u0146\u0148\u0003,\u0016\u0000\u0147\u0145\u0001\u0000\u0000"+ - "\u0000\u0147\u0146\u0001\u0000\u0000\u0000\u0148)\u0001\u0000\u0000\u0000"+ - "\u0149\u014a\u0005L\u0000\u0000\u014a\u014f\u0005M\u0000\u0000\u014b\u014c"+ - "\u0005#\u0000\u0000\u014c\u014e\u0005M\u0000\u0000\u014d\u014b\u0001\u0000"+ - "\u0000\u0000\u014e\u0151\u0001\u0000\u0000\u0000\u014f\u014d\u0001\u0000"+ - "\u0000\u0000\u014f\u0150\u0001\u0000\u0000\u0000\u0150+\u0001\u0000\u0000"+ - "\u0000\u0151\u014f\u0001\u0000\u0000\u0000\u0152\u0153\u0005B\u0000\u0000"+ - "\u0153\u0154\u0003*\u0015\u0000\u0154\u0155\u0005C\u0000\u0000\u0155-"+ - "\u0001\u0000\u0000\u0000\u0156\u0157\u0005\u0015\u0000\u0000\u0157\u015c"+ - "\u0003\"\u0011\u0000\u0158\u0159\u0005#\u0000\u0000\u0159\u015b\u0003"+ - "\"\u0011\u0000\u015a\u0158\u0001\u0000\u0000\u0000\u015b\u015e\u0001\u0000"+ - "\u0000\u0000\u015c\u015a\u0001\u0000\u0000\u0000\u015c\u015d\u0001\u0000"+ - "\u0000\u0000\u015d\u0160\u0001\u0000\u0000\u0000\u015e\u015c\u0001\u0000"+ - "\u0000\u0000\u015f\u0161\u0003\u001c\u000e\u0000\u0160\u015f\u0001\u0000"+ - "\u0000\u0000\u0160\u0161\u0001\u0000\u0000\u0000\u0161\u0164\u0001\u0000"+ - "\u0000\u0000\u0162\u0163\u0005\u001e\u0000\u0000\u0163\u0165\u0003\u001c"+ - "\u000e\u0000\u0164\u0162\u0001\u0000\u0000\u0000\u0164\u0165\u0001\u0000"+ - "\u0000\u0000\u0165/\u0001\u0000\u0000\u0000\u0166\u0167\u0005\u0004\u0000"+ - "\u0000\u0167\u0168\u0003\u001c\u000e\u0000\u01681\u0001\u0000\u0000\u0000"+ - "\u0169\u016b\u0005\u0010\u0000\u0000\u016a\u016c\u0003\u001c\u000e\u0000"+ - "\u016b\u016a\u0001\u0000\u0000\u0000\u016b\u016c\u0001\u0000\u0000\u0000"+ - "\u016c\u016f\u0001\u0000\u0000\u0000\u016d\u016e\u0005\u001e\u0000\u0000"+ - "\u016e\u0170\u0003\u001c\u000e\u0000\u016f\u016d\u0001\u0000\u0000\u0000"+ - "\u016f\u0170\u0001\u0000\u0000\u0000\u01703\u0001\u0000\u0000\u0000\u0171"+ - "\u0176\u0003:\u001d\u0000\u0172\u0173\u0005%\u0000\u0000\u0173\u0175\u0003"+ - ":\u001d\u0000\u0174\u0172\u0001\u0000\u0000\u0000\u0175\u0178\u0001\u0000"+ - "\u0000\u0000\u0176\u0174\u0001\u0000\u0000\u0000\u0176\u0177\u0001\u0000"+ - "\u0000\u0000\u01775\u0001\u0000\u0000\u0000\u0178\u0176\u0001\u0000\u0000"+ - "\u0000\u0179\u017e\u0003<\u001e\u0000\u017a\u017b\u0005%\u0000\u0000\u017b"+ - "\u017d\u0003<\u001e\u0000\u017c\u017a\u0001\u0000\u0000\u0000\u017d\u0180"+ - "\u0001\u0000\u0000\u0000\u017e\u017c\u0001\u0000\u0000\u0000\u017e\u017f"+ - "\u0001\u0000\u0000\u0000\u017f7\u0001\u0000\u0000\u0000\u0180\u017e\u0001"+ - "\u0000\u0000\u0000\u0181\u0186\u00036\u001b\u0000\u0182\u0183\u0005#\u0000"+ - "\u0000\u0183\u0185\u00036\u001b\u0000\u0184\u0182\u0001\u0000\u0000\u0000"+ - "\u0185\u0188\u0001\u0000\u0000\u0000\u0186\u0184\u0001\u0000\u0000\u0000"+ - "\u0186\u0187\u0001\u0000\u0000\u0000\u01879\u0001\u0000\u0000\u0000\u0188"+ - "\u0186\u0001\u0000\u0000\u0000\u0189\u018a\u0007\u0003\u0000\u0000\u018a"+ - ";\u0001\u0000\u0000\u0000\u018b\u018c\u0005Q\u0000\u0000\u018c=\u0001"+ - "\u0000\u0000\u0000\u018d\u01b8\u0005.\u0000\u0000\u018e\u018f\u0003`0"+ - "\u0000\u018f\u0190\u0005D\u0000\u0000\u0190\u01b8\u0001\u0000\u0000\u0000"+ - "\u0191\u01b8\u0003^/\u0000\u0192\u01b8\u0003`0\u0000\u0193\u01b8\u0003"+ - "Z-\u0000\u0194\u01b8\u0003@ \u0000\u0195\u01b8\u0003b1\u0000\u0196\u0197"+ - "\u0005B\u0000\u0000\u0197\u019c\u0003\\.\u0000\u0198\u0199\u0005#\u0000"+ - "\u0000\u0199\u019b\u0003\\.\u0000\u019a\u0198\u0001\u0000\u0000\u0000"+ - "\u019b\u019e\u0001\u0000\u0000\u0000\u019c\u019a\u0001\u0000\u0000\u0000"+ - "\u019c\u019d\u0001\u0000\u0000\u0000\u019d\u019f\u0001\u0000\u0000\u0000"+ - "\u019e\u019c\u0001\u0000\u0000\u0000\u019f\u01a0\u0005C\u0000\u0000\u01a0"+ - "\u01b8\u0001\u0000\u0000\u0000\u01a1\u01a2\u0005B\u0000\u0000\u01a2\u01a7"+ - "\u0003Z-\u0000\u01a3\u01a4\u0005#\u0000\u0000\u01a4\u01a6\u0003Z-\u0000"+ - "\u01a5\u01a3\u0001\u0000\u0000\u0000\u01a6\u01a9\u0001\u0000\u0000\u0000"+ - "\u01a7\u01a5\u0001\u0000\u0000\u0000\u01a7\u01a8\u0001\u0000\u0000\u0000"+ - "\u01a8\u01aa\u0001\u0000\u0000\u0000\u01a9\u01a7\u0001\u0000\u0000\u0000"+ - "\u01aa\u01ab\u0005C\u0000\u0000\u01ab\u01b8\u0001\u0000\u0000\u0000\u01ac"+ - "\u01ad\u0005B\u0000\u0000\u01ad\u01b2\u0003b1\u0000\u01ae\u01af\u0005"+ - "#\u0000\u0000\u01af\u01b1\u0003b1\u0000\u01b0\u01ae\u0001\u0000\u0000"+ - "\u0000\u01b1\u01b4\u0001\u0000\u0000\u0000\u01b2\u01b0\u0001\u0000\u0000"+ - "\u0000\u01b2\u01b3\u0001\u0000\u0000\u0000\u01b3\u01b5\u0001\u0000\u0000"+ - "\u0000\u01b4\u01b2\u0001\u0000\u0000\u0000\u01b5\u01b6\u0005C\u0000\u0000"+ - "\u01b6\u01b8\u0001\u0000\u0000\u0000\u01b7\u018d\u0001\u0000\u0000\u0000"+ - "\u01b7\u018e\u0001\u0000\u0000\u0000\u01b7\u0191\u0001\u0000\u0000\u0000"+ - "\u01b7\u0192\u0001\u0000\u0000\u0000\u01b7\u0193\u0001\u0000\u0000\u0000"+ - "\u01b7\u0194\u0001\u0000\u0000\u0000\u01b7\u0195\u0001\u0000\u0000\u0000"+ - "\u01b7\u0196\u0001\u0000\u0000\u0000\u01b7\u01a1\u0001\u0000\u0000\u0000"+ - "\u01b7\u01ac\u0001\u0000\u0000\u0000\u01b8?\u0001\u0000\u0000\u0000\u01b9"+ - "\u01bc\u00051\u0000\u0000\u01ba\u01bc\u0005A\u0000\u0000\u01bb\u01b9\u0001"+ - "\u0000\u0000\u0000\u01bb\u01ba\u0001\u0000\u0000\u0000\u01bcA\u0001\u0000"+ - "\u0000\u0000\u01bd\u01be\u0005\t\u0000\u0000\u01be\u01bf\u0005\u001c\u0000"+ - "\u0000\u01bfC\u0001\u0000\u0000\u0000\u01c0\u01c1\u0005\u000f\u0000\u0000"+ - "\u01c1\u01c6\u0003F#\u0000\u01c2\u01c3\u0005#\u0000\u0000\u01c3\u01c5"+ - "\u0003F#\u0000\u01c4\u01c2\u0001\u0000\u0000\u0000\u01c5\u01c8\u0001\u0000"+ - "\u0000\u0000\u01c6\u01c4\u0001\u0000\u0000\u0000\u01c6\u01c7\u0001\u0000"+ - "\u0000\u0000\u01c7E\u0001\u0000\u0000\u0000\u01c8\u01c6\u0001\u0000\u0000"+ - "\u0000\u01c9\u01cb\u0003\n\u0005\u0000\u01ca\u01cc\u0007\u0004\u0000\u0000"+ - "\u01cb\u01ca\u0001\u0000\u0000\u0000\u01cb\u01cc\u0001\u0000\u0000\u0000"+ - "\u01cc\u01cf\u0001\u0000\u0000\u0000\u01cd\u01ce\u0005/\u0000\u0000\u01ce"+ - "\u01d0\u0007\u0005\u0000\u0000\u01cf\u01cd\u0001\u0000\u0000\u0000\u01cf"+ - "\u01d0\u0001\u0000\u0000\u0000\u01d0G\u0001\u0000\u0000\u0000\u01d1\u01d2"+ - "\u0005\b\u0000\u0000\u01d2\u01d3\u00038\u001c\u0000\u01d3I\u0001\u0000"+ - "\u0000\u0000\u01d4\u01d5\u0005\u0002\u0000\u0000\u01d5\u01d6\u00038\u001c"+ - "\u0000\u01d6K\u0001\u0000\u0000\u0000\u01d7\u01d8\u0005\f\u0000\u0000"+ - "\u01d8\u01dd\u0003N\'\u0000\u01d9\u01da\u0005#\u0000\u0000\u01da\u01dc"+ - "\u0003N\'\u0000\u01db\u01d9\u0001\u0000\u0000\u0000\u01dc\u01df\u0001"+ - "\u0000\u0000\u0000\u01dd\u01db\u0001\u0000\u0000\u0000\u01dd\u01de\u0001"+ - "\u0000\u0000\u0000\u01deM\u0001\u0000\u0000\u0000\u01df\u01dd\u0001\u0000"+ - "\u0000\u0000\u01e0\u01e1\u00036\u001b\u0000\u01e1\u01e2\u0005U\u0000\u0000"+ - "\u01e2\u01e3\u00036\u001b\u0000\u01e3O\u0001\u0000\u0000\u0000\u01e4\u01e5"+ - "\u0005\u0001\u0000\u0000\u01e5\u01e6\u0003\u0014\n\u0000\u01e6\u01e8\u0003"+ - "b1\u0000\u01e7\u01e9\u0003V+\u0000\u01e8\u01e7\u0001\u0000\u0000\u0000"+ - "\u01e8\u01e9\u0001\u0000\u0000\u0000\u01e9Q\u0001\u0000\u0000\u0000\u01ea"+ - "\u01eb\u0005\u0007\u0000\u0000\u01eb\u01ec\u0003\u0014\n\u0000\u01ec\u01ed"+ - "\u0003b1\u0000\u01edS\u0001\u0000\u0000\u0000\u01ee\u01ef\u0005\u000b"+ - "\u0000\u0000\u01ef\u01f0\u00034\u001a\u0000\u01f0U\u0001\u0000\u0000\u0000"+ - "\u01f1\u01f6\u0003X,\u0000\u01f2\u01f3\u0005#\u0000\u0000\u01f3\u01f5"+ - "\u0003X,\u0000\u01f4\u01f2\u0001\u0000\u0000\u0000\u01f5\u01f8\u0001\u0000"+ - "\u0000\u0000\u01f6\u01f4\u0001\u0000\u0000\u0000\u01f6\u01f7\u0001\u0000"+ - "\u0000\u0000\u01f7W\u0001\u0000\u0000\u0000\u01f8\u01f6\u0001\u0000\u0000"+ - "\u0000\u01f9\u01fa\u0003:\u001d\u0000\u01fa\u01fb\u0005!\u0000\u0000\u01fb"+ - "\u01fc\u0003>\u001f\u0000\u01fcY\u0001\u0000\u0000\u0000\u01fd\u01fe\u0007"+ - "\u0006\u0000\u0000\u01fe[\u0001\u0000\u0000\u0000\u01ff\u0202\u0003^/"+ - "\u0000\u0200\u0202\u0003`0\u0000\u0201\u01ff\u0001\u0000\u0000\u0000\u0201"+ - "\u0200\u0001\u0000\u0000\u0000\u0202]\u0001\u0000\u0000\u0000\u0203\u0205"+ - "\u0007\u0000\u0000\u0000\u0204\u0203\u0001\u0000\u0000\u0000\u0204\u0205"+ - "\u0001\u0000\u0000\u0000\u0205\u0206\u0001\u0000\u0000\u0000\u0206\u0207"+ - "\u0005\u001d\u0000\u0000\u0207_\u0001\u0000\u0000\u0000\u0208\u020a\u0007"+ - "\u0000\u0000\u0000\u0209\u0208\u0001\u0000\u0000\u0000\u0209\u020a\u0001"+ - "\u0000\u0000\u0000\u020a\u020b\u0001\u0000\u0000\u0000\u020b\u020c\u0005"+ - "\u001c\u0000\u0000\u020ca\u0001\u0000\u0000\u0000\u020d\u020e\u0005\u001b"+ - "\u0000\u0000\u020ec\u0001\u0000\u0000\u0000\u020f\u0210\u0007\u0007\u0000"+ - "\u0000\u0210e\u0001\u0000\u0000\u0000\u0211\u0212\u0005\u0005\u0000\u0000"+ - "\u0212\u0213\u0003h4\u0000\u0213g\u0001\u0000\u0000\u0000\u0214\u0215"+ - "\u0005B\u0000\u0000\u0215\u0216\u0003\u0002\u0001\u0000\u0216\u0217\u0005"+ - "C\u0000\u0000\u0217i\u0001\u0000\u0000\u0000\u0218\u0219\u0005\u000e\u0000"+ - "\u0000\u0219\u021a\u0005e\u0000\u0000\u021ak\u0001\u0000\u0000\u0000\u021b"+ - "\u021c\u0005\n\u0000\u0000\u021c\u021d\u0005i\u0000\u0000\u021dm\u0001"+ - "\u0000\u0000\u0000\u021e\u021f\u0005\u0003\u0000\u0000\u021f\u0222\u0005"+ - "[\u0000\u0000\u0220\u0221\u0005Y\u0000\u0000\u0221\u0223\u00036\u001b"+ - "\u0000\u0222\u0220\u0001\u0000\u0000\u0000\u0222\u0223\u0001\u0000\u0000"+ - "\u0000\u0223\u022d\u0001\u0000\u0000\u0000\u0224\u0225\u0005Z\u0000\u0000"+ - "\u0225\u022a\u0003p8\u0000\u0226\u0227\u0005#\u0000\u0000\u0227\u0229"+ - "\u0003p8\u0000\u0228\u0226\u0001\u0000\u0000\u0000\u0229\u022c\u0001\u0000"+ - "\u0000\u0000\u022a\u0228\u0001\u0000\u0000\u0000\u022a\u022b\u0001\u0000"+ - "\u0000\u0000\u022b\u022e\u0001\u0000\u0000\u0000\u022c\u022a\u0001\u0000"+ - "\u0000\u0000\u022d\u0224\u0001\u0000\u0000\u0000\u022d\u022e\u0001\u0000"+ - "\u0000\u0000\u022eo\u0001\u0000\u0000\u0000\u022f\u0230\u00036\u001b\u0000"+ - "\u0230\u0231\u0005!\u0000\u0000\u0231\u0233\u0001\u0000\u0000\u0000\u0232"+ - "\u022f\u0001\u0000\u0000\u0000\u0232\u0233\u0001\u0000\u0000\u0000\u0233"+ - "\u0234\u0001\u0000\u0000\u0000\u0234\u0235\u00036\u001b\u0000\u0235q\u0001"+ - "\u0000\u0000\u0000\u0236\u0237\u0005\u0013\u0000\u0000\u0237\u0238\u0003"+ - "\"\u0011\u0000\u0238\u0239\u0005Y\u0000\u0000\u0239\u023a\u00038\u001c"+ - "\u0000\u023as\u0001\u0000\u0000\u0000\u023b\u023c\u0005\u0012\u0000\u0000"+ - "\u023c\u023f\u0003\u001c\u000e\u0000\u023d\u023e\u0005\u001e\u0000\u0000"+ - "\u023e\u0240\u0003\u001c\u000e\u0000\u023f\u023d\u0001\u0000\u0000\u0000"+ - "\u023f\u0240\u0001\u0000\u0000\u0000\u0240u\u0001\u0000\u0000\u00006\u0081"+ - "\u008b\u009d\u00a9\u00b2\u00ba\u00c0\u00c8\u00ca\u00cf\u00d6\u00db\u00e6"+ - "\u00ec\u00f4\u00f6\u0101\u0108\u0113\u0116\u0124\u012c\u0134\u0138\u013f"+ - "\u0147\u014f\u015c\u0160\u0164\u016b\u016f\u0176\u017e\u0186\u019c\u01a7"+ - "\u01b2\u01b7\u01bb\u01c6\u01cb\u01cf\u01dd\u01e8\u01f6\u0201\u0204\u0209"+ - "\u0222\u022a\u022d\u0232\u023f"; + "\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u0198\b\u001f"+ + "\n\u001f\f\u001f\u019b\t\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001"+ + "\u001f\u0001\u001f\u0001\u001f\u0005\u001f\u01a3\b\u001f\n\u001f\f\u001f"+ + "\u01a6\t\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f\u0001\u001f"+ + "\u0001\u001f\u0005\u001f\u01ae\b\u001f\n\u001f\f\u001f\u01b1\t\u001f\u0001"+ + "\u001f\u0001\u001f\u0003\u001f\u01b5\b\u001f\u0001 \u0001 \u0003 \u01b9"+ + "\b \u0001!\u0001!\u0001!\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01c2"+ + "\b\"\n\"\f\"\u01c5\t\"\u0001#\u0001#\u0003#\u01c9\b#\u0001#\u0001#\u0003"+ + "#\u01cd\b#\u0001$\u0001$\u0001$\u0001%\u0001%\u0001%\u0001&\u0001&\u0001"+ + "&\u0001&\u0005&\u01d9\b&\n&\f&\u01dc\t&\u0001\'\u0001\'\u0001\'\u0001"+ + "\'\u0001(\u0001(\u0001(\u0001(\u0003(\u01e6\b(\u0001)\u0001)\u0001)\u0001"+ + ")\u0001*\u0001*\u0001*\u0001+\u0001+\u0001+\u0005+\u01f2\b+\n+\f+\u01f5"+ + "\t+\u0001,\u0001,\u0001,\u0001,\u0001-\u0001-\u0001.\u0001.\u0003.\u01ff"+ + "\b.\u0001/\u0003/\u0202\b/\u0001/\u0001/\u00010\u00030\u0207\b0\u0001"+ + "0\u00010\u00011\u00011\u00012\u00012\u00013\u00013\u00013\u00014\u0001"+ + "4\u00014\u00014\u00015\u00015\u00015\u00016\u00016\u00016\u00016\u0003"+ + "6\u021d\b6\u00016\u00016\u00016\u00016\u00056\u0223\b6\n6\f6\u0226\t6"+ + "\u00036\u0228\b6\u00017\u00017\u00017\u00037\u022d\b7\u00017\u00017\u0001"+ + "8\u00018\u00018\u00018\u00018\u00019\u00019\u00019\u00019\u00039\u023a"+ + "\b9\u00019\u0000\u0004\u0002\n\u0012\u0014:\u0000\u0002\u0004\u0006\b"+ + "\n\f\u000e\u0010\u0012\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02"+ + "468:<>@BDFHJLNPRTVXZ\\^`bdfhjlnpr\u0000\b\u0001\u0000;<\u0001\u0000=?"+ + "\u0002\u0000\u001a\u001aLL\u0001\u0000CD\u0002\u0000\u001f\u001f##\u0002"+ + "\u0000&&))\u0002\u0000%%33\u0002\u0000446:\u0255\u0000t\u0001\u0000\u0000"+ + "\u0000\u0002w\u0001\u0000\u0000\u0000\u0004\u0088\u0001\u0000\u0000\u0000"+ + "\u0006\u009a\u0001\u0000\u0000\u0000\b\u009c\u0001\u0000\u0000\u0000\n"+ + "\u00bd\u0001\u0000\u0000\u0000\f\u00d8\u0001\u0000\u0000\u0000\u000e\u00da"+ + "\u0001\u0000\u0000\u0000\u0010\u00e3\u0001\u0000\u0000\u0000\u0012\u00e9"+ + "\u0001\u0000\u0000\u0000\u0014\u00fe\u0001\u0000\u0000\u0000\u0016\u0108"+ + "\u0001\u0000\u0000\u0000\u0018\u0117\u0001\u0000\u0000\u0000\u001a\u0119"+ + "\u0001\u0000\u0000\u0000\u001c\u011c\u0001\u0000\u0000\u0000\u001e\u0129"+ + "\u0001\u0000\u0000\u0000 \u012b\u0001\u0000\u0000\u0000\"\u013c\u0001"+ + "\u0000\u0000\u0000$\u013e\u0001\u0000\u0000\u0000&\u0140\u0001\u0000\u0000"+ + "\u0000(\u0144\u0001\u0000\u0000\u0000*\u0146\u0001\u0000\u0000\u0000,"+ + "\u014f\u0001\u0000\u0000\u0000.\u0153\u0001\u0000\u0000\u00000\u0163\u0001"+ + "\u0000\u0000\u00002\u0166\u0001\u0000\u0000\u00004\u016e\u0001\u0000\u0000"+ + "\u00006\u0176\u0001\u0000\u0000\u00008\u017e\u0001\u0000\u0000\u0000:"+ + "\u0186\u0001\u0000\u0000\u0000<\u0188\u0001\u0000\u0000\u0000>\u01b4\u0001"+ + "\u0000\u0000\u0000@\u01b8\u0001\u0000\u0000\u0000B\u01ba\u0001\u0000\u0000"+ + "\u0000D\u01bd\u0001\u0000\u0000\u0000F\u01c6\u0001\u0000\u0000\u0000H"+ + "\u01ce\u0001\u0000\u0000\u0000J\u01d1\u0001\u0000\u0000\u0000L\u01d4\u0001"+ + "\u0000\u0000\u0000N\u01dd\u0001\u0000\u0000\u0000P\u01e1\u0001\u0000\u0000"+ + "\u0000R\u01e7\u0001\u0000\u0000\u0000T\u01eb\u0001\u0000\u0000\u0000V"+ + "\u01ee\u0001\u0000\u0000\u0000X\u01f6\u0001\u0000\u0000\u0000Z\u01fa\u0001"+ + "\u0000\u0000\u0000\\\u01fe\u0001\u0000\u0000\u0000^\u0201\u0001\u0000"+ + "\u0000\u0000`\u0206\u0001\u0000\u0000\u0000b\u020a\u0001\u0000\u0000\u0000"+ + "d\u020c\u0001\u0000\u0000\u0000f\u020e\u0001\u0000\u0000\u0000h\u0211"+ + "\u0001\u0000\u0000\u0000j\u0215\u0001\u0000\u0000\u0000l\u0218\u0001\u0000"+ + "\u0000\u0000n\u022c\u0001\u0000\u0000\u0000p\u0230\u0001\u0000\u0000\u0000"+ + "r\u0235\u0001\u0000\u0000\u0000tu\u0003\u0002\u0001\u0000uv\u0005\u0000"+ + "\u0000\u0001v\u0001\u0001\u0000\u0000\u0000wx\u0006\u0001\uffff\uffff"+ + "\u0000xy\u0003\u0004\u0002\u0000y\u007f\u0001\u0000\u0000\u0000z{\n\u0001"+ + "\u0000\u0000{|\u0005\u0019\u0000\u0000|~\u0003\u0006\u0003\u0000}z\u0001"+ + "\u0000\u0000\u0000~\u0081\u0001\u0000\u0000\u0000\u007f}\u0001\u0000\u0000"+ + "\u0000\u007f\u0080\u0001\u0000\u0000\u0000\u0080\u0003\u0001\u0000\u0000"+ + "\u0000\u0081\u007f\u0001\u0000\u0000\u0000\u0082\u0089\u0003f3\u0000\u0083"+ + "\u0089\u0003 \u0010\u0000\u0084\u0089\u0003\u001a\r\u0000\u0085\u0089"+ + "\u0003j5\u0000\u0086\u0087\u0004\u0002\u0001\u0000\u0087\u0089\u0003."+ + "\u0017\u0000\u0088\u0082\u0001\u0000\u0000\u0000\u0088\u0083\u0001\u0000"+ + "\u0000\u0000\u0088\u0084\u0001\u0000\u0000\u0000\u0088\u0085\u0001\u0000"+ + "\u0000\u0000\u0088\u0086\u0001\u0000\u0000\u0000\u0089\u0005\u0001\u0000"+ + "\u0000\u0000\u008a\u009b\u00030\u0018\u0000\u008b\u009b\u0003\b\u0004"+ + "\u0000\u008c\u009b\u0003H$\u0000\u008d\u009b\u0003B!\u0000\u008e\u009b"+ + "\u00032\u0019\u0000\u008f\u009b\u0003D\"\u0000\u0090\u009b\u0003J%\u0000"+ + "\u0091\u009b\u0003L&\u0000\u0092\u009b\u0003P(\u0000\u0093\u009b\u0003"+ + "R)\u0000\u0094\u009b\u0003l6\u0000\u0095\u009b\u0003T*\u0000\u0096\u0097"+ + "\u0004\u0003\u0002\u0000\u0097\u009b\u0003r9\u0000\u0098\u0099\u0004\u0003"+ + "\u0003\u0000\u0099\u009b\u0003p8\u0000\u009a\u008a\u0001\u0000\u0000\u0000"+ + "\u009a\u008b\u0001\u0000\u0000\u0000\u009a\u008c\u0001\u0000\u0000\u0000"+ + "\u009a\u008d\u0001\u0000\u0000\u0000\u009a\u008e\u0001\u0000\u0000\u0000"+ + "\u009a\u008f\u0001\u0000\u0000\u0000\u009a\u0090\u0001\u0000\u0000\u0000"+ + "\u009a\u0091\u0001\u0000\u0000\u0000\u009a\u0092\u0001\u0000\u0000\u0000"+ + "\u009a\u0093\u0001\u0000\u0000\u0000\u009a\u0094\u0001\u0000\u0000\u0000"+ + "\u009a\u0095\u0001\u0000\u0000\u0000\u009a\u0096\u0001\u0000\u0000\u0000"+ + "\u009a\u0098\u0001\u0000\u0000\u0000\u009b\u0007\u0001\u0000\u0000\u0000"+ + "\u009c\u009d\u0005\u0010\u0000\u0000\u009d\u009e\u0003\n\u0005\u0000\u009e"+ + "\t\u0001\u0000\u0000\u0000\u009f\u00a0\u0006\u0005\uffff\uffff\u0000\u00a0"+ + "\u00a1\u0005,\u0000\u0000\u00a1\u00be\u0003\n\u0005\b\u00a2\u00be\u0003"+ + "\u0010\b\u0000\u00a3\u00be\u0003\f\u0006\u0000\u00a4\u00a6\u0003\u0010"+ + "\b\u0000\u00a5\u00a7\u0005,\u0000\u0000\u00a6\u00a5\u0001\u0000\u0000"+ + "\u0000\u00a6\u00a7\u0001\u0000\u0000\u0000\u00a7\u00a8\u0001\u0000\u0000"+ + "\u0000\u00a8\u00a9\u0005\'\u0000\u0000\u00a9\u00aa\u0005+\u0000\u0000"+ + "\u00aa\u00af\u0003\u0010\b\u0000\u00ab\u00ac\u0005\"\u0000\u0000\u00ac"+ + "\u00ae\u0003\u0010\b\u0000\u00ad\u00ab\u0001\u0000\u0000\u0000\u00ae\u00b1"+ + "\u0001\u0000\u0000\u0000\u00af\u00ad\u0001\u0000\u0000\u0000\u00af\u00b0"+ + "\u0001\u0000\u0000\u0000\u00b0\u00b2\u0001\u0000\u0000\u0000\u00b1\u00af"+ + "\u0001\u0000\u0000\u0000\u00b2\u00b3\u00052\u0000\u0000\u00b3\u00be\u0001"+ + "\u0000\u0000\u0000\u00b4\u00b5\u0003\u0010\b\u0000\u00b5\u00b7\u0005("+ + "\u0000\u0000\u00b6\u00b8\u0005,\u0000\u0000\u00b7\u00b6\u0001\u0000\u0000"+ + "\u0000\u00b7\u00b8\u0001\u0000\u0000\u0000\u00b8\u00b9\u0001\u0000\u0000"+ + "\u0000\u00b9\u00ba\u0005-\u0000\u0000\u00ba\u00be\u0001\u0000\u0000\u0000"+ + "\u00bb\u00bc\u0004\u0005\u0004\u0000\u00bc\u00be\u0003\u000e\u0007\u0000"+ + "\u00bd\u009f\u0001\u0000\u0000\u0000\u00bd\u00a2\u0001\u0000\u0000\u0000"+ + "\u00bd\u00a3\u0001\u0000\u0000\u0000\u00bd\u00a4\u0001\u0000\u0000\u0000"+ + "\u00bd\u00b4\u0001\u0000\u0000\u0000\u00bd\u00bb\u0001\u0000\u0000\u0000"+ + "\u00be\u00c7\u0001\u0000\u0000\u0000\u00bf\u00c0\n\u0005\u0000\u0000\u00c0"+ + "\u00c1\u0005\u001e\u0000\u0000\u00c1\u00c6\u0003\n\u0005\u0006\u00c2\u00c3"+ + "\n\u0004\u0000\u0000\u00c3\u00c4\u0005/\u0000\u0000\u00c4\u00c6\u0003"+ + "\n\u0005\u0005\u00c5\u00bf\u0001\u0000\u0000\u0000\u00c5\u00c2\u0001\u0000"+ + "\u0000\u0000\u00c6\u00c9\u0001\u0000\u0000\u0000\u00c7\u00c5\u0001\u0000"+ + "\u0000\u0000\u00c7\u00c8\u0001\u0000\u0000\u0000\u00c8\u000b\u0001\u0000"+ + "\u0000\u0000\u00c9\u00c7\u0001\u0000\u0000\u0000\u00ca\u00cc\u0003\u0010"+ + "\b\u0000\u00cb\u00cd\u0005,\u0000\u0000\u00cc\u00cb\u0001\u0000\u0000"+ + "\u0000\u00cc\u00cd\u0001\u0000\u0000\u0000\u00cd\u00ce\u0001\u0000\u0000"+ + "\u0000\u00ce\u00cf\u0005*\u0000\u0000\u00cf\u00d0\u0003b1\u0000\u00d0"+ + "\u00d9\u0001\u0000\u0000\u0000\u00d1\u00d3\u0003\u0010\b\u0000\u00d2\u00d4"+ + "\u0005,\u0000\u0000\u00d3\u00d2\u0001\u0000\u0000\u0000\u00d3\u00d4\u0001"+ + "\u0000\u0000\u0000\u00d4\u00d5\u0001\u0000\u0000\u0000\u00d5\u00d6\u0005"+ + "1\u0000\u0000\u00d6\u00d7\u0003b1\u0000\u00d7\u00d9\u0001\u0000\u0000"+ + "\u0000\u00d8\u00ca\u0001\u0000\u0000\u0000\u00d8\u00d1\u0001\u0000\u0000"+ + "\u0000\u00d9\r\u0001\u0000\u0000\u0000\u00da\u00db\u0003\u0010\b\u0000"+ + "\u00db\u00dc\u0005\u0013\u0000\u0000\u00dc\u00dd\u0003b1\u0000\u00dd\u000f"+ + "\u0001\u0000\u0000\u0000\u00de\u00e4\u0003\u0012\t\u0000\u00df\u00e0\u0003"+ + "\u0012\t\u0000\u00e0\u00e1\u0003d2\u0000\u00e1\u00e2\u0003\u0012\t\u0000"+ + "\u00e2\u00e4\u0001\u0000\u0000\u0000\u00e3\u00de\u0001\u0000\u0000\u0000"+ + "\u00e3\u00df\u0001\u0000\u0000\u0000\u00e4\u0011\u0001\u0000\u0000\u0000"+ + "\u00e5\u00e6\u0006\t\uffff\uffff\u0000\u00e6\u00ea\u0003\u0014\n\u0000"+ + "\u00e7\u00e8\u0007\u0000\u0000\u0000\u00e8\u00ea\u0003\u0012\t\u0003\u00e9"+ + "\u00e5\u0001\u0000\u0000\u0000\u00e9\u00e7\u0001\u0000\u0000\u0000\u00ea"+ + "\u00f3\u0001\u0000\u0000\u0000\u00eb\u00ec\n\u0002\u0000\u0000\u00ec\u00ed"+ + "\u0007\u0001\u0000\u0000\u00ed\u00f2\u0003\u0012\t\u0003\u00ee\u00ef\n"+ + "\u0001\u0000\u0000\u00ef\u00f0\u0007\u0000\u0000\u0000\u00f0\u00f2\u0003"+ + "\u0012\t\u0002\u00f1\u00eb\u0001\u0000\u0000\u0000\u00f1\u00ee\u0001\u0000"+ + "\u0000\u0000\u00f2\u00f5\u0001\u0000\u0000\u0000\u00f3\u00f1\u0001\u0000"+ + "\u0000\u0000\u00f3\u00f4\u0001\u0000\u0000\u0000\u00f4\u0013\u0001\u0000"+ + "\u0000\u0000\u00f5\u00f3\u0001\u0000\u0000\u0000\u00f6\u00f7\u0006\n\uffff"+ + "\uffff\u0000\u00f7\u00ff\u0003>\u001f\u0000\u00f8\u00ff\u00034\u001a\u0000"+ + "\u00f9\u00ff\u0003\u0016\u000b\u0000\u00fa\u00fb\u0005+\u0000\u0000\u00fb"+ + "\u00fc\u0003\n\u0005\u0000\u00fc\u00fd\u00052\u0000\u0000\u00fd\u00ff"+ + "\u0001\u0000\u0000\u0000\u00fe\u00f6\u0001\u0000\u0000\u0000\u00fe\u00f8"+ + "\u0001\u0000\u0000\u0000\u00fe\u00f9\u0001\u0000\u0000\u0000\u00fe\u00fa"+ + "\u0001\u0000\u0000\u0000\u00ff\u0105\u0001\u0000\u0000\u0000\u0100\u0101"+ + "\n\u0001\u0000\u0000\u0101\u0102\u0005!\u0000\u0000\u0102\u0104\u0003"+ + "\u0018\f\u0000\u0103\u0100\u0001\u0000\u0000\u0000\u0104\u0107\u0001\u0000"+ + "\u0000\u0000\u0105\u0103\u0001\u0000\u0000\u0000\u0105\u0106\u0001\u0000"+ + "\u0000\u0000\u0106\u0015\u0001\u0000\u0000\u0000\u0107\u0105\u0001\u0000"+ + "\u0000\u0000\u0108\u0109\u0003:\u001d\u0000\u0109\u0113\u0005+\u0000\u0000"+ + "\u010a\u0114\u0005=\u0000\u0000\u010b\u0110\u0003\n\u0005\u0000\u010c"+ + "\u010d\u0005\"\u0000\u0000\u010d\u010f\u0003\n\u0005\u0000\u010e\u010c"+ + "\u0001\u0000\u0000\u0000\u010f\u0112\u0001\u0000\u0000\u0000\u0110\u010e"+ + "\u0001\u0000\u0000\u0000\u0110\u0111\u0001\u0000\u0000\u0000\u0111\u0114"+ + "\u0001\u0000\u0000\u0000\u0112\u0110\u0001\u0000\u0000\u0000\u0113\u010a"+ + "\u0001\u0000\u0000\u0000\u0113\u010b\u0001\u0000\u0000\u0000\u0113\u0114"+ + "\u0001\u0000\u0000\u0000\u0114\u0115\u0001\u0000\u0000\u0000\u0115\u0116"+ + "\u00052\u0000\u0000\u0116\u0017\u0001\u0000\u0000\u0000\u0117\u0118\u0003"+ + ":\u001d\u0000\u0118\u0019\u0001\u0000\u0000\u0000\u0119\u011a\u0005\f"+ + "\u0000\u0000\u011a\u011b\u0003\u001c\u000e\u0000\u011b\u001b\u0001\u0000"+ + "\u0000\u0000\u011c\u0121\u0003\u001e\u000f\u0000\u011d\u011e\u0005\"\u0000"+ + "\u0000\u011e\u0120\u0003\u001e\u000f\u0000\u011f\u011d\u0001\u0000\u0000"+ + "\u0000\u0120\u0123\u0001\u0000\u0000\u0000\u0121\u011f\u0001\u0000\u0000"+ + "\u0000\u0121\u0122\u0001\u0000\u0000\u0000\u0122\u001d\u0001\u0000\u0000"+ + "\u0000\u0123\u0121\u0001\u0000\u0000\u0000\u0124\u012a\u0003\n\u0005\u0000"+ + "\u0125\u0126\u00034\u001a\u0000\u0126\u0127\u0005 \u0000\u0000\u0127\u0128"+ + "\u0003\n\u0005\u0000\u0128\u012a\u0001\u0000\u0000\u0000\u0129\u0124\u0001"+ + "\u0000\u0000\u0000\u0129\u0125\u0001\u0000\u0000\u0000\u012a\u001f\u0001"+ + "\u0000\u0000\u0000\u012b\u012c\u0005\u0006\u0000\u0000\u012c\u0131\u0003"+ + "\"\u0011\u0000\u012d\u012e\u0005\"\u0000\u0000\u012e\u0130\u0003\"\u0011"+ + "\u0000\u012f\u012d\u0001\u0000\u0000\u0000\u0130\u0133\u0001\u0000\u0000"+ + "\u0000\u0131\u012f\u0001\u0000\u0000\u0000\u0131\u0132\u0001\u0000\u0000"+ + "\u0000\u0132\u0135\u0001\u0000\u0000\u0000\u0133\u0131\u0001\u0000\u0000"+ + "\u0000\u0134\u0136\u0003(\u0014\u0000\u0135\u0134\u0001\u0000\u0000\u0000"+ + "\u0135\u0136\u0001\u0000\u0000\u0000\u0136!\u0001\u0000\u0000\u0000\u0137"+ + "\u0138\u0003$\u0012\u0000\u0138\u0139\u0005h\u0000\u0000\u0139\u013a\u0003"+ + "&\u0013\u0000\u013a\u013d\u0001\u0000\u0000\u0000\u013b\u013d\u0003&\u0013"+ + "\u0000\u013c\u0137\u0001\u0000\u0000\u0000\u013c\u013b\u0001\u0000\u0000"+ + "\u0000\u013d#\u0001\u0000\u0000\u0000\u013e\u013f\u0005L\u0000\u0000\u013f"+ + "%\u0001\u0000\u0000\u0000\u0140\u0141\u0007\u0002\u0000\u0000\u0141\'"+ + "\u0001\u0000\u0000\u0000\u0142\u0145\u0003*\u0015\u0000\u0143\u0145\u0003"+ + ",\u0016\u0000\u0144\u0142\u0001\u0000\u0000\u0000\u0144\u0143\u0001\u0000"+ + "\u0000\u0000\u0145)\u0001\u0000\u0000\u0000\u0146\u0147\u0005K\u0000\u0000"+ + "\u0147\u014c\u0005L\u0000\u0000\u0148\u0149\u0005\"\u0000\u0000\u0149"+ + "\u014b\u0005L\u0000\u0000\u014a\u0148\u0001\u0000\u0000\u0000\u014b\u014e"+ + "\u0001\u0000\u0000\u0000\u014c\u014a\u0001\u0000\u0000\u0000\u014c\u014d"+ + "\u0001\u0000\u0000\u0000\u014d+\u0001\u0000\u0000\u0000\u014e\u014c\u0001"+ + "\u0000\u0000\u0000\u014f\u0150\u0005A\u0000\u0000\u0150\u0151\u0003*\u0015"+ + "\u0000\u0151\u0152\u0005B\u0000\u0000\u0152-\u0001\u0000\u0000\u0000\u0153"+ + "\u0154\u0005\u0014\u0000\u0000\u0154\u0159\u0003\"\u0011\u0000\u0155\u0156"+ + "\u0005\"\u0000\u0000\u0156\u0158\u0003\"\u0011\u0000\u0157\u0155\u0001"+ + "\u0000\u0000\u0000\u0158\u015b\u0001\u0000\u0000\u0000\u0159\u0157\u0001"+ + "\u0000\u0000\u0000\u0159\u015a\u0001\u0000\u0000\u0000\u015a\u015d\u0001"+ + "\u0000\u0000\u0000\u015b\u0159\u0001\u0000\u0000\u0000\u015c\u015e\u0003"+ + "\u001c\u000e\u0000\u015d\u015c\u0001\u0000\u0000\u0000\u015d\u015e\u0001"+ + "\u0000\u0000\u0000\u015e\u0161\u0001\u0000\u0000\u0000\u015f\u0160\u0005"+ + "\u001d\u0000\u0000\u0160\u0162\u0003\u001c\u000e\u0000\u0161\u015f\u0001"+ + "\u0000\u0000\u0000\u0161\u0162\u0001\u0000\u0000\u0000\u0162/\u0001\u0000"+ + "\u0000\u0000\u0163\u0164\u0005\u0004\u0000\u0000\u0164\u0165\u0003\u001c"+ + "\u000e\u0000\u01651\u0001\u0000\u0000\u0000\u0166\u0168\u0005\u000f\u0000"+ + "\u0000\u0167\u0169\u0003\u001c\u000e\u0000\u0168\u0167\u0001\u0000\u0000"+ + "\u0000\u0168\u0169\u0001\u0000\u0000\u0000\u0169\u016c\u0001\u0000\u0000"+ + "\u0000\u016a\u016b\u0005\u001d\u0000\u0000\u016b\u016d\u0003\u001c\u000e"+ + "\u0000\u016c\u016a\u0001\u0000\u0000\u0000\u016c\u016d\u0001\u0000\u0000"+ + "\u0000\u016d3\u0001\u0000\u0000\u0000\u016e\u0173\u0003:\u001d\u0000\u016f"+ + "\u0170\u0005$\u0000\u0000\u0170\u0172\u0003:\u001d\u0000\u0171\u016f\u0001"+ + "\u0000\u0000\u0000\u0172\u0175\u0001\u0000\u0000\u0000\u0173\u0171\u0001"+ + "\u0000\u0000\u0000\u0173\u0174\u0001\u0000\u0000\u0000\u01745\u0001\u0000"+ + "\u0000\u0000\u0175\u0173\u0001\u0000\u0000\u0000\u0176\u017b\u0003<\u001e"+ + "\u0000\u0177\u0178\u0005$\u0000\u0000\u0178\u017a\u0003<\u001e\u0000\u0179"+ + "\u0177\u0001\u0000\u0000\u0000\u017a\u017d\u0001\u0000\u0000\u0000\u017b"+ + "\u0179\u0001\u0000\u0000\u0000\u017b\u017c\u0001\u0000\u0000\u0000\u017c"+ + "7\u0001\u0000\u0000\u0000\u017d\u017b\u0001\u0000\u0000\u0000\u017e\u0183"+ + "\u00036\u001b\u0000\u017f\u0180\u0005\"\u0000\u0000\u0180\u0182\u0003"+ + "6\u001b\u0000\u0181\u017f\u0001\u0000\u0000\u0000\u0182\u0185\u0001\u0000"+ + "\u0000\u0000\u0183\u0181\u0001\u0000\u0000\u0000\u0183\u0184\u0001\u0000"+ + "\u0000\u0000\u01849\u0001\u0000\u0000\u0000\u0185\u0183\u0001\u0000\u0000"+ + "\u0000\u0186\u0187\u0007\u0003\u0000\u0000\u0187;\u0001\u0000\u0000\u0000"+ + "\u0188\u0189\u0005P\u0000\u0000\u0189=\u0001\u0000\u0000\u0000\u018a\u01b5"+ + "\u0005-\u0000\u0000\u018b\u018c\u0003`0\u0000\u018c\u018d\u0005C\u0000"+ + "\u0000\u018d\u01b5\u0001\u0000\u0000\u0000\u018e\u01b5\u0003^/\u0000\u018f"+ + "\u01b5\u0003`0\u0000\u0190\u01b5\u0003Z-\u0000\u0191\u01b5\u0003@ \u0000"+ + "\u0192\u01b5\u0003b1\u0000\u0193\u0194\u0005A\u0000\u0000\u0194\u0199"+ + "\u0003\\.\u0000\u0195\u0196\u0005\"\u0000\u0000\u0196\u0198\u0003\\.\u0000"+ + "\u0197\u0195\u0001\u0000\u0000\u0000\u0198\u019b\u0001\u0000\u0000\u0000"+ + "\u0199\u0197\u0001\u0000\u0000\u0000\u0199\u019a\u0001\u0000\u0000\u0000"+ + "\u019a\u019c\u0001\u0000\u0000\u0000\u019b\u0199\u0001\u0000\u0000\u0000"+ + "\u019c\u019d\u0005B\u0000\u0000\u019d\u01b5\u0001\u0000\u0000\u0000\u019e"+ + "\u019f\u0005A\u0000\u0000\u019f\u01a4\u0003Z-\u0000\u01a0\u01a1\u0005"+ + "\"\u0000\u0000\u01a1\u01a3\u0003Z-\u0000\u01a2\u01a0\u0001\u0000\u0000"+ + "\u0000\u01a3\u01a6\u0001\u0000\u0000\u0000\u01a4\u01a2\u0001\u0000\u0000"+ + "\u0000\u01a4\u01a5\u0001\u0000\u0000\u0000\u01a5\u01a7\u0001\u0000\u0000"+ + "\u0000\u01a6\u01a4\u0001\u0000\u0000\u0000\u01a7\u01a8\u0005B\u0000\u0000"+ + "\u01a8\u01b5\u0001\u0000\u0000\u0000\u01a9\u01aa\u0005A\u0000\u0000\u01aa"+ + "\u01af\u0003b1\u0000\u01ab\u01ac\u0005\"\u0000\u0000\u01ac\u01ae\u0003"+ + "b1\u0000\u01ad\u01ab\u0001\u0000\u0000\u0000\u01ae\u01b1\u0001\u0000\u0000"+ + "\u0000\u01af\u01ad\u0001\u0000\u0000\u0000\u01af\u01b0\u0001\u0000\u0000"+ + "\u0000\u01b0\u01b2\u0001\u0000\u0000\u0000\u01b1\u01af\u0001\u0000\u0000"+ + "\u0000\u01b2\u01b3\u0005B\u0000\u0000\u01b3\u01b5\u0001\u0000\u0000\u0000"+ + "\u01b4\u018a\u0001\u0000\u0000\u0000\u01b4\u018b\u0001\u0000\u0000\u0000"+ + "\u01b4\u018e\u0001\u0000\u0000\u0000\u01b4\u018f\u0001\u0000\u0000\u0000"+ + "\u01b4\u0190\u0001\u0000\u0000\u0000\u01b4\u0191\u0001\u0000\u0000\u0000"+ + "\u01b4\u0192\u0001\u0000\u0000\u0000\u01b4\u0193\u0001\u0000\u0000\u0000"+ + "\u01b4\u019e\u0001\u0000\u0000\u0000\u01b4\u01a9\u0001\u0000\u0000\u0000"+ + "\u01b5?\u0001\u0000\u0000\u0000\u01b6\u01b9\u00050\u0000\u0000\u01b7\u01b9"+ + "\u0005@\u0000\u0000\u01b8\u01b6\u0001\u0000\u0000\u0000\u01b8\u01b7\u0001"+ + "\u0000\u0000\u0000\u01b9A\u0001\u0000\u0000\u0000\u01ba\u01bb\u0005\t"+ + "\u0000\u0000\u01bb\u01bc\u0005\u001b\u0000\u0000\u01bcC\u0001\u0000\u0000"+ + "\u0000\u01bd\u01be\u0005\u000e\u0000\u0000\u01be\u01c3\u0003F#\u0000\u01bf"+ + "\u01c0\u0005\"\u0000\u0000\u01c0\u01c2\u0003F#\u0000\u01c1\u01bf\u0001"+ + "\u0000\u0000\u0000\u01c2\u01c5\u0001\u0000\u0000\u0000\u01c3\u01c1\u0001"+ + "\u0000\u0000\u0000\u01c3\u01c4\u0001\u0000\u0000\u0000\u01c4E\u0001\u0000"+ + "\u0000\u0000\u01c5\u01c3\u0001\u0000\u0000\u0000\u01c6\u01c8\u0003\n\u0005"+ + "\u0000\u01c7\u01c9\u0007\u0004\u0000\u0000\u01c8\u01c7\u0001\u0000\u0000"+ + "\u0000\u01c8\u01c9\u0001\u0000\u0000\u0000\u01c9\u01cc\u0001\u0000\u0000"+ + "\u0000\u01ca\u01cb\u0005.\u0000\u0000\u01cb\u01cd\u0007\u0005\u0000\u0000"+ + "\u01cc\u01ca\u0001\u0000\u0000\u0000\u01cc\u01cd\u0001\u0000\u0000\u0000"+ + "\u01cdG\u0001\u0000\u0000\u0000\u01ce\u01cf\u0005\b\u0000\u0000\u01cf"+ + "\u01d0\u00038\u001c\u0000\u01d0I\u0001\u0000\u0000\u0000\u01d1\u01d2\u0005"+ + "\u0002\u0000\u0000\u01d2\u01d3\u00038\u001c\u0000\u01d3K\u0001\u0000\u0000"+ + "\u0000\u01d4\u01d5\u0005\u000b\u0000\u0000\u01d5\u01da\u0003N\'\u0000"+ + "\u01d6\u01d7\u0005\"\u0000\u0000\u01d7\u01d9\u0003N\'\u0000\u01d8\u01d6"+ + "\u0001\u0000\u0000\u0000\u01d9\u01dc\u0001\u0000\u0000\u0000\u01da\u01d8"+ + "\u0001\u0000\u0000\u0000\u01da\u01db\u0001\u0000\u0000\u0000\u01dbM\u0001"+ + "\u0000\u0000\u0000\u01dc\u01da\u0001\u0000\u0000\u0000\u01dd\u01de\u0003"+ + "6\u001b\u0000\u01de\u01df\u0005T\u0000\u0000\u01df\u01e0\u00036\u001b"+ + "\u0000\u01e0O\u0001\u0000\u0000\u0000\u01e1\u01e2\u0005\u0001\u0000\u0000"+ + "\u01e2\u01e3\u0003\u0014\n\u0000\u01e3\u01e5\u0003b1\u0000\u01e4\u01e6"+ + "\u0003V+\u0000\u01e5\u01e4\u0001\u0000\u0000\u0000\u01e5\u01e6\u0001\u0000"+ + "\u0000\u0000\u01e6Q\u0001\u0000\u0000\u0000\u01e7\u01e8\u0005\u0007\u0000"+ + "\u0000\u01e8\u01e9\u0003\u0014\n\u0000\u01e9\u01ea\u0003b1\u0000\u01ea"+ + "S\u0001\u0000\u0000\u0000\u01eb\u01ec\u0005\n\u0000\u0000\u01ec\u01ed"+ + "\u00034\u001a\u0000\u01edU\u0001\u0000\u0000\u0000\u01ee\u01f3\u0003X"+ + ",\u0000\u01ef\u01f0\u0005\"\u0000\u0000\u01f0\u01f2\u0003X,\u0000\u01f1"+ + "\u01ef\u0001\u0000\u0000\u0000\u01f2\u01f5\u0001\u0000\u0000\u0000\u01f3"+ + "\u01f1\u0001\u0000\u0000\u0000\u01f3\u01f4\u0001\u0000\u0000\u0000\u01f4"+ + "W\u0001\u0000\u0000\u0000\u01f5\u01f3\u0001\u0000\u0000\u0000\u01f6\u01f7"+ + "\u0003:\u001d\u0000\u01f7\u01f8\u0005 \u0000\u0000\u01f8\u01f9\u0003>"+ + "\u001f\u0000\u01f9Y\u0001\u0000\u0000\u0000\u01fa\u01fb\u0007\u0006\u0000"+ + "\u0000\u01fb[\u0001\u0000\u0000\u0000\u01fc\u01ff\u0003^/\u0000\u01fd"+ + "\u01ff\u0003`0\u0000\u01fe\u01fc\u0001\u0000\u0000\u0000\u01fe\u01fd\u0001"+ + "\u0000\u0000\u0000\u01ff]\u0001\u0000\u0000\u0000\u0200\u0202\u0007\u0000"+ + "\u0000\u0000\u0201\u0200\u0001\u0000\u0000\u0000\u0201\u0202\u0001\u0000"+ + "\u0000\u0000\u0202\u0203\u0001\u0000\u0000\u0000\u0203\u0204\u0005\u001c"+ + "\u0000\u0000\u0204_\u0001\u0000\u0000\u0000\u0205\u0207\u0007\u0000\u0000"+ + "\u0000\u0206\u0205\u0001\u0000\u0000\u0000\u0206\u0207\u0001\u0000\u0000"+ + "\u0000\u0207\u0208\u0001\u0000\u0000\u0000\u0208\u0209\u0005\u001b\u0000"+ + "\u0000\u0209a\u0001\u0000\u0000\u0000\u020a\u020b\u0005\u001a\u0000\u0000"+ + "\u020bc\u0001\u0000\u0000\u0000\u020c\u020d\u0007\u0007\u0000\u0000\u020d"+ + "e\u0001\u0000\u0000\u0000\u020e\u020f\u0005\u0005\u0000\u0000\u020f\u0210"+ + "\u0003h4\u0000\u0210g\u0001\u0000\u0000\u0000\u0211\u0212\u0005A\u0000"+ + "\u0000\u0212\u0213\u0003\u0002\u0001\u0000\u0213\u0214\u0005B\u0000\u0000"+ + "\u0214i\u0001\u0000\u0000\u0000\u0215\u0216\u0005\r\u0000\u0000\u0216"+ + "\u0217\u0005d\u0000\u0000\u0217k\u0001\u0000\u0000\u0000\u0218\u0219\u0005"+ + "\u0003\u0000\u0000\u0219\u021c\u0005Z\u0000\u0000\u021a\u021b\u0005X\u0000"+ + "\u0000\u021b\u021d\u00036\u001b\u0000\u021c\u021a\u0001\u0000\u0000\u0000"+ + "\u021c\u021d\u0001\u0000\u0000\u0000\u021d\u0227\u0001\u0000\u0000\u0000"+ + "\u021e\u021f\u0005Y\u0000\u0000\u021f\u0224\u0003n7\u0000\u0220\u0221"+ + "\u0005\"\u0000\u0000\u0221\u0223\u0003n7\u0000\u0222\u0220\u0001\u0000"+ + "\u0000\u0000\u0223\u0226\u0001\u0000\u0000\u0000\u0224\u0222\u0001\u0000"+ + "\u0000\u0000\u0224\u0225\u0001\u0000\u0000\u0000\u0225\u0228\u0001\u0000"+ + "\u0000\u0000\u0226\u0224\u0001\u0000\u0000\u0000\u0227\u021e\u0001\u0000"+ + "\u0000\u0000\u0227\u0228\u0001\u0000\u0000\u0000\u0228m\u0001\u0000\u0000"+ + "\u0000\u0229\u022a\u00036\u001b\u0000\u022a\u022b\u0005 \u0000\u0000\u022b"+ + "\u022d\u0001\u0000\u0000\u0000\u022c\u0229\u0001\u0000\u0000\u0000\u022c"+ + "\u022d\u0001\u0000\u0000\u0000\u022d\u022e\u0001\u0000\u0000\u0000\u022e"+ + "\u022f\u00036\u001b\u0000\u022fo\u0001\u0000\u0000\u0000\u0230\u0231\u0005"+ + "\u0012\u0000\u0000\u0231\u0232\u0003\"\u0011\u0000\u0232\u0233\u0005X"+ + "\u0000\u0000\u0233\u0234\u00038\u001c\u0000\u0234q\u0001\u0000\u0000\u0000"+ + "\u0235\u0236\u0005\u0011\u0000\u0000\u0236\u0239\u0003\u001c\u000e\u0000"+ + "\u0237\u0238\u0005\u001d\u0000\u0000\u0238\u023a\u0003\u001c\u000e\u0000"+ + "\u0239\u0237\u0001\u0000\u0000\u0000\u0239\u023a\u0001\u0000\u0000\u0000"+ + "\u023as\u0001\u0000\u0000\u00006\u007f\u0088\u009a\u00a6\u00af\u00b7\u00bd"+ + "\u00c5\u00c7\u00cc\u00d3\u00d8\u00e3\u00e9\u00f1\u00f3\u00fe\u0105\u0110"+ + "\u0113\u0121\u0129\u0131\u0135\u013c\u0144\u014c\u0159\u015d\u0161\u0168"+ + "\u016c\u0173\u017b\u0183\u0199\u01a4\u01af\u01b4\u01b8\u01c3\u01c8\u01cc"+ + "\u01da\u01e5\u01f3\u01fe\u0201\u0206\u021c\u0224\u0227\u022c\u0239"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java index 192b169cc9587..1442aaa99a929 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java @@ -956,18 +956,6 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener { *

    The default implementation does nothing.

    */ @Override public void exitShowInfo(EsqlBaseParser.ShowInfoContext ctx) { } - /** - * {@inheritDoc} - * - *

    The default implementation does nothing.

    - */ - @Override public void enterMetaFunctions(EsqlBaseParser.MetaFunctionsContext ctx) { } - /** - * {@inheritDoc} - * - *

    The default implementation does nothing.

    - */ - @Override public void exitMetaFunctions(EsqlBaseParser.MetaFunctionsContext ctx) { } /** * {@inheritDoc} * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java index de98d4333c1d4..3a3ef05c7a465 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java @@ -566,13 +566,6 @@ public class EsqlBaseParserBaseVisitor extends AbstractParseTreeVisitor im * {@link #visitChildren} on {@code ctx}.

    */ @Override public T visitShowInfo(EsqlBaseParser.ShowInfoContext ctx) { return visitChildren(ctx); } - /** - * {@inheritDoc} - * - *

    The default implementation returns the result of calling - * {@link #visitChildren} on {@code ctx}.

    - */ - @Override public T visitMetaFunctions(EsqlBaseParser.MetaFunctionsContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java index 4348c641d9f69..5d2d417f30c50 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java @@ -861,18 +861,6 @@ public interface EsqlBaseParserListener extends ParseTreeListener { * @param ctx the parse tree */ void exitShowInfo(EsqlBaseParser.ShowInfoContext ctx); - /** - * Enter a parse tree produced by the {@code metaFunctions} - * labeled alternative in {@link EsqlBaseParser#metaCommand}. - * @param ctx the parse tree - */ - void enterMetaFunctions(EsqlBaseParser.MetaFunctionsContext ctx); - /** - * Exit a parse tree produced by the {@code metaFunctions} - * labeled alternative in {@link EsqlBaseParser#metaCommand}. - * @param ctx the parse tree - */ - void exitMetaFunctions(EsqlBaseParser.MetaFunctionsContext ctx); /** * Enter a parse tree produced by {@link EsqlBaseParser#enrichCommand}. * @param ctx the parse tree diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java index c334526abfe39..51f2e845bcc55 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java @@ -519,13 +519,6 @@ public interface EsqlBaseParserVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitShowInfo(EsqlBaseParser.ShowInfoContext ctx); - /** - * Visit a parse tree produced by the {@code metaFunctions} - * labeled alternative in {@link EsqlBaseParser#metaCommand}. - * @param ctx the parse tree - * @return the visitor result - */ - T visitMetaFunctions(EsqlBaseParser.MetaFunctionsContext ctx); /** * Visit a parse tree produced by {@link EsqlBaseParser#enrichCommand}. * @param ctx the parse tree diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index 8dc07e2e1017f..d97c1aefd5487 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -53,7 +53,6 @@ import org.elasticsearch.xpack.esql.plan.logical.Rename; import org.elasticsearch.xpack.esql.plan.logical.Row; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; -import org.elasticsearch.xpack.esql.plan.logical.meta.MetaFunctions; import org.elasticsearch.xpack.esql.plan.logical.show.ShowInfo; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.joni.exception.SyntaxException; @@ -412,11 +411,6 @@ public LogicalPlan visitShowInfo(EsqlBaseParser.ShowInfoContext ctx) { return new ShowInfo(source(ctx)); } - @Override - public LogicalPlan visitMetaFunctions(EsqlBaseParser.MetaFunctionsContext ctx) { - return new MetaFunctions(source(ctx)); - } - @Override public PlanFactory visitEnrichCommand(EsqlBaseParser.EnrichCommandContext ctx) { return p -> { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/meta/MetaFunctions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/meta/MetaFunctions.java deleted file mode 100644 index 029cb6164167c..0000000000000 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/meta/MetaFunctions.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.plan.logical.meta; - -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.core.expression.Attribute; -import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; -import org.elasticsearch.xpack.esql.core.tree.NodeInfo; -import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; -import org.elasticsearch.xpack.esql.plan.logical.LeafPlan; -import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; -import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; - -public class MetaFunctions extends LeafPlan { - - private final List attributes; - - public MetaFunctions(Source source) { - super(source); - - attributes = new ArrayList<>(); - for (var name : List.of("name", "synopsis", "argNames", "argTypes", "argDescriptions", "returnType", "description")) { - attributes.add(new ReferenceAttribute(Source.EMPTY, name, KEYWORD)); - } - for (var name : List.of("optionalArgs", "variadic", "isAggregation")) { - attributes.add(new ReferenceAttribute(Source.EMPTY, name, BOOLEAN)); - } - } - - @Override - public void writeTo(StreamOutput out) { - throw new UnsupportedOperationException("not serialized"); - } - - @Override - public String getWriteableName() { - throw new UnsupportedOperationException("not serialized"); - } - - @Override - public List output() { - return attributes; - } - - public List> values(EsqlFunctionRegistry functionRegistry) { - List> rows = new ArrayList<>(); - for (var def : functionRegistry.listFunctions(null)) { - EsqlFunctionRegistry.FunctionDescription signature = EsqlFunctionRegistry.description(def); - List row = new ArrayList<>(); - row.add(asBytesRefOrNull(signature.name())); - row.add(new BytesRef(signature.fullSignature())); - row.add(collect(signature, EsqlFunctionRegistry.ArgSignature::name)); - row.add(collect(signature, EsqlFunctionRegistry.ArgSignature::type)); - row.add(collect(signature, EsqlFunctionRegistry.ArgSignature::description)); - row.add(withPipes(signature.returnType())); - row.add(signature.description()); - row.add(collect(signature, EsqlFunctionRegistry.ArgSignature::optional)); - row.add(signature.variadic()); - row.add(signature.isAggregation()); - rows.add(row); - } - rows.sort(Comparator.comparing(x -> ((BytesRef) x.get(0)))); - return rows; - } - - private Object collect(EsqlFunctionRegistry.FunctionDescription signature, Function x) { - if (signature.args().size() == 0) { - return null; - } - if (signature.args().size() == 1) { - Object result = x.apply(signature.args().get(0)); - if (result instanceof String[] r) { - return withPipes(r); - } - return result; - } - - List args = signature.args(); - List result = signature.args().stream().map(x).collect(Collectors.toList()); - boolean withPipes = result.get(0) instanceof String[]; - if (result.isEmpty() == false) { - List newResult = new ArrayList<>(); - for (int i = 0; i < result.size(); i++) { - if (signature.variadic() && args.get(i).optional()) { - continue; - } - newResult.add(withPipes ? withPipes((String[]) result.get(i)) : result.get(i)); - } - return newResult; - } - return result; - } - - public static String withPipes(String[] items) { - return Arrays.stream(items).collect(Collectors.joining("|")); - } - - private static BytesRef asBytesRefOrNull(String string) { - return Strings.hasText(string) ? new BytesRef(string) : null; - } - - @Override - public String commandName() { - return "META FUNCTIONS"; - } - - @Override - public boolean expressionsResolved() { - return true; - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this); - } - - @Override - public int hashCode() { - return getClass().hashCode(); - } - - @Override - public boolean equals(Object obj) { - return this == obj || obj != null && getClass() == obj.getClass(); - } -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Mapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Mapper.java index 9613fa1f3fcde..e571be54692c4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Mapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/Mapper.java @@ -32,7 +32,6 @@ import org.elasticsearch.xpack.esql.plan.logical.join.JoinConfig; import org.elasticsearch.xpack.esql.plan.logical.join.JoinType; import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; -import org.elasticsearch.xpack.esql.plan.logical.meta.MetaFunctions; import org.elasticsearch.xpack.esql.plan.logical.show.ShowInfo; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; import org.elasticsearch.xpack.esql.plan.physical.DissectExec; @@ -98,9 +97,6 @@ public PhysicalPlan map(LogicalPlan p) { } // Commands - if (p instanceof MetaFunctions metaFunctions) { - return new ShowExec(metaFunctions.source(), metaFunctions.output(), metaFunctions.values(functionRegistry)); - } if (p instanceof ShowInfo showInfo) { return new ShowExec(showInfo.source(), showInfo.output(), showInfo.values()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/FeatureMetric.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/FeatureMetric.java index c4d890a818ec7..4cae2a9c247f3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/FeatureMetric.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/FeatureMetric.java @@ -21,7 +21,6 @@ import org.elasticsearch.xpack.esql.plan.logical.OrderBy; import org.elasticsearch.xpack.esql.plan.logical.Rename; import org.elasticsearch.xpack.esql.plan.logical.Row; -import org.elasticsearch.xpack.esql.plan.logical.meta.MetaFunctions; import org.elasticsearch.xpack.esql.plan.logical.show.ShowInfo; import java.util.BitSet; @@ -29,11 +28,6 @@ import java.util.function.Predicate; public enum FeatureMetric { - /** - * The order of these enum values is important, do not change it. - * For any new values added to it, they should go at the end of the list. - * see {@link org.elasticsearch.xpack.esql.analysis.Verifier#gatherMetrics} - */ DISSECT(Dissect.class::isInstance), EVAL(Eval.class::isInstance), GROK(Grok.class::isInstance), @@ -48,8 +42,7 @@ public enum FeatureMetric { FROM(EsRelation.class::isInstance), DROP(Drop.class::isInstance), KEEP(Keep.class::isInstance), - RENAME(Rename.class::isInstance), - META(MetaFunctions.class::isInstance); + RENAME(Rename.class::isInstance); private Predicate planCheck; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index adf31ca983067..27656c8122e30 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -921,7 +921,6 @@ public void testDeprecatedIsNullFunction() { public void testMetadataFieldOnOtherSources() { expectError("row a = 1 metadata _index", "line 1:20: extraneous input '_index' expecting "); - expectError("meta functions metadata _index", "line 1:16: token recognition error at: 'm'"); expectError("show info metadata _index", "line 1:11: token recognition error at: 'm'"); expectError( "explain [from foo] metadata _index", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/VerifierMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/VerifierMetricsTests.java index ab004a3a055ce..203e5c3bd37ee 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/VerifierMetricsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/VerifierMetricsTests.java @@ -24,7 +24,6 @@ import static org.elasticsearch.xpack.esql.stats.FeatureMetric.GROK; import static org.elasticsearch.xpack.esql.stats.FeatureMetric.KEEP; import static org.elasticsearch.xpack.esql.stats.FeatureMetric.LIMIT; -import static org.elasticsearch.xpack.esql.stats.FeatureMetric.META; import static org.elasticsearch.xpack.esql.stats.FeatureMetric.MV_EXPAND; import static org.elasticsearch.xpack.esql.stats.FeatureMetric.RENAME; import static org.elasticsearch.xpack.esql.stats.FeatureMetric.ROW; @@ -55,7 +54,6 @@ public void testDissectQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - assertEquals(0, meta(c)); } public void testEvalQuery() { @@ -75,7 +73,6 @@ public void testEvalQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - assertEquals(0, meta(c)); } public void testGrokQuery() { @@ -95,7 +92,6 @@ public void testGrokQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - assertEquals(0, meta(c)); } public void testLimitQuery() { @@ -115,7 +111,6 @@ public void testLimitQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - assertEquals(0, meta(c)); } public void testSortQuery() { @@ -135,7 +130,6 @@ public void testSortQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - assertEquals(0, meta(c)); } public void testStatsQuery() { @@ -155,7 +149,6 @@ public void testStatsQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - assertEquals(0, meta(c)); } public void testWhereQuery() { @@ -175,7 +168,6 @@ public void testWhereQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - assertEquals(0, meta(c)); } public void testTwoWhereQuery() { @@ -195,7 +187,6 @@ public void testTwoWhereQuery() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - assertEquals(0, meta(c)); } public void testTwoQueriesExecuted() { @@ -235,7 +226,6 @@ public void testTwoQueriesExecuted() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - assertEquals(0, meta(c)); } public void testEnrich() { @@ -261,7 +251,6 @@ public void testEnrich() { assertEquals(0, drop(c)); assertEquals(1L, keep(c)); assertEquals(0, rename(c)); - assertEquals(0, meta(c)); } public void testMvExpand() { @@ -290,27 +279,6 @@ public void testMvExpand() { assertEquals(0, drop(c)); assertEquals(1L, keep(c)); assertEquals(0, rename(c)); - assertEquals(0, meta(c)); - } - - public void testMetaFunctions() { - Counters c = esql("meta functions | stats a = count(*) | mv_expand a"); - assertEquals(0, dissect(c)); - assertEquals(0, eval(c)); - assertEquals(0, grok(c)); - assertEquals(0, limit(c)); - assertEquals(0, sort(c)); - assertEquals(1L, stats(c)); - assertEquals(0, where(c)); - assertEquals(0, enrich(c)); - assertEquals(1L, mvExpand(c)); - assertEquals(0, show(c)); - assertEquals(0, row(c)); - assertEquals(0, from(c)); - assertEquals(0, drop(c)); - assertEquals(0, keep(c)); - assertEquals(0, rename(c)); - assertEquals(1L, meta(c)); } public void testShowInfo() { @@ -330,7 +298,6 @@ public void testShowInfo() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - assertEquals(0, meta(c)); } public void testRow() { @@ -350,7 +317,6 @@ public void testRow() { assertEquals(0, drop(c)); assertEquals(0, keep(c)); assertEquals(0, rename(c)); - assertEquals(0, meta(c)); } public void testDropAndRename() { @@ -370,7 +336,6 @@ public void testDropAndRename() { assertEquals(1L, drop(c)); assertEquals(0, keep(c)); assertEquals(1L, rename(c)); - assertEquals(0, meta(c)); } public void testKeep() { @@ -395,7 +360,6 @@ public void testKeep() { assertEquals(0, drop(c)); assertEquals(1L, keep(c)); assertEquals(0, rename(c)); - assertEquals(0, meta(c)); } private long dissect(Counters c) { @@ -458,10 +422,6 @@ private long rename(Counters c) { return c.get(FPREFIX + RENAME); } - private long meta(Counters c) { - return c.get(FPREFIX + META); - } - private Counters esql(String esql) { return esql(esql, null); } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index 74c0e9ef1bb31..8bbdb27a87d1a 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -1,8 +1,13 @@ --- setup: - requires: - cluster_features: ["gte_v8.14.0"] - reason: "Introduction of META tracking in 8.14+" + capabilities: + - method: POST + path: /_query + parameters: [ method, path, parameters, capabilities ] + capabilities: [ no_meta ] + reason: "META command removed which changes the count of the data returned" + test_runner_features: [capabilities] - do: indices.create: @@ -23,7 +28,7 @@ setup: - do: {xpack.usage: {}} - match: { esql.available: true } - match: { esql.enabled: true } - - length: { esql.features: 16 } + - length: { esql.features: 15 } - set: {esql.features.dissect: dissect_counter} - set: {esql.features.drop: drop_counter} - set: {esql.features.eval: eval_counter} @@ -32,7 +37,6 @@ setup: - set: {esql.features.grok: grok_counter} - set: {esql.features.keep: keep_counter} - set: {esql.features.limit: limit_counter} - - set: {esql.features.meta: meta_counter} - set: {esql.features.mv_expand: mv_expand_counter} - set: {esql.features.rename: rename_counter} - set: {esql.features.row: row_counter} From 43e5258b3c3b5e468b6177d1347e20ea98104513 Mon Sep 17 00:00:00 2001 From: Pete Gillin Date: Tue, 8 Oct 2024 17:39:53 +0100 Subject: [PATCH 183/194] Add a `terminate` ingest processor (#114157) This processor simply causes any remaining processors in the pipeline to be skipped. It will normally be executed conditionally using the `if` option. (If this pipeline is being called from another pipeline, the calling pipeline is *not* terminated.) For example, this: ``` POST /_ingest/pipeline/_simulate { "pipeline": { "description": "Appends just 'before' to the steps field if the number field is present, or both 'before' and 'after' if not", "processors": [ { "append": { "field": "steps", "value": "before" } }, { "terminate": { "if": "ctx.error != null" } }, { "append": { "field": "steps", "value": "after" } } ] }, "docs": [ { "_index": "index", "_id": "doc1", "_source": { "name": "okay", "steps": [] } }, { "_index": "index", "_id": "doc2", "_source": { "name": "bad", "error": "oh no", "steps": [] } } ] } ``` returns something like this: ``` { "docs": [ { "doc": { "_index": "index", "_version": "-3", "_id": "doc1", "_source": { "name": "okay", "steps": [ "before", "after" ] }, "_ingest": { "timestamp": "2024-10-04T16:25:20.448881Z" } } }, { "doc": { "_index": "index", "_version": "-3", "_id": "doc2", "_source": { "name": "bad", "error": "oh no", "steps": [ "before" ] }, "_ingest": { "timestamp": "2024-10-04T16:25:20.448932Z" } } } ] } ``` --- docs/changelog/114157.yaml | 6 + .../ingest/processors/terminate.asciidoc | 30 ++++ .../ingest/common/IngestCommonPlugin.java | 1 + .../ingest/common/TerminateProcessor.java | 53 +++++++ .../common/TerminateProcessorTests.java | 70 +++++++++ .../test/ingest/330_terminate_processor.yml | 138 ++++++++++++++++++ .../ingest/CompoundProcessor.java | 7 +- .../elasticsearch/ingest/IngestDocument.java | 22 +++ .../org/elasticsearch/ingest/Pipeline.java | 3 + 9 files changed, 327 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/114157.yaml create mode 100644 docs/reference/ingest/processors/terminate.asciidoc create mode 100644 modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TerminateProcessor.java create mode 100644 modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TerminateProcessorTests.java create mode 100644 modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/330_terminate_processor.yml diff --git a/docs/changelog/114157.yaml b/docs/changelog/114157.yaml new file mode 100644 index 0000000000000..22e0fda173e98 --- /dev/null +++ b/docs/changelog/114157.yaml @@ -0,0 +1,6 @@ +pr: 114157 +summary: Add a `terminate` ingest processor +area: Ingest Node +type: feature +issues: + - 110218 diff --git a/docs/reference/ingest/processors/terminate.asciidoc b/docs/reference/ingest/processors/terminate.asciidoc new file mode 100644 index 0000000000000..a2643fbd955f1 --- /dev/null +++ b/docs/reference/ingest/processors/terminate.asciidoc @@ -0,0 +1,30 @@ +[[terminate-processor]] +=== Terminate processor + +++++ +Terminate +++++ + +Terminates the current ingest pipeline, causing no further processors to be run. +This will normally be executed conditionally, using the `if` option. + +If this pipeline is being called from another pipeline, the calling pipeline is *not* terminated. + +[[terminate-options]] +.Terminate Options +[options="header"] +|====== +| Name | Required | Default | Description +include::common-options.asciidoc[] +|====== + +[source,js] +-------------------------------------------------- +{ + "description" : "terminates the current pipeline if the error field is present", + "terminate": { + "if": "ctx.error != null" + } +} +-------------------------------------------------- +// NOTCONSOLE diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java index 6ef636847e2df..d585c6217202f 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java @@ -72,6 +72,7 @@ public Map getProcessors(Processor.Parameters paramet entry(SetProcessor.TYPE, new SetProcessor.Factory(parameters.scriptService)), entry(SortProcessor.TYPE, new SortProcessor.Factory()), entry(SplitProcessor.TYPE, new SplitProcessor.Factory()), + entry(TerminateProcessor.TYPE, new TerminateProcessor.Factory()), entry(TrimProcessor.TYPE, new TrimProcessor.Factory()), entry(URLDecodeProcessor.TYPE, new URLDecodeProcessor.Factory()), entry(UppercaseProcessor.TYPE, new UppercaseProcessor.Factory()), diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TerminateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TerminateProcessor.java new file mode 100644 index 0000000000000..5b6144ba8eab9 --- /dev/null +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TerminateProcessor.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.common; + +import org.elasticsearch.ingest.AbstractProcessor; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Processor; + +import java.util.Map; + +/** + * A {@link Processor} which simply prevents subsequent processors in the pipeline from running (without failing, like {@link FailProcessor} + * does). This will normally be run conditionally, using the {@code if} option. + */ +public class TerminateProcessor extends AbstractProcessor { + + static final String TYPE = "terminate"; + + TerminateProcessor(String tag, String description) { + super(tag, description); + } + + @Override + public IngestDocument execute(IngestDocument ingestDocument) { + ingestDocument.terminate(); + return ingestDocument; + } + + @Override + public String getType() { + return TYPE; + } + + public static final class Factory implements Processor.Factory { + + @Override + public Processor create( + Map processorFactories, + String tag, + String description, + Map config + ) { + return new TerminateProcessor(tag, description); + } + } +} diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TerminateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TerminateProcessorTests.java new file mode 100644 index 0000000000000..1888f8366edd3 --- /dev/null +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TerminateProcessorTests.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.common; + +import org.elasticsearch.ingest.CompoundProcessor; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Pipeline; +import org.elasticsearch.ingest.TestTemplateService; +import org.elasticsearch.ingest.ValueSource; +import org.elasticsearch.test.ESTestCase; + +import java.util.Map; + +import static org.elasticsearch.ingest.RandomDocumentPicks.randomIngestDocument; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +public class TerminateProcessorTests extends ESTestCase { + + public void testTerminateInPipeline() throws Exception { + Pipeline pipeline = new Pipeline( + "my-pipeline", + null, + null, + null, + new CompoundProcessor( + new SetProcessor( + "before-set", + "Sets before field to true", + new TestTemplateService.MockTemplateScript.Factory("before"), + ValueSource.wrap(true, TestTemplateService.instance()), + null + ), + new TerminateProcessor("terminate", "terminates the pipeline"), + new SetProcessor( + "after-set", + "Sets after field to true", + new TestTemplateService.MockTemplateScript.Factory("after"), + ValueSource.wrap(true, TestTemplateService.instance()), + null + ) + ) + ); + IngestDocument input = randomIngestDocument(random(), Map.of("foo", "bar")); + PipelineOutput output = new PipelineOutput(); + + pipeline.execute(input, output::set); + + assertThat(output.exception, nullValue()); + // We expect the before-set processor to have run, but not the after-set one: + assertThat(output.document.getSource(), is(Map.of("foo", "bar", "before", true))); + } + + private static class PipelineOutput { + IngestDocument document; + Exception exception; + + void set(IngestDocument document, Exception exception) { + this.document = document; + this.exception = exception; + } + } +} diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/330_terminate_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/330_terminate_processor.yml new file mode 100644 index 0000000000000..7a46d7bb272d8 --- /dev/null +++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/330_terminate_processor.yml @@ -0,0 +1,138 @@ +--- +setup: + - do: + ingest.put_pipeline: + id: "test-pipeline" + body: > + { + "description": "Appends just 'before' to the steps field if the number field is less than 50, or both 'before' and 'after' if not", + "processors": [ + { + "append": { + "field": "steps", + "value": "before" + } + }, + { + "terminate": { + "if": "ctx.number < 50" + } + }, + { + "append": { + "field": "steps", + "value": "after" + } + } + ] + } + - do: + ingest.put_pipeline: + id: "test-final-pipeline" + body: > + { + "description": "Appends 'final' to the steps field", + "processors": [ + { + "append": { + "field": "steps", + "value": "final" + } + } + ] + } + - do: + ingest.put_pipeline: + id: "test-outer-pipeline" + body: > + { + "description": "Runs test-pipeline and then append 'outer' to the steps field", + "processors": [ + { + "pipeline": { + "name": "test-pipeline" + } + }, + { + "append": { + "field": "steps", + "value": "outer" + } + } + ] + } + - do: + indices.create: + index: "test-index-with-default-and-final-pipelines" + body: + settings: + index: + default_pipeline: "test-pipeline" + final_pipeline: "test-final-pipeline" + - do: + indices.create: + index: "test-vanilla-index" + +--- +teardown: + - do: + indices.delete: + index: "test-index-with-default-and-final-pipelines" + ignore_unavailable: true + - do: + indices.delete: + index: "test-vanilla-index" + ignore_unavailable: true + - do: + ingest.delete_pipeline: + id: "test-pipeline" + ignore: 404 + - do: + ingest.delete_pipeline: + id: "test-outer-pipeline" + ignore: 404 + +--- +"Test pipeline including conditional terminate pipeline": + + - do: + bulk: + refresh: true + body: + - '{ "index": {"_index": "test-index-with-default-and-final-pipelines" } }' + - '{ "comment": "should terminate", "number": 40, "steps": [] }' + - '{ "index": {"_index": "test-index-with-default-and-final-pipelines" } }' + - '{ "comment": "should continue to end", "number": 60, "steps": [] }' + + - do: + search: + rest_total_hits_as_int: true + index: "test-index-with-default-and-final-pipelines" + body: + sort: "number" + - match: { hits.total: 2 } + - match: { hits.hits.0._source.number: 40 } + - match: { hits.hits.1._source.number: 60 } + - match: { hits.hits.0._source.steps: ["before", "final"] } + - match: { hits.hits.1._source.steps: ["before", "after", "final"] } + +--- +"Test pipeline with terminate invoked from an outer pipeline": + + - do: + bulk: + refresh: true + pipeline: "test-outer-pipeline" + body: + - '{ "index": {"_index": "test-vanilla-index" } }' + - '{ "comment": "should terminate inner pipeline but not outer", "number": 40, "steps": [] }' + + - do: + search: + rest_total_hits_as_int: true + index: "test-vanilla-index" + body: + sort: "number" + - match: { hits.total: 1 } + - match: { hits.hits.0._source.number: 40 } + - match: { hits.hits.0._source.steps: ["before", "outer"] } diff --git a/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java b/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java index 9620becd49d59..873f334d0a650 100644 --- a/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java @@ -148,7 +148,7 @@ public void execute(IngestDocument ingestDocument, BiConsumer handler) { assert currentProcessor <= processorsWithMetrics.size(); - if (currentProcessor == processorsWithMetrics.size() || ingestDocument.isReroute()) { + if (currentProcessor == processorsWithMetrics.size() || ingestDocument.isReroute() || ingestDocument.isTerminate()) { handler.accept(ingestDocument, null); return; } @@ -159,7 +159,8 @@ void innerExecute(int currentProcessor, IngestDocument ingestDocument, final BiC // iteratively execute any sync processors while (currentProcessor < processorsWithMetrics.size() && processorsWithMetrics.get(currentProcessor).v1().isAsync() == false - && ingestDocument.isReroute() == false) { + && ingestDocument.isReroute() == false + && ingestDocument.isTerminate() == false) { processorWithMetric = processorsWithMetrics.get(currentProcessor); processor = processorWithMetric.v1(); metric = processorWithMetric.v2(); @@ -185,7 +186,7 @@ void innerExecute(int currentProcessor, IngestDocument ingestDocument, final BiC } assert currentProcessor <= processorsWithMetrics.size(); - if (currentProcessor == processorsWithMetrics.size() || ingestDocument.isReroute()) { + if (currentProcessor == processorsWithMetrics.size() || ingestDocument.isReroute() || ingestDocument.isTerminate()) { handler.accept(ingestDocument, null); return; } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index 0bc1c0d2932df..280c7684a8553 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -82,6 +82,7 @@ public final class IngestDocument { private boolean doNoSelfReferencesCheck = false; private boolean reroute = false; + private boolean terminate = false; public IngestDocument(String index, String id, long version, String routing, VersionType versionType, Map source) { this.ctxMap = new IngestCtxMap(index, id, version, routing, versionType, ZonedDateTime.now(ZoneOffset.UTC), source); @@ -935,6 +936,27 @@ void resetReroute() { reroute = false; } + /** + * Sets the terminate flag to true, to indicate that no further processors in the current pipeline should be run for this document. + */ + public void terminate() { + terminate = true; + } + + /** + * Returns whether the {@link #terminate()} flag was set. + */ + boolean isTerminate() { + return terminate; + } + + /** + * Resets the {@link #terminate()} flag. + */ + void resetTerminate() { + terminate = false; + } + public enum Metadata { INDEX(IndexFieldMapper.NAME), TYPE("_type"), diff --git a/server/src/main/java/org/elasticsearch/ingest/Pipeline.java b/server/src/main/java/org/elasticsearch/ingest/Pipeline.java index 6153d45bce779..a8e8fbb5d3217 100644 --- a/server/src/main/java/org/elasticsearch/ingest/Pipeline.java +++ b/server/src/main/java/org/elasticsearch/ingest/Pipeline.java @@ -133,6 +133,9 @@ public void execute(IngestDocument ingestDocument, BiConsumer Date: Tue, 8 Oct 2024 09:53:49 -0700 Subject: [PATCH 184/194] Collect query metrics on search nodes (#114267) When I added the query/fetch metrics, I overlooked that non-primary shards were being skipped during metrics collection, and the stateful tests didn't catch it. This change ensures that search metrics are now collected from every shard copy. --- .../monitor/metrics/IndicesMetricsIT.java | 157 ++++++++---------- .../monitor/metrics/IndicesMetrics.java | 69 ++++---- 2 files changed, 111 insertions(+), 115 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java index 4a060eadc735b..fb563ee333d07 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/monitor/metrics/IndicesMetricsIT.java @@ -107,14 +107,14 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { static final String LOGSDB_INDEXING_TIME = "es.indices.logsdb.indexing.time"; static final String LOGSDB_INDEXING_FAILURE = "es.indices.logsdb.indexing.failure.total"; - public void testIndicesMetrics() throws Exception { - String node = internalCluster().startNode(); + public void testIndicesMetrics() { + String indexNode = internalCluster().startNode(); ensureStableCluster(1); - final TestTelemetryPlugin telemetry = internalCluster().getInstance(PluginsService.class, node) + TestTelemetryPlugin telemetry = internalCluster().getInstance(PluginsService.class, indexNode) .filterPlugins(TestTelemetryPlugin.class) .findFirst() .orElseThrow(); - final IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, indexNode); var indexing0 = indicesService.stats(CommonStatsFlags.ALL, false).getIndexing().getTotal(); telemetry.resetMeter(); long numStandardIndices = randomIntBetween(1, 5); @@ -131,19 +131,12 @@ public void testIndicesMetrics() throws Exception { STANDARD_BYTES_SIZE, greaterThan(0L), - TIME_SERIES_INDEX_COUNT, - equalTo(0L), - TIME_SERIES_DOCS_COUNT, - equalTo(0L), - TIME_SERIES_BYTES_SIZE, - equalTo(0L), - - LOGSDB_INDEX_COUNT, - equalTo(0L), - LOGSDB_DOCS_COUNT, - equalTo(0L), - LOGSDB_BYTES_SIZE, - equalTo(0L) + STANDARD_INDEXING_COUNT, + equalTo(numStandardDocs), + STANDARD_INDEXING_TIME, + greaterThanOrEqualTo(0L), + STANDARD_INDEXING_FAILURE, + equalTo(indexing1.getIndexFailedCount() - indexing0.getIndexCount()) ) ); @@ -154,13 +147,6 @@ public void testIndicesMetrics() throws Exception { telemetry, 2, Map.of( - STANDARD_INDEX_COUNT, - equalTo(numStandardIndices), - STANDARD_DOCS_COUNT, - equalTo(numStandardDocs), - STANDARD_BYTES_SIZE, - greaterThan(0L), - TIME_SERIES_INDEX_COUNT, equalTo(numTimeSeriesIndices), TIME_SERIES_DOCS_COUNT, @@ -168,12 +154,12 @@ public void testIndicesMetrics() throws Exception { TIME_SERIES_BYTES_SIZE, greaterThan(20L), - LOGSDB_INDEX_COUNT, - equalTo(0L), - LOGSDB_DOCS_COUNT, - equalTo(0L), - LOGSDB_BYTES_SIZE, - equalTo(0L) + TIME_SERIES_INDEXING_COUNT, + equalTo(numTimeSeriesDocs), + TIME_SERIES_INDEXING_TIME, + greaterThanOrEqualTo(0L), + TIME_SERIES_INDEXING_FAILURE, + equalTo(indexing2.getIndexFailedCount() - indexing1.getIndexFailedCount()) ) ); @@ -184,60 +170,58 @@ public void testIndicesMetrics() throws Exception { telemetry, 3, Map.of( - STANDARD_INDEX_COUNT, - equalTo(numStandardIndices), - STANDARD_DOCS_COUNT, - equalTo(numStandardDocs), - STANDARD_BYTES_SIZE, - greaterThan(0L), - - TIME_SERIES_INDEX_COUNT, - equalTo(numTimeSeriesIndices), - TIME_SERIES_DOCS_COUNT, - equalTo(numTimeSeriesDocs), - TIME_SERIES_BYTES_SIZE, - greaterThan(20L), - LOGSDB_INDEX_COUNT, equalTo(numLogsdbIndices), LOGSDB_DOCS_COUNT, equalTo(numLogsdbDocs), LOGSDB_BYTES_SIZE, - greaterThan(0L) + greaterThan(0L), + LOGSDB_INDEXING_COUNT, + equalTo(numLogsdbDocs), + LOGSDB_INDEXING_TIME, + greaterThanOrEqualTo(0L), + LOGSDB_INDEXING_FAILURE, + equalTo(indexing3.getIndexFailedCount() - indexing2.getIndexFailedCount()) ) ); - // indexing stats + // already collected indexing stats collectThenAssertMetrics( telemetry, 4, Map.of( STANDARD_INDEXING_COUNT, - equalTo(numStandardDocs), + equalTo(0L), STANDARD_INDEXING_TIME, - greaterThanOrEqualTo(0L), + equalTo(0L), STANDARD_INDEXING_FAILURE, - equalTo(indexing1.getIndexFailedCount() - indexing0.getIndexCount()), + equalTo(0L), TIME_SERIES_INDEXING_COUNT, - equalTo(numTimeSeriesDocs), + equalTo(0L), TIME_SERIES_INDEXING_TIME, - greaterThanOrEqualTo(0L), + equalTo(0L), TIME_SERIES_INDEXING_FAILURE, - equalTo(indexing2.getIndexFailedCount() - indexing1.getIndexFailedCount()), + equalTo(0L), LOGSDB_INDEXING_COUNT, - equalTo(numLogsdbDocs), + equalTo(0L), LOGSDB_INDEXING_TIME, - greaterThanOrEqualTo(0L), + equalTo(0L), LOGSDB_INDEXING_FAILURE, - equalTo(indexing3.getIndexFailedCount() - indexing2.getIndexFailedCount()) + equalTo(0L) ) ); - telemetry.resetMeter(); - + String searchNode = internalCluster().startDataOnlyNode(); + indicesService = internalCluster().getInstance(IndicesService.class, searchNode); + telemetry = internalCluster().getInstance(PluginsService.class, searchNode) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + ensureGreen("st*", "log*", "time*"); // search and fetch - client().prepareSearch("standard*").setSize(100).get().decRef(); - var nodeStats1 = indicesService.stats(CommonStatsFlags.ALL, false).getSearch().getTotal(); + String preference = "_only_local"; + client(searchNode).prepareSearch("standard*").setPreference(preference).setSize(100).get().decRef(); + var search1 = indicesService.stats(CommonStatsFlags.ALL, false).getSearch().getTotal(); collectThenAssertMetrics( telemetry, 1, @@ -245,11 +229,11 @@ public void testIndicesMetrics() throws Exception { STANDARD_QUERY_COUNT, equalTo(numStandardIndices), STANDARD_QUERY_TIME, - equalTo(nodeStats1.getQueryTimeInMillis()), + equalTo(search1.getQueryTimeInMillis()), STANDARD_FETCH_COUNT, - equalTo(nodeStats1.getFetchCount()), + equalTo(search1.getFetchCount()), STANDARD_FETCH_TIME, - equalTo(nodeStats1.getFetchTimeInMillis()), + equalTo(search1.getFetchTimeInMillis()), TIME_SERIES_QUERY_COUNT, equalTo(0L), @@ -263,25 +247,25 @@ public void testIndicesMetrics() throws Exception { ) ); - client().prepareSearch("time*").setSize(100).get().decRef(); - var nodeStats2 = indicesService.stats(CommonStatsFlags.ALL, false).getSearch().getTotal(); + client(searchNode).prepareSearch("time*").setPreference(preference).setSize(100).get().decRef(); + var search2 = indicesService.stats(CommonStatsFlags.ALL, false).getSearch().getTotal(); collectThenAssertMetrics( telemetry, 2, Map.of( STANDARD_QUERY_COUNT, - equalTo(numStandardIndices), + equalTo(0L), STANDARD_QUERY_TIME, - equalTo(nodeStats1.getQueryTimeInMillis()), + equalTo(0L), TIME_SERIES_QUERY_COUNT, equalTo(numTimeSeriesIndices), TIME_SERIES_QUERY_TIME, - equalTo(nodeStats2.getQueryTimeInMillis() - nodeStats1.getQueryTimeInMillis()), + equalTo(search2.getQueryTimeInMillis() - search1.getQueryTimeInMillis()), TIME_SERIES_FETCH_COUNT, - equalTo(nodeStats2.getFetchCount() - nodeStats1.getFetchCount()), + equalTo(search2.getFetchCount() - search1.getFetchCount()), TIME_SERIES_FETCH_TIME, - equalTo(nodeStats2.getFetchTimeInMillis() - nodeStats1.getFetchTimeInMillis()), + equalTo(search2.getFetchTimeInMillis() - search1.getFetchTimeInMillis()), LOGSDB_QUERY_COUNT, equalTo(0L), @@ -289,41 +273,44 @@ public void testIndicesMetrics() throws Exception { equalTo(0L) ) ); - client().prepareSearch("logs*").setSize(100).get().decRef(); - var nodeStats3 = indicesService.stats(CommonStatsFlags.ALL, false).getSearch().getTotal(); + client(searchNode).prepareSearch("logs*").setPreference(preference).setSize(100).get().decRef(); + var search3 = indicesService.stats(CommonStatsFlags.ALL, false).getSearch().getTotal(); collectThenAssertMetrics( telemetry, 3, Map.of( STANDARD_QUERY_COUNT, - equalTo(numStandardIndices), + equalTo(0L), STANDARD_QUERY_TIME, - equalTo(nodeStats1.getQueryTimeInMillis()), + equalTo(0L), TIME_SERIES_QUERY_COUNT, - equalTo(numTimeSeriesIndices), + equalTo(0L), TIME_SERIES_QUERY_TIME, - equalTo(nodeStats2.getQueryTimeInMillis() - nodeStats1.getQueryTimeInMillis()), + equalTo(0L), LOGSDB_QUERY_COUNT, equalTo(numLogsdbIndices), LOGSDB_QUERY_TIME, - equalTo(nodeStats3.getQueryTimeInMillis() - nodeStats2.getQueryTimeInMillis()), + equalTo(search3.getQueryTimeInMillis() - search2.getQueryTimeInMillis()), LOGSDB_FETCH_COUNT, - equalTo(nodeStats3.getFetchCount() - nodeStats2.getFetchCount()), + equalTo(search3.getFetchCount() - search2.getFetchCount()), LOGSDB_FETCH_TIME, - equalTo(nodeStats3.getFetchTimeInMillis() - nodeStats2.getFetchTimeInMillis()) + equalTo(search3.getFetchTimeInMillis() - search2.getFetchTimeInMillis()) ) ); // search failures - expectThrows(Exception.class, () -> { client().prepareSearch("logs*").setRuntimeMappings(parseMapping(""" - { - "fail_me": { - "type": "long", - "script": {"source": "<>", "lang": "failing_field"} + expectThrows( + Exception.class, + () -> { client(searchNode).prepareSearch("logs*").setPreference(preference).setRuntimeMappings(parseMapping(""" + { + "fail_me": { + "type": "long", + "script": {"source": "<>", "lang": "failing_field"} + } } - } - """)).setQuery(new RangeQueryBuilder("fail_me").gte(0)).setAllowPartialSearchResults(true).get(); }); + """)).setQuery(new RangeQueryBuilder("fail_me").gte(0)).setAllowPartialSearchResults(true).get(); } + ); collectThenAssertMetrics( telemetry, 4, diff --git a/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java b/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java index e07f6908330df..11df8710fad6c 100644 --- a/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java +++ b/server/src/main/java/org/elasticsearch/monitor/metrics/IndicesMetrics.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.search.stats.SearchStats; +import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.shard.IllegalIndexShardStateException; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexingStats; @@ -31,6 +32,8 @@ import java.util.EnumMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; /** * {@link IndicesMetrics} monitors index statistics on an Elasticsearch node and exposes them as metrics @@ -84,75 +87,75 @@ private static List registerAsyncMetrics(MeterRegistry registry, metrics.add( registry.registerLongGauge( "es.indices." + name + ".query.total", - "total queries of " + name + " indices", + "current queries of " + name + " indices", "unit", - () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).search.getQueryCount()) + diffGauge(() -> cache.getOrRefresh().get(indexMode).search.getQueryCount()) ) ); metrics.add( registry.registerLongGauge( "es.indices." + name + ".query.time", - "total query time of " + name + " indices", + "current query time of " + name + " indices", "ms", - () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).search.getQueryTimeInMillis()) + diffGauge(() -> cache.getOrRefresh().get(indexMode).search.getQueryTimeInMillis()) ) ); metrics.add( registry.registerLongGauge( "es.indices." + name + ".query.failure.total", - "total query failures of " + name + " indices", + "current query failures of " + name + " indices", "unit", - () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).search.getQueryFailure()) + diffGauge(() -> cache.getOrRefresh().get(indexMode).search.getQueryFailure()) ) ); // fetch (count, took, failures) - use gauges as shards can be removed metrics.add( registry.registerLongGauge( "es.indices." + name + ".fetch.total", - "total fetches of " + name + " indices", + "current fetches of " + name + " indices", "unit", - () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).search.getFetchCount()) + diffGauge(() -> cache.getOrRefresh().get(indexMode).search.getFetchCount()) ) ); metrics.add( registry.registerLongGauge( "es.indices." + name + ".fetch.time", - "total fetch time of " + name + " indices", + "current fetch time of " + name + " indices", "ms", - () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).search.getFetchTimeInMillis()) + diffGauge(() -> cache.getOrRefresh().get(indexMode).search.getFetchTimeInMillis()) ) ); metrics.add( registry.registerLongGauge( "es.indices." + name + ".fetch.failure.total", - "total fetch failures of " + name + " indices", + "current fetch failures of " + name + " indices", "unit", - () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).search.getFetchFailure()) + diffGauge(() -> cache.getOrRefresh().get(indexMode).search.getFetchFailure()) ) ); // indexing metrics.add( registry.registerLongGauge( "es.indices." + name + ".indexing.total", - "total indexing operations of " + name + " indices", + "current indexing operations of " + name + " indices", "unit", - () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).indexing.getIndexCount()) + diffGauge(() -> cache.getOrRefresh().get(indexMode).indexing.getIndexCount()) ) ); metrics.add( registry.registerLongGauge( "es.indices." + name + ".indexing.time", - "total indexing time of " + name + " indices", + "current indexing time of " + name + " indices", "ms", - () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).indexing.getIndexTime().millis()) + diffGauge(() -> cache.getOrRefresh().get(indexMode).indexing.getIndexTime().millis()) ) ); metrics.add( registry.registerLongGauge( "es.indices." + name + ".indexing.failure.total", - "total indexing failures of " + name + " indices", + "current indexing failures of " + name + " indices", "unit", - () -> new LongWithAttributes(cache.getOrRefresh().get(indexMode).indexing.getIndexFailedCount()) + diffGauge(() -> cache.getOrRefresh().get(indexMode).indexing.getIndexFailedCount()) ) ); } @@ -160,6 +163,15 @@ private static List registerAsyncMetrics(MeterRegistry registry, return metrics; } + static Supplier diffGauge(Supplier currentValue) { + final AtomicLong counter = new AtomicLong(); + return () -> { + var curr = currentValue.get(); + long prev = counter.getAndUpdate(v -> Math.max(curr, v)); + return new LongWithAttributes(Math.max(0, curr - prev)); + }; + } + @Override protected void doStart() { metrics.addAll(registerAsyncMetrics(registry, stateCache)); @@ -218,22 +230,19 @@ private Map internalGetIndicesStats() { continue; // skip system indices } final ShardRouting shardRouting = indexShard.routingEntry(); - if (shardRouting.primary() == false) { - continue; // count primaries only - } - if (shardRouting.recoverySource() != null) { - continue; // exclude relocating shards - } final IndexMode indexMode = indexShard.indexSettings().getMode(); final IndexStats indexStats = stats.get(indexMode); - if (shardRouting.shardId().id() == 0) { - indexStats.numIndices++; - } try { - indexStats.numDocs += indexShard.commitStats().getNumDocs(); - indexStats.numBytes += indexShard.storeStats().sizeInBytes(); + if (shardRouting.primary() && shardRouting.recoverySource() == null) { + if (shardRouting.shardId().id() == 0) { + indexStats.numIndices++; + } + final DocsStats docStats = indexShard.docStats(); + indexStats.numDocs += docStats.getCount(); + indexStats.numBytes += docStats.getTotalSizeInBytes(); + indexStats.indexing.add(indexShard.indexingStats().getTotal()); + } indexStats.search.add(indexShard.searchStats().getTotal()); - indexStats.indexing.add(indexShard.indexingStats().getTotal()); } catch (IllegalIndexShardStateException | AlreadyClosedException ignored) { // ignored } From 5b1b889b97912d9c7c84b5433b26af4bfd5a797f Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Tue, 8 Oct 2024 19:54:19 +0200 Subject: [PATCH 185/194] Authenticate to elastic docker registry for resolving wolfi image (#114347) We need to resolve the latest wolfi image from our docker registry --- .buildkite/pipelines/dra-workflow.yml | 2 ++ .buildkite/pipelines/periodic-packaging.template.yml | 3 ++- .buildkite/pipelines/periodic-packaging.yml | 3 ++- .../pipelines/pull-request/packaging-tests-unix-sample.yml | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipelines/dra-workflow.yml b/.buildkite/pipelines/dra-workflow.yml index 25477c8541fa9..36828a6512db8 100644 --- a/.buildkite/pipelines/dra-workflow.yml +++ b/.buildkite/pipelines/dra-workflow.yml @@ -2,6 +2,7 @@ steps: - command: .buildkite/scripts/dra-workflow.sh env: USE_DRA_CREDENTIALS: "true" + USE_PROD_DOCKER_CREDENTIALS: "true" agents: provider: gcp image: family/elasticsearch-ubuntu-2204 @@ -18,4 +19,5 @@ steps: branch: "${BUILDKITE_BRANCH}" env: DRA_WORKFLOW: staging + USE_PROD_DOCKER_CREDENTIALS: "true" if: build.env('DRA_WORKFLOW') == 'staging' diff --git a/.buildkite/pipelines/periodic-packaging.template.yml b/.buildkite/pipelines/periodic-packaging.template.yml index e0da1f46486ea..dfedfac9d5b04 100644 --- a/.buildkite/pipelines/periodic-packaging.template.yml +++ b/.buildkite/pipelines/periodic-packaging.template.yml @@ -27,7 +27,8 @@ steps: image: family/elasticsearch-{{matrix.image}} diskSizeGb: 350 machineType: n1-standard-8 - env: {} + env: + USE_PROD_DOCKER_CREDENTIALS: "true" - group: packaging-tests-upgrade steps: $BWC_STEPS - group: packaging-tests-windows diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 5a05d75cf95ac..b29747c60617e 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -28,7 +28,8 @@ steps: image: family/elasticsearch-{{matrix.image}} diskSizeGb: 350 machineType: n1-standard-8 - env: {} + env: + USE_PROD_DOCKER_CREDENTIALS: "true" - group: packaging-tests-upgrade steps: - label: "{{matrix.image}} / 8.0.1 / packaging-tests-upgrade" diff --git a/.buildkite/pipelines/pull-request/packaging-tests-unix-sample.yml b/.buildkite/pipelines/pull-request/packaging-tests-unix-sample.yml index 98bc61ea33738..97558381d0a01 100644 --- a/.buildkite/pipelines/pull-request/packaging-tests-unix-sample.yml +++ b/.buildkite/pipelines/pull-request/packaging-tests-unix-sample.yml @@ -24,4 +24,5 @@ steps: diskSizeGb: 350 machineType: custom-16-32768 env: + USE_PROD_DOCKER_CREDENTIALS: "true" PACKAGING_TASK: "{{matrix.PACKAGING_TASK}}" From ebe3c0f10d6e5180f2aa9593734e9311d8ce4c48 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 8 Oct 2024 14:04:36 -0400 Subject: [PATCH 186/194] ESQL: Document MV_SLICE limitations (#114162) `MV_SLICE` is useful, but loading values from lucene frequently sorts them so `MV_SLICE` is not as useful as you think it is. It's mostly for after, say, a `SPLIT`. This documents that and adds a link to the section on multivalues. It also moves similar docs to a separate paragraph in the docs for easier reading. --- .../esql/functions/description/mv_first.asciidoc | 8 +++++++- .../esql/functions/description/mv_last.asciidoc | 8 +++++++- .../esql/functions/description/mv_slice.asciidoc | 6 +++++- .../esql/functions/kibana/definition/mv_first.json | 2 +- .../esql/functions/kibana/definition/mv_last.json | 2 +- .../esql/functions/kibana/definition/mv_slice.json | 2 +- docs/reference/esql/functions/kibana/docs/mv_first.md | 6 ------ docs/reference/esql/functions/kibana/docs/mv_last.md | 6 ------ docs/reference/esql/functions/kibana/docs/mv_slice.md | 2 ++ .../xpack/esql/expression/function/FunctionInfo.java | 11 ++++++++--- .../function/scalar/multivalue/MvFirst.java | 4 ++-- .../expression/function/scalar/multivalue/MvLast.java | 4 ++-- .../function/scalar/multivalue/MvSlice.java | 9 ++++++++- 13 files changed, 44 insertions(+), 26 deletions(-) diff --git a/docs/reference/esql/functions/description/mv_first.asciidoc b/docs/reference/esql/functions/description/mv_first.asciidoc index 99223e2c02d9f..13c433ce209d0 100644 --- a/docs/reference/esql/functions/description/mv_first.asciidoc +++ b/docs/reference/esql/functions/description/mv_first.asciidoc @@ -2,4 +2,10 @@ *Description* -Converts a multivalued expression into a single valued column containing the first value. This is most useful when reading from a function that emits multivalued columns in a known order like <>. The order that <> are read from underlying storage is not guaranteed. It is *frequently* ascending, but don't rely on that. If you need the minimum value use <> instead of `MV_FIRST`. `MV_MIN` has optimizations for sorted values so there isn't a performance benefit to `MV_FIRST`. +Converts a multivalued expression into a single valued column containing the first value. This is most useful when reading from a function that emits multivalued columns in a known order like <>. + +The order that <> are read from +underlying storage is not guaranteed. It is *frequently* ascending, but don't +rely on that. If you need the minimum value use <> instead of +`MV_FIRST`. `MV_MIN` has optimizations for sorted values so there isn't a +performance benefit to `MV_FIRST`. diff --git a/docs/reference/esql/functions/description/mv_last.asciidoc b/docs/reference/esql/functions/description/mv_last.asciidoc index 4b4b4336588d1..beba7b5a402c9 100644 --- a/docs/reference/esql/functions/description/mv_last.asciidoc +++ b/docs/reference/esql/functions/description/mv_last.asciidoc @@ -2,4 +2,10 @@ *Description* -Converts a multivalue expression into a single valued column containing the last value. This is most useful when reading from a function that emits multivalued columns in a known order like <>. The order that <> are read from underlying storage is not guaranteed. It is *frequently* ascending, but don't rely on that. If you need the maximum value use <> instead of `MV_LAST`. `MV_MAX` has optimizations for sorted values so there isn't a performance benefit to `MV_LAST`. +Converts a multivalue expression into a single valued column containing the last value. This is most useful when reading from a function that emits multivalued columns in a known order like <>. + +The order that <> are read from +underlying storage is not guaranteed. It is *frequently* ascending, but don't +rely on that. If you need the maximum value use <> instead of +`MV_LAST`. `MV_MAX` has optimizations for sorted values so there isn't a +performance benefit to `MV_LAST`. diff --git a/docs/reference/esql/functions/description/mv_slice.asciidoc b/docs/reference/esql/functions/description/mv_slice.asciidoc index 24d3183b6f40e..98438ae097fe7 100644 --- a/docs/reference/esql/functions/description/mv_slice.asciidoc +++ b/docs/reference/esql/functions/description/mv_slice.asciidoc @@ -2,4 +2,8 @@ *Description* -Returns a subset of the multivalued field using the start and end index values. +Returns a subset of the multivalued field using the start and end index values. This is most useful when reading from a function that emits multivalued columns in a known order like <> or <>. + +The order that <> are read from +underlying storage is not guaranteed. It is *frequently* ascending, but don't +rely on that. diff --git a/docs/reference/esql/functions/kibana/definition/mv_first.json b/docs/reference/esql/functions/kibana/definition/mv_first.json index 480c0af2f0918..80e761faafab9 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_first.json +++ b/docs/reference/esql/functions/kibana/definition/mv_first.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "mv_first", - "description" : "Converts a multivalued expression into a single valued column containing the\nfirst value. This is most useful when reading from a function that emits\nmultivalued columns in a known order like <>.\n\nThe order that <> are read from\nunderlying storage is not guaranteed. It is *frequently* ascending, but don't\nrely on that. If you need the minimum value use <> instead of\n`MV_FIRST`. `MV_MIN` has optimizations for sorted values so there isn't a\nperformance benefit to `MV_FIRST`.", + "description" : "Converts a multivalued expression into a single valued column containing the\nfirst value. This is most useful when reading from a function that emits\nmultivalued columns in a known order like <>.", "signatures" : [ { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/mv_last.json b/docs/reference/esql/functions/kibana/definition/mv_last.json index 0918e46454265..fb16400f86e62 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_last.json +++ b/docs/reference/esql/functions/kibana/definition/mv_last.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "mv_last", - "description" : "Converts a multivalue expression into a single valued column containing the last\nvalue. This is most useful when reading from a function that emits multivalued\ncolumns in a known order like <>.\n\nThe order that <> are read from\nunderlying storage is not guaranteed. It is *frequently* ascending, but don't\nrely on that. If you need the maximum value use <> instead of\n`MV_LAST`. `MV_MAX` has optimizations for sorted values so there isn't a\nperformance benefit to `MV_LAST`.", + "description" : "Converts a multivalue expression into a single valued column containing the last\nvalue. This is most useful when reading from a function that emits multivalued\ncolumns in a known order like <>.", "signatures" : [ { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/mv_slice.json b/docs/reference/esql/functions/kibana/definition/mv_slice.json index dcae77f1545a0..399a6145b040e 100644 --- a/docs/reference/esql/functions/kibana/definition/mv_slice.json +++ b/docs/reference/esql/functions/kibana/definition/mv_slice.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "mv_slice", - "description" : "Returns a subset of the multivalued field using the start and end index values.", + "description" : "Returns a subset of the multivalued field using the start and end index values.\nThis is most useful when reading from a function that emits multivalued columns\nin a known order like <> or <>.", "signatures" : [ { "params" : [ diff --git a/docs/reference/esql/functions/kibana/docs/mv_first.md b/docs/reference/esql/functions/kibana/docs/mv_first.md index 4faea6edd9162..c50ed7d764020 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_first.md +++ b/docs/reference/esql/functions/kibana/docs/mv_first.md @@ -7,12 +7,6 @@ Converts a multivalued expression into a single valued column containing the first value. This is most useful when reading from a function that emits multivalued columns in a known order like <>. -The order that <> are read from -underlying storage is not guaranteed. It is *frequently* ascending, but don't -rely on that. If you need the minimum value use <> instead of -`MV_FIRST`. `MV_MIN` has optimizations for sorted values so there isn't a -performance benefit to `MV_FIRST`. - ``` ROW a="foo;bar;baz" | EVAL first_a = MV_FIRST(SPLIT(a, ";")) diff --git a/docs/reference/esql/functions/kibana/docs/mv_last.md b/docs/reference/esql/functions/kibana/docs/mv_last.md index a8c3bf25eb51b..eeefd929c1359 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_last.md +++ b/docs/reference/esql/functions/kibana/docs/mv_last.md @@ -7,12 +7,6 @@ Converts a multivalue expression into a single valued column containing the last value. This is most useful when reading from a function that emits multivalued columns in a known order like <>. -The order that <> are read from -underlying storage is not guaranteed. It is *frequently* ascending, but don't -rely on that. If you need the maximum value use <> instead of -`MV_LAST`. `MV_MAX` has optimizations for sorted values so there isn't a -performance benefit to `MV_LAST`. - ``` ROW a="foo;bar;baz" | EVAL last_a = MV_LAST(SPLIT(a, ";")) diff --git a/docs/reference/esql/functions/kibana/docs/mv_slice.md b/docs/reference/esql/functions/kibana/docs/mv_slice.md index 3daf0de930a7f..bba7a219960ef 100644 --- a/docs/reference/esql/functions/kibana/docs/mv_slice.md +++ b/docs/reference/esql/functions/kibana/docs/mv_slice.md @@ -4,6 +4,8 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ ### MV_SLICE Returns a subset of the multivalued field using the start and end index values. +This is most useful when reading from a function that emits multivalued columns +in a known order like <> or <>. ``` row a = [1, 2, 2, 3] diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionInfo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionInfo.java index f275496c6787a..1491f5643e4f5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionInfo.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/FunctionInfo.java @@ -29,13 +29,18 @@ boolean preview() default false; /** - * The description of the function rendered in {@code META FUNCTIONS} - * and the docs. These should be complete sentences. + * The description of the function rendered in the docs and kibana's + * json files that drive their IDE-like experience. These should be + * complete sentences but can contain asciidoc syntax. It is rendered + * as a single paragraph. */ String description() default ""; /** - * Detailed descriptions of the function rendered in the docs. + * Detailed descriptions of the function rendered in the docs. This is + * rendered as a single paragraph following {@link #description()} in + * the docs and is excluded from Kibana's IDE-like + * experience. It can contain asciidoc syntax. */ String detailedDescription() default ""; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java index 6e76888f72b1d..d5d203e7bb3d1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java @@ -59,8 +59,8 @@ public class MvFirst extends AbstractMultivalueFunction { description = """ Converts a multivalued expression into a single valued column containing the first value. This is most useful when reading from a function that emits - multivalued columns in a known order like <>. - + multivalued columns in a known order like <>.""", + detailedDescription = """ The order that <> are read from underlying storage is not guaranteed. It is *frequently* ascending, but don't rely on that. If you need the minimum value use <> instead of diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java index 198731ca601f4..21487f14817cd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java @@ -59,8 +59,8 @@ public class MvLast extends AbstractMultivalueFunction { description = """ Converts a multivalue expression into a single valued column containing the last value. This is most useful when reading from a function that emits multivalued - columns in a known order like <>. - + columns in a known order like <>.""", + detailedDescription = """ The order that <> are read from underlying storage is not guaranteed. It is *frequently* ascending, but don't rely on that. If you need the maximum value use <> instead of diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java index 9846ebe4111c0..a829b6f1417b9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java @@ -68,7 +68,14 @@ public class MvSlice extends EsqlScalarFunction implements OptionalArgument, Eva "long", "text", "version" }, - description = "Returns a subset of the multivalued field using the start and end index values.", + description = """ + Returns a subset of the multivalued field using the start and end index values. + This is most useful when reading from a function that emits multivalued columns + in a known order like <> or <>.""", + detailedDescription = """ + The order that <> are read from + underlying storage is not guaranteed. It is *frequently* ascending, but don't + rely on that.""", examples = { @Example(file = "ints", tag = "mv_slice_positive"), @Example(file = "ints", tag = "mv_slice_negative") } ) public MvSlice( From a9ea432ac59960babe5cb75c1aa0c9f51b72860b Mon Sep 17 00:00:00 2001 From: Brendan Cully Date: Tue, 8 Oct 2024 11:15:35 -0700 Subject: [PATCH 187/194] Set SlowLog logging to TRACE in tests (#114344) The tests depend on the SlowLog loggers running at TRACE level but were not setting the level themselves. Instead they relied on the SlowLog setting the level to trace internally when it was created. If something else globally adjusted log levels between the time the SlowLog loggers were created and the tests ran, the tests could fail. And in fact, `ScopedSettingsTest.testFallbackToLoggerLevel` was updating the root log level, which had the side effect of updating the SlowLog level. In #112183 SlowLog's log initialization was made static, which opened up its test to failure when ScopedSettingsTest ran before a SlowLog test in the same JVM. I do not know if the intention of the SlowLog is that it overrides the global log level and should always be set at TRACE, in which case this fix is incorrect. It seems surprising, but I don't know why else SlowLog would explicitly initialize itself to TRACE. However, if that was the intention, the code was already at risk due to having no guard against being changed by Loggers.setLevel on an ancestor log. The change in this PR is at least not a regression in that behaviour. It does no longer start out at TRACE however, which is a change in behaviour. --- .../java/org/elasticsearch/index/IndexingSlowLog.java | 5 ----- .../java/org/elasticsearch/index/SearchSlowLog.java | 7 ------- .../org/elasticsearch/index/IndexingSlowLogTests.java | 7 ++++++- .../org/elasticsearch/index/SearchSlowLogTests.java | 10 +++++++++- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java b/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java index 70ecef375498e..3ae4c0eb82ad0 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java +++ b/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java @@ -9,13 +9,11 @@ package org.elasticsearch.index; -import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.util.StringBuilders; import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.ESLogMessage; -import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.xcontent.XContentHelper; @@ -92,9 +90,6 @@ public final class IndexingSlowLog implements IndexingOperationListener { ); private static final Logger indexLogger = LogManager.getLogger(INDEX_INDEXING_SLOWLOG_PREFIX + ".index"); - static { - Loggers.setLevel(indexLogger, Level.TRACE); - } private final Index index; diff --git a/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java b/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java index 2a2d650e20aa2..e4836a391bfec 100644 --- a/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java +++ b/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java @@ -9,11 +9,9 @@ package org.elasticsearch.index; -import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.common.logging.ESLogMessage; -import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.core.TimeValue; @@ -47,11 +45,6 @@ public final class SearchSlowLog implements SearchOperationListener { private static final Logger queryLogger = LogManager.getLogger(INDEX_SEARCH_SLOWLOG_PREFIX + ".query"); private static final Logger fetchLogger = LogManager.getLogger(INDEX_SEARCH_SLOWLOG_PREFIX + ".fetch"); - static { - Loggers.setLevel(queryLogger, Level.TRACE); - Loggers.setLevel(fetchLogger, Level.TRACE); - } - private final SlowLogFieldProvider slowLogFieldProvider; public static final Setting INDEX_SEARCH_SLOWLOG_INCLUDE_USER_SETTING = Setting.boolSetting( diff --git a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java index 03550ca7fc03f..753602e73a30a 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java @@ -58,18 +58,23 @@ public class IndexingSlowLogTests extends ESTestCase { static MockAppender appender; static Releasable appenderRelease; static Logger testLogger1 = LogManager.getLogger(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_PREFIX + ".index"); + static Level origLogLevel = testLogger1.getLevel(); @BeforeClass public static void init() throws IllegalAccessException { appender = new MockAppender("trace_appender"); appender.start(); Loggers.addAppender(testLogger1, appender); + + Loggers.setLevel(testLogger1, Level.TRACE); } @AfterClass public static void cleanup() { - appender.stop(); Loggers.removeAppender(testLogger1, appender); + appender.stop(); + + Loggers.setLevel(testLogger1, origLogLevel); } public void testLevelPrecedence() { diff --git a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java index dd1790cc786af..50e3269a6b9ba 100644 --- a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java +++ b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java @@ -50,6 +50,8 @@ public class SearchSlowLogTests extends ESSingleNodeTestCase { static MockAppender appender; static Logger queryLog = LogManager.getLogger(SearchSlowLog.INDEX_SEARCH_SLOWLOG_PREFIX + ".query"); static Logger fetchLog = LogManager.getLogger(SearchSlowLog.INDEX_SEARCH_SLOWLOG_PREFIX + ".fetch"); + static Level origQueryLogLevel = queryLog.getLevel(); + static Level origFetchLogLevel = fetchLog.getLevel(); @BeforeClass public static void init() throws IllegalAccessException { @@ -57,13 +59,19 @@ public static void init() throws IllegalAccessException { appender.start(); Loggers.addAppender(queryLog, appender); Loggers.addAppender(fetchLog, appender); + + Loggers.setLevel(queryLog, Level.TRACE); + Loggers.setLevel(fetchLog, Level.TRACE); } @AfterClass public static void cleanup() { - appender.stop(); Loggers.removeAppender(queryLog, appender); Loggers.removeAppender(fetchLog, appender); + appender.stop(); + + Loggers.setLevel(queryLog, origQueryLogLevel); + Loggers.setLevel(fetchLog, origFetchLogLevel); } @Override From 216d2de877828d868b9864ea5371df8239142605 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 8 Oct 2024 14:38:15 -0400 Subject: [PATCH 188/194] ESQL: Weaken test assertion (#114336) Weaken the assertion when testing breakers: it's ok to break while building a block in addition to topn. --- .../org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java index a24bd91206ac0..38b3dd4bd7e30 100644 --- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java +++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java @@ -166,7 +166,7 @@ public void testSortByManyLongsTooMuchMemoryAsync() throws IOException { "error", matchesMap().extraOk() .entry("bytes_wanted", greaterThan(1000)) - .entry("reason", matchesRegex("\\[request] Data too large, data for \\[topn] would .+")) + .entry("reason", matchesRegex("\\[request] Data too large, data for \\[(topn|esql_block_factory)] would .+")) .entry("durability", "TRANSIENT") .entry("type", "circuit_breaking_exception") .entry("bytes_limit", greaterThan(1000)) From 19d7028631a284f81c52d7ecdfe152320da3f30e Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Tue, 8 Oct 2024 20:44:24 +0200 Subject: [PATCH 189/194] Run fail formatting yaml test with 1 shard (#114214) --- .../test/aggregations/stats_metric_fail_formatting.yml | 2 ++ muted-tests.yml | 6 ------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml index 82371c973407c..1ff376eac61d1 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/stats_metric_fail_formatting.yml @@ -3,6 +3,8 @@ setup: indices.create: index: test_date body: + settings: + number_of_shards: 1 mappings: properties: date_field: diff --git a/muted-tests.yml b/muted-tests.yml index 696d7a4496e66..ac3730c08b852 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -354,9 +354,6 @@ tests: - class: org.elasticsearch.action.bulk.IncrementalBulkIT method: testIncrementalBulkLowWatermarkBackOff issue: https://github.com/elastic/elasticsearch/issues/114182 -- class: org.elasticsearch.aggregations.AggregationsClientYamlTestSuiteIT - method: test {yaml=aggregations/stats_metric_fail_formatting/fail formatting} - issue: https://github.com/elastic/elasticsearch/issues/114187 - class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT issue: https://github.com/elastic/elasticsearch/issues/114194 - class: org.elasticsearch.xpack.ilm.ExplainLifecycleIT @@ -373,9 +370,6 @@ tests: - class: org.elasticsearch.index.SearchSlowLogTests method: testTwoLoggersDifferentLevel issue: https://github.com/elastic/elasticsearch/issues/114301 -- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT - method: test {p0=aggregations/stats_metric_fail_formatting/fail formatting} - issue: https://github.com/elastic/elasticsearch/issues/114320 - class: org.elasticsearch.xpack.inference.services.cohere.CohereServiceTests method: testInfer_StreamRequest_ErrorResponse issue: https://github.com/elastic/elasticsearch/issues/114327 From 6955bc18a2e5e15f30b89191136596a605a2d808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Halil=20B=C3=BClent=20Orhon?= Date: Tue, 8 Oct 2024 21:47:37 +0300 Subject: [PATCH 190/194] Fix analyzed wildcard query in simple_query_string when disjunctions is empty (#114264) This change fixes analyzed wildcard query in simple_query_string when disjunctions is empty. Closes #114185 --- docs/changelog/114264.yaml | 5 ++++ .../search/query/SimpleQueryStringIT.java | 26 +++++++++++++++++++ .../search/SimpleQueryStringQueryParser.java | 3 +++ 3 files changed, 34 insertions(+) create mode 100644 docs/changelog/114264.yaml diff --git a/docs/changelog/114264.yaml b/docs/changelog/114264.yaml new file mode 100644 index 0000000000000..fe421f6422830 --- /dev/null +++ b/docs/changelog/114264.yaml @@ -0,0 +1,5 @@ +pr: 114264 +summary: "Fix analyzed wildcard query in simple_query_string when disjunctions is empty" +area: Search +type: bug +issues: [114185] diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java index 2fe7931d64c81..35f11eb1429b4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SimpleQueryStringIT.java @@ -582,6 +582,32 @@ public void testFieldAliasOnDisallowedFieldType() throws Exception { }); } + public void testSimpleQueryStringWithAnalysisStopWords() throws Exception { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject("body") + .field("type", "text") + .field("analyzer", "stop") + .endObject() + .endObject() + .endObject() + ); + + CreateIndexRequestBuilder mappingRequest = indicesAdmin().prepareCreate("test1").setMapping(mapping); + mappingRequest.get(); + indexRandom(true, prepareIndex("test1").setId("1").setSource("body", "Some Text")); + refresh(); + + assertHitCount( + prepareSearch().setQuery( + simpleQueryStringQuery("the* text*").analyzeWildcard(true).defaultOperator(Operator.AND).field("body") + ), + 1 + ); + } + private void assertHits(SearchHits hits, String... ids) { assertThat(hits.getTotalHits().value, equalTo((long) ids.length)); Set hitIds = new HashSet<>(); diff --git a/server/src/main/java/org/elasticsearch/index/search/SimpleQueryStringQueryParser.java b/server/src/main/java/org/elasticsearch/index/search/SimpleQueryStringQueryParser.java index 59394c0a8e6f9..5eaf79ad42bde 100644 --- a/server/src/main/java/org/elasticsearch/index/search/SimpleQueryStringQueryParser.java +++ b/server/src/main/java/org/elasticsearch/index/search/SimpleQueryStringQueryParser.java @@ -199,6 +199,9 @@ public Query newPrefixQuery(String text) { if (disjuncts.size() == 1) { return disjuncts.get(0); } + if (disjuncts.size() == 0) { + return null; + } return new DisjunctionMaxQuery(disjuncts, 1.0f); } From 30aef7e99049ffba0d7bd1f2c972ed38f138a548 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Tue, 8 Oct 2024 21:29:11 +0200 Subject: [PATCH 191/194] Revert wolfi image update (#114350) Latest wolfi update caused our wolfi-ess image build to fail --- .../main/java/org/elasticsearch/gradle/internal/DockerBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index 95f279bfa5162..ac83a01ffc294 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -31,7 +31,7 @@ public enum DockerBase { // Chainguard based wolfi image with latest jdk // This is usually updated via renovatebot // spotless:off - WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:90888b190da54062f67f3fef1372eb0ae7d81ea55f5a1f56d748b13e4853d984", + WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:c16d3ad6cebf387e8dd2ad769f54320c4819fbbaa21e729fad087c7ae223b4d0", "-wolfi", "apk" ), From 7bf97da286ee700d9ed52d382674fdc57bfe18e4 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 8 Oct 2024 14:32:29 -0500 Subject: [PATCH 192/194] Fixing IpinfoIpDataLookupsTests for Windows (#114340) --- .../geoip/IpinfoIpDataLookupsTests.java | 27 ++++++++++++------- muted-tests.yml | 2 -- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java index 905eb027626a1..5689693d6c295 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java @@ -13,9 +13,11 @@ import com.maxmind.db.Networks; import com.maxmind.db.Reader; +import org.apache.lucene.util.Constants; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; @@ -48,17 +50,22 @@ public class IpinfoIpDataLookupsTests extends ESTestCase { private ThreadPool threadPool; private ResourceWatcherService resourceWatcherService; + // a temporary directory that mmdb files can be copied to and read from + private Path tmpDir; + @Before public void setup() { threadPool = new TestThreadPool(ConfigDatabases.class.getSimpleName()); Settings settings = Settings.builder().put("resource.reload.interval.high", TimeValue.timeValueMillis(100)).build(); resourceWatcherService = new ResourceWatcherService(settings, threadPool); + tmpDir = createTempDir(); } @After - public void cleanup() { + public void cleanup() throws IOException { resourceWatcherService.close(); threadPool.shutdownNow(); + IOUtils.rm(tmpDir); } public void testDatabasePropertyInvariants() { @@ -82,7 +89,8 @@ public void testParseAsn() { } public void testAsn() throws IOException { - Path configDir = createTempDir(); + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + Path configDir = tmpDir; copyDatabase("ipinfo/ip_asn_sample.mmdb", configDir.resolve("ip_asn_sample.mmdb")); copyDatabase("ipinfo/asn_sample.mmdb", configDir.resolve("asn_sample.mmdb")); @@ -91,8 +99,7 @@ public void testAsn() throws IOException { configDatabases.initialize(resourceWatcherService); // this is the 'free' ASN database (sample) - { - DatabaseReaderLazyLoader loader = configDatabases.getDatabase("ip_asn_sample.mmdb"); + try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("ip_asn_sample.mmdb")) { IpDataLookup lookup = new IpinfoIpDataLookups.Asn(Set.of(Database.Property.values())); Map data = lookup.getData(loader, "5.182.109.0"); assertThat( @@ -110,8 +117,7 @@ public void testAsn() throws IOException { } // this is the non-free or 'standard' ASN database (sample) - { - DatabaseReaderLazyLoader loader = configDatabases.getDatabase("asn_sample.mmdb"); + try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("asn_sample.mmdb")) { IpDataLookup lookup = new IpinfoIpDataLookups.Asn(Set.of(Database.Property.values())); Map data = lookup.getData(loader, "23.53.116.0"); assertThat( @@ -132,7 +138,8 @@ public void testAsn() throws IOException { } public void testAsnInvariants() { - Path configDir = createTempDir(); + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + Path configDir = tmpDir; copyDatabase("ipinfo/ip_asn_sample.mmdb", configDir.resolve("ip_asn_sample.mmdb")); copyDatabase("ipinfo/asn_sample.mmdb", configDir.resolve("asn_sample.mmdb")); @@ -168,7 +175,8 @@ public void testAsnInvariants() { } public void testCountry() throws IOException { - Path configDir = createTempDir(); + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + Path configDir = tmpDir; copyDatabase("ipinfo/ip_country_sample.mmdb", configDir.resolve("ip_country_sample.mmdb")); GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload @@ -176,8 +184,7 @@ public void testCountry() throws IOException { configDatabases.initialize(resourceWatcherService); // this is the 'free' Country database (sample) - { - DatabaseReaderLazyLoader loader = configDatabases.getDatabase("ip_country_sample.mmdb"); + try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("ip_country_sample.mmdb")) { IpDataLookup lookup = new IpinfoIpDataLookups.Country(Set.of(Database.Property.values())); Map data = lookup.getData(loader, "4.221.143.168"); assertThat( diff --git a/muted-tests.yml b/muted-tests.yml index ac3730c08b852..88379d4533a5b 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -362,8 +362,6 @@ tests: - class: org.elasticsearch.xpack.inference.services.openai.OpenAiServiceTests method: testInfer_StreamRequest issue: https://github.com/elastic/elasticsearch/issues/114232 -- class: org.elasticsearch.ingest.geoip.IpinfoIpDataLookupsTests - issue: https://github.com/elastic/elasticsearch/issues/114266 - class: org.elasticsearch.index.SearchSlowLogTests method: testLevelPrecedence issue: https://github.com/elastic/elasticsearch/issues/114300 From 2ba9bc98c5c88caad7e69143d6cb459c267426aa Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Tue, 8 Oct 2024 15:49:26 -0400 Subject: [PATCH 193/194] Adds an advanced binary quantization format copied from Lucene (#113491) This copies the work from https://github.com/apache/lucene/pull/13651 into Elasticsearch. The main reason for the copy is to simply allow it to be deployed & used in Elasticsearch prior to Elasticsearch upgrading to Lucene 10. At which point, we will then use the format as provided by Lucene. This is currently blocked by two pieces of work: blocked by: https://github.com/elastic/elasticsearch/pull/112933 blocked by: https://github.com/elastic/elasticsearch/pull/113333 After the format is merged, then code will be added for integration tests & integration with Elasticsearch through new index format types in the API. --- .../index/codec/vectors/BQSpaceUtils.java | 78 + .../index/codec/vectors/BQVectorUtils.java | 97 + .../vectors/BinarizedByteVectorValues.java | 58 + .../index/codec/vectors/BinaryQuantizer.java | 385 ++++ .../vectors/ES816BinaryFlatVectorsScorer.java | 273 +++ .../ES816BinaryQuantizedVectorsFormat.java | 75 + .../ES816BinaryQuantizedVectorsReader.java | 412 ++++ .../ES816BinaryQuantizedVectorsWriter.java | 987 +++++++++ ...ES816HnswBinaryQuantizedVectorsFormat.java | 144 ++ .../vectors/OffHeapBinarizedVectorValues.java | 456 ++++ ...RandomAccessBinarizedByteVectorValues.java | 70 + .../org.apache.lucene.codecs.KnnVectorsFormat | 2 + .../codec/vectors/BQVectorUtilsTests.java | 90 + .../vectors/BinaryQuantizationTests.java | 1856 +++++++++++++++++ .../ES816BinaryFlatVectorsScorerTests.java | 1746 ++++++++++++++++ ...S816BinaryQuantizedVectorsFormatTests.java | 175 ++ ...HnswBinaryQuantizedVectorsFormatTests.java | 126 ++ 17 files changed, 7030 insertions(+) create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/BQSpaceUtils.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/BQVectorUtils.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/BinarizedByteVectorValues.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/BinaryQuantizer.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorer.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsFormat.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsReader.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsWriter.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/ES816HnswBinaryQuantizedVectorsFormat.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/OffHeapBinarizedVectorValues.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/RandomAccessBinarizedByteVectorValues.java create mode 100644 server/src/test/java/org/elasticsearch/index/codec/vectors/BQVectorUtilsTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/codec/vectors/BinaryQuantizationTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorerTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsFormatTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/codec/vectors/ES816HnswBinaryQuantizedVectorsFormatTests.java diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/BQSpaceUtils.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/BQSpaceUtils.java new file mode 100644 index 0000000000000..68363b5926a6b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/BQSpaceUtils.java @@ -0,0 +1,78 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors; + +/** Utility class for quantization calculations */ +public class BQSpaceUtils { + + public static final short B_QUERY = 4; + // the first four bits masked + private static final int B_QUERY_MASK = 15; + + /** + * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 + * @param q the query vector, assumed to be half-byte quantized with values between 0 and 15 + * @param dimensions the number of dimensions in the query vector + * @param quantQueryByte the byte array to store the transposed query vector + */ + public static void transposeBin(byte[] q, int dimensions, byte[] quantQueryByte) { + // TODO: rewrite this in Panama Vector API + int qOffset = 0; + final byte[] v1 = new byte[4]; + final byte[] v = new byte[32]; + for (int i = 0; i < dimensions; i += 32) { + // for every four bytes we shift left (with remainder across those bytes) + for (int j = 0; j < v.length; j += 4) { + v[j] = (byte) (q[qOffset + j] << B_QUERY | ((q[qOffset + j] >>> B_QUERY) & B_QUERY_MASK)); + v[j + 1] = (byte) (q[qOffset + j + 1] << B_QUERY | ((q[qOffset + j + 1] >>> B_QUERY) & B_QUERY_MASK)); + v[j + 2] = (byte) (q[qOffset + j + 2] << B_QUERY | ((q[qOffset + j + 2] >>> B_QUERY) & B_QUERY_MASK)); + v[j + 3] = (byte) (q[qOffset + j + 3] << B_QUERY | ((q[qOffset + j + 3] >>> B_QUERY) & B_QUERY_MASK)); + } + for (int j = 0; j < B_QUERY; j++) { + moveMaskEpi8Byte(v, v1); + for (int k = 0; k < 4; k++) { + quantQueryByte[(B_QUERY - j - 1) * (dimensions / 8) + i / 8 + k] = v1[k]; + v1[k] = 0; + } + for (int k = 0; k < v.length; k += 4) { + v[k] = (byte) (v[k] + v[k]); + v[k + 1] = (byte) (v[k + 1] + v[k + 1]); + v[k + 2] = (byte) (v[k + 2] + v[k + 2]); + v[k + 3] = (byte) (v[k + 3] + v[k + 3]); + } + } + qOffset += 32; + } + } + + private static void moveMaskEpi8Byte(byte[] v, byte[] v1b) { + int m = 0; + for (int k = 0; k < v.length; k++) { + if ((v[k] & 0b10000000) == 0b10000000) { + v1b[m] |= 0b00000001; + } + if (k % 8 == 7) { + m++; + } else { + v1b[m] <<= 1; + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/BQVectorUtils.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/BQVectorUtils.java new file mode 100644 index 0000000000000..3d2acb533e26d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/BQVectorUtils.java @@ -0,0 +1,97 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BitUtil; +import org.apache.lucene.util.VectorUtil; + +/** Utility class for vector quantization calculations */ +public class BQVectorUtils { + private static final float EPSILON = 1e-4f; + + public static boolean isUnitVector(float[] v) { + double l1norm = VectorUtil.dotProduct(v, v); + return Math.abs(l1norm - 1.0d) <= EPSILON; + } + + public static int discretize(int value, int bucket) { + return ((value + (bucket - 1)) / bucket) * bucket; + } + + public static float[] pad(float[] vector, int dimensions) { + if (vector.length >= dimensions) { + return vector; + } + return ArrayUtil.growExact(vector, dimensions); + } + + public static byte[] pad(byte[] vector, int dimensions) { + if (vector.length >= dimensions) { + return vector; + } + return ArrayUtil.growExact(vector, dimensions); + } + + /** + * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 + * @param d the byte array to count the number of set bits in + * @return count of flipped bits in the byte array + */ + public static int popcount(byte[] d) { + int r = 0; + int cnt = 0; + for (final int upperBound = d.length & -Integer.BYTES; r < upperBound; r += Integer.BYTES) { + cnt += Integer.bitCount((int) BitUtil.VH_NATIVE_INT.get(d, r)); + } + for (; r < d.length; r++) { + cnt += Integer.bitCount(d[r] & 0xFF); + } + return cnt; + } + + // TODO: move to VectorUtil & vectorize? + public static void divideInPlace(float[] a, float b) { + for (int j = 0; j < a.length; j++) { + a[j] /= b; + } + } + + public static float[] subtract(float[] a, float[] b) { + float[] result = new float[a.length]; + subtract(a, b, result); + return result; + } + + public static void subtractInPlace(float[] target, float[] other) { + subtract(target, other, target); + } + + private static void subtract(float[] a, float[] b, float[] result) { + for (int j = 0; j < a.length; j++) { + result[j] = a[j] - b[j]; + } + } + + public static float norm(float[] vector) { + float magnitude = VectorUtil.dotProduct(vector, vector); + return (float) Math.sqrt(magnitude); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/BinarizedByteVectorValues.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/BinarizedByteVectorValues.java new file mode 100644 index 0000000000000..73dd4273a794e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/BinarizedByteVectorValues.java @@ -0,0 +1,58 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.VectorScorer; + +import java.io.IOException; + +/** + * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 + */ +public abstract class BinarizedByteVectorValues extends DocIdSetIterator { + + public abstract float[] getCorrectiveTerms(); + + public abstract byte[] vectorValue() throws IOException; + + /** Return the dimension of the vectors */ + public abstract int dimension(); + + /** + * Return the number of vectors for this field. + * + * @return the number of vectors returned by this iterator + */ + public abstract int size(); + + @Override + public final long cost() { + return size(); + } + + /** + * Return a {@link VectorScorer} for the given query vector. + * + * @param query the query vector + * @return a {@link VectorScorer} instance or null + */ + public abstract VectorScorer scorer(float[] query) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/BinaryQuantizer.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/BinaryQuantizer.java new file mode 100644 index 0000000000000..192fb9092ac3a --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/BinaryQuantizer.java @@ -0,0 +1,385 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.VectorUtil; + +import static org.apache.lucene.index.VectorSimilarityFunction.COSINE; +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; +import static org.elasticsearch.index.codec.vectors.BQVectorUtils.isUnitVector; + +/** + * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 + * Quantized that quantizes raw vector values to binary. The algorithm is based on the paper RaBitQ. + */ +public class BinaryQuantizer { + private final int discretizedDimensions; + + private final VectorSimilarityFunction similarityFunction; + private final float sqrtDimensions; + + public BinaryQuantizer(int dimensions, int discretizedDimensions, VectorSimilarityFunction similarityFunction) { + if (dimensions <= 0) { + throw new IllegalArgumentException("dimensions must be > 0 but was: " + dimensions); + } + assert discretizedDimensions % 64 == 0 : "discretizedDimensions must be a multiple of 64 but was: " + discretizedDimensions; + this.discretizedDimensions = discretizedDimensions; + this.similarityFunction = similarityFunction; + this.sqrtDimensions = (float) Math.sqrt(dimensions); + } + + BinaryQuantizer(int dimensions, VectorSimilarityFunction similarityFunction) { + this(dimensions, dimensions, similarityFunction); + } + + private static void removeSignAndDivide(float[] a, float divisor) { + for (int i = 0; i < a.length; i++) { + a[i] = Math.abs(a[i]) / divisor; + } + } + + private static float sumAndNormalize(float[] a, float norm) { + float aDivided = 0f; + + for (int i = 0; i < a.length; i++) { + aDivided += a[i]; + } + + aDivided = aDivided / norm; + if (Float.isFinite(aDivided) == false) { + aDivided = 0.8f; // can be anything + } + + return aDivided; + } + + private static void packAsBinary(float[] vector, byte[] packedVector) { + for (int h = 0; h < vector.length; h += 8) { + byte result = 0; + int q = 0; + for (int i = 7; i >= 0; i--) { + if (vector[h + i] > 0) { + result |= (byte) (1 << q); + } + q++; + } + packedVector[h / 8] = result; + } + } + + public VectorSimilarityFunction getSimilarity() { + return this.similarityFunction; + } + + private record SubspaceOutput(float projection) {} + + private SubspaceOutput generateSubSpace(float[] vector, float[] centroid, byte[] quantizedVector) { + // typically no-op if dimensions/64 + float[] paddedCentroid = BQVectorUtils.pad(centroid, discretizedDimensions); + float[] paddedVector = BQVectorUtils.pad(vector, discretizedDimensions); + + BQVectorUtils.subtractInPlace(paddedVector, paddedCentroid); + + // The inner product between the data vector and the quantized data vector + float norm = BQVectorUtils.norm(paddedVector); + + packAsBinary(paddedVector, quantizedVector); + + removeSignAndDivide(paddedVector, sqrtDimensions); + float projection = sumAndNormalize(paddedVector, norm); + + return new SubspaceOutput(projection); + } + + record SubspaceOutputMIP(float OOQ, float normOC, float oDotC) {} + + private SubspaceOutputMIP generateSubSpaceMIP(float[] vector, float[] centroid, byte[] quantizedVector) { + + // typically no-op if dimensions/64 + float[] paddedCentroid = BQVectorUtils.pad(centroid, discretizedDimensions); + float[] paddedVector = BQVectorUtils.pad(vector, discretizedDimensions); + + float oDotC = VectorUtil.dotProduct(paddedVector, paddedCentroid); + BQVectorUtils.subtractInPlace(paddedVector, paddedCentroid); + + float normOC = BQVectorUtils.norm(paddedVector); + packAsBinary(paddedVector, quantizedVector); + BQVectorUtils.divideInPlace(paddedVector, normOC); // OmC / norm(OmC) + + float OOQ = computerOOQ(vector.length, paddedVector, quantizedVector); + + return new SubspaceOutputMIP(OOQ, normOC, oDotC); + } + + private float computerOOQ(int originalLength, float[] normOMinusC, byte[] packedBinaryVector) { + float OOQ = 0f; + for (int j = 0; j < originalLength / 8; j++) { + for (int r = 0; r < 8; r++) { + int sign = ((packedBinaryVector[j] >> (7 - r)) & 0b00000001); + OOQ += (normOMinusC[j * 8 + r] * (2 * sign - 1)); + } + } + OOQ = OOQ / sqrtDimensions; + return OOQ; + } + + private static float[] range(float[] q) { + float vl = 1e20f; + float vr = -1e20f; + for (int i = 0; i < q.length; i++) { + if (q[i] < vl) { + vl = q[i]; + } + if (q[i] > vr) { + vr = q[i]; + } + } + + return new float[] { vl, vr }; + } + + /** Results of quantizing a vector for both querying and indexing */ + public record QueryAndIndexResults(float[] indexFeatures, QueryFactors queryFeatures) {} + + public QueryAndIndexResults quantizeQueryAndIndex(float[] vector, byte[] indexDestination, byte[] queryDestination, float[] centroid) { + assert similarityFunction != COSINE || isUnitVector(vector); + assert similarityFunction != COSINE || isUnitVector(centroid); + assert this.discretizedDimensions == BQVectorUtils.discretize(vector.length, 64); + + if (this.discretizedDimensions != indexDestination.length * 8) { + throw new IllegalArgumentException( + "vector and quantized vector destination must be compatible dimensions: " + + BQVectorUtils.discretize(vector.length, 64) + + " [ " + + this.discretizedDimensions + + " ]" + + "!= " + + indexDestination.length + + " * 8" + ); + } + + if (this.discretizedDimensions != (queryDestination.length * 8) / BQSpaceUtils.B_QUERY) { + throw new IllegalArgumentException( + "vector and quantized vector destination must be compatible dimensions: " + + vector.length + + " [ " + + this.discretizedDimensions + + " ]" + + "!= (" + + queryDestination.length + + " * 8) / " + + BQSpaceUtils.B_QUERY + ); + } + + if (vector.length != centroid.length) { + throw new IllegalArgumentException( + "vector and centroid dimensions must be the same: " + vector.length + "!= " + centroid.length + ); + } + vector = ArrayUtil.copyArray(vector); + float distToC = VectorUtil.squareDistance(vector, centroid); + // only need vdotc for dot-products similarity, but not for euclidean + float vDotC = similarityFunction != EUCLIDEAN ? VectorUtil.dotProduct(vector, centroid) : 0f; + BQVectorUtils.subtractInPlace(vector, centroid); + // both euclidean and dot-product need the norm of the vector, just at different times + float normVmC = BQVectorUtils.norm(vector); + // quantize for index + packAsBinary(BQVectorUtils.pad(vector, discretizedDimensions), indexDestination); + if (similarityFunction != EUCLIDEAN) { + BQVectorUtils.divideInPlace(vector, normVmC); + } + + // Quantize for query + float[] range = range(vector); + float lower = range[0]; + float upper = range[1]; + // Δ := (𝑣𝑟 − 𝑣𝑙)/(2𝐵𝑞 − 1) + float width = (upper - lower) / ((1 << BQSpaceUtils.B_QUERY) - 1); + + QuantResult quantResult = quantize(vector, lower, width); + byte[] byteQuery = quantResult.result(); + + // q¯ = Δ · q¯𝑢 + 𝑣𝑙 · 1𝐷 + // q¯ is an approximation of q′ (scalar quantized approximation) + // FIXME: vectors need to be padded but that's expensive; update transponseBin to deal + byteQuery = BQVectorUtils.pad(byteQuery, discretizedDimensions); + BQSpaceUtils.transposeBin(byteQuery, discretizedDimensions, queryDestination); + QueryFactors factors = new QueryFactors(quantResult.quantizedSum, distToC, lower, width, normVmC, vDotC); + final float[] indexCorrections; + if (similarityFunction == EUCLIDEAN) { + indexCorrections = new float[2]; + indexCorrections[0] = (float) Math.sqrt(distToC); + removeSignAndDivide(vector, sqrtDimensions); + indexCorrections[1] = sumAndNormalize(vector, normVmC); + } else { + indexCorrections = new float[3]; + indexCorrections[0] = computerOOQ(vector.length, vector, indexDestination); + indexCorrections[1] = normVmC; + indexCorrections[2] = vDotC; + } + return new QueryAndIndexResults(indexCorrections, factors); + } + + public float[] quantizeForIndex(float[] vector, byte[] destination, float[] centroid) { + assert similarityFunction != COSINE || isUnitVector(vector); + assert similarityFunction != COSINE || isUnitVector(centroid); + assert this.discretizedDimensions == BQVectorUtils.discretize(vector.length, 64); + + if (this.discretizedDimensions != destination.length * 8) { + throw new IllegalArgumentException( + "vector and quantized vector destination must be compatible dimensions: " + + BQVectorUtils.discretize(vector.length, 64) + + " [ " + + this.discretizedDimensions + + " ]" + + "!= " + + destination.length + + " * 8" + ); + } + + if (vector.length != centroid.length) { + throw new IllegalArgumentException( + "vector and centroid dimensions must be the same: " + vector.length + "!= " + centroid.length + ); + } + + float[] corrections; + + // FIXME: make a copy of vector so we don't overwrite it here? + // ... (could trade subtractInPlace w subtract in genSubSpace) + vector = ArrayUtil.copyArray(vector); + + switch (similarityFunction) { + case EUCLIDEAN: + float distToCentroid = (float) Math.sqrt(VectorUtil.squareDistance(vector, centroid)); + + SubspaceOutput subspaceOutput = generateSubSpace(vector, centroid, destination); + corrections = new float[2]; + // FIXME: quantize these values so we are passing back 1 byte values for all three of these + corrections[0] = distToCentroid; + corrections[1] = subspaceOutput.projection(); + break; + case MAXIMUM_INNER_PRODUCT: + case COSINE: + case DOT_PRODUCT: + SubspaceOutputMIP subspaceOutputMIP = generateSubSpaceMIP(vector, centroid, destination); + corrections = new float[3]; + // FIXME: quantize these values so we are passing back 1 byte values for all three of these + corrections[0] = subspaceOutputMIP.OOQ(); + corrections[1] = subspaceOutputMIP.normOC(); + corrections[2] = subspaceOutputMIP.oDotC(); + break; + default: + throw new UnsupportedOperationException("Unsupported similarity function: " + similarityFunction); + } + + return corrections; + } + + private record QuantResult(byte[] result, int quantizedSum) {} + + private static QuantResult quantize(float[] vector, float lower, float width) { + // FIXME: speed up with panama? and/or use existing scalar quantization utils in Lucene? + byte[] result = new byte[vector.length]; + float oneOverWidth = 1.0f / width; + int sumQ = 0; + for (int i = 0; i < vector.length; i++) { + byte res = (byte) ((vector[i] - lower) * oneOverWidth); + result[i] = res; + sumQ += res; + } + + return new QuantResult(result, sumQ); + } + + /** Factors for quantizing query */ + public record QueryFactors(int quantizedSum, float distToC, float lower, float width, float normVmC, float vDotC) {} + + public QueryFactors quantizeForQuery(float[] vector, byte[] destination, float[] centroid) { + assert similarityFunction != COSINE || isUnitVector(vector); + assert similarityFunction != COSINE || isUnitVector(centroid); + assert this.discretizedDimensions == BQVectorUtils.discretize(vector.length, 64); + + if (this.discretizedDimensions != (destination.length * 8) / BQSpaceUtils.B_QUERY) { + throw new IllegalArgumentException( + "vector and quantized vector destination must be compatible dimensions: " + + vector.length + + " [ " + + this.discretizedDimensions + + " ]" + + "!= (" + + destination.length + + " * 8) / " + + BQSpaceUtils.B_QUERY + ); + } + + if (vector.length != centroid.length) { + throw new IllegalArgumentException( + "vector and centroid dimensions must be the same: " + vector.length + "!= " + centroid.length + ); + } + + float distToC = VectorUtil.squareDistance(vector, centroid); + + // FIXME: make a copy of vector so we don't overwrite it here? + // ... (could subtractInPlace but the passed vector is modified) <<--- + float[] vmC = BQVectorUtils.subtract(vector, centroid); + + // FIXME: should other similarity functions behave like MIP on query like COSINE + float normVmC = 0f; + if (similarityFunction != EUCLIDEAN) { + normVmC = BQVectorUtils.norm(vmC); + BQVectorUtils.divideInPlace(vmC, normVmC); + } + float[] range = range(vmC); + float lower = range[0]; + float upper = range[1]; + // Δ := (𝑣𝑟 − 𝑣𝑙)/(2𝐵𝑞 − 1) + float width = (upper - lower) / ((1 << BQSpaceUtils.B_QUERY) - 1); + + QuantResult quantResult = quantize(vmC, lower, width); + byte[] byteQuery = quantResult.result(); + + // q¯ = Δ · q¯𝑢 + 𝑣𝑙 · 1𝐷 + // q¯ is an approximation of q′ (scalar quantized approximation) + // FIXME: vectors need to be padded but that's expensive; update transponseBin to deal + byteQuery = BQVectorUtils.pad(byteQuery, discretizedDimensions); + BQSpaceUtils.transposeBin(byteQuery, discretizedDimensions, destination); + + QueryFactors factors; + if (similarityFunction != EUCLIDEAN) { + float vDotC = VectorUtil.dotProduct(vector, centroid); + // FIXME: quantize the corrections as well so we store less + factors = new QueryFactors(quantResult.quantizedSum, distToC, lower, width, normVmC, vDotC); + } else { + // FIXME: quantize the corrections as well so we store less + factors = new QueryFactors(quantResult.quantizedSum, distToC, lower, width, 0f, 0f); + } + + return factors; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorer.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorer.java new file mode 100644 index 0000000000000..78fa282709098 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorer.java @@ -0,0 +1,273 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.codecs.hnsw.FlatVectorsScorer; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.VectorUtil; +import org.apache.lucene.util.hnsw.RandomAccessVectorValues; +import org.apache.lucene.util.hnsw.RandomVectorScorer; +import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier; +import org.elasticsearch.simdvec.ESVectorUtil; + +import java.io.IOException; + +import static org.apache.lucene.index.VectorSimilarityFunction.COSINE; +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; +import static org.apache.lucene.index.VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT; + +/** Vector scorer over binarized vector values */ +public class ES816BinaryFlatVectorsScorer implements FlatVectorsScorer { + private final FlatVectorsScorer nonQuantizedDelegate; + + public ES816BinaryFlatVectorsScorer(FlatVectorsScorer nonQuantizedDelegate) { + this.nonQuantizedDelegate = nonQuantizedDelegate; + } + + @Override + public RandomVectorScorerSupplier getRandomVectorScorerSupplier( + VectorSimilarityFunction similarityFunction, + RandomAccessVectorValues vectorValues + ) throws IOException { + if (vectorValues instanceof RandomAccessBinarizedByteVectorValues) { + throw new UnsupportedOperationException( + "getRandomVectorScorerSupplier(VectorSimilarityFunction,RandomAccessVectorValues) not implemented for binarized format" + ); + } + return nonQuantizedDelegate.getRandomVectorScorerSupplier(similarityFunction, vectorValues); + } + + @Override + public RandomVectorScorer getRandomVectorScorer( + VectorSimilarityFunction similarityFunction, + RandomAccessVectorValues vectorValues, + float[] target + ) throws IOException { + if (vectorValues instanceof RandomAccessBinarizedByteVectorValues binarizedVectors) { + BinaryQuantizer quantizer = binarizedVectors.getQuantizer(); + float[] centroid = binarizedVectors.getCentroid(); + // FIXME: precompute this once? + int discretizedDimensions = BQVectorUtils.discretize(target.length, 64); + if (similarityFunction == COSINE) { + float[] copy = ArrayUtil.copyOfSubArray(target, 0, target.length); + VectorUtil.l2normalize(copy); + target = copy; + } + byte[] quantized = new byte[BQSpaceUtils.B_QUERY * discretizedDimensions / 8]; + BinaryQuantizer.QueryFactors factors = quantizer.quantizeForQuery(target, quantized, centroid); + BinaryQueryVector queryVector = new BinaryQueryVector(quantized, factors); + return new BinarizedRandomVectorScorer(queryVector, binarizedVectors, similarityFunction); + } + return nonQuantizedDelegate.getRandomVectorScorer(similarityFunction, vectorValues, target); + } + + @Override + public RandomVectorScorer getRandomVectorScorer( + VectorSimilarityFunction similarityFunction, + RandomAccessVectorValues vectorValues, + byte[] target + ) throws IOException { + return nonQuantizedDelegate.getRandomVectorScorer(similarityFunction, vectorValues, target); + } + + RandomVectorScorerSupplier getRandomVectorScorerSupplier( + VectorSimilarityFunction similarityFunction, + ES816BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues scoringVectors, + RandomAccessBinarizedByteVectorValues targetVectors + ) { + return new BinarizedRandomVectorScorerSupplier(scoringVectors, targetVectors, similarityFunction); + } + + @Override + public String toString() { + return "ES816BinaryFlatVectorsScorer(nonQuantizedDelegate=" + nonQuantizedDelegate + ")"; + } + + /** Vector scorer supplier over binarized vector values */ + static class BinarizedRandomVectorScorerSupplier implements RandomVectorScorerSupplier { + private final ES816BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues queryVectors; + private final RandomAccessBinarizedByteVectorValues targetVectors; + private final VectorSimilarityFunction similarityFunction; + + BinarizedRandomVectorScorerSupplier( + ES816BinaryQuantizedVectorsWriter.OffHeapBinarizedQueryVectorValues queryVectors, + RandomAccessBinarizedByteVectorValues targetVectors, + VectorSimilarityFunction similarityFunction + ) { + this.queryVectors = queryVectors; + this.targetVectors = targetVectors; + this.similarityFunction = similarityFunction; + } + + @Override + public RandomVectorScorer scorer(int ord) throws IOException { + byte[] vector = queryVectors.vectorValue(ord); + int quantizedSum = queryVectors.sumQuantizedValues(ord); + float distanceToCentroid = queryVectors.getCentroidDistance(ord); + float lower = queryVectors.getLower(ord); + float width = queryVectors.getWidth(ord); + float normVmC = 0f; + float vDotC = 0f; + if (similarityFunction != EUCLIDEAN) { + normVmC = queryVectors.getNormVmC(ord); + vDotC = queryVectors.getVDotC(ord); + } + BinaryQueryVector binaryQueryVector = new BinaryQueryVector( + vector, + new BinaryQuantizer.QueryFactors(quantizedSum, distanceToCentroid, lower, width, normVmC, vDotC) + ); + return new BinarizedRandomVectorScorer(binaryQueryVector, targetVectors, similarityFunction); + } + + @Override + public RandomVectorScorerSupplier copy() throws IOException { + return new BinarizedRandomVectorScorerSupplier(queryVectors.copy(), targetVectors.copy(), similarityFunction); + } + } + + /** A binarized query representing its quantized form along with factors */ + public record BinaryQueryVector(byte[] vector, BinaryQuantizer.QueryFactors factors) {} + + /** Vector scorer over binarized vector values */ + public static class BinarizedRandomVectorScorer extends RandomVectorScorer.AbstractRandomVectorScorer { + private final BinaryQueryVector queryVector; + private final RandomAccessBinarizedByteVectorValues targetVectors; + private final VectorSimilarityFunction similarityFunction; + + private final float sqrtDimensions; + + public BinarizedRandomVectorScorer( + BinaryQueryVector queryVectors, + RandomAccessBinarizedByteVectorValues targetVectors, + VectorSimilarityFunction similarityFunction + ) { + super(targetVectors); + this.queryVector = queryVectors; + this.targetVectors = targetVectors; + this.similarityFunction = similarityFunction; + // FIXME: precompute this once? + this.sqrtDimensions = (float) Utils.constSqrt(targetVectors.dimension()); + } + + // FIXME: utils class; pull this out + private static class Utils { + public static double sqrtNewtonRaphson(double x, double curr, double prev) { + return (curr == prev) ? curr : sqrtNewtonRaphson(x, 0.5 * (curr + x / curr), curr); + } + + public static double constSqrt(double x) { + return x >= 0 && Double.isInfinite(x) == false ? sqrtNewtonRaphson(x, x, 0) : Double.NaN; + } + } + + @Override + public float score(int targetOrd) throws IOException { + // FIXME: implement fastscan in the future? + + byte[] quantizedQuery = queryVector.vector(); + int quantizedSum = queryVector.factors().quantizedSum(); + float lower = queryVector.factors().lower(); + float width = queryVector.factors().width(); + float distanceToCentroid = queryVector.factors().distToC(); + if (similarityFunction == EUCLIDEAN) { + return euclideanScore(targetOrd, sqrtDimensions, quantizedQuery, distanceToCentroid, lower, quantizedSum, width); + } + + float vmC = queryVector.factors().normVmC(); + float vDotC = queryVector.factors().vDotC(); + float cDotC = targetVectors.getCentroidDP(); + byte[] binaryCode = targetVectors.vectorValue(targetOrd); + float ooq = targetVectors.getOOQ(targetOrd); + float normOC = targetVectors.getNormOC(targetOrd); + float oDotC = targetVectors.getODotC(targetOrd); + + float qcDist = ESVectorUtil.ipByteBinByte(quantizedQuery, binaryCode); + + // FIXME: pre-compute these only once for each target vector + // ... pull this out or use a similar cache mechanism as do in score + float xbSum = (float) BQVectorUtils.popcount(binaryCode); + final float dist; + // If ||o-c|| == 0, so, it's ok to throw the rest of the equation away + // and simply use `oDotC + vDotC - cDotC` as centroid == doc vector + if (normOC == 0 || ooq == 0) { + dist = oDotC + vDotC - cDotC; + } else { + // If ||o-c|| != 0, we should assume that `ooq` is finite + assert Float.isFinite(ooq); + float estimatedDot = (2 * width / sqrtDimensions * qcDist + 2 * lower / sqrtDimensions * xbSum - width / sqrtDimensions + * quantizedSum - sqrtDimensions * lower) / ooq; + dist = vmC * normOC * estimatedDot + oDotC + vDotC - cDotC; + } + assert Float.isFinite(dist); + + // TODO: this is useful for mandatory rescoring by accounting for bias + // However, for just oversampling & rescoring, it isn't strictly useful. + // We should consider utilizing this bias in the future to determine which vectors need to + // be rescored + // float ooqSqr = (float) Math.pow(ooq, 2); + // float errorBound = (float) (normVmC * normOC * (maxX1 * Math.sqrt((1 - ooqSqr) / ooqSqr))); + // float score = dist - errorBound; + if (similarityFunction == MAXIMUM_INNER_PRODUCT) { + return VectorUtil.scaleMaxInnerProductScore(dist); + } + return Math.max((1f + dist) / 2f, 0); + } + + private float euclideanScore( + int targetOrd, + float sqrtDimensions, + byte[] quantizedQuery, + float distanceToCentroid, + float lower, + int quantizedSum, + float width + ) throws IOException { + byte[] binaryCode = targetVectors.vectorValue(targetOrd); + + // FIXME: pre-compute these only once for each target vector + // .. not sure how to enumerate the target ordinals but that's what we did in PoC + float targetDistToC = targetVectors.getCentroidDistance(targetOrd); + float x0 = targetVectors.getVectorMagnitude(targetOrd); + float sqrX = targetDistToC * targetDistToC; + double xX0 = targetDistToC / x0; + + // TODO maybe store? + float xbSum = (float) BQVectorUtils.popcount(binaryCode); + float factorPPC = (float) (-2.0 / sqrtDimensions * xX0 * (xbSum * 2.0 - targetVectors.dimension())); + float factorIP = (float) (-2.0 / sqrtDimensions * xX0); + + long qcDist = ESVectorUtil.ipByteBinByte(quantizedQuery, binaryCode); + float score = sqrX + distanceToCentroid + factorPPC * lower + (qcDist * 2 - quantizedSum) * factorIP * width; + // TODO: this is useful for mandatory rescoring by accounting for bias + // However, for just oversampling & rescoring, it isn't strictly useful. + // We should consider utilizing this bias in the future to determine which vectors need to + // be rescored + // float projectionDist = (float) Math.sqrt(xX0 * xX0 - targetDistToC * targetDistToC); + // float error = 2.0f * maxX1 * projectionDist; + // float y = (float) Math.sqrt(distanceToCentroid); + // float errorBound = y * error; + // if (Float.isFinite(errorBound)) { + // score = dist + errorBound; + // } + return Math.max(1 / (1f + score), 0); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsFormat.java new file mode 100644 index 0000000000000..523d5f6c4a91f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsFormat.java @@ -0,0 +1,75 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; + +import java.io.IOException; + +/** + * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 + */ +public class ES816BinaryQuantizedVectorsFormat extends FlatVectorsFormat { + + public static final String BINARIZED_VECTOR_COMPONENT = "BVEC"; + public static final String NAME = "ES816BinaryQuantizedVectorsFormat"; + + static final int VERSION_START = 0; + static final int VERSION_CURRENT = VERSION_START; + static final String META_CODEC_NAME = "ES816BinaryQuantizedVectorsFormatMeta"; + static final String VECTOR_DATA_CODEC_NAME = "ES816BinaryQuantizedVectorsFormatData"; + static final String META_EXTENSION = "vemb"; + static final String VECTOR_DATA_EXTENSION = "veb"; + static final int DIRECT_MONOTONIC_BLOCK_SHIFT = 16; + + private static final FlatVectorsFormat rawVectorFormat = new Lucene99FlatVectorsFormat( + FlatVectorScorerUtil.getLucene99FlatVectorsScorer() + ); + + private static final ES816BinaryFlatVectorsScorer scorer = new ES816BinaryFlatVectorsScorer( + FlatVectorScorerUtil.getLucene99FlatVectorsScorer() + ); + + /** Creates a new instance with the default number of vectors per cluster. */ + public ES816BinaryQuantizedVectorsFormat() { + super(NAME); + } + + @Override + public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return new ES816BinaryQuantizedVectorsWriter(scorer, rawVectorFormat.fieldsWriter(state), state); + } + + @Override + public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException { + return new ES816BinaryQuantizedVectorsReader(state, rawVectorFormat.fieldsReader(state), scorer); + } + + @Override + public String toString() { + return "ES816BinaryQuantizedVectorsFormat(name=" + NAME + ", flatVectorScorer=" + scorer + ")"; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsReader.java new file mode 100644 index 0000000000000..b0378fee6793d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsReader.java @@ -0,0 +1,412 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.search.VectorScorer; +import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.RamUsageEstimator; +import org.apache.lucene.util.SuppressForbidden; +import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector; +import org.apache.lucene.util.hnsw.RandomVectorScorer; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding; + +/** + * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 + */ +@SuppressForbidden(reason = "Lucene classes") +public class ES816BinaryQuantizedVectorsReader extends FlatVectorsReader { + + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(ES816BinaryQuantizedVectorsReader.class); + + private final Map fields = new HashMap<>(); + private final IndexInput quantizedVectorData; + private final FlatVectorsReader rawVectorsReader; + private final ES816BinaryFlatVectorsScorer vectorScorer; + + public ES816BinaryQuantizedVectorsReader( + SegmentReadState state, + FlatVectorsReader rawVectorsReader, + ES816BinaryFlatVectorsScorer vectorsScorer + ) throws IOException { + super(vectorsScorer); + this.vectorScorer = vectorsScorer; + this.rawVectorsReader = rawVectorsReader; + int versionMeta = -1; + String metaFileName = IndexFileNames.segmentFileName( + state.segmentInfo.name, + state.segmentSuffix, + ES816BinaryQuantizedVectorsFormat.META_EXTENSION + ); + boolean success = false; + try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName, state.context)) { + Throwable priorE = null; + try { + versionMeta = CodecUtil.checkIndexHeader( + meta, + ES816BinaryQuantizedVectorsFormat.META_CODEC_NAME, + ES816BinaryQuantizedVectorsFormat.VERSION_START, + ES816BinaryQuantizedVectorsFormat.VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix + ); + readFields(meta, state.fieldInfos); + } catch (Throwable exception) { + priorE = exception; + } finally { + CodecUtil.checkFooter(meta, priorE); + } + quantizedVectorData = openDataInput( + state, + versionMeta, + ES816BinaryQuantizedVectorsFormat.VECTOR_DATA_EXTENSION, + ES816BinaryQuantizedVectorsFormat.VECTOR_DATA_CODEC_NAME, + // Quantized vectors are accessed randomly from their node ID stored in the HNSW + // graph. + state.context.withRandomAccess() + ); + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } + } + + private void readFields(ChecksumIndexInput meta, FieldInfos infos) throws IOException { + for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) { + FieldInfo info = infos.fieldInfo(fieldNumber); + if (info == null) { + throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta); + } + FieldEntry fieldEntry = readField(meta, info); + validateFieldEntry(info, fieldEntry); + fields.put(info.name, fieldEntry); + } + } + + static void validateFieldEntry(FieldInfo info, FieldEntry fieldEntry) { + int dimension = info.getVectorDimension(); + if (dimension != fieldEntry.dimension) { + throw new IllegalStateException( + "Inconsistent vector dimension for field=\"" + info.name + "\"; " + dimension + " != " + fieldEntry.dimension + ); + } + + int binaryDims = BQVectorUtils.discretize(dimension, 64) / 8; + int correctionsCount = fieldEntry.similarityFunction != VectorSimilarityFunction.EUCLIDEAN ? 3 : 2; + long numQuantizedVectorBytes = Math.multiplyExact((binaryDims + (Float.BYTES * correctionsCount)), (long) fieldEntry.size); + if (numQuantizedVectorBytes != fieldEntry.vectorDataLength) { + throw new IllegalStateException( + "Binarized vector data length " + + fieldEntry.vectorDataLength + + " not matching size = " + + fieldEntry.size + + " * (binaryBytes=" + + binaryDims + + " + 8" + + ") = " + + numQuantizedVectorBytes + ); + } + } + + @Override + public RandomVectorScorer getRandomVectorScorer(String field, float[] target) throws IOException { + FieldEntry fi = fields.get(field); + if (fi == null) { + return null; + } + return vectorScorer.getRandomVectorScorer( + fi.similarityFunction, + OffHeapBinarizedVectorValues.load( + fi.ordToDocDISIReaderConfiguration, + fi.dimension, + fi.size, + new BinaryQuantizer(fi.dimension, fi.descritizedDimension, fi.similarityFunction), + fi.similarityFunction, + vectorScorer, + fi.centroid, + fi.centroidDP, + fi.vectorDataOffset, + fi.vectorDataLength, + quantizedVectorData + ), + target + ); + } + + @Override + public RandomVectorScorer getRandomVectorScorer(String field, byte[] target) throws IOException { + return rawVectorsReader.getRandomVectorScorer(field, target); + } + + @Override + public void checkIntegrity() throws IOException { + rawVectorsReader.checkIntegrity(); + CodecUtil.checksumEntireFile(quantizedVectorData); + } + + @Override + public FloatVectorValues getFloatVectorValues(String field) throws IOException { + FieldEntry fi = fields.get(field); + if (fi == null) { + return null; + } + if (fi.vectorEncoding != VectorEncoding.FLOAT32) { + throw new IllegalArgumentException( + "field=\"" + field + "\" is encoded as: " + fi.vectorEncoding + " expected: " + VectorEncoding.FLOAT32 + ); + } + OffHeapBinarizedVectorValues bvv = OffHeapBinarizedVectorValues.load( + fi.ordToDocDISIReaderConfiguration, + fi.dimension, + fi.size, + new BinaryQuantizer(fi.dimension, fi.descritizedDimension, fi.similarityFunction), + fi.similarityFunction, + vectorScorer, + fi.centroid, + fi.centroidDP, + fi.vectorDataOffset, + fi.vectorDataLength, + quantizedVectorData + ); + return new BinarizedVectorValues(rawVectorsReader.getFloatVectorValues(field), bvv); + } + + @Override + public ByteVectorValues getByteVectorValues(String field) throws IOException { + return rawVectorsReader.getByteVectorValues(field); + } + + @Override + public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + rawVectorsReader.search(field, target, knnCollector, acceptDocs); + } + + @Override + public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + if (knnCollector.k() == 0) return; + final RandomVectorScorer scorer = getRandomVectorScorer(field, target); + if (scorer == null) return; + OrdinalTranslatedKnnCollector collector = new OrdinalTranslatedKnnCollector(knnCollector, scorer::ordToDoc); + Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs); + for (int i = 0; i < scorer.maxOrd(); i++) { + if (acceptedOrds == null || acceptedOrds.get(i)) { + collector.collect(i, scorer.score(i)); + collector.incVisitedCount(1); + } + } + } + + @Override + public void close() throws IOException { + IOUtils.close(quantizedVectorData, rawVectorsReader); + } + + @Override + public long ramBytesUsed() { + long size = SHALLOW_SIZE; + size += RamUsageEstimator.sizeOfMap(fields, RamUsageEstimator.shallowSizeOfInstance(FieldEntry.class)); + size += rawVectorsReader.ramBytesUsed(); + return size; + } + + public float[] getCentroid(String field) { + FieldEntry fieldEntry = fields.get(field); + if (fieldEntry != null) { + return fieldEntry.centroid; + } + return null; + } + + private static IndexInput openDataInput( + SegmentReadState state, + int versionMeta, + String fileExtension, + String codecName, + IOContext context + ) throws IOException { + String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, fileExtension); + IndexInput in = state.directory.openInput(fileName, context); + boolean success = false; + try { + int versionVectorData = CodecUtil.checkIndexHeader( + in, + codecName, + ES816BinaryQuantizedVectorsFormat.VERSION_START, + ES816BinaryQuantizedVectorsFormat.VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix + ); + if (versionMeta != versionVectorData) { + throw new CorruptIndexException( + "Format versions mismatch: meta=" + versionMeta + ", " + codecName + "=" + versionVectorData, + in + ); + } + CodecUtil.retrieveChecksum(in); + success = true; + return in; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(in); + } + } + } + + private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException { + VectorEncoding vectorEncoding = readVectorEncoding(input); + VectorSimilarityFunction similarityFunction = readSimilarityFunction(input); + if (similarityFunction != info.getVectorSimilarityFunction()) { + throw new IllegalStateException( + "Inconsistent vector similarity function for field=\"" + + info.name + + "\"; " + + similarityFunction + + " != " + + info.getVectorSimilarityFunction() + ); + } + return FieldEntry.create(input, vectorEncoding, info.getVectorSimilarityFunction()); + } + + private record FieldEntry( + VectorSimilarityFunction similarityFunction, + VectorEncoding vectorEncoding, + int dimension, + int descritizedDimension, + long vectorDataOffset, + long vectorDataLength, + int size, + float[] centroid, + float centroidDP, + OrdToDocDISIReaderConfiguration ordToDocDISIReaderConfiguration + ) { + + static FieldEntry create(IndexInput input, VectorEncoding vectorEncoding, VectorSimilarityFunction similarityFunction) + throws IOException { + int dimension = input.readVInt(); + long vectorDataOffset = input.readVLong(); + long vectorDataLength = input.readVLong(); + int size = input.readVInt(); + final float[] centroid; + float centroidDP = 0; + if (size > 0) { + centroid = new float[dimension]; + input.readFloats(centroid, 0, dimension); + centroidDP = Float.intBitsToFloat(input.readInt()); + } else { + centroid = null; + } + OrdToDocDISIReaderConfiguration conf = OrdToDocDISIReaderConfiguration.fromStoredMeta(input, size); + return new FieldEntry( + similarityFunction, + vectorEncoding, + dimension, + BQVectorUtils.discretize(dimension, 64), + vectorDataOffset, + vectorDataLength, + size, + centroid, + centroidDP, + conf + ); + } + } + + /** Binarized vector values holding row and quantized vector values */ + protected static final class BinarizedVectorValues extends FloatVectorValues { + private final FloatVectorValues rawVectorValues; + private final OffHeapBinarizedVectorValues quantizedVectorValues; + + BinarizedVectorValues(FloatVectorValues rawVectorValues, OffHeapBinarizedVectorValues quantizedVectorValues) { + this.rawVectorValues = rawVectorValues; + this.quantizedVectorValues = quantizedVectorValues; + } + + @Override + public int dimension() { + return rawVectorValues.dimension(); + } + + @Override + public int size() { + return rawVectorValues.size(); + } + + @Override + public float[] vectorValue() throws IOException { + return rawVectorValues.vectorValue(); + } + + @Override + public int docID() { + return rawVectorValues.docID(); + } + + @Override + public int nextDoc() throws IOException { + int rawDocId = rawVectorValues.nextDoc(); + int quantizedDocId = quantizedVectorValues.nextDoc(); + assert rawDocId == quantizedDocId; + return quantizedDocId; + } + + @Override + public int advance(int target) throws IOException { + int rawDocId = rawVectorValues.advance(target); + int quantizedDocId = quantizedVectorValues.advance(target); + assert rawDocId == quantizedDocId; + return quantizedDocId; + } + + @Override + public VectorScorer scorer(float[] query) throws IOException { + return quantizedVectorValues.scorer(query); + } + + protected OffHeapBinarizedVectorValues getQuantizedVectorValues() throws IOException { + return quantizedVectorValues; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsWriter.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsWriter.java new file mode 100644 index 0000000000000..92837a8ffce45 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsWriter.java @@ -0,0 +1,987 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.apache.lucene.index.DocsWithFieldSet; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.MergeState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.Sorter; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.internal.hppc.FloatArrayList; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.VectorScorer; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.RamUsageEstimator; +import org.apache.lucene.util.VectorUtil; +import org.apache.lucene.util.hnsw.CloseableRandomVectorScorerSupplier; +import org.apache.lucene.util.hnsw.RandomAccessVectorValues; +import org.apache.lucene.util.hnsw.RandomVectorScorer; +import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier; +import org.elasticsearch.core.SuppressForbidden; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.apache.lucene.index.VectorSimilarityFunction.COSINE; +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; +import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; +import static org.elasticsearch.index.codec.vectors.ES816BinaryQuantizedVectorsFormat.BINARIZED_VECTOR_COMPONENT; +import static org.elasticsearch.index.codec.vectors.ES816BinaryQuantizedVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; + +/** + * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 + */ +@SuppressForbidden(reason = "Lucene classes") +public class ES816BinaryQuantizedVectorsWriter extends FlatVectorsWriter { + private static final long SHALLOW_RAM_BYTES_USED = shallowSizeOfInstance(ES816BinaryQuantizedVectorsWriter.class); + + private final SegmentWriteState segmentWriteState; + private final List fields = new ArrayList<>(); + private final IndexOutput meta, binarizedVectorData; + private final FlatVectorsWriter rawVectorDelegate; + private final ES816BinaryFlatVectorsScorer vectorsScorer; + private boolean finished; + + /** + * Sole constructor + * + * @param vectorsScorer the scorer to use for scoring vectors + */ + protected ES816BinaryQuantizedVectorsWriter( + ES816BinaryFlatVectorsScorer vectorsScorer, + FlatVectorsWriter rawVectorDelegate, + SegmentWriteState state + ) throws IOException { + super(vectorsScorer); + this.vectorsScorer = vectorsScorer; + this.segmentWriteState = state; + String metaFileName = IndexFileNames.segmentFileName( + state.segmentInfo.name, + state.segmentSuffix, + ES816BinaryQuantizedVectorsFormat.META_EXTENSION + ); + + String binarizedVectorDataFileName = IndexFileNames.segmentFileName( + state.segmentInfo.name, + state.segmentSuffix, + ES816BinaryQuantizedVectorsFormat.VECTOR_DATA_EXTENSION + ); + this.rawVectorDelegate = rawVectorDelegate; + boolean success = false; + try { + meta = state.directory.createOutput(metaFileName, state.context); + binarizedVectorData = state.directory.createOutput(binarizedVectorDataFileName, state.context); + + CodecUtil.writeIndexHeader( + meta, + ES816BinaryQuantizedVectorsFormat.META_CODEC_NAME, + ES816BinaryQuantizedVectorsFormat.VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix + ); + CodecUtil.writeIndexHeader( + binarizedVectorData, + ES816BinaryQuantizedVectorsFormat.VECTOR_DATA_CODEC_NAME, + ES816BinaryQuantizedVectorsFormat.VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix + ); + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } + } + + public FlatFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException { + FlatFieldVectorsWriter rawVectorDelegate = this.rawVectorDelegate.addField(fieldInfo); + if (fieldInfo.getVectorEncoding().equals(VectorEncoding.FLOAT32)) { + @SuppressWarnings("unchecked") + FieldWriter fieldWriter = new FieldWriter(fieldInfo, (FlatFieldVectorsWriter) rawVectorDelegate); + fields.add(fieldWriter); + return fieldWriter; + } + return rawVectorDelegate; + } + + @Override + public void flush(int maxDoc, Sorter.DocMap sortMap) throws IOException { + rawVectorDelegate.flush(maxDoc, sortMap); + for (FieldWriter field : fields) { + // after raw vectors are written, normalize vectors for clustering and quantization + if (VectorSimilarityFunction.COSINE == field.fieldInfo.getVectorSimilarityFunction()) { + field.normalizeVectors(); + } + + final float[] clusterCenter; + int vectorCount = field.flatFieldVectorsWriter.getVectors().size(); + clusterCenter = new float[field.dimensionSums.length]; + if (vectorCount > 0) { + for (int i = 0; i < field.dimensionSums.length; i++) { + clusterCenter[i] = field.dimensionSums[i] / vectorCount; + } + if (VectorSimilarityFunction.COSINE == field.fieldInfo.getVectorSimilarityFunction()) { + VectorUtil.l2normalize(clusterCenter); + } + } + if (segmentWriteState.infoStream.isEnabled(BINARIZED_VECTOR_COMPONENT)) { + segmentWriteState.infoStream.message(BINARIZED_VECTOR_COMPONENT, "Vectors' count:" + vectorCount); + } + int descritizedDimension = BQVectorUtils.discretize(field.fieldInfo.getVectorDimension(), 64); + BinaryQuantizer quantizer = new BinaryQuantizer( + field.fieldInfo.getVectorDimension(), + descritizedDimension, + field.fieldInfo.getVectorSimilarityFunction() + ); + if (sortMap == null) { + writeField(field, clusterCenter, maxDoc, quantizer); + } else { + writeSortingField(field, clusterCenter, maxDoc, sortMap, quantizer); + } + field.finish(); + } + } + + private void writeField(FieldWriter fieldData, float[] clusterCenter, int maxDoc, BinaryQuantizer quantizer) throws IOException { + // write vector values + long vectorDataOffset = binarizedVectorData.alignFilePointer(Float.BYTES); + writeBinarizedVectors(fieldData, clusterCenter, quantizer); + long vectorDataLength = binarizedVectorData.getFilePointer() - vectorDataOffset; + float centroidDp = fieldData.getVectors().size() > 0 ? VectorUtil.dotProduct(clusterCenter, clusterCenter) : 0; + + writeMeta( + fieldData.fieldInfo, + maxDoc, + vectorDataOffset, + vectorDataLength, + clusterCenter, + centroidDp, + fieldData.getDocsWithFieldSet() + ); + } + + private void writeBinarizedVectors(FieldWriter fieldData, float[] clusterCenter, BinaryQuantizer scalarQuantizer) throws IOException { + byte[] vector = new byte[BQVectorUtils.discretize(fieldData.fieldInfo.getVectorDimension(), 64) / 8]; + int correctionsCount = scalarQuantizer.getSimilarity() != EUCLIDEAN ? 3 : 2; + final ByteBuffer correctionsBuffer = ByteBuffer.allocate(Float.BYTES * correctionsCount).order(ByteOrder.LITTLE_ENDIAN); + for (int i = 0; i < fieldData.getVectors().size(); i++) { + float[] v = fieldData.getVectors().get(i); + float[] corrections = scalarQuantizer.quantizeForIndex(v, vector, clusterCenter); + binarizedVectorData.writeBytes(vector, vector.length); + for (int j = 0; j < corrections.length; j++) { + correctionsBuffer.putFloat(corrections[j]); + } + binarizedVectorData.writeBytes(correctionsBuffer.array(), correctionsBuffer.array().length); + correctionsBuffer.rewind(); + } + } + + private void writeSortingField( + FieldWriter fieldData, + float[] clusterCenter, + int maxDoc, + Sorter.DocMap sortMap, + BinaryQuantizer scalarQuantizer + ) throws IOException { + final int[] ordMap = new int[fieldData.getDocsWithFieldSet().cardinality()]; // new ord to old ord + + DocsWithFieldSet newDocsWithField = new DocsWithFieldSet(); + mapOldOrdToNewOrd(fieldData.getDocsWithFieldSet(), sortMap, null, ordMap, newDocsWithField); + + // write vector values + long vectorDataOffset = binarizedVectorData.alignFilePointer(Float.BYTES); + writeSortedBinarizedVectors(fieldData, clusterCenter, ordMap, scalarQuantizer); + long quantizedVectorLength = binarizedVectorData.getFilePointer() - vectorDataOffset; + + float centroidDp = VectorUtil.dotProduct(clusterCenter, clusterCenter); + writeMeta(fieldData.fieldInfo, maxDoc, vectorDataOffset, quantizedVectorLength, clusterCenter, centroidDp, newDocsWithField); + } + + private void writeSortedBinarizedVectors(FieldWriter fieldData, float[] clusterCenter, int[] ordMap, BinaryQuantizer scalarQuantizer) + throws IOException { + byte[] vector = new byte[BQVectorUtils.discretize(fieldData.fieldInfo.getVectorDimension(), 64) / 8]; + int correctionsCount = scalarQuantizer.getSimilarity() != EUCLIDEAN ? 3 : 2; + final ByteBuffer correctionsBuffer = ByteBuffer.allocate(Float.BYTES * correctionsCount).order(ByteOrder.LITTLE_ENDIAN); + for (int ordinal : ordMap) { + float[] v = fieldData.getVectors().get(ordinal); + float[] corrections = scalarQuantizer.quantizeForIndex(v, vector, clusterCenter); + binarizedVectorData.writeBytes(vector, vector.length); + for (int i = 0; i < corrections.length; i++) { + correctionsBuffer.putFloat(corrections[i]); + } + binarizedVectorData.writeBytes(correctionsBuffer.array(), correctionsBuffer.array().length); + correctionsBuffer.rewind(); + } + } + + private void writeMeta( + FieldInfo field, + int maxDoc, + long vectorDataOffset, + long vectorDataLength, + float[] clusterCenter, + float centroidDp, + DocsWithFieldSet docsWithField + ) throws IOException { + meta.writeInt(field.number); + meta.writeInt(field.getVectorEncoding().ordinal()); + meta.writeInt(field.getVectorSimilarityFunction().ordinal()); + meta.writeVInt(field.getVectorDimension()); + meta.writeVLong(vectorDataOffset); + meta.writeVLong(vectorDataLength); + int count = docsWithField.cardinality(); + meta.writeVInt(count); + if (count > 0) { + final ByteBuffer buffer = ByteBuffer.allocate(field.getVectorDimension() * Float.BYTES).order(ByteOrder.LITTLE_ENDIAN); + buffer.asFloatBuffer().put(clusterCenter); + meta.writeBytes(buffer.array(), buffer.array().length); + meta.writeInt(Float.floatToIntBits(centroidDp)); + } + OrdToDocDISIReaderConfiguration.writeStoredMeta( + DIRECT_MONOTONIC_BLOCK_SHIFT, + meta, + binarizedVectorData, + count, + maxDoc, + docsWithField + ); + } + + @Override + public void finish() throws IOException { + if (finished) { + throw new IllegalStateException("already finished"); + } + finished = true; + rawVectorDelegate.finish(); + if (meta != null) { + // write end of fields marker + meta.writeInt(-1); + CodecUtil.writeFooter(meta); + } + if (binarizedVectorData != null) { + CodecUtil.writeFooter(binarizedVectorData); + } + } + + @Override + public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + if (fieldInfo.getVectorEncoding().equals(VectorEncoding.FLOAT32)) { + final float[] centroid; + final float[] mergedCentroid = new float[fieldInfo.getVectorDimension()]; + int vectorCount = mergeAndRecalculateCentroids(mergeState, fieldInfo, mergedCentroid); + // Don't need access to the random vectors, we can just use the merged + rawVectorDelegate.mergeOneField(fieldInfo, mergeState); + centroid = mergedCentroid; + if (segmentWriteState.infoStream.isEnabled(BINARIZED_VECTOR_COMPONENT)) { + segmentWriteState.infoStream.message(BINARIZED_VECTOR_COMPONENT, "Vectors' count:" + vectorCount); + } + int descritizedDimension = BQVectorUtils.discretize(fieldInfo.getVectorDimension(), 64); + FloatVectorValues floatVectorValues = KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); + if (fieldInfo.getVectorSimilarityFunction() == COSINE) { + floatVectorValues = new NormalizedFloatVectorValues(floatVectorValues); + } + BinarizedFloatVectorValues binarizedVectorValues = new BinarizedFloatVectorValues( + floatVectorValues, + new BinaryQuantizer(fieldInfo.getVectorDimension(), descritizedDimension, fieldInfo.getVectorSimilarityFunction()), + centroid + ); + long vectorDataOffset = binarizedVectorData.alignFilePointer(Float.BYTES); + DocsWithFieldSet docsWithField = writeBinarizedVectorData(binarizedVectorData, binarizedVectorValues); + long vectorDataLength = binarizedVectorData.getFilePointer() - vectorDataOffset; + float centroidDp = docsWithField.cardinality() > 0 ? VectorUtil.dotProduct(centroid, centroid) : 0; + writeMeta( + fieldInfo, + segmentWriteState.segmentInfo.maxDoc(), + vectorDataOffset, + vectorDataLength, + centroid, + centroidDp, + docsWithField + ); + } else { + rawVectorDelegate.mergeOneField(fieldInfo, mergeState); + } + } + + static DocsWithFieldSet writeBinarizedVectorAndQueryData( + IndexOutput binarizedVectorData, + IndexOutput binarizedQueryData, + FloatVectorValues floatVectorValues, + float[] centroid, + BinaryQuantizer binaryQuantizer + ) throws IOException { + DocsWithFieldSet docsWithField = new DocsWithFieldSet(); + byte[] toIndex = new byte[BQVectorUtils.discretize(floatVectorValues.dimension(), 64) / 8]; + byte[] toQuery = new byte[(BQVectorUtils.discretize(floatVectorValues.dimension(), 64) / 8) * BQSpaceUtils.B_QUERY]; + int queryCorrectionCount = binaryQuantizer.getSimilarity() != EUCLIDEAN ? 5 : 3; + final ByteBuffer queryCorrectionsBuffer = ByteBuffer.allocate(Float.BYTES * queryCorrectionCount + Short.BYTES) + .order(ByteOrder.LITTLE_ENDIAN); + for (int docV = floatVectorValues.nextDoc(); docV != NO_MORE_DOCS; docV = floatVectorValues.nextDoc()) { + // write index vector + BinaryQuantizer.QueryAndIndexResults r = binaryQuantizer.quantizeQueryAndIndex( + floatVectorValues.vectorValue(), + toIndex, + toQuery, + centroid + ); + binarizedVectorData.writeBytes(toIndex, toIndex.length); + float[] corrections = r.indexFeatures(); + for (int i = 0; i < corrections.length; i++) { + binarizedVectorData.writeInt(Float.floatToIntBits(corrections[i])); + } + docsWithField.add(docV); + + // write query vector + binarizedQueryData.writeBytes(toQuery, toQuery.length); + BinaryQuantizer.QueryFactors factors = r.queryFeatures(); + queryCorrectionsBuffer.putFloat(factors.distToC()); + queryCorrectionsBuffer.putFloat(factors.lower()); + queryCorrectionsBuffer.putFloat(factors.width()); + + if (binaryQuantizer.getSimilarity() != EUCLIDEAN) { + queryCorrectionsBuffer.putFloat(factors.normVmC()); + queryCorrectionsBuffer.putFloat(factors.vDotC()); + } + // ensure we are positive and fit within an unsigned short value. + assert factors.quantizedSum() >= 0 && factors.quantizedSum() <= 0xffff; + queryCorrectionsBuffer.putShort((short) factors.quantizedSum()); + + binarizedQueryData.writeBytes(queryCorrectionsBuffer.array(), queryCorrectionsBuffer.array().length); + queryCorrectionsBuffer.rewind(); + } + return docsWithField; + } + + static DocsWithFieldSet writeBinarizedVectorData(IndexOutput output, BinarizedByteVectorValues binarizedByteVectorValues) + throws IOException { + DocsWithFieldSet docsWithField = new DocsWithFieldSet(); + for (int docV = binarizedByteVectorValues.nextDoc(); docV != NO_MORE_DOCS; docV = binarizedByteVectorValues.nextDoc()) { + // write vector + byte[] binaryValue = binarizedByteVectorValues.vectorValue(); + output.writeBytes(binaryValue, binaryValue.length); + float[] corrections = binarizedByteVectorValues.getCorrectiveTerms(); + for (int i = 0; i < corrections.length; i++) { + output.writeInt(Float.floatToIntBits(corrections[i])); + } + docsWithField.add(docV); + } + return docsWithField; + } + + @Override + public CloseableRandomVectorScorerSupplier mergeOneFieldToIndex(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + if (fieldInfo.getVectorEncoding().equals(VectorEncoding.FLOAT32)) { + final float[] centroid; + final float cDotC; + final float[] mergedCentroid = new float[fieldInfo.getVectorDimension()]; + int vectorCount = mergeAndRecalculateCentroids(mergeState, fieldInfo, mergedCentroid); + + // Don't need access to the random vectors, we can just use the merged + rawVectorDelegate.mergeOneField(fieldInfo, mergeState); + centroid = mergedCentroid; + cDotC = vectorCount > 0 ? VectorUtil.dotProduct(centroid, centroid) : 0; + if (segmentWriteState.infoStream.isEnabled(BINARIZED_VECTOR_COMPONENT)) { + segmentWriteState.infoStream.message(BINARIZED_VECTOR_COMPONENT, "Vectors' count:" + vectorCount); + } + return mergeOneFieldToIndex(segmentWriteState, fieldInfo, mergeState, centroid, cDotC); + } + return rawVectorDelegate.mergeOneFieldToIndex(fieldInfo, mergeState); + } + + private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( + SegmentWriteState segmentWriteState, + FieldInfo fieldInfo, + MergeState mergeState, + float[] centroid, + float cDotC + ) throws IOException { + long vectorDataOffset = binarizedVectorData.alignFilePointer(Float.BYTES); + final IndexOutput tempQuantizedVectorData = segmentWriteState.directory.createTempOutput( + binarizedVectorData.getName(), + "temp", + segmentWriteState.context + ); + final IndexOutput tempScoreQuantizedVectorData = segmentWriteState.directory.createTempOutput( + binarizedVectorData.getName(), + "score_temp", + segmentWriteState.context + ); + IndexInput binarizedDataInput = null; + IndexInput binarizedScoreDataInput = null; + boolean success = false; + int descritizedDimension = BQVectorUtils.discretize(fieldInfo.getVectorDimension(), 64); + BinaryQuantizer quantizer = new BinaryQuantizer( + fieldInfo.getVectorDimension(), + descritizedDimension, + fieldInfo.getVectorSimilarityFunction() + ); + try { + FloatVectorValues floatVectorValues = KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); + if (fieldInfo.getVectorSimilarityFunction() == COSINE) { + floatVectorValues = new NormalizedFloatVectorValues(floatVectorValues); + } + DocsWithFieldSet docsWithField = writeBinarizedVectorAndQueryData( + tempQuantizedVectorData, + tempScoreQuantizedVectorData, + floatVectorValues, + centroid, + quantizer + ); + CodecUtil.writeFooter(tempQuantizedVectorData); + IOUtils.close(tempQuantizedVectorData); + binarizedDataInput = segmentWriteState.directory.openInput(tempQuantizedVectorData.getName(), segmentWriteState.context); + binarizedVectorData.copyBytes(binarizedDataInput, binarizedDataInput.length() - CodecUtil.footerLength()); + long vectorDataLength = binarizedVectorData.getFilePointer() - vectorDataOffset; + CodecUtil.retrieveChecksum(binarizedDataInput); + CodecUtil.writeFooter(tempScoreQuantizedVectorData); + IOUtils.close(tempScoreQuantizedVectorData); + binarizedScoreDataInput = segmentWriteState.directory.openInput( + tempScoreQuantizedVectorData.getName(), + segmentWriteState.context + ); + writeMeta( + fieldInfo, + segmentWriteState.segmentInfo.maxDoc(), + vectorDataOffset, + vectorDataLength, + centroid, + cDotC, + docsWithField + ); + success = true; + final IndexInput finalBinarizedDataInput = binarizedDataInput; + final IndexInput finalBinarizedScoreDataInput = binarizedScoreDataInput; + OffHeapBinarizedVectorValues vectorValues = new OffHeapBinarizedVectorValues.DenseOffHeapVectorValues( + fieldInfo.getVectorDimension(), + docsWithField.cardinality(), + centroid, + cDotC, + quantizer, + fieldInfo.getVectorSimilarityFunction(), + vectorsScorer, + finalBinarizedDataInput + ); + RandomVectorScorerSupplier scorerSupplier = vectorsScorer.getRandomVectorScorerSupplier( + fieldInfo.getVectorSimilarityFunction(), + new OffHeapBinarizedQueryVectorValues( + finalBinarizedScoreDataInput, + fieldInfo.getVectorDimension(), + docsWithField.cardinality(), + fieldInfo.getVectorSimilarityFunction() + ), + vectorValues + ); + return new BinarizedCloseableRandomVectorScorerSupplier(scorerSupplier, vectorValues, () -> { + IOUtils.close(finalBinarizedDataInput, finalBinarizedScoreDataInput); + IOUtils.deleteFilesIgnoringExceptions( + segmentWriteState.directory, + tempQuantizedVectorData.getName(), + tempScoreQuantizedVectorData.getName() + ); + }); + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException( + tempQuantizedVectorData, + tempScoreQuantizedVectorData, + binarizedDataInput, + binarizedScoreDataInput + ); + IOUtils.deleteFilesIgnoringExceptions( + segmentWriteState.directory, + tempQuantizedVectorData.getName(), + tempScoreQuantizedVectorData.getName() + ); + } + } + } + + @Override + public void close() throws IOException { + IOUtils.close(meta, binarizedVectorData, rawVectorDelegate); + } + + static float[] getCentroid(KnnVectorsReader vectorsReader, String fieldName) { + if (vectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader candidateReader) { + vectorsReader = candidateReader.getFieldReader(fieldName); + } + if (vectorsReader instanceof ES816BinaryQuantizedVectorsReader reader) { + return reader.getCentroid(fieldName); + } + return null; + } + + static int mergeAndRecalculateCentroids(MergeState mergeState, FieldInfo fieldInfo, float[] mergedCentroid) throws IOException { + boolean recalculate = false; + int totalVectorCount = 0; + for (int i = 0; i < mergeState.knnVectorsReaders.length; i++) { + KnnVectorsReader knnVectorsReader = mergeState.knnVectorsReaders[i]; + if (knnVectorsReader == null || knnVectorsReader.getFloatVectorValues(fieldInfo.name) == null) { + continue; + } + float[] centroid = getCentroid(knnVectorsReader, fieldInfo.name); + int vectorCount = knnVectorsReader.getFloatVectorValues(fieldInfo.name).size(); + if (vectorCount == 0) { + continue; + } + totalVectorCount += vectorCount; + // If there aren't centroids, or previously clustered with more than one cluster + // or if there are deleted docs, we must recalculate the centroid + if (centroid == null || mergeState.liveDocs[i] != null) { + recalculate = true; + break; + } + for (int j = 0; j < centroid.length; j++) { + mergedCentroid[j] += centroid[j] * vectorCount; + } + } + if (recalculate) { + return calculateCentroid(mergeState, fieldInfo, mergedCentroid); + } else { + for (int j = 0; j < mergedCentroid.length; j++) { + mergedCentroid[j] = mergedCentroid[j] / totalVectorCount; + } + if (fieldInfo.getVectorSimilarityFunction() == COSINE) { + VectorUtil.l2normalize(mergedCentroid); + } + return totalVectorCount; + } + } + + static int calculateCentroid(MergeState mergeState, FieldInfo fieldInfo, float[] centroid) throws IOException { + assert fieldInfo.getVectorEncoding().equals(VectorEncoding.FLOAT32); + // clear out the centroid + Arrays.fill(centroid, 0); + int count = 0; + for (int i = 0; i < mergeState.knnVectorsReaders.length; i++) { + KnnVectorsReader knnVectorsReader = mergeState.knnVectorsReaders[i]; + if (knnVectorsReader == null) continue; + FloatVectorValues vectorValues = mergeState.knnVectorsReaders[i].getFloatVectorValues(fieldInfo.name); + if (vectorValues == null) { + continue; + } + for (int doc = vectorValues.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = vectorValues.nextDoc()) { + float[] vector = vectorValues.vectorValue(); + // TODO Panama sum + for (int j = 0; j < vector.length; j++) { + centroid[j] += vector[j]; + } + } + count += vectorValues.size(); + } + if (count == 0) { + return count; + } + // TODO Panama div + for (int i = 0; i < centroid.length; i++) { + centroid[i] /= count; + } + if (fieldInfo.getVectorSimilarityFunction() == COSINE) { + VectorUtil.l2normalize(centroid); + } + return count; + } + + @Override + public long ramBytesUsed() { + long total = SHALLOW_RAM_BYTES_USED; + for (FieldWriter field : fields) { + // the field tracks the delegate field usage + total += field.ramBytesUsed(); + } + return total; + } + + static class FieldWriter extends FlatFieldVectorsWriter { + private static final long SHALLOW_SIZE = shallowSizeOfInstance(FieldWriter.class); + private final FieldInfo fieldInfo; + private boolean finished; + private final FlatFieldVectorsWriter flatFieldVectorsWriter; + private final float[] dimensionSums; + private final FloatArrayList magnitudes = new FloatArrayList(); + + FieldWriter(FieldInfo fieldInfo, FlatFieldVectorsWriter flatFieldVectorsWriter) { + this.fieldInfo = fieldInfo; + this.flatFieldVectorsWriter = flatFieldVectorsWriter; + this.dimensionSums = new float[fieldInfo.getVectorDimension()]; + } + + @Override + public List getVectors() { + return flatFieldVectorsWriter.getVectors(); + } + + public void normalizeVectors() { + for (int i = 0; i < flatFieldVectorsWriter.getVectors().size(); i++) { + float[] vector = flatFieldVectorsWriter.getVectors().get(i); + float magnitude = magnitudes.get(i); + for (int j = 0; j < vector.length; j++) { + vector[j] /= magnitude; + } + } + } + + @Override + public DocsWithFieldSet getDocsWithFieldSet() { + return flatFieldVectorsWriter.getDocsWithFieldSet(); + } + + @Override + public void finish() throws IOException { + if (finished) { + return; + } + assert flatFieldVectorsWriter.isFinished(); + finished = true; + } + + @Override + public boolean isFinished() { + return finished && flatFieldVectorsWriter.isFinished(); + } + + @Override + public void addValue(int docID, float[] vectorValue) throws IOException { + flatFieldVectorsWriter.addValue(docID, vectorValue); + if (fieldInfo.getVectorSimilarityFunction() == COSINE) { + float dp = VectorUtil.dotProduct(vectorValue, vectorValue); + float divisor = (float) Math.sqrt(dp); + magnitudes.add(divisor); + for (int i = 0; i < vectorValue.length; i++) { + dimensionSums[i] += (vectorValue[i] / divisor); + } + } else { + for (int i = 0; i < vectorValue.length; i++) { + dimensionSums[i] += vectorValue[i]; + } + } + } + + @Override + public float[] copyValue(float[] vectorValue) { + throw new UnsupportedOperationException(); + } + + @Override + public long ramBytesUsed() { + long size = SHALLOW_SIZE; + size += flatFieldVectorsWriter.ramBytesUsed(); + size += RamUsageEstimator.sizeOf(dimensionSums); + size += magnitudes.ramBytesUsed(); + return size; + } + } + + // When accessing vectorValue method, targerOrd here means a row ordinal. + static class OffHeapBinarizedQueryVectorValues { + private final IndexInput slice; + private final int dimension; + private final int size; + protected final byte[] binaryValue; + protected final ByteBuffer byteBuffer; + private final int byteSize; + protected final float[] correctiveValues; + private int sumQuantizationValues; + private int lastOrd = -1; + private final int correctiveValuesSize; + private final VectorSimilarityFunction vectorSimilarityFunction; + + OffHeapBinarizedQueryVectorValues(IndexInput data, int dimension, int size, VectorSimilarityFunction vectorSimilarityFunction) { + this.slice = data; + this.dimension = dimension; + this.size = size; + this.vectorSimilarityFunction = vectorSimilarityFunction; + this.correctiveValuesSize = vectorSimilarityFunction != EUCLIDEAN ? 5 : 3; + // 4x the quantized binary dimensions + int binaryDimensions = (BQVectorUtils.discretize(dimension, 64) / 8) * BQSpaceUtils.B_QUERY; + this.byteBuffer = ByteBuffer.allocate(binaryDimensions); + this.binaryValue = byteBuffer.array(); + this.correctiveValues = new float[correctiveValuesSize]; + this.byteSize = binaryDimensions + Float.BYTES * correctiveValuesSize + Short.BYTES; + } + + public float getCentroidDistance(int targetOrd) throws IOException { + if (lastOrd == targetOrd) { + return correctiveValues[0]; + } + readCorrectiveValues(targetOrd); + return correctiveValues[0]; + } + + public float getLower(int targetOrd) throws IOException { + if (lastOrd == targetOrd) { + return correctiveValues[1]; + } + readCorrectiveValues(targetOrd); + return correctiveValues[1]; + } + + public float getWidth(int targetOrd) throws IOException { + if (lastOrd == targetOrd) { + return correctiveValues[2]; + } + readCorrectiveValues(targetOrd); + return correctiveValues[2]; + } + + public float getNormVmC(int targetOrd) throws IOException { + if (lastOrd == targetOrd) { + return correctiveValues[3]; + } + readCorrectiveValues(targetOrd); + return correctiveValues[3]; + } + + public float getVDotC(int targetOrd) throws IOException { + if (lastOrd == targetOrd) { + return correctiveValues[4]; + } + readCorrectiveValues(targetOrd); + return correctiveValues[4]; + } + + private void readCorrectiveValues(int targetOrd) throws IOException { + // load values + vectorValue(targetOrd); + } + + public int sumQuantizedValues(int targetOrd) throws IOException { + if (lastOrd == targetOrd) { + return sumQuantizationValues; + } + // load values + vectorValue(targetOrd); + return sumQuantizationValues; + } + + public int size() { + return size; + } + + public int dimension() { + return dimension; + } + + public OffHeapBinarizedQueryVectorValues copy() throws IOException { + return new OffHeapBinarizedQueryVectorValues(slice.clone(), dimension, size, vectorSimilarityFunction); + } + + public IndexInput getSlice() { + return slice; + } + + public byte[] vectorValue(int targetOrd) throws IOException { + if (lastOrd == targetOrd) { + return binaryValue; + } + slice.seek((long) targetOrd * byteSize); + slice.readBytes(binaryValue, 0, binaryValue.length); + slice.readFloats(correctiveValues, 0, correctiveValuesSize); + sumQuantizationValues = Short.toUnsignedInt(slice.readShort()); + lastOrd = targetOrd; + return binaryValue; + } + } + + static class BinarizedFloatVectorValues extends BinarizedByteVectorValues { + private float[] corrections; + private final byte[] binarized; + private final float[] centroid; + private final FloatVectorValues values; + private final BinaryQuantizer quantizer; + private int lastDoc; + + BinarizedFloatVectorValues(FloatVectorValues delegate, BinaryQuantizer quantizer, float[] centroid) { + this.values = delegate; + this.quantizer = quantizer; + this.binarized = new byte[BQVectorUtils.discretize(delegate.dimension(), 64) / 8]; + this.centroid = centroid; + lastDoc = -1; + } + + @Override + public float[] getCorrectiveTerms() { + return corrections; + } + + @Override + public byte[] vectorValue() throws IOException { + return binarized; + } + + @Override + public int dimension() { + return values.dimension(); + } + + @Override + public int size() { + return values.size(); + } + + @Override + public int docID() { + return values.docID(); + } + + @Override + public int nextDoc() throws IOException { + int doc = values.nextDoc(); + if (doc != NO_MORE_DOCS) { + binarize(); + } + lastDoc = doc; + return doc; + } + + @Override + public int advance(int target) throws IOException { + int doc = values.advance(target); + if (doc != NO_MORE_DOCS) { + binarize(); + } + lastDoc = doc; + return doc; + } + + @Override + public VectorScorer scorer(float[] target) throws IOException { + throw new UnsupportedOperationException(); + } + + private void binarize() throws IOException { + if (lastDoc == docID()) return; + corrections = quantizer.quantizeForIndex(values.vectorValue(), binarized, centroid); + } + } + + static class BinarizedCloseableRandomVectorScorerSupplier implements CloseableRandomVectorScorerSupplier { + private final RandomVectorScorerSupplier supplier; + private final RandomAccessVectorValues vectorValues; + private final Closeable onClose; + + BinarizedCloseableRandomVectorScorerSupplier( + RandomVectorScorerSupplier supplier, + RandomAccessVectorValues vectorValues, + Closeable onClose + ) { + this.supplier = supplier; + this.onClose = onClose; + this.vectorValues = vectorValues; + } + + @Override + public RandomVectorScorer scorer(int ord) throws IOException { + return supplier.scorer(ord); + } + + @Override + public RandomVectorScorerSupplier copy() throws IOException { + return supplier.copy(); + } + + @Override + public void close() throws IOException { + onClose.close(); + } + + @Override + public int totalVectorCount() { + return vectorValues.size(); + } + } + + static final class NormalizedFloatVectorValues extends FloatVectorValues { + private final FloatVectorValues values; + private final float[] normalizedVector; + int curDoc = -1; + + NormalizedFloatVectorValues(FloatVectorValues values) { + this.values = values; + this.normalizedVector = new float[values.dimension()]; + } + + @Override + public int dimension() { + return values.dimension(); + } + + @Override + public int size() { + return values.size(); + } + + @Override + public float[] vectorValue() { + return normalizedVector; + } + + @Override + public VectorScorer scorer(float[] query) { + throw new UnsupportedOperationException(); + } + + @Override + public int docID() { + return values.docID(); + } + + @Override + public int nextDoc() throws IOException { + curDoc = values.nextDoc(); + if (curDoc != NO_MORE_DOCS) { + System.arraycopy(values.vectorValue(), 0, normalizedVector, 0, normalizedVector.length); + VectorUtil.l2normalize(normalizedVector); + } + return curDoc; + } + + @Override + public int advance(int target) throws IOException { + curDoc = values.advance(target); + if (curDoc != NO_MORE_DOCS) { + System.arraycopy(values.vectorValue(), 0, normalizedVector, 0, normalizedVector.length); + VectorUtil.l2normalize(normalizedVector); + } + return curDoc; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816HnswBinaryQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816HnswBinaryQuantizedVectorsFormat.java new file mode 100644 index 0000000000000..989f88e0a7857 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES816HnswBinaryQuantizedVectorsFormat.java @@ -0,0 +1,144 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.search.TaskExecutor; +import org.apache.lucene.util.hnsw.HnswGraph; + +import java.io.IOException; +import java.util.concurrent.ExecutorService; + +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.MAXIMUM_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.MAXIMUM_MAX_CONN; + +/** + * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 + */ +public class ES816HnswBinaryQuantizedVectorsFormat extends KnnVectorsFormat { + + public static final String NAME = "ES816HnswBinaryQuantizedVectorsFormat"; + + /** + * Controls how many of the nearest neighbor candidates are connected to the new node. Defaults to + * {@link Lucene99HnswVectorsFormat#DEFAULT_MAX_CONN}. See {@link HnswGraph} for more details. + */ + private final int maxConn; + + /** + * The number of candidate neighbors to track while searching the graph for each newly inserted + * node. Defaults to {@link Lucene99HnswVectorsFormat#DEFAULT_BEAM_WIDTH}. See {@link HnswGraph} + * for details. + */ + private final int beamWidth; + + /** The format for storing, reading, merging vectors on disk */ + private static final FlatVectorsFormat flatVectorsFormat = new ES816BinaryQuantizedVectorsFormat(); + + private final int numMergeWorkers; + private final TaskExecutor mergeExec; + + /** Constructs a format using default graph construction parameters */ + public ES816HnswBinaryQuantizedVectorsFormat() { + this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, DEFAULT_NUM_MERGE_WORKER, null); + } + + /** + * Constructs a format using the given graph construction parameters. + * + * @param maxConn the maximum number of connections to a node in the HNSW graph + * @param beamWidth the size of the queue maintained during graph construction. + */ + public ES816HnswBinaryQuantizedVectorsFormat(int maxConn, int beamWidth) { + this(maxConn, beamWidth, DEFAULT_NUM_MERGE_WORKER, null); + } + + /** + * Constructs a format using the given graph construction parameters and scalar quantization. + * + * @param maxConn the maximum number of connections to a node in the HNSW graph + * @param beamWidth the size of the queue maintained during graph construction. + * @param numMergeWorkers number of workers (threads) that will be used when doing merge. If + * larger than 1, a non-null {@link ExecutorService} must be passed as mergeExec + * @param mergeExec the {@link ExecutorService} that will be used by ALL vector writers that are + * generated by this format to do the merge + */ + public ES816HnswBinaryQuantizedVectorsFormat(int maxConn, int beamWidth, int numMergeWorkers, ExecutorService mergeExec) { + super(NAME); + if (maxConn <= 0 || maxConn > MAXIMUM_MAX_CONN) { + throw new IllegalArgumentException( + "maxConn must be positive and less than or equal to " + MAXIMUM_MAX_CONN + "; maxConn=" + maxConn + ); + } + if (beamWidth <= 0 || beamWidth > MAXIMUM_BEAM_WIDTH) { + throw new IllegalArgumentException( + "beamWidth must be positive and less than or equal to " + MAXIMUM_BEAM_WIDTH + "; beamWidth=" + beamWidth + ); + } + this.maxConn = maxConn; + this.beamWidth = beamWidth; + if (numMergeWorkers == 1 && mergeExec != null) { + throw new IllegalArgumentException("No executor service is needed as we'll use single thread to merge"); + } + this.numMergeWorkers = numMergeWorkers; + if (mergeExec != null) { + this.mergeExec = new TaskExecutor(mergeExec); + } else { + this.mergeExec = null; + } + } + + @Override + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return new Lucene99HnswVectorsWriter(state, maxConn, beamWidth, flatVectorsFormat.fieldsWriter(state), numMergeWorkers, mergeExec); + } + + @Override + public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { + return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state)); + } + + @Override + public int getMaxDimensions(String fieldName) { + return 1024; + } + + @Override + public String toString() { + return "ES816HnswBinaryQuantizedVectorsFormat(name=ES816HnswBinaryQuantizedVectorsFormat, maxConn=" + + maxConn + + ", beamWidth=" + + beamWidth + + ", flatVectorFormat=" + + flatVectorsFormat + + ")"; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/OffHeapBinarizedVectorValues.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/OffHeapBinarizedVectorValues.java new file mode 100644 index 0000000000000..2a3c3aca60e54 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/OffHeapBinarizedVectorValues.java @@ -0,0 +1,456 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.codecs.hnsw.FlatVectorsScorer; +import org.apache.lucene.codecs.lucene90.IndexedDISI; +import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.VectorScorer; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.hnsw.RandomVectorScorer; +import org.apache.lucene.util.packed.DirectMonotonicReader; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; + +/** Binarized vector values loaded from off-heap */ +public abstract class OffHeapBinarizedVectorValues extends BinarizedByteVectorValues implements RandomAccessBinarizedByteVectorValues { + + protected final int dimension; + protected final int size; + protected final int numBytes; + protected final VectorSimilarityFunction similarityFunction; + protected final FlatVectorsScorer vectorsScorer; + + protected final IndexInput slice; + protected final byte[] binaryValue; + protected final ByteBuffer byteBuffer; + protected final int byteSize; + private int lastOrd = -1; + protected final float[] correctiveValues; + protected final BinaryQuantizer binaryQuantizer; + protected final float[] centroid; + protected final float centroidDp; + private final int correctionsCount; + + OffHeapBinarizedVectorValues( + int dimension, + int size, + float[] centroid, + float centroidDp, + BinaryQuantizer quantizer, + VectorSimilarityFunction similarityFunction, + FlatVectorsScorer vectorsScorer, + IndexInput slice + ) { + this.dimension = dimension; + this.size = size; + this.similarityFunction = similarityFunction; + this.vectorsScorer = vectorsScorer; + this.slice = slice; + this.centroid = centroid; + this.centroidDp = centroidDp; + this.numBytes = BQVectorUtils.discretize(dimension, 64) / 8; + this.correctionsCount = similarityFunction != EUCLIDEAN ? 3 : 2; + this.correctiveValues = new float[this.correctionsCount]; + this.byteSize = numBytes + (Float.BYTES * correctionsCount); + this.byteBuffer = ByteBuffer.allocate(numBytes); + this.binaryValue = byteBuffer.array(); + this.binaryQuantizer = quantizer; + } + + @Override + public int dimension() { + return dimension; + } + + @Override + public int size() { + return size; + } + + @Override + public byte[] vectorValue(int targetOrd) throws IOException { + if (lastOrd == targetOrd) { + return binaryValue; + } + slice.seek((long) targetOrd * byteSize); + slice.readBytes(byteBuffer.array(), byteBuffer.arrayOffset(), numBytes); + slice.readFloats(correctiveValues, 0, correctionsCount); + lastOrd = targetOrd; + return binaryValue; + } + + @Override + public float getCentroidDP() { + return centroidDp; + } + + @Override + public float[] getCorrectiveTerms() { + return correctiveValues; + } + + @Override + public float getCentroidDistance(int targetOrd) throws IOException { + if (lastOrd == targetOrd) { + return correctiveValues[0]; + } + slice.seek(((long) targetOrd * byteSize) + numBytes); + slice.readFloats(correctiveValues, 0, correctionsCount); + return correctiveValues[0]; + } + + @Override + public float getVectorMagnitude(int targetOrd) throws IOException { + if (lastOrd == targetOrd) { + return correctiveValues[1]; + } + slice.seek(((long) targetOrd * byteSize) + numBytes); + slice.readFloats(correctiveValues, 0, correctionsCount); + return correctiveValues[1]; + } + + @Override + public float getOOQ(int targetOrd) throws IOException { + if (lastOrd == targetOrd) { + return correctiveValues[0]; + } + slice.seek(((long) targetOrd * byteSize) + numBytes); + slice.readFloats(correctiveValues, 0, correctionsCount); + return correctiveValues[0]; + } + + @Override + public float getNormOC(int targetOrd) throws IOException { + if (lastOrd == targetOrd) { + return correctiveValues[1]; + } + slice.seek(((long) targetOrd * byteSize) + numBytes); + slice.readFloats(correctiveValues, 0, correctionsCount); + return correctiveValues[1]; + } + + @Override + public float getODotC(int targetOrd) throws IOException { + if (lastOrd == targetOrd) { + return correctiveValues[2]; + } + slice.seek(((long) targetOrd * byteSize) + numBytes); + slice.readFloats(correctiveValues, 0, correctionsCount); + return correctiveValues[2]; + } + + @Override + public BinaryQuantizer getQuantizer() { + return binaryQuantizer; + } + + @Override + public float[] getCentroid() { + return centroid; + } + + @Override + public IndexInput getSlice() { + return slice; + } + + @Override + public int getVectorByteLength() { + return numBytes; + } + + public static OffHeapBinarizedVectorValues load( + OrdToDocDISIReaderConfiguration configuration, + int dimension, + int size, + BinaryQuantizer binaryQuantizer, + VectorSimilarityFunction similarityFunction, + FlatVectorsScorer vectorsScorer, + float[] centroid, + float centroidDp, + long quantizedVectorDataOffset, + long quantizedVectorDataLength, + IndexInput vectorData + ) throws IOException { + if (configuration.isEmpty()) { + return new EmptyOffHeapVectorValues(dimension, similarityFunction, vectorsScorer); + } + assert centroid != null; + IndexInput bytesSlice = vectorData.slice("quantized-vector-data", quantizedVectorDataOffset, quantizedVectorDataLength); + if (configuration.isDense()) { + return new DenseOffHeapVectorValues( + dimension, + size, + centroid, + centroidDp, + binaryQuantizer, + similarityFunction, + vectorsScorer, + bytesSlice + ); + } else { + return new SparseOffHeapVectorValues( + configuration, + dimension, + size, + centroid, + centroidDp, + binaryQuantizer, + vectorData, + similarityFunction, + vectorsScorer, + bytesSlice + ); + } + } + + /** Dense off-heap binarized vector values */ + public static class DenseOffHeapVectorValues extends OffHeapBinarizedVectorValues { + private int doc = -1; + + public DenseOffHeapVectorValues( + int dimension, + int size, + float[] centroid, + float centroidDp, + BinaryQuantizer binaryQuantizer, + VectorSimilarityFunction similarityFunction, + FlatVectorsScorer vectorsScorer, + IndexInput slice + ) { + super(dimension, size, centroid, centroidDp, binaryQuantizer, similarityFunction, vectorsScorer, slice); + } + + @Override + public byte[] vectorValue() throws IOException { + return vectorValue(doc); + } + + @Override + public int docID() { + return doc; + } + + @Override + public int nextDoc() { + return advance(doc + 1); + } + + @Override + public int advance(int target) { + assert docID() < target; + if (target >= size) { + return doc = NO_MORE_DOCS; + } + return doc = target; + } + + @Override + public DenseOffHeapVectorValues copy() throws IOException { + return new DenseOffHeapVectorValues( + dimension, + size, + centroid, + centroidDp, + binaryQuantizer, + similarityFunction, + vectorsScorer, + slice.clone() + ); + } + + @Override + public Bits getAcceptOrds(Bits acceptDocs) { + return acceptDocs; + } + + @Override + public VectorScorer scorer(float[] target) throws IOException { + DenseOffHeapVectorValues copy = copy(); + RandomVectorScorer scorer = vectorsScorer.getRandomVectorScorer(similarityFunction, copy, target); + return new VectorScorer() { + @Override + public float score() throws IOException { + return scorer.score(copy.doc); + } + + @Override + public DocIdSetIterator iterator() { + return copy; + } + }; + } + } + + /** Sparse off-heap binarized vector values */ + private static class SparseOffHeapVectorValues extends OffHeapBinarizedVectorValues { + private final DirectMonotonicReader ordToDoc; + private final IndexedDISI disi; + // dataIn was used to init a new IndexedDIS for #randomAccess() + private final IndexInput dataIn; + private final OrdToDocDISIReaderConfiguration configuration; + + SparseOffHeapVectorValues( + OrdToDocDISIReaderConfiguration configuration, + int dimension, + int size, + float[] centroid, + float centroidDp, + BinaryQuantizer binaryQuantizer, + IndexInput dataIn, + VectorSimilarityFunction similarityFunction, + FlatVectorsScorer vectorsScorer, + IndexInput slice + ) throws IOException { + super(dimension, size, centroid, centroidDp, binaryQuantizer, similarityFunction, vectorsScorer, slice); + this.configuration = configuration; + this.dataIn = dataIn; + this.ordToDoc = configuration.getDirectMonotonicReader(dataIn); + this.disi = configuration.getIndexedDISI(dataIn); + } + + @Override + public byte[] vectorValue() throws IOException { + return vectorValue(disi.index()); + } + + @Override + public int docID() { + return disi.docID(); + } + + @Override + public int nextDoc() throws IOException { + return disi.nextDoc(); + } + + @Override + public int advance(int target) throws IOException { + assert docID() < target; + return disi.advance(target); + } + + @Override + public SparseOffHeapVectorValues copy() throws IOException { + return new SparseOffHeapVectorValues( + configuration, + dimension, + size, + centroid, + centroidDp, + binaryQuantizer, + dataIn, + similarityFunction, + vectorsScorer, + slice.clone() + ); + } + + @Override + public int ordToDoc(int ord) { + return (int) ordToDoc.get(ord); + } + + @Override + public Bits getAcceptOrds(Bits acceptDocs) { + if (acceptDocs == null) { + return null; + } + return new Bits() { + @Override + public boolean get(int index) { + return acceptDocs.get(ordToDoc(index)); + } + + @Override + public int length() { + return size; + } + }; + } + + @Override + public VectorScorer scorer(float[] target) throws IOException { + SparseOffHeapVectorValues copy = copy(); + RandomVectorScorer scorer = vectorsScorer.getRandomVectorScorer(similarityFunction, copy, target); + return new VectorScorer() { + @Override + public float score() throws IOException { + return scorer.score(copy.disi.index()); + } + + @Override + public DocIdSetIterator iterator() { + return copy; + } + }; + } + } + + private static class EmptyOffHeapVectorValues extends OffHeapBinarizedVectorValues { + private int doc = -1; + + EmptyOffHeapVectorValues(int dimension, VectorSimilarityFunction similarityFunction, FlatVectorsScorer vectorsScorer) { + super(dimension, 0, null, Float.NaN, null, similarityFunction, vectorsScorer, null); + } + + @Override + public int docID() { + return doc; + } + + @Override + public int nextDoc() { + return advance(doc + 1); + } + + @Override + public int advance(int target) { + return doc = NO_MORE_DOCS; + } + + @Override + public byte[] vectorValue() { + throw new UnsupportedOperationException(); + } + + @Override + public DenseOffHeapVectorValues copy() { + throw new UnsupportedOperationException(); + } + + @Override + public Bits getAcceptOrds(Bits acceptDocs) { + return null; + } + + @Override + public VectorScorer scorer(float[] target) throws IOException { + return null; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/RandomAccessBinarizedByteVectorValues.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/RandomAccessBinarizedByteVectorValues.java new file mode 100644 index 0000000000000..2417353373ba5 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/RandomAccessBinarizedByteVectorValues.java @@ -0,0 +1,70 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.util.VectorUtil; +import org.apache.lucene.util.hnsw.RandomAccessVectorValues; + +import java.io.IOException; + +/** + * Copied from Lucene, replace with Lucene's implementation sometime after Lucene 10 + */ +public interface RandomAccessBinarizedByteVectorValues extends RandomAccessVectorValues.Bytes { + /** Returns the centroid distance for the vector */ + float getCentroidDistance(int vectorOrd) throws IOException; + + /** Returns the vector magnitude for the vector */ + float getVectorMagnitude(int vectorOrd) throws IOException; + + /** Returns OOQ corrective factor for the given vector ordinal */ + float getOOQ(int targetOrd) throws IOException; + + /** + * Returns the norm of the target vector w the centroid corrective factor for the given vector + * ordinal + */ + float getNormOC(int targetOrd) throws IOException; + + /** + * Returns the target vector dot product the centroid corrective factor for the given vector + * ordinal + */ + float getODotC(int targetOrd) throws IOException; + + /** + * @return the quantizer used to quantize the vectors + */ + BinaryQuantizer getQuantizer(); + + /** + * @return coarse grained centroids for the vectors + */ + float[] getCentroid() throws IOException; + + @Override + RandomAccessBinarizedByteVectorValues copy() throws IOException; + + default float getCentroidDP() throws IOException { + // this only gets executed on-merge + float[] centroid = getCentroid(); + return VectorUtil.dotProduct(centroid, centroid); + } +} diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat index da2a0c4b90f30..c2201f5b1c319 100644 --- a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat +++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat @@ -3,3 +3,5 @@ org.elasticsearch.index.codec.vectors.ES813Int8FlatVectorFormat org.elasticsearch.index.codec.vectors.ES814HnswScalarQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.ES815HnswBitVectorsFormat org.elasticsearch.index.codec.vectors.ES815BitFlatVectorFormat +org.elasticsearch.index.codec.vectors.ES816BinaryQuantizedVectorsFormat +org.elasticsearch.index.codec.vectors.ES816HnswBinaryQuantizedVectorsFormat diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/BQVectorUtilsTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/BQVectorUtilsTests.java new file mode 100644 index 0000000000000..9f9114c70b6db --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/BQVectorUtilsTests.java @@ -0,0 +1,90 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.tests.util.LuceneTestCase; + +public class BQVectorUtilsTests extends LuceneTestCase { + + public static int popcount(byte[] a, int aOffset, byte[] b, int length) { + int res = 0; + for (int j = 0; j < length; j++) { + int value = (a[aOffset + j] & b[j]) & 0xFF; + for (int k = 0; k < Byte.SIZE; k++) { + if ((value & (1 << k)) != 0) { + ++res; + } + } + } + return res; + } + + private static float DELTA = Float.MIN_VALUE; + + public void testPadFloat() { + assertArrayEquals(new float[] { 1, 2, 3, 4 }, BQVectorUtils.pad(new float[] { 1, 2, 3, 4 }, 4), DELTA); + assertArrayEquals(new float[] { 1, 2, 3, 4 }, BQVectorUtils.pad(new float[] { 1, 2, 3, 4 }, 3), DELTA); + assertArrayEquals(new float[] { 1, 2, 3, 4, 0 }, BQVectorUtils.pad(new float[] { 1, 2, 3, 4 }, 5), DELTA); + } + + public void testPadByte() { + assertArrayEquals(new byte[] { 1, 2, 3, 4 }, BQVectorUtils.pad(new byte[] { 1, 2, 3, 4 }, 4)); + assertArrayEquals(new byte[] { 1, 2, 3, 4 }, BQVectorUtils.pad(new byte[] { 1, 2, 3, 4 }, 3)); + assertArrayEquals(new byte[] { 1, 2, 3, 4, 0 }, BQVectorUtils.pad(new byte[] { 1, 2, 3, 4 }, 5)); + } + + public void testPopCount() { + assertEquals(0, BQVectorUtils.popcount(new byte[] {})); + assertEquals(1, BQVectorUtils.popcount(new byte[] { 1 })); + assertEquals(2, BQVectorUtils.popcount(new byte[] { 2, 1 })); + assertEquals(2, BQVectorUtils.popcount(new byte[] { 8, 0, 1 })); + assertEquals(4, BQVectorUtils.popcount(new byte[] { 7, 1 })); + + int iterations = atLeast(50); + for (int i = 0; i < iterations; i++) { + int size = random().nextInt(5000); + var a = new byte[size]; + random().nextBytes(a); + assertEquals(popcount(a, 0, a, size), BQVectorUtils.popcount(a)); + } + } + + public void testNorm() { + assertEquals(3.0f, BQVectorUtils.norm(new float[] { 3 }), DELTA); + assertEquals(5.0f, BQVectorUtils.norm(new float[] { 5 }), DELTA); + assertEquals(4.0f, BQVectorUtils.norm(new float[] { 2, 2, 2, 2 }), DELTA); + assertEquals(9.0f, BQVectorUtils.norm(new float[] { 3, 3, 3, 3, 3, 3, 3, 3, 3 }), DELTA); + } + + public void testSubtract() { + assertArrayEquals(new float[] { 1 }, BQVectorUtils.subtract(new float[] { 3 }, new float[] { 2 }), DELTA); + assertArrayEquals(new float[] { 2, 1, 0 }, BQVectorUtils.subtract(new float[] { 3, 3, 3 }, new float[] { 1, 2, 3 }), DELTA); + } + + public void testSubtractInPlace() { + var a = new float[] { 3 }; + BQVectorUtils.subtractInPlace(a, new float[] { 2 }); + assertArrayEquals(new float[] { 1 }, a, DELTA); + + a = new float[] { 3, 3, 3 }; + BQVectorUtils.subtractInPlace(a, new float[] { 1, 2, 3 }); + assertArrayEquals(new float[] { 2, 1, 0 }, a, DELTA); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/BinaryQuantizationTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/BinaryQuantizationTests.java new file mode 100644 index 0000000000000..32d717bd76f91 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/BinaryQuantizationTests.java @@ -0,0 +1,1856 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.util.VectorUtil; + +import java.util.Random; + +public class BinaryQuantizationTests extends LuceneTestCase { + + public void testQuantizeForIndex() { + int dimensions = random().nextInt(1, 4097); + int discretizedDimensions = BQVectorUtils.discretize(dimensions, 64); + + int randIdx = random().nextInt(VectorSimilarityFunction.values().length); + VectorSimilarityFunction similarityFunction = VectorSimilarityFunction.values()[randIdx]; + + BinaryQuantizer quantizer = new BinaryQuantizer(discretizedDimensions, similarityFunction); + + float[] centroid = new float[dimensions]; + for (int i = 0; i < dimensions; i++) { + centroid[i] = random().nextFloat(-50f, 50f); + } + + float[] vector = new float[dimensions]; + for (int i = 0; i < dimensions; i++) { + vector[i] = random().nextFloat(-50f, 50f); + } + if (similarityFunction == VectorSimilarityFunction.COSINE) { + VectorUtil.l2normalize(vector); + VectorUtil.l2normalize(centroid); + } + + byte[] destination = new byte[discretizedDimensions / 8]; + float[] corrections = quantizer.quantizeForIndex(vector, destination, centroid); + + for (float correction : corrections) { + assertFalse(Float.isNaN(correction)); + } + + if (similarityFunction != VectorSimilarityFunction.EUCLIDEAN) { + assertEquals(3, corrections.length); + assertTrue(corrections[0] >= 0); + assertTrue(corrections[1] > 0); + } else { + assertEquals(2, corrections.length); + assertTrue(corrections[0] > 0); + assertTrue(corrections[1] > 0); + } + } + + public void testQuantizeForQuery() { + int dimensions = random().nextInt(1, 4097); + int discretizedDimensions = BQVectorUtils.discretize(dimensions, 64); + + int randIdx = random().nextInt(VectorSimilarityFunction.values().length); + VectorSimilarityFunction similarityFunction = VectorSimilarityFunction.values()[randIdx]; + + BinaryQuantizer quantizer = new BinaryQuantizer(discretizedDimensions, similarityFunction); + + float[] centroid = new float[dimensions]; + for (int i = 0; i < dimensions; i++) { + centroid[i] = random().nextFloat(-50f, 50f); + } + + float[] vector = new float[dimensions]; + for (int i = 0; i < dimensions; i++) { + vector[i] = random().nextFloat(-50f, 50f); + } + if (similarityFunction == VectorSimilarityFunction.COSINE) { + VectorUtil.l2normalize(vector); + VectorUtil.l2normalize(centroid); + } + float cDotC = VectorUtil.dotProduct(centroid, centroid); + byte[] destination = new byte[discretizedDimensions / 8 * BQSpaceUtils.B_QUERY]; + BinaryQuantizer.QueryFactors corrections = quantizer.quantizeForQuery(vector, destination, centroid); + + if (similarityFunction != VectorSimilarityFunction.EUCLIDEAN) { + int sumQ = corrections.quantizedSum(); + float distToC = corrections.distToC(); + float lower = corrections.lower(); + float width = corrections.width(); + float normVmC = corrections.normVmC(); + float vDotC = corrections.vDotC(); + assertTrue(sumQ >= 0); + assertTrue(distToC >= 0); + assertFalse(Float.isNaN(lower)); + assertTrue(width >= 0); + assertTrue(normVmC >= 0); + assertFalse(Float.isNaN(vDotC)); + assertTrue(cDotC >= 0); + } else { + int sumQ = corrections.quantizedSum(); + float distToC = corrections.distToC(); + float lower = corrections.lower(); + float width = corrections.width(); + assertTrue(sumQ >= 0); + assertTrue(distToC >= 0); + assertFalse(Float.isNaN(lower)); + assertTrue(width >= 0); + assertEquals(corrections.normVmC(), 0.0f, 0.01f); + assertEquals(corrections.vDotC(), 0.0f, 0.01f); + } + } + + public void testQuantizeForIndexEuclidean() { + int dimensions = 128; + + BinaryQuantizer quantizer = new BinaryQuantizer(dimensions, VectorSimilarityFunction.EUCLIDEAN); + float[] vector = new float[] { + 0f, + 0.0f, + 16.0f, + 35.0f, + 5.0f, + 32.0f, + 31.0f, + 14.0f, + 10.0f, + 11.0f, + 78.0f, + 55.0f, + 10.0f, + 45.0f, + 83.0f, + 11.0f, + 6.0f, + 14.0f, + 57.0f, + 102.0f, + 75.0f, + 20.0f, + 8.0f, + 3.0f, + 5.0f, + 67.0f, + 17.0f, + 19.0f, + 26.0f, + 5.0f, + 0.0f, + 1.0f, + 22.0f, + 60.0f, + 26.0f, + 7.0f, + 1.0f, + 18.0f, + 22.0f, + 84.0f, + 53.0f, + 85.0f, + 119.0f, + 119.0f, + 4.0f, + 24.0f, + 18.0f, + 7.0f, + 7.0f, + 1.0f, + 81.0f, + 106.0f, + 102.0f, + 72.0f, + 30.0f, + 6.0f, + 0.0f, + 9.0f, + 1.0f, + 9.0f, + 119.0f, + 72.0f, + 1.0f, + 4.0f, + 33.0f, + 119.0f, + 29.0f, + 6.0f, + 1.0f, + 0.0f, + 1.0f, + 14.0f, + 52.0f, + 119.0f, + 30.0f, + 3.0f, + 0.0f, + 0.0f, + 55.0f, + 92.0f, + 111.0f, + 2.0f, + 5.0f, + 4.0f, + 9.0f, + 22.0f, + 89.0f, + 96.0f, + 14.0f, + 1.0f, + 0.0f, + 1.0f, + 82.0f, + 59.0f, + 16.0f, + 20.0f, + 5.0f, + 25.0f, + 14.0f, + 11.0f, + 4.0f, + 0.0f, + 0.0f, + 1.0f, + 26.0f, + 47.0f, + 23.0f, + 4.0f, + 0.0f, + 0.0f, + 4.0f, + 38.0f, + 83.0f, + 30.0f, + 14.0f, + 9.0f, + 4.0f, + 9.0f, + 17.0f, + 23.0f, + 41.0f, + 0.0f, + 0.0f, + 2.0f, + 8.0f, + 19.0f, + 25.0f, + 23.0f }; + byte[] destination = new byte[dimensions / 8]; + float[] centroid = new float[] { + 27.054054f, + 22.252253f, + 25.027027f, + 23.55856f, + 31.099098f, + 28.765766f, + 31.64865f, + 30.981981f, + 24.675676f, + 21.81982f, + 26.72973f, + 25.486486f, + 30.504505f, + 35.216217f, + 28.306307f, + 24.486486f, + 29.675676f, + 26.153152f, + 31.315315f, + 25.225225f, + 29.234234f, + 30.855856f, + 24.495495f, + 29.828829f, + 31.54955f, + 24.36937f, + 25.108109f, + 24.873875f, + 22.918919f, + 24.918919f, + 29.027027f, + 25.513514f, + 27.64865f, + 28.405405f, + 23.603603f, + 17.900902f, + 22.522522f, + 24.855856f, + 31.396397f, + 32.585587f, + 26.297297f, + 27.468468f, + 19.675676f, + 19.018019f, + 24.801802f, + 30.27928f, + 27.945946f, + 25.324324f, + 29.918919f, + 27.864864f, + 28.081081f, + 23.45946f, + 28.828829f, + 28.387388f, + 25.387388f, + 27.90991f, + 25.621622f, + 21.585585f, + 26.378378f, + 24.144144f, + 21.666666f, + 22.72973f, + 26.837837f, + 22.747747f, + 29.0f, + 28.414415f, + 24.612612f, + 21.594595f, + 19.117117f, + 24.045046f, + 30.612612f, + 27.55856f, + 25.117117f, + 27.783783f, + 21.639639f, + 19.36937f, + 21.252253f, + 29.153152f, + 29.216217f, + 24.747747f, + 28.252253f, + 25.288288f, + 25.738739f, + 23.44144f, + 24.423424f, + 23.693693f, + 26.306307f, + 29.162163f, + 28.684685f, + 34.648647f, + 25.576576f, + 25.288288f, + 29.63063f, + 20.225225f, + 25.72973f, + 29.009008f, + 28.666666f, + 29.243244f, + 26.36937f, + 25.864864f, + 21.522522f, + 21.414415f, + 25.963964f, + 26.054054f, + 25.099098f, + 30.477478f, + 29.55856f, + 24.837837f, + 24.801802f, + 21.18018f, + 24.027027f, + 26.360361f, + 33.153152f, + 29.135136f, + 30.486486f, + 28.639639f, + 27.576576f, + 24.486486f, + 26.297297f, + 21.774775f, + 25.936937f, + 35.36937f, + 25.171171f, + 30.405405f, + 31.522522f, + 29.765766f, + 22.324324f, + 26.09009f }; + float[] corrections = quantizer.quantizeForIndex(vector, destination, centroid); + + assertEquals(2, corrections.length); + float distToCentroid = corrections[0]; + float magnitude = corrections[1]; + + assertEquals(387.90204f, distToCentroid, 0.0003f); + assertEquals(0.75916624f, magnitude, 0.0000001f); + assertArrayEquals(new byte[] { 20, 54, 56, 72, 97, -16, 62, 12, -32, -29, -125, 12, 0, -63, -63, -126 }, destination); + } + + public void testQuantizeForQueryEuclidean() { + int dimensions = 128; + + BinaryQuantizer quantizer = new BinaryQuantizer(dimensions, VectorSimilarityFunction.EUCLIDEAN); + float[] vector = new float[] { + 0.0f, + 8.0f, + 69.0f, + 45.0f, + 2.0f, + 0f, + 16.0f, + 52.0f, + 32.0f, + 13.0f, + 2.0f, + 6.0f, + 34.0f, + 49.0f, + 45.0f, + 83.0f, + 6.0f, + 2.0f, + 26.0f, + 57.0f, + 14.0f, + 46.0f, + 19.0f, + 9.0f, + 4.0f, + 13.0f, + 53.0f, + 104.0f, + 33.0f, + 11.0f, + 25.0f, + 19.0f, + 30.0f, + 10.0f, + 7.0f, + 2.0f, + 8.0f, + 7.0f, + 25.0f, + 1.0f, + 2.0f, + 25.0f, + 24.0f, + 28.0f, + 61.0f, + 83.0f, + 41.0f, + 9.0f, + 14.0f, + 3.0f, + 7.0f, + 114.0f, + 114.0f, + 114.0f, + 114.0f, + 5.0f, + 5.0f, + 1.0f, + 5.0f, + 114.0f, + 73.0f, + 75.0f, + 106.0f, + 3.0f, + 5.0f, + 6.0f, + 6.0f, + 8.0f, + 15.0f, + 45.0f, + 2.0f, + 15.0f, + 7.0f, + 114.0f, + 103.0f, + 6.0f, + 5.0f, + 4.0f, + 9.0f, + 67.0f, + 47.0f, + 22.0f, + 32.0f, + 27.0f, + 41.0f, + 10.0f, + 114.0f, + 36.0f, + 43.0f, + 42.0f, + 23.0f, + 9.0f, + 7.0f, + 30.0f, + 114.0f, + 19.0f, + 7.0f, + 5.0f, + 6.0f, + 6.0f, + 21.0f, + 48.0f, + 2.0f, + 1.0f, + 0.0f, + 8.0f, + 114.0f, + 13.0f, + 0.0f, + 1.0f, + 53.0f, + 83.0f, + 14.0f, + 8.0f, + 16.0f, + 12.0f, + 16.0f, + 20.0f, + 27.0f, + 87.0f, + 45.0f, + 50.0f, + 15.0f, + 5.0f, + 5.0f, + 6.0f, + 32.0f, + 49.0f }; + byte[] destination = new byte[dimensions / 8 * BQSpaceUtils.B_QUERY]; + float[] centroid = new float[] { + 26.7f, + 16.2f, + 10.913f, + 10.314f, + 12.12f, + 14.045f, + 15.887f, + 16.864f, + 32.232f, + 31.567f, + 34.922f, + 21.624f, + 16.349f, + 29.625f, + 31.994f, + 22.044f, + 37.847f, + 24.622f, + 36.299f, + 27.966f, + 14.368f, + 19.248f, + 30.778f, + 35.927f, + 27.019f, + 16.381f, + 17.325f, + 16.517f, + 13.272f, + 9.154f, + 9.242f, + 17.995f, + 53.777f, + 23.011f, + 12.929f, + 16.128f, + 22.16f, + 28.643f, + 25.861f, + 27.197f, + 59.883f, + 40.878f, + 34.153f, + 22.795f, + 24.402f, + 37.427f, + 34.19f, + 29.288f, + 61.812f, + 26.355f, + 39.071f, + 37.789f, + 23.33f, + 22.299f, + 28.64f, + 47.828f, + 52.457f, + 21.442f, + 24.039f, + 29.781f, + 27.707f, + 19.484f, + 14.642f, + 28.757f, + 54.567f, + 20.936f, + 25.112f, + 25.521f, + 22.077f, + 18.272f, + 14.526f, + 29.054f, + 61.803f, + 24.509f, + 37.517f, + 35.906f, + 24.106f, + 22.64f, + 32.1f, + 48.788f, + 60.102f, + 39.625f, + 34.766f, + 22.497f, + 24.397f, + 41.599f, + 38.419f, + 30.99f, + 55.647f, + 25.115f, + 14.96f, + 18.882f, + 26.918f, + 32.442f, + 26.231f, + 27.107f, + 26.828f, + 15.968f, + 18.668f, + 14.071f, + 10.906f, + 8.989f, + 9.721f, + 17.294f, + 36.32f, + 21.854f, + 35.509f, + 27.106f, + 14.067f, + 19.82f, + 33.582f, + 35.997f, + 33.528f, + 30.369f, + 36.955f, + 21.23f, + 15.2f, + 30.252f, + 34.56f, + 22.295f, + 29.413f, + 16.576f, + 11.226f, + 10.754f, + 12.936f, + 15.525f, + 15.868f, + 16.43f }; + BinaryQuantizer.QueryFactors corrections = quantizer.quantizeForQuery(vector, destination, centroid); + + int sumQ = corrections.quantizedSum(); + float lower = corrections.lower(); + float width = corrections.width(); + + assertEquals(729, sumQ); + assertEquals(-57.883f, lower, 0.001f); + assertEquals(9.972266f, width, 0.000001f); + assertArrayEquals( + new byte[] { + -77, + -49, + 73, + -17, + -89, + 9, + -43, + -27, + 40, + 15, + 42, + 76, + -122, + 38, + -22, + -37, + -96, + 111, + -63, + -102, + -123, + 23, + 110, + 127, + 32, + 95, + 29, + 106, + -120, + -121, + -32, + -94, + 78, + -98, + 42, + 95, + 122, + 114, + 30, + 18, + 91, + 97, + -5, + -9, + 123, + 122, + 31, + -66, + 49, + 1, + 20, + 48, + 0, + 12, + 30, + 30, + 4, + 96, + 2, + 2, + 4, + 33, + 1, + 65 }, + destination + ); + } + + private float[] generateRandomFloatArray(Random random, int dimensions, float lowerBoundInclusive, float upperBoundExclusive) { + float[] data = new float[dimensions]; + for (int i = 0; i < dimensions; i++) { + data[i] = random.nextFloat(lowerBoundInclusive, upperBoundExclusive); + } + return data; + } + + public void testQuantizeForIndexMIP() { + int dimensions = 768; + + // we want fixed values for these arrays so define our own random generation here to track + // quantization changes + Random random = new Random(42); + + float[] mipVectorToIndex = generateRandomFloatArray(random, dimensions, -1f, 1f); + float[] mipCentroid = generateRandomFloatArray(random, dimensions, -1f, 1f); + + VectorSimilarityFunction[] similarityFunctionsActingLikeEucllidean = new VectorSimilarityFunction[] { + VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT, + VectorSimilarityFunction.DOT_PRODUCT }; + int randIdx = random().nextInt(similarityFunctionsActingLikeEucllidean.length); + VectorSimilarityFunction similarityFunction = similarityFunctionsActingLikeEucllidean[randIdx]; + + BinaryQuantizer quantizer = new BinaryQuantizer(dimensions, similarityFunction); + float[] vector = mipVectorToIndex; + byte[] destination = new byte[dimensions / 8]; + float[] centroid = mipCentroid; + float[] corrections = quantizer.quantizeForIndex(vector, destination, centroid); + + assertEquals(3, corrections.length); + float ooq = corrections[0]; + float normOC = corrections[1]; + float oDotC = corrections[2]; + + assertEquals(0.8141399f, ooq, 0.0000001f); + assertEquals(21.847124f, normOC, 0.00001f); + assertEquals(6.4300356f, oDotC, 0.0001f); + assertArrayEquals( + new byte[] { + -83, + -91, + -71, + 97, + 32, + -96, + 89, + -80, + -19, + -108, + 3, + 113, + -111, + 12, + -86, + 32, + -43, + 76, + 122, + -106, + -83, + -37, + -122, + 118, + 84, + -72, + 34, + 20, + 57, + -29, + 119, + -8, + -10, + -100, + -109, + 62, + -54, + 53, + -44, + 8, + -16, + 80, + 58, + 50, + 105, + -25, + 47, + 115, + -106, + -92, + -122, + -44, + 8, + 18, + -23, + 24, + -15, + 62, + 58, + 111, + 99, + -116, + -111, + -5, + 101, + -69, + -32, + -74, + -105, + 113, + -89, + 44, + 100, + -93, + -80, + 82, + -64, + 91, + -87, + -95, + 115, + 6, + 76, + 110, + 101, + 39, + 108, + 72, + 2, + 112, + -63, + -43, + 105, + -42, + 9, + -128 }, + destination + ); + } + + public void testQuantizeForQueryMIP() { + int dimensions = 768; + + // we want fixed values for these arrays so define our own random generation here to track + // quantization changes + Random random = new Random(42); + + float[] mipVectorToQuery = generateRandomFloatArray(random, dimensions, -1f, 1f); + float[] mipCentroid = generateRandomFloatArray(random, dimensions, -1f, 1f); + + VectorSimilarityFunction[] similarityFunctionsActingLikeEucllidean = new VectorSimilarityFunction[] { + VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT, + VectorSimilarityFunction.DOT_PRODUCT }; + int randIdx = random().nextInt(similarityFunctionsActingLikeEucllidean.length); + VectorSimilarityFunction similarityFunction = similarityFunctionsActingLikeEucllidean[randIdx]; + + BinaryQuantizer quantizer = new BinaryQuantizer(dimensions, similarityFunction); + float[] vector = mipVectorToQuery; + byte[] destination = new byte[dimensions / 8 * BQSpaceUtils.B_QUERY]; + float[] centroid = mipCentroid; + float cDotC = VectorUtil.dotProduct(centroid, centroid); + BinaryQuantizer.QueryFactors corrections = quantizer.quantizeForQuery(vector, destination, centroid); + + int sumQ = corrections.quantizedSum(); + float lower = corrections.lower(); + float width = corrections.width(); + float normVmC = corrections.normVmC(); + float vDotC = corrections.vDotC(); + + assertEquals(5272, sumQ); + assertEquals(-0.08603752f, lower, 0.00000001f); + assertEquals(0.011431276f, width, 0.00000001f); + assertEquals(21.847124f, normVmC, 0.00001f); + assertEquals(6.4300356f, vDotC, 0.0001f); + assertEquals(252.37146f, cDotC, 0.0001f); + assertArrayEquals( + new byte[] { + -81, + 19, + 67, + 33, + 112, + 8, + 40, + -5, + -19, + 115, + -87, + -63, + -59, + 12, + -2, + -127, + -23, + 43, + 24, + 16, + -69, + 112, + -22, + 75, + -81, + -50, + 100, + -41, + 3, + -120, + -93, + -4, + 4, + 125, + 34, + -57, + -109, + 89, + -63, + -35, + -116, + 4, + 35, + 93, + -26, + -88, + -55, + -86, + 63, + -46, + -122, + -96, + -26, + 124, + -64, + 21, + 96, + 46, + 98, + 97, + 88, + -98, + -83, + 121, + 16, + -14, + -89, + -118, + 65, + -39, + -111, + -35, + 113, + 108, + 111, + 86, + 17, + -69, + -47, + 72, + 1, + 36, + 17, + 113, + -87, + -5, + -46, + -37, + -2, + 93, + -123, + 118, + 4, + -12, + -33, + 95, + 32, + -63, + -97, + -109, + 27, + 111, + 42, + -57, + -87, + -41, + -73, + -106, + 27, + -31, + 32, + -1, + 9, + -88, + -35, + -11, + -103, + 5, + 27, + -127, + 108, + 127, + -119, + 58, + 38, + 18, + -103, + -27, + -63, + 56, + 77, + -13, + 3, + -40, + -127, + 37, + 82, + -87, + -26, + -45, + -14, + 18, + -50, + 76, + 25, + 37, + -12, + 106, + 17, + 115, + 0, + 23, + -109, + 26, + -110, + 17, + -35, + 111, + 4, + 60, + 58, + -64, + -104, + -125, + 23, + -58, + 89, + -117, + 104, + -71, + 3, + -89, + -26, + 46, + 15, + 82, + -83, + -75, + -72, + -69, + 20, + -38, + -47, + 109, + -66, + -66, + -89, + 108, + -122, + -3, + -69, + -85, + 18, + 59, + 85, + -97, + -114, + 95, + 2, + -84, + -77, + 121, + -6, + 10, + 110, + -13, + -123, + -34, + 106, + -71, + -107, + 123, + 67, + -111, + 58, + 52, + -53, + 87, + -113, + -21, + -44, + 26, + 10, + -62, + 56, + 111, + 36, + -126, + 26, + 94, + -88, + -13, + -113, + -50, + -9, + -115, + 84, + 8, + -32, + -102, + -4, + 89, + 29, + 75, + -73, + -19, + 22, + -90, + 76, + -61, + 4, + -48, + -100, + -11, + 107, + 20, + -39, + -98, + 123, + 77, + 104, + 9, + 9, + 91, + -105, + -40, + -106, + -87, + 38, + 48, + 60, + 29, + -68, + 124, + -78, + -63, + -101, + -115, + 67, + -17, + 101, + -53, + 121, + 44, + -78, + -12, + 110, + 91, + -83, + -92, + -72, + 96, + 32, + -96, + 89, + 48, + 76, + -124, + 3, + 113, + -111, + 12, + -86, + 32, + -43, + 68, + 106, + -122, + -84, + -37, + -124, + 118, + 84, + -72, + 34, + 20, + 57, + -29, + 119, + 56, + -10, + -108, + -109, + 60, + -56, + 37, + 84, + 8, + -16, + 80, + 24, + 50, + 41, + -25, + 47, + 115, + -122, + -92, + -126, + -44, + 8, + 18, + -23, + 24, + -15, + 60, + 58, + 111, + 99, + -120, + -111, + -21, + 101, + 59, + -32, + -74, + -105, + 113, + -90, + 36, + 100, + -93, + -80, + 82, + -64, + 91, + -87, + -95, + 115, + 6, + 76, + 110, + 101, + 39, + 44, + 0, + 2, + 112, + -64, + -47, + 105, + 2, + 1, + -128 }, + destination + ); + } + + public void testQuantizeForIndexCosine() { + int dimensions = 768; + + // we want fixed values for these arrays so define our own random generation here to track + // quantization changes + Random random = new Random(42); + + float[] mipVectorToIndex = generateRandomFloatArray(random, dimensions, -1f, 1f); + float[] mipCentroid = generateRandomFloatArray(random, dimensions, -1f, 1f); + + mipVectorToIndex = VectorUtil.l2normalize(mipVectorToIndex); + mipCentroid = VectorUtil.l2normalize(mipCentroid); + + BinaryQuantizer quantizer = new BinaryQuantizer(dimensions, VectorSimilarityFunction.COSINE); + float[] vector = mipVectorToIndex; + byte[] destination = new byte[dimensions / 8]; + float[] centroid = mipCentroid; + float[] corrections = quantizer.quantizeForIndex(vector, destination, centroid); + + assertEquals(3, corrections.length); + float ooq = corrections[0]; + float normOC = corrections[1]; + float oDotC = corrections[2]; + + assertEquals(0.8145253f, ooq, 0.000001f); + assertEquals(1.3955297f, normOC, 0.00001f); + assertEquals(0.026248248f, oDotC, 0.0001f); + assertArrayEquals( + new byte[] { + -83, + -91, + -71, + 97, + 32, + -96, + 89, + -80, + -20, + -108, + 3, + 113, + -111, + 12, + -86, + 32, + -43, + 76, + 122, + -106, + -83, + -37, + -122, + 118, + 84, + -72, + 34, + 20, + 57, + -29, + 119, + -72, + -10, + -100, + -109, + 62, + -54, + 117, + -44, + 8, + -16, + 80, + 58, + 50, + 41, + -25, + 47, + 115, + -106, + -92, + -122, + -44, + 8, + 18, + -23, + 24, + -15, + 62, + 58, + 111, + 99, + -116, + -111, + -21, + 101, + -69, + -32, + -74, + -105, + 113, + -90, + 44, + 100, + -93, + -80, + 82, + -64, + 91, + -87, + -95, + 115, + 6, + 76, + 110, + 101, + 39, + 44, + 72, + 2, + 112, + -63, + -43, + 105, + -42, + 9, + -126 }, + destination + ); + } + + public void testQuantizeForQueryCosine() { + int dimensions = 768; + + // we want fixed values for these arrays so define our own random generation here to track + // quantization changes + Random random = new Random(42); + + float[] mipVectorToQuery = generateRandomFloatArray(random, dimensions, -1f, 1f); + float[] mipCentroid = generateRandomFloatArray(random, dimensions, -1f, 1f); + + mipVectorToQuery = VectorUtil.l2normalize(mipVectorToQuery); + mipCentroid = VectorUtil.l2normalize(mipCentroid); + + BinaryQuantizer quantizer = new BinaryQuantizer(dimensions, VectorSimilarityFunction.COSINE); + float[] vector = mipVectorToQuery; + byte[] destination = new byte[dimensions / 8 * BQSpaceUtils.B_QUERY]; + float[] centroid = mipCentroid; + float cDotC = VectorUtil.dotProduct(centroid, centroid); + BinaryQuantizer.QueryFactors corrections = quantizer.quantizeForQuery(vector, destination, centroid); + + int sumQ = corrections.quantizedSum(); + float lower = corrections.lower(); + float width = corrections.width(); + float normVmC = corrections.normVmC(); + float vDotC = corrections.vDotC(); + + assertEquals(5277, sumQ); + assertEquals(-0.086002514f, lower, 0.00000001f); + assertEquals(0.011431345f, width, 0.00000001f); + assertEquals(1.3955297f, normVmC, 0.00001f); + assertEquals(0.026248248f, vDotC, 0.0001f); + assertEquals(1.0f, cDotC, 0.0001f); + assertArrayEquals( + new byte[] { + -83, + 18, + 67, + 37, + 80, + 8, + 40, + -1, + -19, + 115, + -87, + -63, + -59, + 12, + -2, + -63, + -19, + 43, + -104, + 16, + -69, + 80, + -22, + 75, + -81, + -50, + 100, + -41, + 7, + -88, + -93, + -4, + 4, + 117, + 34, + -57, + -109, + 89, + -63, + -35, + -116, + 4, + 35, + 93, + -26, + -88, + -56, + -82, + 63, + -46, + -122, + -96, + -26, + 124, + -64, + 21, + 96, + 46, + 114, + 101, + 92, + -98, + -83, + 121, + 48, + -14, + -89, + -118, + 65, + -47, + -79, + -35, + 113, + 110, + 111, + 70, + 17, + -69, + -47, + 64, + 1, + 102, + 19, + 113, + -87, + -5, + -46, + -34, + -2, + 93, + -123, + 102, + 4, + -12, + 127, + 95, + 32, + -64, + -97, + -105, + 59, + 111, + 42, + -57, + -87, + -41, + -73, + -106, + 27, + -31, + 32, + -65, + 9, + -88, + 93, + -11, + -103, + 37, + 27, + -127, + 108, + 127, + -119, + 58, + 38, + 18, + -103, + -27, + -63, + 48, + 77, + -13, + 3, + -40, + -127, + 37, + 82, + -87, + -26, + -45, + -14, + 18, + -49, + 76, + 25, + 37, + -12, + 106, + 17, + 115, + 0, + 23, + -109, + 26, + -126, + 21, + -35, + 111, + 4, + 60, + 58, + -64, + -104, + -125, + 23, + -58, + 121, + -117, + 104, + -69, + 3, + -89, + -26, + 46, + 15, + 90, + -83, + -73, + -72, + -69, + 20, + -38, + -47, + 109, + -66, + -66, + -89, + 108, + -122, + -3, + 59, + -85, + 18, + 58, + 85, + -101, + -114, + 95, + 2, + -84, + -77, + 121, + -6, + 10, + 110, + -13, + -123, + -34, + 106, + -71, + -107, + 123, + 67, + -111, + 58, + 52, + -53, + 87, + -113, + -21, + -44, + 26, + 10, + -62, + 56, + 103, + 36, + -126, + 26, + 94, + -88, + -13, + -113, + -50, + -9, + -115, + 84, + 8, + -32, + -102, + -4, + 89, + 29, + 75, + -73, + -19, + 22, + -90, + 76, + -61, + 4, + -44, + -100, + -11, + 107, + 20, + -39, + -98, + 123, + 77, + 104, + 9, + 41, + 91, + -105, + -38, + -106, + -87, + 38, + 48, + 60, + 29, + -68, + 126, + -78, + -63, + -101, + -115, + 67, + -17, + 101, + -53, + 121, + 44, + -78, + -12, + -18, + 91, + -83, + -91, + -72, + 96, + 32, + -96, + 89, + 48, + 76, + -124, + 3, + 113, + -111, + 12, + -86, + 32, + -43, + 68, + 106, + -122, + -84, + -37, + -124, + 118, + 84, + -72, + 34, + 20, + 57, + -29, + 119, + 56, + -10, + -100, + -109, + 60, + -56, + 37, + 84, + 8, + -16, + 80, + 24, + 50, + 41, + -25, + 47, + 115, + -122, + -92, + -126, + -44, + 8, + 18, + -23, + 24, + -15, + 60, + 58, + 107, + 99, + -120, + -111, + -21, + 101, + 59, + -32, + -74, + -105, + 113, + -122, + 36, + 100, + -95, + -80, + 82, + -64, + 91, + -87, + -95, + 115, + 4, + 76, + 110, + 101, + 39, + 44, + 0, + 2, + 112, + -64, + -47, + 105, + 2, + 1, + -128 }, + destination + ); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorerTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorerTests.java new file mode 100644 index 0000000000000..4ac66a9f63a3f --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryFlatVectorsScorerTests.java @@ -0,0 +1,1746 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.util.VectorUtil; +import org.elasticsearch.common.logging.LogConfigurator; + +import java.io.IOException; + +public class ES816BinaryFlatVectorsScorerTests extends LuceneTestCase { + + static { + LogConfigurator.loadLog4jPlugins(); + LogConfigurator.configureESLogging(); // native access requires logging to be initialized + } + + public void testScore() throws IOException { + int dimensions = random().nextInt(1, 4097); + int discretizedDimensions = BQVectorUtils.discretize(dimensions, 64); + + int randIdx = random().nextInt(VectorSimilarityFunction.values().length); + VectorSimilarityFunction similarityFunction = VectorSimilarityFunction.values()[randIdx]; + + float[] centroid = new float[dimensions]; + for (int j = 0; j < dimensions; j++) { + centroid[j] = random().nextFloat(-50f, 50f); + } + if (similarityFunction == VectorSimilarityFunction.COSINE) { + VectorUtil.l2normalize(centroid); + } + + byte[] vector = new byte[discretizedDimensions / 8 * BQSpaceUtils.B_QUERY]; + random().nextBytes(vector); + float distanceToCentroid = random().nextFloat(0f, 10_000.0f); + float vl = random().nextFloat(-1000f, 1000f); + float width = random().nextFloat(0f, 1000f); + short quantizedSum = (short) random().nextInt(0, 4097); + float normVmC = random().nextFloat(-1000f, 1000f); + float vDotC = random().nextFloat(-1000f, 1000f); + ES816BinaryFlatVectorsScorer.BinaryQueryVector queryVector = new ES816BinaryFlatVectorsScorer.BinaryQueryVector( + vector, + new BinaryQuantizer.QueryFactors(quantizedSum, distanceToCentroid, vl, width, normVmC, vDotC) + ); + + RandomAccessBinarizedByteVectorValues targetVectors = new RandomAccessBinarizedByteVectorValues() { + @Override + public float getCentroidDistance(int vectorOrd) throws IOException { + return random().nextFloat(0f, 1000f); + } + + @Override + public float getVectorMagnitude(int vectorOrd) throws IOException { + return random().nextFloat(0f, 100f); + } + + @Override + public float getOOQ(int targetOrd) throws IOException { + return random().nextFloat(-1000f, 1000f); + } + + @Override + public float getNormOC(int targetOrd) throws IOException { + return random().nextFloat(-1000f, 1000f); + } + + @Override + public float getODotC(int targetOrd) throws IOException { + return random().nextFloat(-1000f, 1000f); + } + + @Override + public BinaryQuantizer getQuantizer() { + int dimensions = 128; + return new BinaryQuantizer(dimensions, dimensions, VectorSimilarityFunction.EUCLIDEAN); + } + + @Override + public float[] getCentroid() throws IOException { + return centroid; + } + + @Override + public RandomAccessBinarizedByteVectorValues copy() throws IOException { + return null; + } + + @Override + public byte[] vectorValue(int targetOrd) throws IOException { + byte[] vectorBytes = new byte[discretizedDimensions / 8]; + random().nextBytes(vectorBytes); + return vectorBytes; + } + + @Override + public int size() { + return 1; + } + + @Override + public int dimension() { + return dimensions; + } + }; + + ES816BinaryFlatVectorsScorer.BinarizedRandomVectorScorer scorer = new ES816BinaryFlatVectorsScorer.BinarizedRandomVectorScorer( + queryVector, + targetVectors, + similarityFunction + ); + + float score = scorer.score(0); + + assertTrue(score >= 0f); + } + + public void testScoreEuclidean() throws IOException { + int dimensions = 128; + + byte[] vector = new byte[] { + -8, + 10, + -27, + 112, + -83, + 36, + -36, + -122, + -114, + 82, + 55, + 33, + -33, + 120, + 55, + -99, + -93, + -86, + -55, + 21, + -121, + 30, + 111, + 30, + 0, + 82, + 21, + 38, + -120, + -127, + 40, + -32, + 78, + -37, + 42, + -43, + 122, + 115, + 30, + 115, + 123, + 108, + -13, + -65, + 123, + 124, + -33, + -68, + 49, + 5, + 20, + 58, + 0, + 12, + 30, + 30, + 4, + 97, + 10, + 66, + 4, + 35, + 1, + 67 }; + float distanceToCentroid = 157799.12f; + float vl = -57.883f; + float width = 9.972266f; + short quantizedSum = 795; + ES816BinaryFlatVectorsScorer.BinaryQueryVector queryVector = new ES816BinaryFlatVectorsScorer.BinaryQueryVector( + vector, + new BinaryQuantizer.QueryFactors(quantizedSum, distanceToCentroid, vl, width, 0f, 0f) + ); + + RandomAccessBinarizedByteVectorValues targetVectors = new RandomAccessBinarizedByteVectorValues() { + @Override + public float getCentroidDistance(int vectorOrd) { + return 355.78073f; + } + + @Override + public float getVectorMagnitude(int vectorOrd) { + return 0.7636705f; + } + + @Override + public float getOOQ(int targetOrd) { + return 0; + } + + @Override + public float getNormOC(int targetOrd) { + return 0; + } + + @Override + public float getODotC(int targetOrd) { + return 0; + } + + @Override + public BinaryQuantizer getQuantizer() { + int dimensions = 128; + return new BinaryQuantizer(dimensions, dimensions, VectorSimilarityFunction.EUCLIDEAN); + } + + @Override + public float[] getCentroid() { + return new float[] { + 26.7f, + 16.2f, + 10.913f, + 10.314f, + 12.12f, + 14.045f, + 15.887f, + 16.864f, + 32.232f, + 31.567f, + 34.922f, + 21.624f, + 16.349f, + 29.625f, + 31.994f, + 22.044f, + 37.847f, + 24.622f, + 36.299f, + 27.966f, + 14.368f, + 19.248f, + 30.778f, + 35.927f, + 27.019f, + 16.381f, + 17.325f, + 16.517f, + 13.272f, + 9.154f, + 9.242f, + 17.995f, + 53.777f, + 23.011f, + 12.929f, + 16.128f, + 22.16f, + 28.643f, + 25.861f, + 27.197f, + 59.883f, + 40.878f, + 34.153f, + 22.795f, + 24.402f, + 37.427f, + 34.19f, + 29.288f, + 61.812f, + 26.355f, + 39.071f, + 37.789f, + 23.33f, + 22.299f, + 28.64f, + 47.828f, + 52.457f, + 21.442f, + 24.039f, + 29.781f, + 27.707f, + 19.484f, + 14.642f, + 28.757f, + 54.567f, + 20.936f, + 25.112f, + 25.521f, + 22.077f, + 18.272f, + 14.526f, + 29.054f, + 61.803f, + 24.509f, + 37.517f, + 35.906f, + 24.106f, + 22.64f, + 32.1f, + 48.788f, + 60.102f, + 39.625f, + 34.766f, + 22.497f, + 24.397f, + 41.599f, + 38.419f, + 30.99f, + 55.647f, + 25.115f, + 14.96f, + 18.882f, + 26.918f, + 32.442f, + 26.231f, + 27.107f, + 26.828f, + 15.968f, + 18.668f, + 14.071f, + 10.906f, + 8.989f, + 9.721f, + 17.294f, + 36.32f, + 21.854f, + 35.509f, + 27.106f, + 14.067f, + 19.82f, + 33.582f, + 35.997f, + 33.528f, + 30.369f, + 36.955f, + 21.23f, + 15.2f, + 30.252f, + 34.56f, + 22.295f, + 29.413f, + 16.576f, + 11.226f, + 10.754f, + 12.936f, + 15.525f, + 15.868f, + 16.43f }; + } + + @Override + public RandomAccessBinarizedByteVectorValues copy() { + return null; + } + + @Override + public byte[] vectorValue(int targetOrd) { + return new byte[] { 44, 108, 120, -15, -61, -32, 124, 25, -63, -57, 6, 24, 1, -61, 1, 14 }; + } + + @Override + public int size() { + return 1; + } + + @Override + public int dimension() { + return dimensions; + } + }; + + VectorSimilarityFunction similarityFunction = VectorSimilarityFunction.EUCLIDEAN; + + ES816BinaryFlatVectorsScorer.BinarizedRandomVectorScorer scorer = new ES816BinaryFlatVectorsScorer.BinarizedRandomVectorScorer( + queryVector, + targetVectors, + similarityFunction + ); + + assertEquals(1f / (1f + 245482.47f), scorer.score(0), 0.1f); + } + + public void testScoreMIP() throws IOException { + int dimensions = 768; + + byte[] vector = new byte[] { + -76, + 44, + 81, + 31, + 30, + -59, + 56, + -118, + -36, + 45, + -11, + 8, + -61, + 95, + -100, + 18, + -91, + -98, + -46, + 31, + -8, + 82, + -42, + 121, + 75, + -61, + 125, + -21, + -82, + 16, + 21, + 40, + -1, + 12, + -92, + -22, + -49, + -92, + -19, + -32, + -56, + -34, + 60, + -100, + 69, + 13, + 60, + -51, + 90, + 4, + -77, + 63, + 124, + 69, + 88, + 73, + -72, + 29, + -96, + 44, + 69, + -123, + -59, + -94, + 84, + 80, + -61, + 27, + -37, + -92, + -51, + -86, + 19, + -55, + -36, + -2, + 68, + -37, + -128, + 59, + -47, + 119, + -53, + 56, + -12, + 37, + 27, + 119, + -37, + 125, + 78, + 19, + 15, + -9, + 94, + 100, + -72, + 55, + 86, + -48, + 26, + 10, + -112, + 28, + -15, + -64, + -34, + 55, + -42, + -31, + -96, + -18, + 60, + -44, + 69, + 106, + -20, + 15, + 47, + 49, + -122, + -45, + 119, + 101, + 22, + 77, + 108, + -15, + -71, + -28, + -43, + -68, + -127, + -86, + -118, + -51, + 121, + -65, + -10, + -49, + 115, + -6, + -61, + -98, + 21, + 41, + 56, + 29, + -16, + -82, + 4, + 72, + -77, + 23, + 23, + -32, + -98, + 112, + 27, + -4, + 91, + -69, + 102, + -114, + 16, + -20, + -76, + -124, + 43, + 12, + 3, + -30, + 42, + -44, + -88, + -72, + -76, + -94, + -73, + 46, + -17, + 4, + -74, + -44, + 53, + -11, + -117, + -105, + -113, + -37, + -43, + -128, + -70, + 56, + -68, + -100, + 56, + -20, + 77, + 12, + 17, + -119, + -17, + 59, + -10, + -26, + 29, + 42, + -59, + -28, + -28, + 60, + -34, + 60, + -24, + 80, + -81, + 24, + 122, + 127, + 62, + 124, + -5, + -11, + 59, + -52, + 74, + -29, + -116, + 3, + -40, + -99, + -24, + 11, + -10, + 95, + 21, + -38, + 59, + -52, + 29, + 58, + 112, + 100, + -106, + -90, + 71, + 72, + 57, + 95, + 98, + 96, + -41, + -16, + 50, + -18, + 123, + -36, + 74, + -101, + 17, + 50, + 48, + 96, + 57, + 7, + 81, + -16, + -32, + -102, + -24, + -71, + -10, + 37, + -22, + 94, + -36, + -52, + -71, + -47, + 47, + -1, + -31, + -10, + -126, + -15, + -123, + -59, + 71, + -49, + 67, + 99, + -57, + 21, + -93, + -13, + -18, + 54, + -112, + -60, + 9, + 25, + -30, + -47, + 26, + 27, + 26, + -63, + 1, + -63, + 18, + -114, + 80, + 110, + -123, + 0, + -63, + -126, + -128, + 10, + -60, + 51, + -71, + 28, + 114, + -4, + 53, + 10, + 23, + -96, + 9, + 32, + -22, + 5, + -108, + 33, + 98, + -59, + -106, + -126, + 73, + 72, + -72, + -73, + -60, + -96, + -99, + 31, + 40, + 15, + -19, + 17, + -128, + 33, + -75, + 96, + -18, + -47, + 75, + 27, + -60, + -16, + -82, + 13, + 21, + 37, + 23, + 70, + 9, + -39, + 16, + -127, + 35, + -78, + 64, + 99, + -46, + 1, + 28, + 65, + 125, + 14, + 42, + 26 }; + float distanceToCentroid = 95.39032f; + float vl = -0.10079563f; + float width = 0.014609014f; + short quantizedSum = 5306; + float normVmC = 9.766797f; + float vDotC = 133.56123f; + float cDotC = 132.20227f; + ES816BinaryFlatVectorsScorer.BinaryQueryVector queryVector = new ES816BinaryFlatVectorsScorer.BinaryQueryVector( + vector, + new BinaryQuantizer.QueryFactors(quantizedSum, distanceToCentroid, vl, width, normVmC, vDotC) + ); + + RandomAccessBinarizedByteVectorValues targetVectors = new RandomAccessBinarizedByteVectorValues() { + @Override + public float getCentroidDistance(int vectorOrd) { + return 0f; + } + + @Override + public float getCentroidDP() { + return cDotC; + } + + @Override + public float getVectorMagnitude(int vectorOrd) { + return 0f; + } + + @Override + public float getOOQ(int targetOrd) { + return 0.7882396f; + } + + @Override + public float getNormOC(int targetOrd) { + return 5.0889387f; + } + + @Override + public float getODotC(int targetOrd) { + return 131.485660f; + } + + @Override + public BinaryQuantizer getQuantizer() { + int dimensions = 768; + return new BinaryQuantizer(dimensions, dimensions, VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT); + } + + @Override + public float[] getCentroid() { + return new float[] { + 0.16672021f, + 0.11700719f, + 0.013227397f, + 0.09305186f, + -0.029422699f, + 0.17622353f, + 0.4267106f, + -0.297038f, + 0.13915674f, + 0.38441318f, + -0.486725f, + -0.15987667f, + -0.19712289f, + 0.1349074f, + -0.19016947f, + -0.026179956f, + 0.4129807f, + 0.14325741f, + -0.09106042f, + 0.06876218f, + -0.19389102f, + 0.4467732f, + 0.03169017f, + -0.066950575f, + -0.044301506f, + -0.0059755715f, + -0.33196586f, + 0.18213534f, + -0.25065416f, + 0.30251458f, + 0.3448419f, + -0.14900115f, + -0.07782894f, + 0.3568707f, + -0.46595258f, + 0.37295088f, + -0.088741764f, + 0.17248306f, + -0.0072736046f, + 0.32928637f, + 0.13216197f, + 0.032092985f, + 0.21553043f, + 0.016091486f, + 0.31958902f, + 0.0133126f, + 0.1579258f, + 0.018537233f, + 0.046248164f, + -0.0048194043f, + -0.2184672f, + -0.26273906f, + -0.110678785f, + -0.04542999f, + -0.41625032f, + 0.46025568f, + -0.16116948f, + 0.4091706f, + 0.18427321f, + 0.004736977f, + 0.16289745f, + -0.05330932f, + -0.2694863f, + -0.14762327f, + 0.17744702f, + 0.2445075f, + 0.14377175f, + 0.37390858f, + 0.16165806f, + 0.17177118f, + 0.097307935f, + 0.36326465f, + 0.23221572f, + 0.15579978f, + -0.065486655f, + -0.29006517f, + -0.009194494f, + 0.009019374f, + 0.32154799f, + -0.23186184f, + 0.46485493f, + -0.110756285f, + -0.18604982f, + 0.35027295f, + 0.19815539f, + 0.47386464f, + -0.031379268f, + 0.124035835f, + 0.11556784f, + 0.4304302f, + -0.24455063f, + 0.1816723f, + 0.034300473f, + -0.034347706f, + 0.040140998f, + 0.1389901f, + 0.22840638f, + -0.19911191f, + 0.07563166f, + -0.2744902f, + 0.13114859f, + -0.23862572f, + -0.31404558f, + 0.41355187f, + 0.12970817f, + -0.35403475f, + -0.2714075f, + 0.07231573f, + 0.043893218f, + 0.30324167f, + 0.38928393f, + -0.1567055f, + -0.0083288215f, + 0.0487653f, + 0.12073729f, + -0.01582117f, + 0.13381198f, + -0.084824145f, + -0.15329859f, + -1.120622f, + 0.3972598f, + 0.36022213f, + -0.29826534f, + -0.09468781f, + 0.03550699f, + -0.21630692f, + 0.55655843f, + -0.14842057f, + 0.5924833f, + 0.38791573f, + 0.1502777f, + 0.111737385f, + 0.1926823f, + 0.66021144f, + 0.25601995f, + 0.28220543f, + 0.10194068f, + 0.013066262f, + -0.09348819f, + -0.24085014f, + -0.17843121f, + -0.012598432f, + 0.18757571f, + 0.48543528f, + -0.059388146f, + 0.1548026f, + 0.041945867f, + 0.3322589f, + 0.012830887f, + 0.16621992f, + 0.22606649f, + 0.13959105f, + -0.16688728f, + 0.47194278f, + -0.12767595f, + 0.037815034f, + 0.441938f, + 0.07875027f, + 0.08625042f, + 0.053454693f, + 0.74093896f, + 0.34662113f, + 0.009829135f, + -0.033400282f, + 0.030965377f, + 0.17645596f, + 0.083803624f, + 0.32578796f, + 0.49538168f, + -0.13212465f, + -0.39596975f, + 0.109529115f, + 0.2815771f, + -0.051440604f, + 0.21889819f, + 0.25598505f, + 0.012208843f, + -0.012405662f, + 0.3248759f, + 0.00997502f, + 0.05999008f, + 0.03562817f, + 0.19007418f, + 0.24805716f, + 0.5926766f, + 0.26937613f, + 0.25856f, + -0.05798439f, + -0.29168302f, + 0.14050555f, + 0.084851265f, + -0.03763504f, + 0.8265359f, + -0.23383066f, + -0.042164285f, + 0.19120507f, + -0.12189065f, + 0.3864055f, + -0.19823311f, + 0.30280992f, + 0.10814344f, + -0.164514f, + -0.22905481f, + 0.13680641f, + 0.4513772f, + -0.514546f, + -0.061746247f, + 0.11598224f, + -0.23093395f, + -0.09735358f, + 0.02767051f, + 0.11594536f, + 0.17106244f, + 0.21301728f, + -0.048222974f, + 0.2212131f, + -0.018857865f, + -0.09783516f, + 0.42156664f, + -0.14032331f, + -0.103861615f, + 0.4190284f, + 0.068923555f, + -0.015083771f, + 0.083590426f, + -0.15759592f, + -0.19096768f, + -0.4275228f, + 0.12626286f, + 0.12192557f, + 0.4157616f, + 0.048780657f, + 0.008426048f, + -0.0869124f, + 0.054927208f, + 0.28417027f, + 0.29765493f, + 0.09203619f, + -0.14446871f, + -0.117514975f, + 0.30662632f, + 0.24904715f, + -0.19551662f, + -0.0045785015f, + 0.4217626f, + -0.31457824f, + 0.23381722f, + 0.089111514f, + -0.27170828f, + -0.06662652f, + 0.10011391f, + -0.090274535f, + 0.101849966f, + 0.26554734f, + -0.1722843f, + 0.23296228f, + 0.25112453f, + -0.16790418f, + 0.010348314f, + 0.05061285f, + 0.38003662f, + 0.0804625f, + 0.3450673f, + 0.364368f, + -0.2529952f, + -0.034065288f, + 0.22796603f, + 0.5457553f, + 0.11120353f, + 0.24596325f, + 0.42822433f, + -0.19215727f, + -0.06974534f, + 0.19388479f, + -0.17598474f, + -0.08769705f, + 0.12769659f, + 0.1371616f, + -0.4636819f, + 0.16870509f, + 0.14217548f, + 0.04412187f, + -0.20930687f, + 0.0075530168f, + 0.10065227f, + 0.45334083f, + -0.1097471f, + -0.11139921f, + -0.31835595f, + -0.057386875f, + 0.16285825f, + 0.5088513f, + -0.06318843f, + -0.34759882f, + 0.21132466f, + 0.33609292f, + 0.04858872f, + -0.058759f, + 0.22845529f, + -0.07641319f, + 0.5452827f, + -0.5050389f, + 0.1788054f, + 0.37428045f, + 0.066334985f, + -0.28162515f, + -0.15629752f, + 0.33783385f, + -0.0832242f, + 0.29144394f, + 0.47892854f, + -0.47006592f, + -0.07867588f, + 0.3872869f, + 0.28053126f, + 0.52399015f, + 0.21979983f, + 0.076880336f, + 0.47866163f, + 0.252952f, + -0.1323851f, + -0.22225754f, + -0.38585815f, + 0.12967427f, + 0.20340872f, + -0.326928f, + 0.09636557f, + -0.35929212f, + 0.5413311f, + 0.019960884f, + 0.33512768f, + 0.15133342f, + -0.14124066f, + -0.1868793f, + -0.07862198f, + 0.22739467f, + 0.19598985f, + 0.34314656f, + -0.05071516f, + -0.21107961f, + 0.19934991f, + 0.04822684f, + 0.15060754f, + 0.26586458f, + -0.15528078f, + 0.123646654f, + 0.14450715f, + -0.12574252f, + 0.30608323f, + 0.018549249f, + 0.36323825f, + 0.06762097f, + 0.08562406f, + -0.07863075f, + 0.15975896f, + 0.008347004f, + 0.37931192f, + 0.22957338f, + 0.33606857f, + -0.25204057f, + 0.18126069f, + 0.41903302f, + 0.20244692f, + -0.053850617f, + 0.23088565f, + 0.16085246f, + 0.1077502f, + -0.12445943f, + 0.115779735f, + 0.124704875f, + 0.13076028f, + -0.11628619f, + -0.12580182f, + 0.065204754f, + -0.26290357f, + -0.23539798f, + -0.1855292f, + 0.39872098f, + 0.44495568f, + 0.05491784f, + 0.05135692f, + 0.624011f, + 0.22839564f, + 0.0022447354f, + -0.27169296f, + -0.1694988f, + -0.19106841f, + 0.0110123325f, + 0.15464798f, + -0.16269256f, + 0.04033836f, + -0.11792753f, + 0.17172396f, + -0.08912173f, + -0.30929542f, + -0.03446989f, + -0.21738084f, + 0.39657044f, + 0.33550346f, + -0.06839139f, + 0.053675443f, + 0.33783767f, + 0.22576828f, + 0.38280004f, + 4.1448855f, + 0.14225426f, + 0.24038498f, + 0.072373435f, + -0.09465926f, + -0.016144043f, + 0.40864578f, + -0.2583055f, + 0.031816103f, + 0.062555805f, + 0.06068663f, + 0.25858644f, + -0.10598804f, + 0.18201788f, + -0.00090025424f, + 0.085680895f, + 0.4304161f, + 0.028686283f, + 0.027298616f, + 0.27473378f, + -0.3888415f, + 0.44825438f, + 0.3600378f, + 0.038944595f, + 0.49292335f, + 0.18556066f, + 0.15779617f, + 0.29989767f, + 0.39233804f, + 0.39759228f, + 0.3850708f, + -0.0526475f, + 0.18572918f, + 0.09667526f, + -0.36111078f, + 0.3439669f, + 0.1724522f, + 0.14074509f, + 0.26097745f, + 0.16626832f, + -0.3062964f, + -0.054877423f, + 0.21702516f, + 0.4736452f, + 0.2298038f, + -0.2983771f, + 0.118479654f, + 0.35940516f, + 0.12212727f, + 0.17234904f, + 0.30632678f, + 0.09207966f, + -0.14084268f, + -0.19737118f, + 0.12442629f, + 0.52454203f, + 0.1266684f, + 0.3062802f, + 0.121598125f, + -0.09156268f, + 0.11491686f, + -0.105715364f, + 0.19831072f, + 0.061421417f, + -0.41778997f, + 0.14488487f, + 0.023310646f, + 0.27257463f, + 0.16821945f, + -0.16702746f, + 0.263203f, + 0.33512688f, + 0.35117313f, + -0.31740817f, + -0.14203706f, + 0.061256267f, + -0.19764185f, + 0.04822579f, + -0.0016218472f, + -0.025792575f, + 0.4885193f, + -0.16942391f, + -0.04156327f, + 0.15908112f, + -0.06998626f, + 0.53907114f, + 0.10317832f, + -0.365468f, + 0.4729886f, + 0.14291425f, + 0.32812154f, + -0.0273262f, + 0.31760117f, + 0.16925456f, + 0.21820979f, + 0.085142255f, + 0.16118735f, + -3.7089362f, + 0.251577f, + 0.18394576f, + 0.027926167f, + 0.15720351f, + 0.13084261f, + 0.16240814f, + 0.23045056f, + -0.3966458f, + 0.22822891f, + -0.061541352f, + 0.028320132f, + -0.14736478f, + 0.184569f, + 0.084853746f, + 0.15172474f, + 0.08277542f, + 0.27751622f, + 0.23450488f, + -0.15349835f, + 0.29665688f, + 0.32045734f, + 0.20012043f, + -0.2749372f, + 0.011832386f, + 0.05976605f, + 0.018300122f, + -0.07855043f, + -0.075900674f, + 0.0384252f, + -0.15101928f, + 0.10922137f, + 0.47396383f, + -0.1771141f, + 0.2203417f, + 0.33174303f, + 0.36640546f, + 0.10906258f, + 0.13765177f, + 0.2488032f, + -0.061588854f, + 0.20347528f, + 0.2574979f, + 0.22369152f, + 0.18777567f, + -0.0772263f, + -0.1353299f, + 0.087077625f, + -0.05409276f, + 0.027534787f, + 0.08053508f, + 0.3403908f, + -0.15362988f, + 0.07499862f, + 0.54367846f, + -0.045938436f, + 0.12206868f, + 0.031069376f, + 0.2972343f, + 0.3235321f, + -0.053970363f, + -0.0042564687f, + 0.21447177f, + 0.023565233f, + -0.1286087f, + -0.047359955f, + 0.23021339f, + 0.059837278f, + 0.19709614f, + -0.17340347f, + 0.11572943f, + 0.21720429f, + 0.29375625f, + -0.045433592f, + 0.033339307f, + 0.24594454f, + -0.021661613f, + -0.12823369f, + 0.41809165f, + 0.093840264f, + -0.007481906f, + 0.22441079f, + -0.45719734f, + 0.2292629f, + 2.675806f, + 0.3690025f, + 2.1311781f, + 0.07818368f, + -0.17055893f, + 0.3162922f, + -0.2983149f, + 0.21211359f, + 0.037087034f, + 0.021580033f, + 0.086415835f, + 0.13541797f, + -0.12453424f, + 0.04563163f, + -0.082379065f, + -0.15938349f, + 0.38595748f, + -0.8796574f, + -0.080991246f, + 0.078572094f, + 0.20274459f, + 0.009252143f, + -0.12719384f, + 0.105845824f, + 0.1592398f, + -0.08656061f, + -0.053054806f, + 0.090986334f, + -0.02223379f, + -0.18215932f, + -0.018316114f, + 0.1806707f, + 0.24788831f, + -0.041049056f, + 0.01839475f, + 0.19160001f, + -0.04827654f, + 4.4070687f, + 0.12640671f, + -0.11171499f, + -0.015480781f, + 0.14313947f, + 0.10024215f, + 0.4129662f, + 0.038836367f, + -0.030228542f, + 0.2948598f, + 0.32946473f, + 0.2237934f, + 0.14260699f, + -0.044821896f, + 0.23791742f, + 0.079720296f, + 0.27059034f, + 0.32129505f, + 0.2725177f, + 0.06883333f, + 0.1478041f, + 0.07598411f, + 0.27230525f, + -0.04704308f, + 0.045167264f, + 0.215413f, + 0.20359069f, + -0.092178136f, + -0.09523752f, + 0.21427691f, + 0.10512272f, + 5.1295033f, + 0.040909242f, + 0.007160441f, + -0.192866f, + -0.102640584f, + 0.21103396f, + -0.006780398f, + -0.049653083f, + -0.29426834f, + -0.0038102255f, + -0.13842082f, + 0.06620181f, + -0.3196518f, + 0.33279592f, + 0.13845938f, + 0.16162738f, + -0.24798508f, + -0.06672485f, + 0.195944f, + -0.11957207f, + 0.44237947f, + -0.07617347f, + 0.13575341f, + -0.35074243f, + -0.093798876f, + 0.072853446f, + -0.20490398f, + 0.26504788f, + -0.046076056f, + 0.16488416f, + 0.36007464f, + 0.20955376f, + -0.3082038f, + 0.46533757f, + -0.27326992f, + -0.14167665f, + 0.25017953f, + 0.062622115f, + 0.14057694f, + -0.102370486f, + 0.33898357f, + 0.36456722f, + -0.10120469f, + -0.27838466f, + -0.11779602f, + 0.18517569f, + -0.05942488f, + 0.076405466f, + 0.007960496f, + 0.0443746f, + 0.098998964f, + -0.01897129f, + 0.8059487f, + 0.06991939f, + 0.26562217f, + 0.26942885f, + 0.11432197f, + -0.0055776504f, + 0.054493718f, + -0.13086213f, + 0.6841702f, + 0.121975765f, + 0.02787146f, + 0.29039973f, + 0.30943078f, + 0.21762547f, + 0.28751117f, + 0.027524523f, + 0.5315654f, + -0.22451901f, + -0.13782433f, + 0.08228316f, + 0.07808882f, + 0.17445615f, + -0.042489477f, + 0.13232234f, + 0.2756272f, + -0.18824948f, + 0.14326479f, + -0.119312495f, + 0.011788091f, + -0.22103515f, + -0.2477118f, + -0.10513839f, + 0.034028634f, + 0.10693818f, + 0.03057979f, + 0.04634646f, + 0.2289361f, + 0.09981585f, + 0.26901972f, + 0.1561221f, + -0.10639886f, + 0.36466748f, + 0.06350991f, + 0.027927283f, + 0.11919768f, + 0.23290513f, + -0.03417105f, + 0.16698854f, + -0.19243467f, + 0.28430334f, + 0.03754995f, + -0.08697018f, + 0.20413163f, + -0.27218238f, + 0.13707504f, + -0.082289375f, + 0.03479585f, + 0.2298305f, + 0.4983682f, + 0.34522808f, + -0.05711886f, + -0.10568684f, + -0.07771385f }; + } + + @Override + public RandomAccessBinarizedByteVectorValues copy() { + return null; + } + + @Override + public byte[] vectorValue(int targetOrd) { + return new byte[] { + -88, + -3, + 60, + -75, + -38, + 79, + 84, + -53, + -116, + -126, + 19, + -19, + -21, + -80, + 69, + 101, + -71, + 53, + 101, + -124, + -24, + -76, + 92, + -45, + 108, + -107, + -18, + 102, + 23, + -80, + -47, + 116, + 87, + -50, + 27, + -31, + -10, + -13, + 117, + -88, + -27, + -93, + -98, + -39, + 30, + -109, + -114, + 5, + -15, + 98, + -82, + 81, + 83, + 118, + 30, + -118, + -12, + -95, + 121, + 125, + -13, + -88, + 75, + -85, + -56, + -126, + 82, + -59, + 48, + -81, + 67, + -63, + 81, + 24, + -83, + 95, + -44, + 103, + 3, + -40, + -13, + -41, + -29, + -60, + 1, + 65, + -4, + -110, + -40, + 34, + 118, + 51, + -76, + 75, + 70, + -51 }; + } + + @Override + public int size() { + return 1; + } + + @Override + public int dimension() { + return dimensions; + } + }; + + VectorSimilarityFunction similarityFunction = VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT; + + ES816BinaryFlatVectorsScorer.BinarizedRandomVectorScorer scorer = new ES816BinaryFlatVectorsScorer.BinarizedRandomVectorScorer( + queryVector, + targetVectors, + similarityFunction + ); + + assertEquals(132.30249f, scorer.score(0), 0.0001f); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsFormatTests.java new file mode 100644 index 0000000000000..0892436891ff1 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816BinaryQuantizedVectorsFormatTests.java @@ -0,0 +1,175 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; +import org.elasticsearch.common.logging.LogConfigurator; + +import java.io.IOException; +import java.util.Locale; + +import static java.lang.String.format; +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.oneOf; + +public class ES816BinaryQuantizedVectorsFormatTests extends BaseKnnVectorsFormatTestCase { + + static { + LogConfigurator.loadLog4jPlugins(); + LogConfigurator.configureESLogging(); // native access requires logging to be initialized + } + + @Override + protected Codec getCodec() { + return new Lucene912Codec() { + @Override + public KnnVectorsFormat getKnnVectorsFormatForField(String field) { + return new ES816BinaryQuantizedVectorsFormat(); + } + }; + } + + public void testSearch() throws Exception { + String fieldName = "field"; + int numVectors = random().nextInt(99, 500); + int dims = random().nextInt(4, 65); + float[] vector = randomVector(dims); + VectorSimilarityFunction similarityFunction = randomSimilarity(); + KnnFloatVectorField knnField = new KnnFloatVectorField(fieldName, vector, similarityFunction); + IndexWriterConfig iwc = newIndexWriterConfig(); + try (Directory dir = newDirectory()) { + try (IndexWriter w = new IndexWriter(dir, iwc)) { + for (int i = 0; i < numVectors; i++) { + Document doc = new Document(); + knnField.setVectorValue(randomVector(dims)); + doc.add(knnField); + w.addDocument(doc); + } + w.commit(); + + try (IndexReader reader = DirectoryReader.open(w)) { + IndexSearcher searcher = new IndexSearcher(reader); + final int k = random().nextInt(5, 50); + float[] queryVector = randomVector(dims); + Query q = new KnnFloatVectorQuery(fieldName, queryVector, k); + TopDocs collectedDocs = searcher.search(q, k); + assertEquals(k, collectedDocs.totalHits.value); + assertEquals(TotalHits.Relation.EQUAL_TO, collectedDocs.totalHits.relation); + } + } + } + } + + public void testToString() { + FilterCodec customCodec = new FilterCodec("foo", Codec.getDefault()) { + @Override + public KnnVectorsFormat knnVectorsFormat() { + return new ES816BinaryQuantizedVectorsFormat(); + } + }; + String expectedPattern = "ES816BinaryQuantizedVectorsFormat(" + + "name=ES816BinaryQuantizedVectorsFormat, " + + "flatVectorScorer=ES816BinaryFlatVectorsScorer(nonQuantizedDelegate=%s()))"; + var defaultScorer = format(Locale.ROOT, expectedPattern, "DefaultFlatVectorScorer"); + var memSegScorer = format(Locale.ROOT, expectedPattern, "Lucene99MemorySegmentFlatVectorsScorer"); + assertThat(customCodec.knnVectorsFormat().toString(), is(oneOf(defaultScorer, memSegScorer))); + } + + @Override + public void testRandomWithUpdatesAndGraph() { + // graph not supported + } + + @Override + public void testSearchWithVisitedLimit() { + // visited limit is not respected, as it is brute force search + } + + public void testQuantizedVectorsWriteAndRead() throws IOException { + String fieldName = "field"; + int numVectors = random().nextInt(99, 500); + int dims = random().nextInt(4, 65); + + float[] vector = randomVector(dims); + VectorSimilarityFunction similarityFunction = randomSimilarity(); + KnnFloatVectorField knnField = new KnnFloatVectorField(fieldName, vector, similarityFunction); + try (Directory dir = newDirectory()) { + try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + for (int i = 0; i < numVectors; i++) { + Document doc = new Document(); + knnField.setVectorValue(randomVector(dims)); + doc.add(knnField); + w.addDocument(doc); + if (i % 101 == 0) { + w.commit(); + } + } + w.commit(); + w.forceMerge(1); + + try (IndexReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + FloatVectorValues vectorValues = r.getFloatVectorValues(fieldName); + assertEquals(vectorValues.size(), numVectors); + OffHeapBinarizedVectorValues qvectorValues = ((ES816BinaryQuantizedVectorsReader.BinarizedVectorValues) vectorValues) + .getQuantizedVectorValues(); + float[] centroid = qvectorValues.getCentroid(); + assertEquals(centroid.length, dims); + + int descritizedDimension = BQVectorUtils.discretize(dims, 64); + BinaryQuantizer quantizer = new BinaryQuantizer(dims, descritizedDimension, similarityFunction); + byte[] expectedVector = new byte[BQVectorUtils.discretize(dims, 64) / 8]; + if (similarityFunction == VectorSimilarityFunction.COSINE) { + vectorValues = new ES816BinaryQuantizedVectorsWriter.NormalizedFloatVectorValues(vectorValues); + } + + while (vectorValues.nextDoc() != NO_MORE_DOCS) { + float[] corrections = quantizer.quantizeForIndex(vectorValues.vectorValue(), expectedVector, centroid); + assertArrayEquals(expectedVector, qvectorValues.vectorValue()); + assertEquals(corrections.length, qvectorValues.getCorrectiveTerms().length); + for (int i = 0; i < corrections.length; i++) { + assertEquals(corrections[i], qvectorValues.getCorrectiveTerms()[i], 0.00001f); + } + } + } + } + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816HnswBinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816HnswBinaryQuantizedVectorsFormatTests.java new file mode 100644 index 0000000000000..f607de57e1fd5 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES816HnswBinaryQuantizedVectorsFormatTests.java @@ -0,0 +1,126 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; +import org.apache.lucene.util.SameThreadExecutorService; +import org.elasticsearch.common.logging.LogConfigurator; + +import java.util.Arrays; +import java.util.Locale; + +import static java.lang.String.format; +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.oneOf; + +public class ES816HnswBinaryQuantizedVectorsFormatTests extends BaseKnnVectorsFormatTestCase { + + static { + LogConfigurator.loadLog4jPlugins(); + LogConfigurator.configureESLogging(); // native access requires logging to be initialized + } + + @Override + protected Codec getCodec() { + return new Lucene912Codec() { + @Override + public KnnVectorsFormat getKnnVectorsFormatForField(String field) { + return new ES816HnswBinaryQuantizedVectorsFormat(); + } + }; + } + + public void testToString() { + FilterCodec customCodec = new FilterCodec("foo", Codec.getDefault()) { + @Override + public KnnVectorsFormat knnVectorsFormat() { + return new ES816HnswBinaryQuantizedVectorsFormat(10, 20, 1, null); + } + }; + String expectedPattern = + "ES816HnswBinaryQuantizedVectorsFormat(name=ES816HnswBinaryQuantizedVectorsFormat, maxConn=10, beamWidth=20," + + " flatVectorFormat=ES816BinaryQuantizedVectorsFormat(name=ES816BinaryQuantizedVectorsFormat," + + " flatVectorScorer=ES816BinaryFlatVectorsScorer(nonQuantizedDelegate=%s())))"; + + var defaultScorer = format(Locale.ROOT, expectedPattern, "DefaultFlatVectorScorer"); + var memSegScorer = format(Locale.ROOT, expectedPattern, "Lucene99MemorySegmentFlatVectorsScorer"); + assertThat(customCodec.knnVectorsFormat().toString(), is(oneOf(defaultScorer, memSegScorer))); + } + + public void testSingleVectorCase() throws Exception { + float[] vector = randomVector(random().nextInt(12, 500)); + for (VectorSimilarityFunction similarityFunction : VectorSimilarityFunction.values()) { + try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + Document doc = new Document(); + doc.add(new KnnFloatVectorField("f", vector, similarityFunction)); + w.addDocument(doc); + w.commit(); + try (IndexReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + FloatVectorValues vectorValues = r.getFloatVectorValues("f"); + assert (vectorValues.size() == 1); + while (vectorValues.nextDoc() != NO_MORE_DOCS) { + assertArrayEquals(vector, vectorValues.vectorValue(), 0.00001f); + } + TopDocs td = r.searchNearestVectors("f", randomVector(vector.length), 1, null, Integer.MAX_VALUE); + assertEquals(1, td.totalHits.value); + assertTrue(td.scoreDocs[0].score >= 0); + } + } + } + } + + public void testLimits() { + expectThrows(IllegalArgumentException.class, () -> new ES816HnswBinaryQuantizedVectorsFormat(-1, 20)); + expectThrows(IllegalArgumentException.class, () -> new ES816HnswBinaryQuantizedVectorsFormat(0, 20)); + expectThrows(IllegalArgumentException.class, () -> new ES816HnswBinaryQuantizedVectorsFormat(20, 0)); + expectThrows(IllegalArgumentException.class, () -> new ES816HnswBinaryQuantizedVectorsFormat(20, -1)); + expectThrows(IllegalArgumentException.class, () -> new ES816HnswBinaryQuantizedVectorsFormat(512 + 1, 20)); + expectThrows(IllegalArgumentException.class, () -> new ES816HnswBinaryQuantizedVectorsFormat(20, 3201)); + expectThrows( + IllegalArgumentException.class, + () -> new ES816HnswBinaryQuantizedVectorsFormat(20, 100, 1, new SameThreadExecutorService()) + ); + } + + // Ensures that all expected vector similarity functions are translatable in the format. + public void testVectorSimilarityFuncs() { + // This does not necessarily have to be all similarity functions, but + // differences should be considered carefully. + var expectedValues = Arrays.stream(VectorSimilarityFunction.values()).toList(); + assertEquals(Lucene99HnswVectorsReader.SIMILARITY_FUNCTIONS, expectedValues); + } +} From 84fe9cf3a303c8c9477abe16c1783cd319e2c89f Mon Sep 17 00:00:00 2001 From: Dianna Hohensee Date: Tue, 8 Oct 2024 22:59:37 +0200 Subject: [PATCH 194/194] Track shard snapshot progress during node shutdown (#112567) Track shard snapshot progress during shutdown to identify any bottlenecks that cause slowness that can ultimately block shard re-allocation. Relates ES-9086 --- docs/changelog/112567.yaml | 5 + .../decider/DiskThresholdDeciderIT.java | 36 +- .../snapshots/SnapshotShutdownIT.java | 255 ++++++++++- .../cluster/node/DiscoveryNodes.java | 5 + .../common/settings/ClusterSettings.java | 2 + .../snapshots/IndexShardSnapshotStatus.java | 31 ++ .../snapshots/SnapshotShardsService.java | 96 ++++- .../SnapshotShutdownProgressTracker.java | 270 ++++++++++++ .../SnapshotShutdownProgressTrackerTests.java | 407 ++++++++++++++++++ .../elasticsearch/test/ESIntegTestCase.java | 32 ++ 10 files changed, 1087 insertions(+), 52 deletions(-) create mode 100644 docs/changelog/112567.yaml create mode 100644 server/src/main/java/org/elasticsearch/snapshots/SnapshotShutdownProgressTracker.java create mode 100644 server/src/test/java/org/elasticsearch/snapshots/SnapshotShutdownProgressTrackerTests.java diff --git a/docs/changelog/112567.yaml b/docs/changelog/112567.yaml new file mode 100644 index 0000000000000..25e3ac8360c2b --- /dev/null +++ b/docs/changelog/112567.yaml @@ -0,0 +1,5 @@ +pr: 112567 +summary: Track shard snapshot progress during node shutdown +area: Snapshot/Restore +type: enhancement +issues: [] diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java index 2a275cf563d86..19b0f0bd73233 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java @@ -253,35 +253,17 @@ private Set getShardIds(final String nodeId, final String indexName) { } /** - * Index documents until all the shards are at least WATERMARK_BYTES in size, and return the one with the smallest size + * Index documents until all the shards are at least WATERMARK_BYTES in size. + * @return the shard sizes. */ private ShardSizes createReasonableSizedShards(final String indexName) { - while (true) { - indexRandom(false, indexName, scaledRandomIntBetween(100, 10000)); - forceMerge(); - refresh(); - - final ShardStats[] shardStates = indicesAdmin().prepareStats(indexName) - .clear() - .setStore(true) - .setTranslog(true) - .get() - .getShards(); - - var smallestShardSize = Arrays.stream(shardStates) - .mapToLong(it -> it.getStats().getStore().sizeInBytes()) - .min() - .orElseThrow(() -> new AssertionError("no shards")); - - if (smallestShardSize > WATERMARK_BYTES) { - var shardSizes = Arrays.stream(shardStates) - .map(it -> new ShardSize(removeIndexUUID(it.getShardRouting().shardId()), it.getStats().getStore().sizeInBytes())) - .sorted(Comparator.comparing(ShardSize::size)) - .toList(); - logger.info("Created shards with sizes {}", shardSizes); - return new ShardSizes(shardSizes); - } - } + ShardStats[] shardStats = indexAllShardsToAnEqualOrGreaterMinimumSize(indexName, WATERMARK_BYTES); + var shardSizes = Arrays.stream(shardStats) + .map(it -> new ShardSize(removeIndexUUID(it.getShardRouting().shardId()), it.getStats().getStore().sizeInBytes())) + .sorted(Comparator.comparing(ShardSize::size)) + .toList(); + logger.info("Created shards with sizes {}", shardSizes); + return new ShardSizes(shardSizes); } private record ShardSizes(List sizes) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java index 3c71b50321c76..980ef2a87c9c2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.snapshots; +import org.apache.logging.log4j.Level; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionResponse; @@ -33,19 +34,26 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.MockLog; +import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.transport.MockTransportService; import java.util.Collection; import java.util.Map; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; +import static org.elasticsearch.snapshots.SnapshotShutdownProgressTracker.SNAPSHOT_PROGRESS_DURING_SHUTDOWN_LOG_INTERVAL_SETTING; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasSize; @@ -55,15 +63,44 @@ public class SnapshotShutdownIT extends AbstractSnapshotIntegTestCase { private static final String REQUIRE_NODE_NAME_SETTING = IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX + "._name"; + private MockLog mockLog; + + public void setUp() throws Exception { + super.setUp(); + mockLog = MockLog.capture(SnapshotShutdownProgressTracker.class); + } + + private void resetMockLog() { + mockLog.close(); + mockLog = MockLog.capture(SnapshotShutdownProgressTracker.class); + } + + public void tearDown() throws Exception { + mockLog.close(); + super.tearDown(); + } + @Override protected Collection> nodePlugins() { return CollectionUtils.appendToCopy(super.nodePlugins(), MockTransportService.TestPlugin.class); } + /** + * Tests that shard snapshots on a node with RESTART shutdown metadata will finish on the same node. + */ + @TestLogging( + value = "org.elasticsearch.snapshots.SnapshotShutdownProgressTracker:DEBUG", + reason = "Testing SnapshotShutdownProgressTracker's progress, which is reported at the DEBUG logging level" + ) public void testRestartNodeDuringSnapshot() throws Exception { // Marking a node for restart has no impact on snapshots (see #71333 for how to handle this case) internalCluster().ensureAtLeastNumDataNodes(1); - final var originalNode = internalCluster().startDataOnlyNode(); + final var originalNode = internalCluster().startDataOnlyNode( + // Speed up the logging frequency, so that the test doesn't have to wait too long to check for log messages. + Settings.builder().put(SNAPSHOT_PROGRESS_DURING_SHUTDOWN_LOG_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(200)).build() + ); + final String originalNodeId = internalCluster().getInstance(NodeEnvironment.class, originalNode).nodeId(); + final var indexName = randomIdentifier(); createIndexWithContent(indexName, indexSettings(1, 0).put(REQUIRE_NODE_NAME_SETTING, originalNode).build()); @@ -88,6 +125,16 @@ public void testRestartNodeDuringSnapshot() throws Exception { }); addUnassignedShardsWatcher(clusterService, indexName); + // Ensure that the SnapshotShutdownProgressTracker does not start logging in RESTART mode. + mockLog.addExpectation( + new MockLog.UnseenEventExpectation( + "SnapshotShutdownProgressTracker start log message", + SnapshotShutdownProgressTracker.class.getCanonicalName(), + Level.DEBUG, + "Starting shutdown snapshot progress logging on node [" + originalNodeId + "]" + ) + ); + safeAwait( (ActionListener listener) -> putShutdownMetadata( clusterService, @@ -100,9 +147,15 @@ public void testRestartNodeDuringSnapshot() throws Exception { ) ); assertFalse(snapshotCompletesWithoutPausingListener.isDone()); + + // Verify no SnapshotShutdownProgressTracker logging in RESTART mode. + mockLog.awaitAllExpectationsMatched(); + resetMockLog(); + unblockAllDataNodes(repoName); // lets the shard snapshot continue so the snapshot can succeed assertEquals(SnapshotState.SUCCESS, snapshotFuture.get(10, TimeUnit.SECONDS).getSnapshotInfo().state()); safeAwait(snapshotCompletesWithoutPausingListener); + clearShutdownMetadata(clusterService); } @@ -117,7 +170,7 @@ public void testRemoveNodeDuringSnapshot() throws Exception { final var clusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class); final var snapshotFuture = startFullSnapshotBlockedOnDataNode(randomIdentifier(), repoName, originalNode); - final var snapshotPausedListener = createSnapshotPausedListener(clusterService, repoName, indexName); + final var snapshotPausedListener = createSnapshotPausedListener(clusterService, repoName, indexName, 1); addUnassignedShardsWatcher(clusterService, indexName); updateIndexSettings(Settings.builder().putNull(REQUIRE_NODE_NAME_SETTING), indexName); @@ -146,7 +199,7 @@ public void testRemoveNodeAndFailoverMasterDuringSnapshot() throws Exception { final var clusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class); final var snapshotFuture = startFullSnapshotBlockedOnDataNode(randomIdentifier(), repoName, originalNode); - final var snapshotPausedListener = createSnapshotPausedListener(clusterService, repoName, indexName); + final var snapshotPausedListener = createSnapshotPausedListener(clusterService, repoName, indexName, 1); addUnassignedShardsWatcher(clusterService, indexName); final var snapshotStatusUpdateBarrier = new CyclicBarrier(2); @@ -264,7 +317,7 @@ public void testRemoveNodeDuringSnapshotWithOtherRunningShardSnapshots() throws final var clusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class); final var snapshotFuture = startFullSnapshotBlockedOnDataNode(randomIdentifier(), repoName, nodeForRemoval); - final var snapshotPausedListener = createSnapshotPausedListener(clusterService, repoName, indexName); + final var snapshotPausedListener = createSnapshotPausedListener(clusterService, repoName, indexName, 1); addUnassignedShardsWatcher(clusterService, indexName); waitForBlock(otherNode, repoName); @@ -320,7 +373,7 @@ public void testStartRemoveNodeButDoNotComplete() throws Exception { final var clusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class); final var snapshotFuture = startFullSnapshotBlockedOnDataNode(randomIdentifier(), repoName, primaryNode); - final var snapshotPausedListener = createSnapshotPausedListener(clusterService, repoName, indexName); + final var snapshotPausedListener = createSnapshotPausedListener(clusterService, repoName, indexName, 1); addUnassignedShardsWatcher(clusterService, indexName); putShutdownForRemovalMetadata(primaryNode, clusterService); @@ -334,6 +387,9 @@ public void testStartRemoveNodeButDoNotComplete() throws Exception { assertEquals(SnapshotState.SUCCESS, snapshotFuture.get(10, TimeUnit.SECONDS).getSnapshotInfo().state()); } + /** + * Tests that deleting a snapshot will abort paused shard snapshots on a node with shutdown metadata. + */ public void testAbortSnapshotWhileRemovingNode() throws Exception { final var primaryNode = internalCluster().startDataOnlyNode(); final var indexName = randomIdentifier(); @@ -363,7 +419,7 @@ public void testAbortSnapshotWhileRemovingNode() throws Exception { final var clusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class); addUnassignedShardsWatcher(clusterService, indexName); putShutdownForRemovalMetadata(primaryNode, clusterService); - unblockAllDataNodes(repoName); // lets the shard snapshot abort, but allocation filtering stops it from moving + unblockAllDataNodes(repoName); // lets the shard snapshot pause, but allocation filtering stops it from moving safeAwait(updateSnapshotStatusBarrier); // wait for data node to notify master that the shard snapshot is paused // abort snapshot (and wait for the abort to land in the cluster state) @@ -414,10 +470,180 @@ public void testShutdownWhileSuccessInFlight() throws Exception { clearShutdownMetadata(clusterService); } + /** + * This test exercises the SnapshotShutdownProgressTracker's log messages reporting the progress of shard snapshots on data nodes. + */ + @TestLogging( + value = "org.elasticsearch.snapshots.SnapshotShutdownProgressTracker:TRACE", + reason = "Testing SnapshotShutdownProgressTracker's progress, which is reported at the TRACE logging level" + ) + public void testSnapshotShutdownProgressTracker() throws Exception { + final var repoName = randomIdentifier(); + final int numShards = randomIntBetween(1, 10); + createRepository(repoName, "mock"); + + // Create another index on another node which will be blocked (remain in state INIT) throughout. + // Not required for this test, just adds some more concurrency. + final var otherNode = internalCluster().startDataOnlyNode(); + final var otherIndex = randomIdentifier(); + createIndexWithContent(otherIndex, indexSettings(numShards, 0).put(REQUIRE_NODE_NAME_SETTING, otherNode).build()); + blockDataNode(repoName, otherNode); + + final var nodeForRemoval = internalCluster().startDataOnlyNode( + // Speed up the logging frequency, so that the test doesn't have to wait too long to check for log messages. + Settings.builder().put(SNAPSHOT_PROGRESS_DURING_SHUTDOWN_LOG_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(200)).build() + ); + final String nodeForRemovalId = internalCluster().getInstance(NodeEnvironment.class, nodeForRemoval).nodeId(); + final var indexName = randomIdentifier(); + createIndexWithContent(indexName, indexSettings(numShards, 0).put(REQUIRE_NODE_NAME_SETTING, nodeForRemoval).build()); + indexAllShardsToAnEqualOrGreaterMinimumSize(indexName, new ByteSizeValue(2, ByteSizeUnit.KB).getBytes()); + + // Start the snapshot with blocking in place on the data node not to allow shard snapshots to finish yet. + final var clusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class); + final var snapshotFuture = startFullSnapshotBlockedOnDataNode(randomIdentifier(), repoName, nodeForRemoval); + final var snapshotPausedListener = createSnapshotPausedListener(clusterService, repoName, indexName, numShards); + addUnassignedShardsWatcher(clusterService, indexName); + + waitForBlock(otherNode, repoName); + + logger.info("---> nodeForRemovalId: " + nodeForRemovalId + ", numShards: " + numShards); + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "SnapshotShutdownProgressTracker start log message", + SnapshotShutdownProgressTracker.class.getCanonicalName(), + Level.DEBUG, + "Starting shutdown snapshot progress logging on node [" + nodeForRemovalId + "]" + ) + ); + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "SnapshotShutdownProgressTracker pause set log message", + SnapshotShutdownProgressTracker.class.getCanonicalName(), + Level.DEBUG, + "Pause signals have been set for all shard snapshots on data node [" + nodeForRemovalId + "]" + ) + ); + + putShutdownForRemovalMetadata(nodeForRemoval, clusterService); + + // Check that the SnapshotShutdownProgressTracker was turned on after the shutdown metadata is set above. + mockLog.awaitAllExpectationsMatched(); + resetMockLog(); + + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "SnapshotShutdownProgressTracker running number of snapshots", + SnapshotShutdownProgressTracker.class.getCanonicalName(), + Level.INFO, + "*Number shard snapshots running [" + numShards + "].*" + ) + ); + + // Check that the SnapshotShutdownProgressTracker is tracking the active (not yet paused) shard snapshots. + mockLog.awaitAllExpectationsMatched(); + resetMockLog(); + + // Block on the master when a shard snapshot request comes in, until we can verify that the Tracker saw the outgoing request. + final CountDownLatch snapshotStatusUpdateLatch = new CountDownLatch(1); + final var masterTransportService = MockTransportService.getInstance(internalCluster().getMasterName()); + masterTransportService.addRequestHandlingBehavior( + SnapshotsService.UPDATE_SNAPSHOT_STATUS_ACTION_NAME, + (handler, request, channel, task) -> masterTransportService.getThreadPool().generic().execute(() -> { + safeAwait(snapshotStatusUpdateLatch); + try { + handler.messageReceived(request, channel, task); + } catch (Exception e) { + fail(e); + } + }) + ); + + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "SnapshotShutdownProgressTracker shard snapshot has paused log message", + SnapshotShutdownProgressTracker.class.getCanonicalName(), + Level.INFO, + "*Number shard snapshots waiting for master node reply to status update request [" + numShards + "]*" + ) + ); + + // Let the shard snapshot proceed. It will still get stuck waiting for the master node to respond. + unblockNode(repoName, nodeForRemoval); + + // Check that the SnapshotShutdownProgressTracker observed the request sent to the master node. + mockLog.awaitAllExpectationsMatched(); + resetMockLog(); + + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "SnapshotShutdownProgressTracker shard snapshot has paused log message", + SnapshotShutdownProgressTracker.class.getCanonicalName(), + Level.INFO, + "Current active shard snapshot stats on data node [" + nodeForRemovalId + "]*Paused [" + numShards + "]" + ) + ); + + // Release the master node to respond + snapshotStatusUpdateLatch.countDown(); + + // Wait for the snapshot to fully pause. + safeAwait(snapshotPausedListener); + + // Check that the SnapshotShutdownProgressTracker observed the shard snapshot finishing as paused. + mockLog.awaitAllExpectationsMatched(); + resetMockLog(); + + // Remove the allocation filter so that the shard moves off of the node shutting down. + updateIndexSettings(Settings.builder().putNull(REQUIRE_NODE_NAME_SETTING), indexName); + + // Wait for the shard snapshot to succeed on the non-shutting down node. + safeAwait( + ClusterServiceUtils.addTemporaryStateListener( + clusterService, + state -> SnapshotsInProgress.get(state) + .asStream() + .allMatch( + e -> e.shards() + .entrySet() + .stream() + .anyMatch( + shardEntry -> shardEntry.getKey().getIndexName().equals(indexName) + && switch (shardEntry.getValue().state()) { + case INIT, PAUSED_FOR_NODE_REMOVAL -> false; + case SUCCESS -> true; + case FAILED, ABORTED, MISSING, QUEUED, WAITING -> throw new AssertionError(shardEntry.toString()); + } + ) + ) + ) + ); + + unblockAllDataNodes(repoName); + + // Snapshot completes when the node vacates even though it hasn't been removed yet + assertEquals(SnapshotState.SUCCESS, snapshotFuture.get(10, TimeUnit.SECONDS).getSnapshotInfo().state()); + + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "SnapshotShutdownProgressTracker cancelled log message", + SnapshotShutdownProgressTracker.class.getCanonicalName(), + Level.DEBUG, + "Cancelling shutdown snapshot progress logging on node [" + nodeForRemovalId + "]" + ) + ); + + clearShutdownMetadata(clusterService); + + // Check that the SnapshotShutdownProgressTracker logging was cancelled by the removal of the shutdown metadata. + mockLog.awaitAllExpectationsMatched(); + resetMockLog(); + } + private static SubscribableListener createSnapshotPausedListener( ClusterService clusterService, String repoName, - String indexName + String indexName, + int numShards ) { return ClusterServiceUtils.addTemporaryStateListener(clusterService, state -> { final var entriesForRepo = SnapshotsInProgress.get(state).forRepo(repoName); @@ -434,10 +660,17 @@ private static SubscribableListener createSnapshotPausedListener( .stream() .flatMap(e -> e.getKey().getIndexName().equals(indexName) ? Stream.of(e.getValue()) : Stream.of()) .toList(); - assertThat(shardSnapshotStatuses, hasSize(1)); - final var shardState = shardSnapshotStatuses.iterator().next().state(); - assertThat(shardState, oneOf(SnapshotsInProgress.ShardState.INIT, SnapshotsInProgress.ShardState.PAUSED_FOR_NODE_REMOVAL)); - return shardState == SnapshotsInProgress.ShardState.PAUSED_FOR_NODE_REMOVAL; + assertThat(shardSnapshotStatuses, hasSize(numShards)); + for (var shardStatus : shardSnapshotStatuses) { + assertThat( + shardStatus.state(), + oneOf(SnapshotsInProgress.ShardState.INIT, SnapshotsInProgress.ShardState.PAUSED_FOR_NODE_REMOVAL) + ); + if (shardStatus.state() == SnapshotsInProgress.ShardState.INIT) { + return false; + } + } + return true; }); } diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java index a7ae17c8dac14..9477f9c6a5cc1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java +++ b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java @@ -628,6 +628,11 @@ public List addedNodes() { return added; } + @Override + public String toString() { + return shortSummary(); + } + public String shortSummary() { final StringBuilder summary = new StringBuilder(); if (masterNodeChanged()) { diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index fbce913dac139..69be30b0b5111 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -122,6 +122,7 @@ import org.elasticsearch.search.fetch.subphase.highlight.FastVectorHighlighter; import org.elasticsearch.snapshots.InternalSnapshotsInfoService; import org.elasticsearch.snapshots.RestoreService; +import org.elasticsearch.snapshots.SnapshotShutdownProgressTracker; import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.ProxyConnectionStrategy; @@ -368,6 +369,7 @@ public void apply(Settings value, Settings current, Settings previous) { SniffConnectionStrategy.REMOTE_NODE_CONNECTIONS, TransportCloseIndexAction.CLUSTER_INDICES_CLOSE_ENABLE_SETTING, ShardsLimitAllocationDecider.CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING, + SnapshotShutdownProgressTracker.SNAPSHOT_PROGRESS_DURING_SHUTDOWN_LOG_INTERVAL_SETTING, NodeConnectionsService.CLUSTER_NODE_RECONNECT_INTERVAL_SETTING, HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_TYPE_SETTING, HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_TYPE_SETTING, diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java b/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java index b5607c31641d3..70ba9950f7689 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/IndexShardSnapshotStatus.java @@ -184,6 +184,10 @@ public synchronized void moveToDone(final long endTime, final ShardSnapshotResul } } + public Stage getStage() { + return stage.get(); + } + public void addAbortListener(ActionListener listener) { abortListeners.addListener(listener); } @@ -429,4 +433,31 @@ public String toString() { + ')'; } } + + @Override + public String toString() { + return "index shard snapshot status (" + + "stage=" + + stage + + ", startTime=" + + startTime + + ", totalTime=" + + totalTime + + ", incrementalFileCount=" + + incrementalFileCount + + ", totalFileCount=" + + totalFileCount + + ", processedFileCount=" + + processedFileCount + + ", incrementalSize=" + + incrementalSize + + ", totalSize=" + + totalSize + + ", processedSize=" + + processedSize + + ", failure='" + + failure + + '\'' + + ')'; + } } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java index abc5f36eef7da..7b2066f243771 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java @@ -21,6 +21,8 @@ import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.SnapshotsInProgress.ShardSnapshotStatus; import org.elasticsearch.cluster.SnapshotsInProgress.ShardState; +import org.elasticsearch.cluster.metadata.NodesShutdownMetadata; +import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -82,6 +84,8 @@ public final class SnapshotShardsService extends AbstractLifecycleComponent impl private final ThreadPool threadPool; + private final SnapshotShutdownProgressTracker snapshotShutdownProgressTracker; + private final Map> shardSnapshots = new HashMap<>(); // A map of snapshots to the shardIds that we already reported to the master as failed @@ -102,6 +106,11 @@ public SnapshotShardsService( this.transportService = transportService; this.clusterService = clusterService; this.threadPool = transportService.getThreadPool(); + this.snapshotShutdownProgressTracker = new SnapshotShutdownProgressTracker( + () -> clusterService.state().nodes().getLocalNodeId(), + clusterService.getClusterSettings(), + threadPool + ); this.remoteFailedRequestDeduplicator = new ResultDeduplicator<>(threadPool.getThreadContext()); if (DiscoveryNode.canContainData(settings)) { // this is only useful on the nodes that can hold data @@ -130,11 +139,38 @@ protected void doClose() { @Override public void clusterChanged(ClusterChangedEvent event) { try { + final var localNodeId = clusterService.localNode().getId(); + + // Track when this node enters and leaves shutdown mode because we pause shard snapshots for shutdown. + // The snapshotShutdownProgressTracker will report (via logging) on the progress shard snapshots make + // towards either completing (successfully or otherwise) or pausing. + NodesShutdownMetadata currentShutdownMetadata = event.state().metadata().custom(NodesShutdownMetadata.TYPE); + NodesShutdownMetadata previousShutdownMetadata = event.previousState().metadata().custom(NodesShutdownMetadata.TYPE); + SingleNodeShutdownMetadata currentLocalNodeShutdownMetadata = currentShutdownMetadata != null + ? currentShutdownMetadata.get(localNodeId) + : null; + SingleNodeShutdownMetadata previousLocalNodeShutdownMetadata = previousShutdownMetadata != null + ? previousShutdownMetadata.get(localNodeId) + : null; + + boolean isLocalNodeAddingShutdown = false; + if (isPausingProgressTrackedShutdown(previousLocalNodeShutdownMetadata) == false + && isPausingProgressTrackedShutdown(currentLocalNodeShutdownMetadata)) { + snapshotShutdownProgressTracker.onClusterStateAddShutdown(); + isLocalNodeAddingShutdown = true; + } else if (isPausingProgressTrackedShutdown(previousLocalNodeShutdownMetadata) + && isPausingProgressTrackedShutdown(currentLocalNodeShutdownMetadata) == false) { + snapshotShutdownProgressTracker.onClusterStateRemoveShutdown(); + } + final var currentSnapshots = SnapshotsInProgress.get(event.state()); + if (SnapshotsInProgress.get(event.previousState()).equals(currentSnapshots) == false) { - final var localNodeId = clusterService.localNode().getId(); synchronized (shardSnapshots) { + // Cancel any snapshots that have been removed from the cluster state. cancelRemoved(currentSnapshots); + + // Update running snapshots or start any snapshots that are set to run. for (final var oneRepoSnapshotsInProgress : currentSnapshots.entriesByRepo()) { for (final var snapshotsInProgressEntry : oneRepoSnapshotsInProgress) { handleUpdatedSnapshotsInProgressEntry( @@ -147,6 +183,11 @@ public void clusterChanged(ClusterChangedEvent event) { } } + if (isLocalNodeAddingShutdown) { + // Any active snapshots would have been signalled to pause in the previous code block. + snapshotShutdownProgressTracker.onClusterStatePausingSetForAllShardSnapshots(); + } + String previousMasterNodeId = event.previousState().nodes().getMasterNodeId(); String currentMasterNodeId = event.state().nodes().getMasterNodeId(); if (currentMasterNodeId != null && currentMasterNodeId.equals(previousMasterNodeId) == false) { @@ -164,6 +205,17 @@ public void clusterChanged(ClusterChangedEvent event) { } } + /** + * Determines whether we want to track this kind of shutdown for snapshot pausing progress. + * We want tracking is shutdown metadata is set, and not type RESTART. + * Note that the Shutdown API is idempotent and the type of shutdown may change to / from RESTART to / from some other type of interest. + * + * @return true if snapshots will be paused during this type of local node shutdown. + */ + private static boolean isPausingProgressTrackedShutdown(@Nullable SingleNodeShutdownMetadata localNodeShutdownMetadata) { + return localNodeShutdownMetadata != null && localNodeShutdownMetadata.getType() != SingleNodeShutdownMetadata.Type.RESTART; + } + @Override public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, Settings indexSettings) { // abort any snapshots occurring on the soon-to-be closed shard @@ -231,6 +283,9 @@ private void cancelRemoved(SnapshotsInProgress snapshotsInProgress) { } } + /** + * Starts new snapshots and pauses or aborts active shard snapshot based on the updated {@link SnapshotsInProgress} entry. + */ private void handleUpdatedSnapshotsInProgressEntry(String localNodeId, boolean removingLocalNode, SnapshotsInProgress.Entry entry) { if (entry.isClone()) { // This is a snapshot clone, it will be executed on the current master @@ -364,8 +419,7 @@ private Runnable newShardSnapshotTask( final IndexVersion entryVersion, final long entryStartTime ) { - // separate method to make sure this lambda doesn't capture any heavy local objects like a SnapshotsInProgress.Entry - return () -> snapshot(shardId, snapshot, indexId, snapshotStatus, entryVersion, entryStartTime, new ActionListener<>() { + ActionListener snapshotResultListener = new ActionListener<>() { @Override public void onResponse(ShardSnapshotResult shardSnapshotResult) { final ShardGeneration newGeneration = shardSnapshotResult.getGeneration(); @@ -405,7 +459,15 @@ public void onFailure(Exception e) { final var shardState = snapshotStatus.moveToUnsuccessful(nextStage, failure, threadPool.absoluteTimeInMillis()); notifyUnsuccessfulSnapshotShard(snapshot, shardId, shardState, failure, snapshotStatus.generation()); } + }; + + snapshotShutdownProgressTracker.incNumberOfShardSnapshotsInProgress(shardId, snapshot); + var decTrackerRunsBeforeResultListener = ActionListener.runAfter(snapshotResultListener, () -> { + snapshotShutdownProgressTracker.decNumberOfShardSnapshotsInProgress(shardId, snapshot, snapshotStatus); }); + + // separate method to make sure this lambda doesn't capture any heavy local objects like a SnapshotsInProgress.Entry + return () -> snapshot(shardId, snapshot, indexId, snapshotStatus, entryVersion, entryStartTime, decTrackerRunsBeforeResultListener); } // package private for testing @@ -665,19 +727,25 @@ private void notifyUnsuccessfulSnapshotShard( /** Updates the shard snapshot status by sending a {@link UpdateIndexShardSnapshotStatusRequest} to the master node */ private void sendSnapshotShardUpdate(final Snapshot snapshot, final ShardId shardId, final ShardSnapshotStatus status) { + ActionListener updateResultListener = new ActionListener<>() { + @Override + public void onResponse(Void aVoid) { + logger.trace("[{}][{}] updated snapshot state to [{}]", shardId, snapshot, status); + } + + @Override + public void onFailure(Exception e) { + logger.warn(() -> format("[%s][%s] failed to update snapshot state to [%s]", shardId, snapshot, status), e); + } + }; + snapshotShutdownProgressTracker.trackRequestSentToMaster(snapshot, shardId); + var releaseTrackerRequestRunsBeforeResultListener = ActionListener.runBefore(updateResultListener, () -> { + snapshotShutdownProgressTracker.releaseRequestSentToMaster(snapshot, shardId); + }); + remoteFailedRequestDeduplicator.executeOnce( new UpdateIndexShardSnapshotStatusRequest(snapshot, shardId, status), - new ActionListener<>() { - @Override - public void onResponse(Void aVoid) { - logger.trace("[{}][{}] updated snapshot state to [{}]", shardId, snapshot, status); - } - - @Override - public void onFailure(Exception e) { - logger.warn(() -> format("[%s][%s] failed to update snapshot state to [%s]", shardId, snapshot, status), e); - } - }, + releaseTrackerRequestRunsBeforeResultListener, (req, reqListener) -> transportService.sendRequest( transportService.getLocalNode(), SnapshotsService.UPDATE_SNAPSHOT_STATUS_ACTION_NAME, diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShutdownProgressTracker.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShutdownProgressTracker.java new file mode 100644 index 0000000000000..5d81e3c4e46af --- /dev/null +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShutdownProgressTracker.java @@ -0,0 +1,270 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.snapshots; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ResultDeduplicator; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; +import org.elasticsearch.threadpool.Scheduler; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +/** + * Tracks progress of shard snapshots during shutdown, on this single data node. Periodically reports progress via logging, the interval for + * which see {@link #SNAPSHOT_PROGRESS_DURING_SHUTDOWN_LOG_INTERVAL_SETTING}. + */ +public class SnapshotShutdownProgressTracker { + + /** How frequently shard snapshot progress is logged after receiving local node shutdown metadata. */ + public static final Setting SNAPSHOT_PROGRESS_DURING_SHUTDOWN_LOG_INTERVAL_SETTING = Setting.timeSetting( + "snapshots.shutdown.progress.interval", + TimeValue.timeValueSeconds(5), + TimeValue.MINUS_ONE, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + private static final Logger logger = LogManager.getLogger(SnapshotShutdownProgressTracker.class); + + private final Supplier getLocalNodeId; + private final ThreadPool threadPool; + + private volatile TimeValue progressLoggerInterval; + private Scheduler.Cancellable scheduledProgressLoggerFuture; + + /** + * The time at which the cluster state update began that found a shutdown signal for this node. Negative value means unset (node is not + * shutting down). + */ + private volatile long shutdownStartMillis = -1; + + /** + * The time at which the cluster state finished setting shard snapshot states to PAUSING, which the shard snapshot operations will + * discover asynchronously. Negative value means unset (node is not shutting down) + */ + private volatile long shutdownFinishedSignallingPausingMillis = -1; + + /** + * Tracks the number of shard snapshots that have started on the data node but not yet finished. + */ + private final AtomicLong numberOfShardSnapshotsInProgressOnDataNode = new AtomicLong(); + + /** + * The logic to track shard snapshot status update requests to master can result in duplicate requests (see + * {@link ResultDeduplicator}), as well as resending requests if the elected master changes. + * Tracking specific requests uniquely by snapshot ID + shard ID de-duplicates requests for tracking. + * Also tracks the absolute start time of registration, to report duration on de-registration. + */ + private final Map shardSnapshotRequests = ConcurrentCollections.newConcurrentMap(); + + /** + * Track how the shard snapshots reach completion during shutdown: did they fail, succeed or pause? + */ + private final AtomicLong doneCount = new AtomicLong(); + private final AtomicLong failureCount = new AtomicLong(); + private final AtomicLong abortedCount = new AtomicLong(); + private final AtomicLong pausedCount = new AtomicLong(); + + public SnapshotShutdownProgressTracker(Supplier localNodeIdSupplier, ClusterSettings clusterSettings, ThreadPool threadPool) { + this.getLocalNodeId = localNodeIdSupplier; + clusterSettings.initializeAndWatch( + SNAPSHOT_PROGRESS_DURING_SHUTDOWN_LOG_INTERVAL_SETTING, + value -> this.progressLoggerInterval = value + ); + this.threadPool = threadPool; + } + + private void scheduleProgressLogger() { + if (progressLoggerInterval.millis() > 0) { + scheduledProgressLoggerFuture = threadPool.scheduleWithFixedDelay( + this::logProgressReport, + progressLoggerInterval, + threadPool.executor(ThreadPool.Names.GENERIC) + ); + logger.debug( + () -> Strings.format( + "Starting shutdown snapshot progress logging on node [%s], runs every [%s]", + getLocalNodeId.get(), + progressLoggerInterval + ) + ); + } else { + logger.debug("Snapshot progress logging during shutdown is disabled"); + } + } + + private void cancelProgressLogger() { + assert scheduledProgressLoggerFuture != null : "Somehow shutdown mode was removed before it was added."; + scheduledProgressLoggerFuture.cancel(); + if (progressLoggerInterval.millis() > 0) { + // Only log cancellation if it was most likely started. Theoretically the interval setting could be updated during shutdown, + // such that the progress logger is already running and ignores the new value, but that does not currently happen. + logger.debug(() -> Strings.format("Cancelling shutdown snapshot progress logging on node [%s]", getLocalNodeId.get())); + } + } + + /** + * Logs some statistics about shard snapshot progress. + */ + private void logProgressReport() { + logger.info( + """ + Current active shard snapshot stats on data node [{}]. \ + Node shutdown cluster state update received at [{}]. \ + Finished signalling shard snapshots to pause at [{}]. \ + Number shard snapshots running [{}]. \ + Number shard snapshots waiting for master node reply to status update request [{}] \ + Shard snapshot completion stats since shutdown began: Done [{}]; Failed [{}]; Aborted [{}]; Paused [{}]\ + """, + getLocalNodeId.get(), + shutdownStartMillis, + shutdownFinishedSignallingPausingMillis, + numberOfShardSnapshotsInProgressOnDataNode.get(), + shardSnapshotRequests.size(), + doneCount.get(), + failureCount.get(), + abortedCount.get(), + pausedCount.get() + ); + } + + /** + * Called as soon as a node shutdown signal is received. + */ + public void onClusterStateAddShutdown() { + assert this.shutdownStartMillis == -1 : "Expected not to be tracking anything. Call shutdown remove before adding shutdown again"; + + // Reset these values when a new shutdown occurs, to minimize/eliminate chances of racing if shutdown is later removed and async + // shard snapshots updates continue to occur. + doneCount.set(0); + failureCount.set(0); + abortedCount.set(0); + pausedCount.set(0); + + // Track the timestamp of shutdown signal, on which to base periodic progress logging. + this.shutdownStartMillis = threadPool.relativeTimeInMillis(); + + // Start logging periodic progress reports. + scheduleProgressLogger(); + } + + /** + * Called when the cluster state update processing a shutdown signal has finished signalling (setting PAUSING) all shard snapshots to + * pause. + */ + public void onClusterStatePausingSetForAllShardSnapshots() { + assert this.shutdownStartMillis != -1 + : "Should not have left shutdown mode before finishing processing the cluster state update with shutdown"; + this.shutdownFinishedSignallingPausingMillis = threadPool.relativeTimeInMillis(); + logger.debug(() -> Strings.format("Pause signals have been set for all shard snapshots on data node [%s]", getLocalNodeId.get())); + } + + /** + * The cluster state indicating that a node is to be shutdown may be cleared instead of following through with node shutdown. In that + * case, no further shutdown shard snapshot progress reporting is desired. + */ + public void onClusterStateRemoveShutdown() { + assert shutdownStartMillis != -1 : "Expected a call to add shutdown mode before a call to remove shutdown mode."; + + // Reset the shutdown specific trackers. + this.shutdownStartMillis = -1; + this.shutdownFinishedSignallingPausingMillis = -1; + + // Turn off the progress logger, which we only want to run during shutdown. + cancelProgressLogger(); + } + + /** + * Tracks how many shard snapshots are started. + */ + public void incNumberOfShardSnapshotsInProgress(ShardId shardId, Snapshot snapshot) { + logger.debug(() -> Strings.format("Started shard (shard ID: [%s]) in snapshot ([%s])", shardId, snapshot)); + numberOfShardSnapshotsInProgressOnDataNode.incrementAndGet(); + } + + /** + * Tracks how many shard snapshots have finished since shutdown mode began. + */ + public void decNumberOfShardSnapshotsInProgress(ShardId shardId, Snapshot snapshot, IndexShardSnapshotStatus shardSnapshotStatus) { + logger.debug( + () -> Strings.format( + "Finished shard (shard ID: [%s]) in snapshot ([%s]) with status ([%s]): ", + shardId, + snapshot, + shardSnapshotStatus.toString() + ) + ); + + numberOfShardSnapshotsInProgressOnDataNode.decrementAndGet(); + if (shutdownStartMillis != -1) { + switch (shardSnapshotStatus.getStage()) { + case DONE -> doneCount.incrementAndGet(); + case FAILURE -> failureCount.incrementAndGet(); + case ABORTED -> abortedCount.incrementAndGet(); + case PAUSED -> pausedCount.incrementAndGet(); + // The other stages are active, we should only see the end result because this method is called upon completion. + default -> { + assert false : "unexpected shard snapshot stage transition during shutdown: " + shardSnapshotStatus.getStage(); + } + } + } + } + + /** + * Uniquely tracks a request to update a shard snapshot status sent to the master node. Idempotent, safe to call multiple times. + * + * @param snapshot first part of a unique tracking identifier + * @param shardId second part of a unique tracking identifier + */ + public void trackRequestSentToMaster(Snapshot snapshot, ShardId shardId) { + logger.debug(() -> Strings.format("Tracking shard (shard ID: [%s]) snapshot ([%s]) request to master", shardId, snapshot)); + shardSnapshotRequests.put(snapshot.toString() + shardId.getIndexName() + shardId.getId(), threadPool.relativeTimeInNanos()); + } + + /** + * Stops tracking a request to update a shard snapshot status sent to the master node. Idempotent, safe to call multiple times. + * + * @param snapshot first part of a unique tracking identifier + * @param shardId second part of a unique tracking identifier + */ + public void releaseRequestSentToMaster(Snapshot snapshot, ShardId shardId) { + var masterRequestStartTime = shardSnapshotRequests.remove(snapshot.toString() + shardId.getIndexName() + shardId.getId()); + // This method is may be called multiple times. Only log if this is the first time, and the entry hasn't already been removed. + if (masterRequestStartTime != null) { + logger.debug( + () -> Strings.format( + "Finished shard (shard ID: [%s]) snapshot ([%s]) update request to master in [%s]", + shardId, + snapshot, + new TimeValue(threadPool.relativeTimeInNanos() - masterRequestStartTime.longValue(), TimeUnit.NANOSECONDS) + ) + ); + } + } + + // Test only + void assertStatsForTesting(long done, long failure, long aborted, long paused) { + assert doneCount.get() == done : "doneCount is " + doneCount.get() + ", expected count was " + done; + assert failureCount.get() == failure : "failureCount is " + doneCount.get() + ", expected count was " + failure; + assert abortedCount.get() == aborted : "abortedCount is " + doneCount.get() + ", expected count was " + aborted; + assert pausedCount.get() == paused : "pausedCount is " + doneCount.get() + ", expected count was " + paused; + } +} diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotShutdownProgressTrackerTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotShutdownProgressTrackerTests.java new file mode 100644 index 0000000000000..fbf742ae2ea57 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotShutdownProgressTrackerTests.java @@ -0,0 +1,407 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.snapshots; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.coordination.Coordinator; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; +import org.elasticsearch.repositories.ShardGeneration; +import org.elasticsearch.repositories.ShardSnapshotResult; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockLog; +import org.elasticsearch.test.junit.annotations.TestLogging; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.Before; + +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; + +public class SnapshotShutdownProgressTrackerTests extends ESTestCase { + private static final Logger logger = LogManager.getLogger(SnapshotShutdownProgressTrackerTests.class); + + final Settings settings = Settings.builder() + .put( + SnapshotShutdownProgressTracker.SNAPSHOT_PROGRESS_DURING_SHUTDOWN_LOG_INTERVAL_SETTING.getKey(), + TimeValue.timeValueMillis(500) + ) + .build(); + final Settings disabledTrackerLoggingSettings = Settings.builder() + .put(SnapshotShutdownProgressTracker.SNAPSHOT_PROGRESS_DURING_SHUTDOWN_LOG_INTERVAL_SETTING.getKey(), TimeValue.MINUS_ONE) + .build(); + ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + + DeterministicTaskQueue deterministicTaskQueue; + + // Construction parameters for the Tracker. + ThreadPool testThreadPool; + private final Supplier getLocalNodeIdSupplier = () -> "local-node-id-for-test"; + private final BiConsumer, Consumer> addSettingsUpdateConsumerNoOp = (setting, updateMethod) -> {}; + + // Set up some dummy shard snapshot information to feed the Tracker. + private final ShardId dummyShardId = new ShardId(new Index("index-name-for-test", "index-uuid-for-test"), 0); + private final Snapshot dummySnapshot = new Snapshot( + "snapshot-repo-name-for-test", + new SnapshotId("snapshot-name-for-test", "snapshot-uuid-for-test") + ); + Function dummyShardSnapshotStatusSupplier = (stage) -> { + var shardGen = new ShardGeneration("shard-gen-string-for-test"); + IndexShardSnapshotStatus newStatus = IndexShardSnapshotStatus.newInitializing(new ShardGeneration("shard-gen-string-for-test")); + switch (stage) { + case DONE -> { + newStatus.moveToStarted(0L, 1, 10, 2L, 20L); + newStatus.moveToFinalize(); + newStatus.moveToDone(10L, new ShardSnapshotResult(shardGen, ByteSizeValue.MINUS_ONE, 2)); + } + case ABORTED -> newStatus.abortIfNotCompleted("snapshot-aborted-for-test", (listener) -> {}); + case FAILURE -> newStatus.moveToFailed(300, "shard-snapshot-failure-string for-test"); + case PAUSED -> { + newStatus.pauseIfNotCompleted((listener) -> {}); + newStatus.moveToUnsuccessful(IndexShardSnapshotStatus.Stage.PAUSED, "shard-paused-string-for-test", 100L); + } + default -> newStatus.pauseIfNotCompleted((listener) -> {}); + } + return newStatus; + }; + + @Before + public void setUpThreadPool() { + deterministicTaskQueue = new DeterministicTaskQueue(); + testThreadPool = deterministicTaskQueue.getThreadPool(); + } + + /** + * Increments the tracker's shard snapshot completion stats. Evenly adds to each type of {@link IndexShardSnapshotStatus.Stage} stat + * supported by the tracker. + */ + void simulateShardSnapshotsCompleting(SnapshotShutdownProgressTracker tracker, int numShardSnapshots) { + for (int i = 0; i < numShardSnapshots; ++i) { + tracker.incNumberOfShardSnapshotsInProgress(dummyShardId, dummySnapshot); + IndexShardSnapshotStatus status; + switch (i % 4) { + case 0 -> status = dummyShardSnapshotStatusSupplier.apply(IndexShardSnapshotStatus.Stage.DONE); + case 1 -> status = dummyShardSnapshotStatusSupplier.apply(IndexShardSnapshotStatus.Stage.ABORTED); + case 2 -> status = dummyShardSnapshotStatusSupplier.apply(IndexShardSnapshotStatus.Stage.FAILURE); + case 3 -> status = dummyShardSnapshotStatusSupplier.apply(IndexShardSnapshotStatus.Stage.PAUSED); + // decNumberOfShardSnapshotsInProgress will throw an assertion if this value is ever set. + default -> status = dummyShardSnapshotStatusSupplier.apply(IndexShardSnapshotStatus.Stage.PAUSING); + } + logger.info("---> Generated shard snapshot status in stage (" + status.getStage() + ") for switch case (" + (i % 4) + ")"); + tracker.decNumberOfShardSnapshotsInProgress(dummyShardId, dummySnapshot, status); + } + } + + public void testTrackerLogsStats() { + SnapshotShutdownProgressTracker tracker = new SnapshotShutdownProgressTracker( + getLocalNodeIdSupplier, + clusterSettings, + testThreadPool + ); + + try (var mockLog = MockLog.capture(Coordinator.class, SnapshotShutdownProgressTracker.class)) { + mockLog.addExpectation( + new MockLog.UnseenEventExpectation( + "unset shard snapshot completion stats", + SnapshotShutdownProgressTracker.class.getCanonicalName(), + Level.INFO, + "*snapshots to pause [-1]*Done [0]; Failed [0]; Aborted [0]; Paused [0]*" + ) + ); + + // Simulate starting shutdown -- should reset the completion stats and start logging + tracker.onClusterStateAddShutdown(); + + // Wait for the initial progress log message with no shard snapshot completions. + deterministicTaskQueue.advanceTime(); + deterministicTaskQueue.runAllRunnableTasks(); + mockLog.awaitAllExpectationsMatched(); + } + + try (var mockLog = MockLog.capture(Coordinator.class, SnapshotShutdownProgressTracker.class)) { + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "shard snapshot completed stats", + SnapshotShutdownProgressTracker.class.getCanonicalName(), + Level.INFO, + "*Shard snapshot completion stats since shutdown began: Done [2]; Failed [1]; Aborted [1]; Paused [1]*" + ) + ); + + // Simulate updating the shard snapshot completion stats. + simulateShardSnapshotsCompleting(tracker, 5); + tracker.assertStatsForTesting(2, 1, 1, 1); + + // Wait for the next periodic log message to include the new completion stats. + deterministicTaskQueue.advanceTime(); + deterministicTaskQueue.runAllRunnableTasks(); + mockLog.awaitAllExpectationsMatched(); + } + } + + /** + * Test that {@link SnapshotShutdownProgressTracker#SNAPSHOT_PROGRESS_DURING_SHUTDOWN_LOG_INTERVAL_SETTING} can be disabled by setting + * a value of {@link TimeValue#MINUS_ONE}. This will disable progress logging, though the Tracker will continue to track things. + */ + @TestLogging( + value = "org.elasticsearch.snapshots.SnapshotShutdownProgressTracker:DEBUG", + reason = "Test checks for DEBUG-level log message" + ) + public void testTrackerProgressLoggingIntervalSettingCanBeDisabled() { + ClusterSettings clusterSettingsDisabledLogging = new ClusterSettings( + disabledTrackerLoggingSettings, + ClusterSettings.BUILT_IN_CLUSTER_SETTINGS + ); + SnapshotShutdownProgressTracker tracker = new SnapshotShutdownProgressTracker( + getLocalNodeIdSupplier, + clusterSettingsDisabledLogging, + testThreadPool + ); + + try (var mockLog = MockLog.capture(Coordinator.class, SnapshotShutdownProgressTracker.class)) { + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "disabled logging message", + SnapshotShutdownProgressTracker.class.getName(), + Level.DEBUG, + "Snapshot progress logging during shutdown is disabled" + ) + ); + mockLog.addExpectation( + new MockLog.UnseenEventExpectation( + "no progress logging message", + SnapshotShutdownProgressTracker.class.getName(), + Level.INFO, + "Current active shard snapshot stats on data node*" + ) + ); + + // Simulate starting shutdown -- no logging will start because the Tracker logging is disabled. + tracker.onClusterStateAddShutdown(); + tracker.onClusterStatePausingSetForAllShardSnapshots(); + + // Wait for the logging disabled message. + deterministicTaskQueue.runAllTasks(); + mockLog.awaitAllExpectationsMatched(); + } + } + + @TestLogging( + value = "org.elasticsearch.snapshots.SnapshotShutdownProgressTracker:DEBUG", + reason = "Test checks for DEBUG-level log message" + ) + public void testTrackerIntervalSettingDynamically() { + ClusterSettings clusterSettingsDisabledLogging = new ClusterSettings( + disabledTrackerLoggingSettings, + ClusterSettings.BUILT_IN_CLUSTER_SETTINGS + ); + SnapshotShutdownProgressTracker tracker = new SnapshotShutdownProgressTracker( + getLocalNodeIdSupplier, + clusterSettingsDisabledLogging, + testThreadPool + ); + // Re-enable the progress logging + clusterSettingsDisabledLogging.applySettings(settings); + + // Check that the logging is active. + try (var mockLog = MockLog.capture(Coordinator.class, SnapshotShutdownProgressTracker.class)) { + mockLog.addExpectation( + new MockLog.UnseenEventExpectation( + "disabled logging message", + SnapshotShutdownProgressTracker.class.getName(), + Level.DEBUG, + "Snapshot progress logging during shutdown is disabled" + ) + ); + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "progress logging message", + SnapshotShutdownProgressTracker.class.getName(), + Level.INFO, + "Current active shard snapshot stats on data node*" + ) + ); + + // Simulate starting shutdown -- progress logging should begin. + tracker.onClusterStateAddShutdown(); + tracker.onClusterStatePausingSetForAllShardSnapshots(); + + // Wait for the progress logging message + deterministicTaskQueue.advanceTime(); + deterministicTaskQueue.runAllRunnableTasks(); + mockLog.awaitAllExpectationsMatched(); + } + } + + public void testTrackerPauseTimestamp() { + SnapshotShutdownProgressTracker tracker = new SnapshotShutdownProgressTracker( + getLocalNodeIdSupplier, + clusterSettings, + testThreadPool + ); + + try (var mockLog = MockLog.capture(Coordinator.class, SnapshotShutdownProgressTracker.class)) { + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "pausing timestamp should be set", + SnapshotShutdownProgressTracker.class.getName(), + Level.INFO, + "*Finished signalling shard snapshots to pause at [" + testThreadPool.relativeTimeInMillis() + "]*" + ) + ); + + // Simulate starting shutdown -- start logging. + tracker.onClusterStateAddShutdown(); + + // Set a pausing complete timestamp. + tracker.onClusterStatePausingSetForAllShardSnapshots(); + + // Wait for the first log message to ensure the pausing timestamp was set. + deterministicTaskQueue.advanceTime(); + deterministicTaskQueue.runAllRunnableTasks(); + mockLog.awaitAllExpectationsMatched(); + } + } + + public void testTrackerRequestsToMaster() { + SnapshotShutdownProgressTracker tracker = new SnapshotShutdownProgressTracker( + getLocalNodeIdSupplier, + clusterSettings, + testThreadPool + ); + Snapshot snapshot = new Snapshot("repositoryName", new SnapshotId("snapshotName", "snapshotUUID")); + ShardId shardId = new ShardId(new Index("indexName", "indexUUID"), 0); + + // Simulate starting shutdown -- start logging. + tracker.onClusterStateAddShutdown(); + + // Set a pausing complete timestamp. + tracker.onClusterStatePausingSetForAllShardSnapshots(); + + try (var mockLog = MockLog.capture(Coordinator.class, SnapshotShutdownProgressTracker.class)) { + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "one master status update request", + SnapshotShutdownProgressTracker.class.getName(), + Level.INFO, + "*master node reply to status update request [1]*" + ) + ); + + tracker.trackRequestSentToMaster(snapshot, shardId); + + // Wait for the first log message to ensure the pausing timestamp was set. + deterministicTaskQueue.advanceTime(); + deterministicTaskQueue.runAllRunnableTasks(); + mockLog.awaitAllExpectationsMatched(); + } + + try (var mockLog = MockLog.capture(Coordinator.class, SnapshotShutdownProgressTracker.class)) { + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "no master status update requests", + SnapshotShutdownProgressTracker.class.getName(), + Level.INFO, + "*master node reply to status update request [0]*" + ) + ); + + tracker.releaseRequestSentToMaster(snapshot, shardId); + + // Wait for the first log message to ensure the pausing timestamp was set. + deterministicTaskQueue.advanceTime(); + deterministicTaskQueue.runAllRunnableTasks(); + mockLog.awaitAllExpectationsMatched(); + } + } + + public void testTrackerClearShutdown() { + SnapshotShutdownProgressTracker tracker = new SnapshotShutdownProgressTracker( + getLocalNodeIdSupplier, + clusterSettings, + testThreadPool + ); + + try (var mockLog = MockLog.capture(Coordinator.class, SnapshotShutdownProgressTracker.class)) { + mockLog.addExpectation( + new MockLog.UnseenEventExpectation( + "pausing timestamp should be unset", + SnapshotShutdownProgressTracker.class.getName(), + Level.INFO, + "*Finished signalling shard snapshots to pause at [-1]*" + ) + ); + + // Simulate starting shutdown -- start logging. + tracker.onClusterStateAddShutdown(); + + // Set a pausing complete timestamp. + tracker.onClusterStatePausingSetForAllShardSnapshots(); + + // Wait for the first log message to ensure the pausing timestamp was set. + deterministicTaskQueue.advanceTime(); + deterministicTaskQueue.runAllRunnableTasks(); + mockLog.awaitAllExpectationsMatched(); + } + + try (var mockLog = MockLog.capture(Coordinator.class, SnapshotShutdownProgressTracker.class)) { + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "logging completed shard snapshot stats", + SnapshotShutdownProgressTracker.class.getName(), + Level.INFO, + "*Done [2]; Failed [2]; Aborted [2]; Paused [1]*" + ) + ); + + // Simulate updating the shard snapshot completion stats. + simulateShardSnapshotsCompleting(tracker, 7); + tracker.assertStatsForTesting(2, 2, 2, 1); + + // Wait for the first log message to ensure the pausing timestamp was set. + deterministicTaskQueue.advanceTime(); + deterministicTaskQueue.runAllRunnableTasks(); + mockLog.awaitAllExpectationsMatched(); + } + + // Clear start and pause timestamps + tracker.onClusterStateRemoveShutdown(); + + try (var mockLog = MockLog.capture(Coordinator.class, SnapshotShutdownProgressTracker.class)) { + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "completed shard snapshot stats are reset", + SnapshotShutdownProgressTracker.class.getName(), + Level.INFO, + "*Done [0]; Failed [0]; Aborted [0]; Paused [0]" + ) + ); + + // Start logging again and check that the pause timestamp was reset from the last time. + tracker.onClusterStateAddShutdown(); + + // Wait for the first log message to ensure the pausing timestamp was set. + deterministicTaskQueue.advanceTime(); + deterministicTaskQueue.runAllRunnableTasks(); + mockLog.awaitAllExpectationsMatched(); + } + + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 5a40816c94beb..87a834d6424b7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -45,6 +45,7 @@ import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse; import org.elasticsearch.action.admin.indices.segments.ShardSegments; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder; +import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; @@ -1545,6 +1546,37 @@ protected final DocWriteResponse indexDoc(String index, String id, Object... sou return prepareIndex(index).setId(id).setSource(source).get(); } + /** + * Runs random indexing until each shard in the given index is at least minBytesPerShard in size. + * Force merges all cluster shards down to one segment, and then invokes refresh to ensure all shard data is visible for readers, + * before returning. + * + * @return The final {@link ShardStats} for all shards of the index. + */ + protected ShardStats[] indexAllShardsToAnEqualOrGreaterMinimumSize(final String indexName, long minBytesPerShard) { + while (true) { + indexRandom(false, indexName, scaledRandomIntBetween(100, 10000)); + forceMerge(); + refresh(); + + final ShardStats[] shardStats = indicesAdmin().prepareStats(indexName) + .clear() + .setStore(true) + .setTranslog(true) + .get() + .getShards(); + + var smallestShardSize = Arrays.stream(shardStats) + .mapToLong(it -> it.getStats().getStore().sizeInBytes()) + .min() + .orElseThrow(() -> new AssertionError("no shards")); + + if (smallestShardSize >= minBytesPerShard) { + return shardStats; + } + } + } + /** * Syntactic sugar for: *