From a21ae458fa2abc554aef74f74fad2ec1d8856418 Mon Sep 17 00:00:00 2001 From: Brendan Cully Date: Tue, 8 Oct 2024 14:17:33 -0700 Subject: [PATCH 01/10] Unmute `SearchSlowLogTests` (#114364) These were fixed in #114344 --- muted-tests.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 88379d4533a5b..91f26674373d0 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -362,12 +362,6 @@ tests: - class: org.elasticsearch.xpack.inference.services.openai.OpenAiServiceTests method: testInfer_StreamRequest issue: https://github.com/elastic/elasticsearch/issues/114232 -- class: org.elasticsearch.index.SearchSlowLogTests - method: testLevelPrecedence - issue: https://github.com/elastic/elasticsearch/issues/114300 -- class: org.elasticsearch.index.SearchSlowLogTests - method: testTwoLoggersDifferentLevel - issue: https://github.com/elastic/elasticsearch/issues/114301 - class: org.elasticsearch.xpack.inference.services.cohere.CohereServiceTests method: testInfer_StreamRequest_ErrorResponse issue: https://github.com/elastic/elasticsearch/issues/114327 From 1a8f50608c981425a2b4bceba6e60d06c8836975 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 9 Oct 2024 08:37:48 +1100 Subject: [PATCH 02/10] Mute org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT test {yaml=cluster.stats/30_ccs_stats/cross-cluster search stats search} #114371 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 91f26674373d0..5ab607fab2090 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -368,6 +368,9 @@ tests: - class: org.elasticsearch.xpack.rank.rrf.RRFRankClientYamlTestSuiteIT method: test {yaml=rrf/700_rrf_retriever_search_api_compatibility/rrf retriever with top-level collapse} issue: https://github.com/elastic/elasticsearch/issues/114331 +- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT + method: test {yaml=cluster.stats/30_ccs_stats/cross-cluster search stats search} + issue: https://github.com/elastic/elasticsearch/issues/114371 # Examples: # From bee1d912c47f7404079d40786ca23599b36b3019 Mon Sep 17 00:00:00 2001 From: Nick Tindall Date: Wed, 9 Oct 2024 09:15:59 +1100 Subject: [PATCH 03/10] Use RestStatus.isSuccessful to deduplicate status code checks (#114279) --- .../AlibabaCloudSearchResponseHandler.java | 3 ++- .../external/anthropic/AnthropicResponseHandler.java | 4 ++-- .../inference/external/cohere/CohereResponseHandler.java | 4 ++-- .../elastic/ElasticInferenceServiceResponseHandler.java | 4 ++-- .../googleaistudio/GoogleAiStudioResponseHandler.java | 4 ++-- .../googlevertexai/GoogleVertexAiResponseHandler.java | 4 ++-- .../xpack/inference/external/http/HttpResult.java | 5 +++++ .../external/huggingface/HuggingFaceResponseHandler.java | 4 ++-- .../external/ibmwatsonx/IbmWatsonxResponseHandler.java | 4 ++-- .../inference/external/openai/OpenAiResponseHandler.java | 4 ++-- .../external/request/ibmwatsonx/IbmWatsonxRequest.java | 2 +- .../response/AzureMistralOpenAiExternalResponseHandler.java | 4 ++-- .../email/attachment/HttpEmailAttachementParser.java | 3 ++- .../xpack/watcher/notification/slack/SentMessages.java | 3 ++- 14 files changed, 30 insertions(+), 22 deletions(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/alibabacloudsearch/AlibabaCloudSearchResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/alibabacloudsearch/AlibabaCloudSearchResponseHandler.java index 05d51372d9cdc..ecfa988b5035e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/alibabacloudsearch/AlibabaCloudSearchResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/alibabacloudsearch/AlibabaCloudSearchResponseHandler.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.inference.external.alibabacloudsearch; import org.apache.logging.log4j.Logger; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.external.http.HttpResult; import org.elasticsearch.xpack.inference.external.http.retry.BaseResponseHandler; import org.elasticsearch.xpack.inference.external.http.retry.ResponseParser; @@ -43,7 +44,7 @@ public void validateResponse(ThrottlerManager throttlerManager, Logger logger, R */ void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { int statusCode = result.response().getStatusLine().getStatusCode(); - if (statusCode >= 200 && statusCode < 300) { + if (RestStatus.isSuccessful(statusCode)) { return; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/anthropic/AnthropicResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/anthropic/AnthropicResponseHandler.java index cab2c655b9ffb..aec47f19b2642 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/anthropic/AnthropicResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/anthropic/AnthropicResponseHandler.java @@ -61,12 +61,12 @@ public void validateResponse(ThrottlerManager throttlerManager, Logger logger, R * @throws RetryException Throws if status code is {@code >= 300 or < 200 } */ void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { - int statusCode = result.response().getStatusLine().getStatusCode(); - if (statusCode >= 200 && statusCode < 300) { + if (result.isSuccessfulResponse()) { return; } // handle error codes + int statusCode = result.response().getStatusLine().getStatusCode(); if (statusCode == 500) { throw new RetryException(true, buildError(SERVER_ERROR, request, result)); } else if (statusCode == 529) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandler.java index 3579cd4100bbb..ac2e1747f8057 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereResponseHandler.java @@ -74,12 +74,12 @@ public InferenceServiceResults parseResult(Request request, Flow.Publisher= 300 or < 200 } */ void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { - int statusCode = result.response().getStatusLine().getStatusCode(); - if (statusCode >= 200 && statusCode < 300) { + if (result.isSuccessfulResponse()) { return; } // handle error codes + int statusCode = result.response().getStatusLine().getStatusCode(); if (statusCode == 500) { throw new RetryException(true, buildError(SERVER_ERROR, request, result)); } else if (statusCode > 500) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandler.java index 15e543fadad71..2b79afb3b56fd 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/elastic/ElasticInferenceServiceResponseHandler.java @@ -33,11 +33,11 @@ public void validateResponse(ThrottlerManager throttlerManager, Logger logger, R } void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { - int statusCode = result.response().getStatusLine().getStatusCode(); - if (statusCode >= 200 && statusCode < 300) { + if (result.isSuccessfulResponse()) { return; } + int statusCode = result.response().getStatusLine().getStatusCode(); if (statusCode == 500) { throw new RetryException(true, buildError(SERVER_ERROR, request, result)); } else if (statusCode == 400) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioResponseHandler.java index 1138cfcb7cdc6..4ba5b552f802a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googleaistudio/GoogleAiStudioResponseHandler.java @@ -43,12 +43,12 @@ public void validateResponse(ThrottlerManager throttlerManager, Logger logger, R * @throws RetryException Throws if status code is {@code >= 300 or < 200 } */ void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { - int statusCode = result.response().getStatusLine().getStatusCode(); - if (statusCode >= 200 && statusCode < 300) { + if (result.isSuccessfulResponse()) { return; } // handle error codes + int statusCode = result.response().getStatusLine().getStatusCode(); if (statusCode == 500) { throw new RetryException(true, buildError(SERVER_ERROR, request, result)); } else if (statusCode == 503) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googlevertexai/GoogleVertexAiResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googlevertexai/GoogleVertexAiResponseHandler.java index 872bf51f3662a..6b1aef9856d33 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googlevertexai/GoogleVertexAiResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/googlevertexai/GoogleVertexAiResponseHandler.java @@ -35,12 +35,12 @@ public void validateResponse(ThrottlerManager throttlerManager, Logger logger, R } void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { - int statusCode = result.response().getStatusLine().getStatusCode(); - if (statusCode >= 200 && statusCode < 300) { + if (result.isSuccessfulResponse()) { return; } // handle error codes + int statusCode = result.response().getStatusLine().getStatusCode(); if (statusCode == 500) { throw new RetryException(true, buildError(SERVER_ERROR, request, result)); } else if (statusCode == 503) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpResult.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpResult.java index 6c79daa2dedc0..68a94ac0b0c0e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpResult.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/HttpResult.java @@ -10,6 +10,7 @@ import org.apache.http.HttpResponse; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Streams; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.inference.common.SizeLimitInputStream; import java.io.ByteArrayOutputStream; @@ -47,4 +48,8 @@ private static byte[] limitBody(ByteSizeValue maxResponseSize, HttpResponse resp public boolean isBodyEmpty() { return body().length == 0; } + + public boolean isSuccessfulResponse() { + return RestStatus.isSuccessful(response.getStatusLine().getStatusCode()); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/huggingface/HuggingFaceResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/huggingface/HuggingFaceResponseHandler.java index 59804b37e465b..f6fd9afabe28d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/huggingface/HuggingFaceResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/huggingface/HuggingFaceResponseHandler.java @@ -41,11 +41,11 @@ public void validateResponse(ThrottlerManager throttlerManager, Logger logger, R * @throws RetryException thrown if status code is {@code >= 300 or < 200} */ void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { - int statusCode = result.response().getStatusLine().getStatusCode(); - if (statusCode >= 200 && statusCode < 300) { + if (result.isSuccessfulResponse()) { return; } + int statusCode = result.response().getStatusLine().getStatusCode(); if (statusCode == 503 || statusCode == 502 || statusCode == 429) { throw new RetryException(true, buildError(RATE_LIMIT, request, result)); } else if (statusCode >= 500) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/ibmwatsonx/IbmWatsonxResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/ibmwatsonx/IbmWatsonxResponseHandler.java index 161ca6966cece..cb686ddb654db 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/ibmwatsonx/IbmWatsonxResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/ibmwatsonx/IbmWatsonxResponseHandler.java @@ -42,11 +42,11 @@ public void validateResponse(ThrottlerManager throttlerManager, Logger logger, R * @throws RetryException thrown if status code is {@code >= 300 or < 200} */ void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { - int statusCode = result.response().getStatusLine().getStatusCode(); - if (statusCode >= 200 && statusCode < 300) { + if (result.isSuccessfulResponse()) { return; } + int statusCode = result.response().getStatusLine().getStatusCode(); if (statusCode == 500) { throw new RetryException(true, buildError(SERVER_ERROR, request, result)); } else if (statusCode == 404) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandler.java index c193280e1978b..6404236d51184 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiResponseHandler.java @@ -67,12 +67,12 @@ public void validateResponse(ThrottlerManager throttlerManager, Logger logger, R * @throws RetryException Throws if status code is {@code >= 300 or < 200 } */ void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { - int statusCode = result.response().getStatusLine().getStatusCode(); - if (statusCode >= 200 && statusCode < 300) { + if (result.isSuccessfulResponse()) { return; } // handle error codes + int statusCode = result.response().getStatusLine().getStatusCode(); if (statusCode == 500) { throw new RetryException(true, buildError(SERVER_ERROR, request, result)); } else if (statusCode == 503) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/IbmWatsonxRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/IbmWatsonxRequest.java index e5ac64624a697..d9dd6c4c8b44d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/IbmWatsonxRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/ibmwatsonx/IbmWatsonxRequest.java @@ -78,7 +78,7 @@ static void decorateWithBearerToken(HttpPost httpPost, DefaultSecretSettings sec static void validateResponse(String bearerTokenGenUrl, String inferenceId, HttpResponse response) { int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode >= 200 && statusCode < 300) { + if (RestStatus.isSuccessful(statusCode)) { return; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureMistralOpenAiExternalResponseHandler.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureMistralOpenAiExternalResponseHandler.java index e4e96ca644c7f..3116bf0f6cd2d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureMistralOpenAiExternalResponseHandler.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/response/AzureMistralOpenAiExternalResponseHandler.java @@ -59,12 +59,12 @@ public void validateResponse(ThrottlerManager throttlerManager, Logger logger, R } public void checkForFailureStatusCode(Request request, HttpResult result) throws RetryException { - int statusCode = result.response().getStatusLine().getStatusCode(); - if (statusCode >= 200 && statusCode < 300) { + if (result.isSuccessfulResponse()) { return; } // handle error codes + int statusCode = result.response().getStatusLine().getStatusCode(); if (statusCode == 500) { throw handle500Error(request, result); } else if (statusCode == 503) { diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/HttpEmailAttachementParser.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/HttpEmailAttachementParser.java index e4d7fcc3a2935..19b9f68ae9619 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/HttpEmailAttachementParser.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/attachment/HttpEmailAttachementParser.java @@ -10,6 +10,7 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.core.watcher.execution.WatchExecutionContext; @@ -84,7 +85,7 @@ public Attachment toAttachment(WatchExecutionContext context, Payload payload, H HttpResponse response = webhookService.modifyAndExecuteHttpRequest(httpRequest).v2(); // check for status 200, only then append attachment - if (response.status() >= 200 && response.status() < 300) { + if (RestStatus.isSuccessful(response.status())) { if (response.hasContent()) { String contentType = attachment.getContentType(); String attachmentContentType = Strings.hasLength(contentType) ? contentType : response.contentType(); diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/SentMessages.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/SentMessages.java index 94d17844f06da..f98bb99525864 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/SentMessages.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/SentMessages.java @@ -8,6 +8,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.core.Nullable; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -110,7 +111,7 @@ public Exception getException() { } public boolean isSuccess() { - return response != null && response.status() >= 200 && response.status() < 300; + return response != null && RestStatus.isSuccessful(response.status()); } @Override From 07e6932405a75aa9062d56fb5f3751965de1d5d4 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Tue, 8 Oct 2024 19:48:05 -0300 Subject: [PATCH 04/10] IPinfo geolocation support (#114311) --- .../elasticsearch/ingest/geoip/Database.java | 14 ++- .../ingest/geoip/IpinfoIpDataLookups.java | 103 ++++++++++++++++++ .../geoip/IpinfoIpDataLookupsTests.java | 91 +++++++++++++++- .../ingest/geoip/MaxMindSupportTests.java | 2 +- .../ipinfo/ip_geolocation_sample.mmdb | Bin 0 -> 33552 bytes 5 files changed, 205 insertions(+), 5 deletions(-) create mode 100644 modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_sample.mmdb diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java index 128c16e163764..4c2f047c35709 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java @@ -169,7 +169,19 @@ enum Database { Property.TYPE ), Set.of(Property.IP, Property.ASN, Property.ORGANIZATION_NAME, Property.NETWORK) - ); + ), + CityV2( + Set.of( + Property.IP, + Property.COUNTRY_ISO_CODE, + Property.REGION_NAME, + Property.CITY_NAME, + Property.TIMEZONE, + Property.LOCATION, + Property.POSTAL_CODE + ), + Set.of(Property.COUNTRY_ISO_CODE, Property.REGION_NAME, Property.CITY_NAME, Property.LOCATION) + ),; private final Set properties; private final Set defaultProperties; diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java index ac7f56468f37e..d2c734cb9bae7 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookups.java @@ -58,6 +58,25 @@ static Long parseAsn(final String asn) { } } + /** + * Lax-ly parses a string that contains a double into a Double (or null, if such parsing isn't possible). + * @param latlon a potentially empty (or null) string that is expected to contain a parsable double + * @return the parsed double + */ + static Double parseLocationDouble(final String latlon) { + if (latlon == null || Strings.hasText(latlon) == false) { + return null; + } else { + String stripped = latlon.trim(); + try { + return Double.parseDouble(stripped); + } catch (NumberFormatException e) { + logger.trace("Unable to parse non-compliant location string [{}]", latlon); + return null; + } + } + } + public record AsnResult( Long asn, @Nullable String country, // not present in the free asn database @@ -88,6 +107,31 @@ public record CountryResult( public CountryResult {} } + public record GeolocationResult( + String city, + String country, + Double latitude, + Double longitude, + String postalCode, + String region, + String timezone + ) { + @SuppressWarnings("checkstyle:RedundantModifier") + @MaxMindDbConstructor + public GeolocationResult( + @MaxMindDbParameter(name = "city") String city, + @MaxMindDbParameter(name = "country") String country, + @MaxMindDbParameter(name = "latitude") String latitude, + @MaxMindDbParameter(name = "longitude") String longitude, + // @MaxMindDbParameter(name = "network") String network, // for now we're not exposing this + @MaxMindDbParameter(name = "postal_code") String postalCode, + @MaxMindDbParameter(name = "region") String region, + @MaxMindDbParameter(name = "timezone") String timezone + ) { + this(city, country, parseLocationDouble(latitude), parseLocationDouble(longitude), postalCode, region, timezone); + } + } + static class Asn extends AbstractBase { Asn(Set properties) { super(properties, AsnResult.class); @@ -183,6 +227,65 @@ protected Map transform(final Result result) { } } + static class Geolocation extends AbstractBase { + Geolocation(final Set properties) { + super(properties, GeolocationResult.class); + } + + @Override + protected Map transform(final Result result) { + GeolocationResult response = result.result; + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", result.ip); + case COUNTRY_ISO_CODE -> { + String countryIsoCode = response.country; + if (countryIsoCode != null) { + data.put("country_iso_code", countryIsoCode); + } + } + case REGION_NAME -> { + String subdivisionName = response.region; + if (subdivisionName != null) { + data.put("region_name", subdivisionName); + } + } + case CITY_NAME -> { + String cityName = response.city; + if (cityName != null) { + data.put("city_name", cityName); + } + } + case TIMEZONE -> { + String locationTimeZone = response.timezone; + if (locationTimeZone != null) { + data.put("timezone", locationTimeZone); + } + } + case POSTAL_CODE -> { + String postalCode = response.postalCode; + if (postalCode != null) { + data.put("postal_code", postalCode); + } + } + case LOCATION -> { + Double latitude = response.latitude; + Double longitude = response.longitude; + if (latitude != null && longitude != null) { + Map locationObject = new HashMap<>(); + locationObject.put("lat", latitude); + locationObject.put("lon", longitude); + data.put("location", locationObject); + } + } + } + } + return data; + } + } + /** * Just a little record holder -- there's the data that we receive via the binding to our record objects from the Reader via the * getRecord call, but then we also need to capture the passed-in ip address that came from the caller as well as the network for diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java index 5689693d6c295..f58f8819e7ed9 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IpinfoIpDataLookupsTests.java @@ -38,6 +38,7 @@ import static java.util.Map.entry; import static org.elasticsearch.ingest.geoip.GeoIpTestUtils.copyDatabase; import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseAsn; +import static org.elasticsearch.ingest.geoip.IpinfoIpDataLookups.parseLocationDouble; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -72,6 +73,10 @@ public void testDatabasePropertyInvariants() { // the second ASN variant database is like a specialization of the ASN database assertThat(Sets.difference(Database.Asn.properties(), Database.AsnV2.properties()), is(empty())); assertThat(Database.Asn.defaultProperties(), equalTo(Database.AsnV2.defaultProperties())); + + // the second City variant database is like a version of the ordinary City database but lacking many fields + assertThat(Sets.difference(Database.CityV2.properties(), Database.City.properties()), is(empty())); + assertThat(Sets.difference(Database.CityV2.defaultProperties(), Database.City.defaultProperties()), is(empty())); } public void testParseAsn() { @@ -88,6 +93,18 @@ public void testParseAsn() { assertThat(parseAsn("anythingelse"), nullValue()); } + public void testParseLocationDouble() { + // expected case: "123.45" is 123.45 + assertThat(parseLocationDouble("123.45"), equalTo(123.45)); + // defensive cases: null and empty becomes null, this is not expected fwiw + assertThat(parseLocationDouble(null), nullValue()); + assertThat(parseLocationDouble(""), nullValue()); + // defensive cases: we strip whitespace + assertThat(parseLocationDouble(" -123.45 "), equalTo(-123.45)); + // bottom case: a non-parsable string is null + assertThat(parseLocationDouble("anythingelse"), nullValue()); + } + public void testAsn() throws IOException { assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); Path configDir = tmpDir; @@ -100,7 +117,7 @@ public void testAsn() throws IOException { // this is the 'free' ASN database (sample) try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("ip_asn_sample.mmdb")) { - IpDataLookup lookup = new IpinfoIpDataLookups.Asn(Set.of(Database.Property.values())); + IpDataLookup lookup = new IpinfoIpDataLookups.Asn(Database.AsnV2.properties()); Map data = lookup.getData(loader, "5.182.109.0"); assertThat( data, @@ -118,7 +135,7 @@ public void testAsn() throws IOException { // this is the non-free or 'standard' ASN database (sample) try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("asn_sample.mmdb")) { - IpDataLookup lookup = new IpinfoIpDataLookups.Asn(Set.of(Database.Property.values())); + IpDataLookup lookup = new IpinfoIpDataLookups.Asn(Database.AsnV2.properties()); Map data = lookup.getData(loader, "23.53.116.0"); assertThat( data, @@ -185,7 +202,7 @@ public void testCountry() throws IOException { // this is the 'free' Country database (sample) try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("ip_country_sample.mmdb")) { - IpDataLookup lookup = new IpinfoIpDataLookups.Country(Set.of(Database.Property.values())); + IpDataLookup lookup = new IpinfoIpDataLookups.Country(Database.Country.properties()); Map data = lookup.getData(loader, "4.221.143.168"); assertThat( data, @@ -202,6 +219,74 @@ public void testCountry() throws IOException { } } + public void testGeolocation() throws IOException { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + Path configDir = tmpDir; + copyDatabase("ipinfo/ip_geolocation_sample.mmdb", configDir.resolve("ip_geolocation_sample.mmdb")); + + GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload + ConfigDatabases configDatabases = new ConfigDatabases(configDir, cache); + configDatabases.initialize(resourceWatcherService); + + // this is the non-free or 'standard' Geolocation database (sample) + try (DatabaseReaderLazyLoader loader = configDatabases.getDatabase("ip_geolocation_sample.mmdb")) { + IpDataLookup lookup = new IpinfoIpDataLookups.Geolocation(Database.CityV2.properties()); + Map data = lookup.getData(loader, "2.124.90.182"); + assertThat( + data, + equalTo( + Map.ofEntries( + entry("ip", "2.124.90.182"), + entry("country_iso_code", "GB"), + entry("region_name", "England"), + entry("city_name", "London"), + entry("timezone", "Europe/London"), + entry("postal_code", "E1W"), + entry("location", Map.of("lat", 51.50853, "lon", -0.12574)) + ) + ) + ); + } + } + + public void testGeolocationInvariants() { + assumeFalse("https://github.com/elastic/elasticsearch/issues/114266", Constants.WINDOWS); + Path configDir = tmpDir; + copyDatabase("ipinfo/ip_geolocation_sample.mmdb", configDir.resolve("ip_geolocation_sample.mmdb")); + + { + final Set expectedColumns = Set.of( + "network", + "city", + "region", + "country", + "postal_code", + "timezone", + "latitude", + "longitude" + ); + + Path databasePath = configDir.resolve("ip_geolocation_sample.mmdb"); + assertDatabaseInvariants(databasePath, (ip, row) -> { + assertThat(row.keySet(), equalTo(expectedColumns)); + { + String latitude = (String) row.get("latitude"); + assertThat(latitude, equalTo(latitude.trim())); + Double parsed = parseLocationDouble(latitude); + assertThat(parsed, notNullValue()); + assertThat(latitude, equalTo(Double.toString(parsed))); // reverse it + } + { + String longitude = (String) row.get("longitude"); + assertThat(longitude, equalTo(longitude.trim())); + Double parsed = parseLocationDouble(longitude); + assertThat(parsed, notNullValue()); + assertThat(longitude, equalTo(Double.toString(parsed))); // reverse it + } + }); + } + } + private static void assertDatabaseInvariants(final Path databasePath, final BiConsumer> rowConsumer) { try (Reader reader = new Reader(pathToFile(databasePath))) { Networks networks = reader.networks(Map.class); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java index 1e05cf2b3ba33..d377a9b97fcc4 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java @@ -361,7 +361,7 @@ public class MaxMindSupportTests extends ESTestCase { private static final Set> KNOWN_UNSUPPORTED_RESPONSE_CLASSES = Set.of(IpRiskResponse.class); - private static final Set KNOWN_UNSUPPORTED_DATABASE_VARIANTS = Set.of(Database.AsnV2); + private static final Set KNOWN_UNSUPPORTED_DATABASE_VARIANTS = Set.of(Database.AsnV2, Database.CityV2); public void testMaxMindSupport() { for (Database databaseType : Database.values()) { diff --git a/modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_sample.mmdb b/modules/ingest-geoip/src/test/resources/ipinfo/ip_geolocation_sample.mmdb new file mode 100644 index 0000000000000000000000000000000000000000..ed738bdde145082d329a40b878618bc5f4595932 GIT binary patch literal 33552 zcmb`N2Y6IP_r^CNfrQ?BS$Zg0wl_r$q!(HOh=_5MY?6g!7dHt_MX~qZE1-yq1;wt| z0L5Ohi(&;E_Fn$)%)FaD@Hd~|_dMb={?2*Nxik09otZoLZY&l{ipA3LH;cuRO186D z(){q#=%#30dFNeQE_($`&W1t@^^p(_) zvsgNDp6Yg2A@2l{cOv|ggnu%RI|cfwLO%`u>B2vQd1pdDi}u+fehzf>wQln~p`TCf z0`fxgBJyJL5*&Xi$`$l4BQH1Xv>LH1sa++st8u-E>2|M0&I>5;E%#Ad1ADEpbCgg{(<~IMf@*gr$AQqroiZOP%@sEQa%*B); zC>X;jLs2jnQ-+~jVB#rO*uzoEQAVHyQARR0O4K)++8D&g(#{g`an!~mK7sZ`GF#-? z5T8WdE_4SqC+R|-Tf{x^y^Q%tKN+xCy0UK)OgRAg2a4k-!{_}?Iauh2K+h5W6xKNv zdals(;O8?|AoN1$MZzzJUn2Zc9#ck6Lwq{QOq3Zo??pyeOqrFm^_Z-Hwh(1DkDWu# zHPktWQk#do!-PH`{^7!30RIT)Rto({Y9Yj{gkDX(h72QKE8=zV>xCb|wd`Q7x|U;6 z8t6BYQ8H$z&wnwsqsTapYogsuwvbE6rH1F%OpJAxa>rCipQ9oPg=TJMBJdZrzV(G@~)W=^){UY*W z)OQK(OAU42Wz;T5{0gC8N&PD5SJS>m#IJ?En)(`{ucdY!;@1m(9rYW?_2iA@P2|lu zZiBJ)`Q2jdZm&bXjXAfA_(p1XAbuz9O(K35wYw3&N9gxbzmL2h@ds!>DDobnwi)q< zX+I+3k3xS;_>VJZ3-qnV?#}t1@&xwDW1&CEob9lGM0rXa|1|t(P)$(+AXhb5_}roe}w>PkhMirP*i)5-Q^2jpeY&NS5d z9iexk-dX5fES4TI;det`cj5QoF+Is%i1()5hwN*p+v$gRf9eB-J`nmK>VwH4B0iMb zFyvWj4<|>EBgs+ZXhYrZ7+eSDfwr?y$9$A=^vA=VN_zr15n8sf^)WW;lSn(_4iR_4 zcQNJ`x(B*f_&)f4<^_Zvq;>$}2MT>M{DT-fm^?(pbEr+RSbE|*)Vb>O&Bbwf!q10a zApAo3MZz!UaV2Ca;$^g_k<$%z{WGY|BxfPNT*NEj&t`0n(C0!wl=?jKuw=aFmDCPL z-U5^*^pAjDN&84LWT?-l3h`>{HDp-CYZ0%bUN7_r^o7(H3B7?@BN;^==RxXXk#`jO z9dkc5j?!dw?Ki`3N&5O6mx|+-A->%BsVjtjG__;MW0AK~#E+wXJoHsUKLP%U!as?5 zCzGcjek$$LMBeGp&!B!Ld6r=>bx+T>STxTu)YoyI#nNsMV&|jWz+)~TFNAgx?TZcd zF_%!g6!FW1emV6kpeOGARU&>h^lOBFE%R5CYsj_8yN>qtu-6&8*IhVo6nzfsq5p(( zBgz((n~bhwH^bk+yjz5RE4ACm+sTc{zk~Lj!?&n?P z2Z(=2`y=vWL!I{t;-6CAO=1kU*7F(Sp9}vB_+K*bE1~bD_BHtp`7QFlqy0VXAB^2w zwfkez*5~pw=8w9jU%)8%D|iryYiidW{GIjuLH=o|kNwNo?JUSmN$TxVsUx8B+NF`{ zWP7p$Vj0HPd71D#GS-RYIBM5L|V5cC-d6%q1Kn|hrIqG zJ^=ne#s&#}Fts7%P~;7xZ54ULsf|E&OH0=8s{TR8JJc^7P>V9mZ){J^vXfGj`io9jimXj-xe>Cl5$YTw4y(?CR@8}?KEc`C|k>Zg&XLpy`^nTGoKv#6bo_&Gv9m->0o&!>HXh+jzU zBE&Bi`X%r$W$ZHYa`FoDO7bd0eI8fixNC%et+D&v0eub1eLQ9@c^!E@xsJR6x$BLs z>%EcsP2|mpZ=ijP$h#H#ZPafU`bKK%Jnj_wChB)VzuRKz-48JcKA;T|7qqu1N~W{KL`JL z#$FKmi?|j!K3_)OE6jV9+(EuZzD~YjsL$g~9QT&+-)7D`au<&Oz}ULJ z52=3y{bQkjg7f_X+NUVHjjnusOk#e2j-Sp4QB%2J!tVfnCFJQU2iVx z;W?(|p}u_h=)1H66wJM}Lgp2b#bgO$QQD=j%VPL{3Lpgdb*3E%Z8}*He#>3lU$0(n!C- za3K0l6)N|Ipn$IdF1)z1>}X~MdZcgB_!s&KG(|(wS76x z^9t%$l2?&elh=^flB>xz}qgypg<#yqVlU-a_6=-bUU|ZY1v@?<6;o zcae9K_mKCJ_mTIL50DR%50RV6hsj6CN6E*?$H^_^R&pEp1o-P3FHvzKy&;X}?3h3vDOu_YC#1??c~3 z{R8qt5&sDBkA?pU{7;3y8~z?1_Zj&)`Gv^)5_w-y-z)U5seOa^x3s?_zbAh%)aUyn z@_w>d26YC1CVwG+C4VD-C;u>1$E)LXJ%3>>w0Lx#SH`*_ue;EDQ11!77s>#X-Z2SNQ#?_cv5|s-zD@8I;u0 z2g4sC{Gp-_tI<_|qz^|KVf^%wi;&kqyOE5NF+*Md zV&okq{5bq3;Wt|>Lx%CVC5WAYvXuTZayhxeP}h4j@{XZ?EV+_Ajy#@RWvGukf!c}W zNvP*!+NX%TQ>mRso^CkgM&zF<^3G!XZ0P3*{apB%ue#0ig?<6_3x$6X{EL})37Ht5 zmx;W~k#_~-R|@?qYFAqo=jE zn`v(_)cP&dZbkeyq2Eq@BlJ6rJv4FrCdTd}?Z9;zn`jf)n4*x0UJxxACJ}dH`L*Dbke}OqKk}o0tGVNDH z-mB1eP=AelUBus@_NK)$%t`xg@*VPBLw&tFsl7+OkNS6s_y_Pm6#hrd`* zCijq^8R~XEN8T64ABKCZwXZCe4wzr*d%+RZzJ~ve#nS#y`rpF;j`sKD59E*JPvpNbAG@xQ_EPycuLf6!(dI`1!|wYMNQg?6f;innivk|z9g=HP(#9cX8enPf+@ z6WN*ULUuLO^>;%(-KqBwdQWP-$lhchXzl!1&7GSux1hCYP)P;!`v zTM-{FeBPJ#Bbhr2@zEka2L4#avdD2FJ|6K2!k-8~TlhBklX#q+bdVgs`n+A#+=kY3 zah#W$kMu)}paf86q6Ceuk3RtZfy|ps9z-52@(w{>4)rPIR5DlOY{>Ck5wdwAkr&tj~ctRQC_>f`57n@b*wdgh7vVescOb~w2}#E+m> ziM%6eheW&zdbRLt;D?!4OV*L~$-Lp}{w$=vD5nSCYqxyyK~@vRFo>(>@W$Y(+VV{>kJiR*op&+vE}?!Yc^P>*Vrx*Y5cyX! zb`|uig?N95g$y!)u%FZ2hfJxD%8ZbtsYv>!3l?L7+pF_bMRk6SDw zIW`p&cKdBS?g`|4fbt~$?V_Hipg&Fh8S+{3Ir4e(1@cAmCGutR74lU>eV#k0y@qyP z7y29U-xU5^@ZVS;nxhu&WJI9TUrkeP^gq}_?^Om-o=8tUV^QR`0jAbXO%$lhch)ZdqO zKSNzlf9L~*KM?*P;SYvCgvSjfhmlrtxS^_V6vj`7ktm0ujADE=IffidW|8B_@#F+@ zqM<&|Y-%>tJBhZPbdXL%o$o@Po4SYel0MQ;2FReHKK=k4e;`T;%4Eh5A`d1H5&1dP zrXYW+&~xGEF_tg%0&0bb7YV)C*rUVHONCwrf12>88@s~{=rd{0BFo7Na<-wqt~tn` zOZ`xCUNSx!_fO}|Cl5zH$9;z*s8y0jk|DB+tR`znT#Kqx*ISGF>S))K5pp59h-@Gm z$tW2!)b%Y!eMeD`3%vBhES52e zemW7kmry^6JQ><4v`-~ZBTpyKFx1y|CbhH3v&nPFbIJ2i@A<~o$6WybLdGr<`o+c` z^C;q%qCAUo8Onnwmoxtg@=Ee5@@hj}|25RE#c`{JzJ~f*=-1J{o?J)XK&~fmByS>b zHq`BGpmqy+D|s7vJGqg(1MS|4axeW&aIxU+UOnLVq0k7U6GY&Nk>z(0)?Hw^Mrx@uz7&V>p)gtiyBEpC?}+UnE~5UnXB6 zUqziiqU#XS5^J$aJ#3p{hTl0}3iow)Qj0j$|jYGuegg zisQOjELn@e?qmF1LLWFd|(GPbU- zn0g8HQlXc@pT^j9at4{Gdlvn2vVxp#sOz6YZ7z8z>YYdXFmgV5xS>970ktELUrGB& zG9>b?I;;D)cI;293}iXbDGFz zvV~kiF2!-njIHZgZtU?Ks~JZlevF783xB2XkAr`_@K?bIl6 z6Yzry$#SCUtW z_|?>|F&w{({%RbzhW1+6PoZ3gau2oZVXvcogQ4#4_0Vq={!PrencP6$Lf%T=Mq*55 zY&5p6?+)sBlAFl8$h!?EaPDT@OZ`67b3g3|40U}ELVt+*W}!b!?GeNu75ZcFA7^Zf z(6>V0Cj2Lu^CY?5Vwu3c)$Kk_{TUqhtT^sD>d%uekS~%iA^&A#>v~?H{wlcx@z-d- zPQF3DNxntCZK&&c$Ji5o!|^*Af6rpcM7Ve=l>sCch!SCBGxTCx5{4Khpk*{F(g4P_;909`e9U%#DoS z8UF*mny-J-{|o*hC>E4aM$1egQ=zppwyG~P4JBRp?csM2eg=G0q;orxokYAd;$5hB z6?!*n-4X9WyQhfvqShPnKD7Ii{mA}?y6pkT8z}ri%oz-Q2<@TdFw#m6Cr6MY4JYy* zWR5nzZf6X0#*$g6f1HSqr#=DtM4@L>w?Uso+fF)2C+Q;HhPqu3j`LFYk$y5j1`T!o z0n`p8C*$~oMEqc5XRm;sBlIcor=o;;TrQbM<|9@>yO1n0)a@1{UP8T;EEDl*)TWa& z$eH9UvYf0yeY1_N>z_k?F7!i%j&ZN!he4k&^uysVVC)F8l01?OkyT_hS!0;}hM}&% z7WLH$zn-}f=nI9u2!4a`8{tQp8zUEs_)&<*sW*|$WQ)jKg1n{FmkE72HQY~qo<}3^ z7~02*yp`0BL;QG)#fCOASCJ=>Cz2;ooWO%uVE7w$iJ8N zedPV*1LT9`L*!=iVe%0}bsr}sj(v==$5H>5WIS^#{B6R20{)Z2-wyvN;XiHc%xB1F z$>)&wJna|A7Y+4zeF^cGh5w4hVjsY~9pr1|>m=u8=9}=tIQ}hgIQTZ$1$@Wk>H6P= zzti}c?~(75yF}gx)IKCX!f_wd{)GIL+)eH=)a`tR{Lh8|1^h3W_Z7L9{F?lR{FeOA zP}lc8wI9eIQQuFreK#{;ofTgj{HA_{wMsul72@E?G!MTb~_PIgPl&lJ=uZG zATvo+W?v4w6WN)>xoBOTQ&*I3%vXMQlpgeZlD){@q{{01`HoT%&VI1ptJ zUx0Wt_4AmRtYo-F)>Xdet7LOX|?LaKJBqT~uc4|cxr z3t$)0FA{n&>=OE=WEnY)oKDUlReLj0X3;MvD@YZejWUP+T=GzI9(fo!pH#;kPJ01( z1Z~xaDt4ry;}I}qv1nGot~Pqd8WC6dwJ0y6)S;Y>Qjena2+Gm47m|y>2HGm8k#-b} z34Jl`qsTbfL^hKx*&jZ)cJ|Db*wmOdsQ7)o?F?k88;+LXa zM*ni5UqSmy@+$Ib@*475axQCi}9ZltfYn`qxmZXj$5qr2i`E$_+ir6>JwA6qG~B#27GE^a z3EyoCK=wPcrBkvv*3=wosH~3Fgs0{?=H|!4brCd_7p-duMQaM1BaPt|v1qs~uO%K^ z9G+y(W4AS>I5*s6EsI5>K$j`A%dPzj-42`AZuhvYdm)&QLYALMfZH0gerjE5{R%bbmxfy6;Re=Z%D@=Z2dS9gH*a`(0ks!5HuXyIqeRROEDcY)(u9htrJAS&-tYFR znQh0(yRnu`o`;U{xNR7=PLD(NQmcbToNl`pQ@gM<)>sva*DSZLEaA+a6>AKmy{Q#p zbzk$aKIP*~eF047{A`Q|jF`#u{4N_hSxvTpGrQGsVnM3Rb9nrIjL3q5P(!3z4gJIt zSQe_m1*xvjX%TbGjU~?Rb778|aqo2c-I&)4ECEb1ta?s2)-AU;dmmRd6!6<|!(zpA zLrwKq=hV$yIpQbOUofLS7LG=iB_^@UhH>t~jX(nqtaCoMU5`suqaSx6fbr=w4bU}L z;>U7>1`2ZG5iI4nlGat04WO;UDPhdvB`sX~3UDEgz`{)QGIx!y2I0Ohf!6)?+o~GhkZ_Tyd&VnZE)DJHlT}K z7*41&>0tGC`JGM&ZgD|D%fe7R)NEZ@n$r{sO)7~sEDAM;aOy=f>UTZ3-deh4Gg_IW zTe0K*dR%H2IP4x<09T+NMC#e&$8c8j8(kgR>s?7neJExv$5f8ip~;FuTo~?$%Y*;)&~@6}F3c}n7$%4x0~d>k8x6KP zs;itXhYQ!0TcIy%?1`wjcxo(yaoik><2H=huBvs|@I*yj=r6A=;PpCi8|LXXTR%}< z=r6T^YX`TeGT-aQO;N+CLM=78lA0Fl%CS9BdtOCs(Q-8zif35+7oe}bel;749C#pk z16Y*x4b}?}29(cj^WvO^>$6)cF)tn2ey<1jILlIOU0JN}g{~Maps6;HgFGGJ=}STx)`#ba6HS!HS|XSrYM$#Risr`R)mYBMaWv2eW6FcB@dwndrVv{}r&IN| zI)AUti5um1+uXP@!o&GbojVI>Z(V{aPerd3O^Jt^w%n@vt;>`tv(yd2*!KAXW`}{b zBY-(>77VQ0xZMs9YIk^0uW+$dD)QlW=sS*?mmjVP<8D}UBe*M(YE1Q+iCsW>C{{Tm z)Y70ggy_U@Bpy@yPByL>nBnkJ07)IZdiD_V>VcC=1hVgPs6kG{r@$D_pKN(^cB#s-&(avY`raBo7d;}>dlCH_<3A@>>N-xE({x8Y<8WvC&JTr z#NB%KS?wZ;GcG9$FN;*itW)tWg!;=4Rcj;e2cEHJBH~<~KCB07qWIN=7AqoF8FkZz zt2aG(;c#LJ$VzW7wwb2>N>#s__A@Yl&^v5y26lx7Si@cFfy=ELZiX7*sSfp0g*O28 z@_`3c_I^GlCfc)OV&+ecHEh|0XZy6opf-)5xvbQS*lAg*>#XCmQtvP|V*`uT02ih@ z3pGpMZ>`Kq-HFRc{VdT*#Zy}v>RUq9Th?HIY5E8&k{^RZZ{7la+zza`s%~uTu|8ta zb>Xb-p6va6oVDA97f3vI3$b@=sKa(PEBy$ZYJP60p#ekKydYt?V0(`zzFG_MV72Qh zG1qCwMjY>Acon0GTcP{j=Rr%>SYAbV87eUgRNnNa(4xdcqYx{*H{kN&^#Lop+MD6s z5oer&zQs!y3i>$^$d)diNf?wKw+l}WFM2ww{RfFbsUA}1ftO{Oi=MTFP}fkk2-NO{j;Tc{t^jMew9eEZ%AvbUXE!#H4Wq zut=cAuDWuM2QAuz_Mn=e!JywXxt|(F-@9Dul47GC<)*oybUC_6E#On8O~+8g8!&d^ zSilR=DPE`EW)9O$xV>&q(Czd(g4hKlE-ESh%?0V}!*kK$(i`aEe1ldfde3 zOv7tky>%9L!QpK62C}%`Jew0cZ@e)yM4}jl({h?Zb)n^<>_)6=s`K@prV!%=FUop2 z>P?Nyr@N-FKDCguCY(4cZM**e-IZrmh%JlSim0U`dAEyl8r7q;1{*}&j8gMBMLTwM zMS&Y{N&&BWJ5o)$f=;*JVVb1<4^8IcwqyL3nSM}j)%XOH{8-W2G&2;!-3zPw6Wp^$q{s8v;?t~R0o|QGc5L3|KB+0u*mn}8;;t3&&-Lp zL_^iMA=ns)YQjzR+*Ov8gyLH_G|E%KCk(eM;7}V3^=xwbv0dEfRPfU22v*nbcPdod zJQa`KqdyfDRbVrS2eW#f>5Xl{l*q!+;+8~DIIs|58-o|ILLUYMR=oXs!i8?a^xv;} zs%Y!XPCs!b&_daEzRZXF^`4RygB%=zxUvQi(YVH#Z;43 zEq!jRcut&1!0ikN-L0LlkJx4(xUn#)XIV*Es2)u<)noU}UdZaSmOYTwX(LwJP7mp6 zfcNvDD-_n#pw<3|tIq252j*PoG_0_l)%O{EHE9Y(lQ-C@-aYX;tu74@7QY)`j?DN9 z1aWzoYt?pV(1%slboIV!n-}M_yVR$`SlR4m^y7ZReEJMrZ^PDAc$sZ%!B@CMFXL2v zUWYmrzuRUHc=Q{&nsusCuQTWj`$IuLcDcz@Nvdt03aWIfv6er*Cel=&=;D&X<=7x^ zT~)PpRZXIOwavjbo33zsy!&(o9yV^L!xaoy%l4CMo9(NYBsK7|I`vGJiWji;zzjqEIoKQXQ6;)4?Up35sf zGrBN|OtYT-n#FR4?~L}Kcb|?6<$a{Kx}wl)cVY{LI~dFAxgfD&o>mqL z<6~`0c6I%>4cpFcZC!nf4LHmv9rae?P@iZ;>!Io(wj#Xo=7hA=X6yK62p`>0W2_** zWl>8^J+aN6y^Yrbx4kM@jTerfJLCz5!il!_5!-5Q|JQ=QeI5=%SJnjmu39(RNmNgv zt-U#SV*l@J~!yRFSA%+yCQ3z6e|T`pBXXeg))kR6oPucNi=~Me3I} z7Z!H4wN`yti=9i*5vs0AyyhgO>i=*_g?4=R#RdjXw!*^LqDAhWoj3velp`{g~U&cK3tuCVZ;iL$$~G7>LM|9g3c-@#v%tw ztKe&6&U2vTsd=xiu(dh!%S%?Dr_D=hR-c{NpY_?pX52w3|r6Aag2^oJdO{YHj0UI=Zqi}ytQ z(5Ii#?RW6gL0i{^_iTqREUt+{TXp~aHMLcHAQ%c&sYSueVEoQ)gny^@P%wZGn4)$H z|4QvHeD|mg*CuOE2>(v)u3*6BuHyHNY_I69%hl{jHNQ;s4_tR8J~v zY_@1CH@r9$Z*B}n@m+gX;uBypj@7lOARbzRcepLL;>66CZS`XSwut7$)Q2ypPdya- z;l$j*8VnPc3(qXK{HlUgu)L)KE1~}Kot1VI+s{h7 zA1g)LHg<(Oh@S#@;QP3JVK4PdkS zaYtQrVYd3y5xn53n}Kh*0sYP0+zh<7S7CQw8?Ff@E;=c+dDUL0SFiiHA^B6oq4<_H z;+KVdd~?D(^$PV%;xyGK=#S!%{(&KR7x1ek{vbe~5uQr;u~qL8hv{B-hHKn-&GNYt zj|X)tl49FEQ=}iT>Jt+>Xi&2I2h2pnYFuUws7Ch>i0M^R)f%Yr>209i1GNhOG`R-c zjrs>{#_b)jLp{UWs^1&*`?2pyENj?47@@5zNGzKFZT*3uM}6{1JkR`CHns2%b1Cb; zt-m(t@r0eNi;o|>1TFlJ{Zm*G!|zNjO=>*mtMQ0Cj)$TX8`A=8tMIWwJ)j5UOB&vC zu`zb}y+J&u^%su)w7*^R)c0NWh{l(s>DBn#u23|heq-jJE#NbQ{sW$@%yZGc`oNf# zxf-J&bAuiQA$3!{xJ!5$(4(ML_@}GP`~ssO^E-@!j_uSaXsiAJzMeSoO4(XJg@3wB z`_-SHi`{_!$x;0w6@FpH+q3#2h+hivyH7yQ`Nlx72Dpr*zlT{kvhMLOo z;#hTkT54VD_HJc0*qu~`n!=UM%NK_iq(v4-qO~z=WN~F(7;~^1KbXg&l}(|>#SLLw zV`EKK34SQ3jz<2oWBot0JaBRV literal 0 HcmV?d00001 From fb482f863d5430702b19bd3dd23e9d8652f12ddd Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 8 Oct 2024 18:15:37 -0500 Subject: [PATCH 05/10] Adding index_template_substitutions to the simulate ingest API (#114128) This adds support for a new `index_template_substitutions` field to the body of an ingest simulate API request. These substitutions can be used to change the pipeline(s) used for ingest, or to change the mappings used for validation. It is similar to the `component_template_substitutions` added in #113276. Here is an example that shows both of those usages working together: ``` ## First, add a couple of pipelines that set a field to a boolean: PUT /_ingest/pipeline/foo-pipeline?pretty { "processors": [ { "set": { "field": "foo", "value": true } } ] } PUT /_ingest/pipeline/bar-pipeline?pretty { "processors": [ { "set": { "field": "bar", "value": true } } ] } ## Now, create three component templates. One provides a mapping enforces that the only field is "foo" ## and that field is a keyword. The next is similar, but adds a `bar` field. The final one provides a setting ## that makes "foo-pipeline" the default pipeline. ## Remember that the "foo-pipeline" sets the "foo" field to a boolean, so using both of these templates ## together would cause a validation exception. These could be in the same template, but are provided ## separately just so that later we can show how multiple templates can be overridden. PUT _component_template/mappings_template { "template": { "mappings": { "dynamic": "strict", "properties": { "foo": { "type": "keyword" } } } } } PUT _component_template/mappings_template_with_bar { "template": { "mappings": { "dynamic": "strict", "properties": { "foo": { "type": "keyword" }, "bar": { "type": "boolean" } } } } } PUT _component_template/settings_template { "template": { "settings": { "index": { "default_pipeline": "foo-pipeline" } } } } ## Here we create an index template pulling in both of the component templates above PUT _index_template/template_1 { "index_patterns": ["foo*"], "composed_of": ["mappings_template", "settings_template"] } ## We can index a document here to create the index, or not. Either way the simulate call ought to work the same POST foo-1/_doc { "foo": "FOO" } ## This will not blow up with validation exceptions because the substitute "index_template_substitutions" ## uses `mappings_template_with_bar`, which adds the bar field. ## And the bar-pipeline is executed rather than the foo-pipeline because the substitute ## "index_template_substitutions" uses a substitute `settings_template`, so the value of "foo" ## does not get set to an invalid type. POST _ingest/_simulate?pretty&index=foo-1 { "docs": [ { "_id": "asdf", "_source": { "foo": "foo", "bar": "bar" } } ], "component_template_substitutions": { "settings_template": { "template": { "settings": { "index": { "default_pipeline": "bar-pipeline" } } } } }, "index_template_substitutions": { "template_1": { "index_patterns": ["foo*"], "composed_of": ["mappings_template_with_bar", "settings_template"] } } } ``` --- docs/changelog/114128.yaml | 5 + .../indices/put-index-template.asciidoc | 11 +- .../ingest/apis/simulate-ingest.asciidoc | 21 + .../test/ingest/80_ingest_simulate.yml | 413 +++++++++++++++++- .../bulk/TransportSimulateBulkActionIT.java | 64 ++- .../org/elasticsearch/TransportVersions.java | 1 + .../action/bulk/BulkFeatures.java | 8 +- .../action/bulk/BulkRequest.java | 5 + .../action/bulk/SimulateBulkRequest.java | 56 ++- .../bulk/TransportAbstractBulkAction.java | 11 +- .../bulk/TransportSimulateBulkAction.java | 19 +- .../MetadataIndexTemplateService.java | 24 +- .../ingest/RestSimulateIngestAction.java | 3 +- .../action/bulk/SimulateBulkRequestTests.java | 115 ++++- .../TransportSimulateBulkActionTests.java | 4 +- .../ingest/SimulateIngestServiceTests.java | 6 +- 16 files changed, 699 insertions(+), 67 deletions(-) create mode 100644 docs/changelog/114128.yaml diff --git a/docs/changelog/114128.yaml b/docs/changelog/114128.yaml new file mode 100644 index 0000000000000..721649d0d6fe0 --- /dev/null +++ b/docs/changelog/114128.yaml @@ -0,0 +1,5 @@ +pr: 114128 +summary: Adding `index_template_substitutions` to the simulate ingest API +area: Ingest Node +type: enhancement +issues: [] diff --git a/docs/reference/indices/put-index-template.asciidoc b/docs/reference/indices/put-index-template.asciidoc index 772bd51afdce8..36fc66ecb90b8 100644 --- a/docs/reference/indices/put-index-template.asciidoc +++ b/docs/reference/indices/put-index-template.asciidoc @@ -85,6 +85,8 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[put-index-template-api-request-body]] ==== {api-request-body-title} +// tag::request-body[] + `composed_of`:: (Optional, array of strings) An ordered list of component template names. Component templates are merged in the order @@ -102,7 +104,7 @@ See <>. + .Properties of `data_stream` [%collapsible%open] -==== +===== `allow_custom_routing`:: (Optional, Boolean) If `true`, the data stream supports <>. Defaults to `false`. @@ -117,7 +119,7 @@ See <>. + If `time_series`, each backing index has an `index.mode` index setting of `time_series`. -==== +===== `index_patterns`:: (Required, array of strings) @@ -146,7 +148,7 @@ Template to be applied. It may optionally include an `aliases`, `mappings`, or + .Properties of `template` [%collapsible%open] -==== +===== `aliases`:: (Optional, object of objects) Aliases to add. + @@ -161,7 +163,7 @@ include::{es-ref-dir}/indices/create-index.asciidoc[tag=aliases-props] include::{docdir}/rest-api/common-parms.asciidoc[tag=mappings] include::{docdir}/rest-api/common-parms.asciidoc[tag=settings] -==== +===== `version`:: (Optional, integer) @@ -174,6 +176,7 @@ Marks this index template as deprecated. When creating or updating a non-deprecated index template that uses deprecated components, {es} will emit a deprecation warning. // end::index-template-api-body[] +// end::request-body[] [[put-index-template-api-example]] ==== {api-examples-title} diff --git a/docs/reference/ingest/apis/simulate-ingest.asciidoc b/docs/reference/ingest/apis/simulate-ingest.asciidoc index ac6da515402bb..1bee03ea3e58a 100644 --- a/docs/reference/ingest/apis/simulate-ingest.asciidoc +++ b/docs/reference/ingest/apis/simulate-ingest.asciidoc @@ -102,6 +102,12 @@ POST /_ingest/_simulate } } } + }, + "index_template_substitutions": { <3> + "my-index-template": { + "index_patterns": ["my-index-*"], + "composed_of": ["component_template_1", "component_template_2"] + } } } ---- @@ -109,6 +115,8 @@ POST /_ingest/_simulate <1> This replaces the existing `my-pipeline` pipeline with the contents given here for the duration of this request. <2> This replaces the existing `my-component-template` component template with the contents given here for the duration of this request. These templates can be used to change the pipeline(s) used, or to modify the mapping that will be used to validate the result. +<3> This replaces the existing `my-index-template` index template with the contents given here for the duration of this request. +These templates can be used to change the pipeline(s) used, or to modify the mapping that will be used to validate the result. [[simulate-ingest-api-request]] ==== {api-request-title} @@ -225,6 +233,19 @@ include::{es-ref-dir}/indices/put-component-template.asciidoc[tag=template] ==== +`index_template_substitutions`:: +(Optional, map of strings to objects) +Map of index template names to substitute index template definition objects. ++ +.Properties of index template definition objects +[%collapsible%open] + +==== + +include::{es-ref-dir}/indices/put-index-template.asciidoc[tag=request-body] + +==== + [[simulate-ingest-api-example]] ==== {api-examples-title} diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index b4672b1d8924d..18eb401aaa0fe 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -371,7 +371,7 @@ setup: template: settings: index: - default_pipeline: "foo_pipeline" + default_pipeline: "foo-pipeline" - do: allowed_warnings: @@ -523,7 +523,7 @@ setup: template: settings: index: - default_pipeline: "foo_pipeline" + default_pipeline: "foo-pipeline" - do: allowed_warnings: @@ -807,3 +807,412 @@ setup: - match: { docs.0.doc._source.foo: "FOO" } - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } - not_exists: docs.0.doc.error + +--- +"Test ingest simulate with index template substitutions": + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.index.template.substitutions"] + reason: "ingest simulate index template substitutions added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo-pipeline" + + - do: + cluster.put_component_template: + name: mappings_template + body: + template: + mappings: + dynamic: strict + properties: + foo: + type: keyword + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - mappings_template + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO" + } + } + ], + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": null + } + } + } + } + }, + "index_template_substitutions": { + "foo_index_template": { + "index_patterns":[ + "foo*" + ], + "composed_of": ["settings_template"] + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: [] } + - not_exists: docs.0.doc.error + + - do: + indices.create: + index: foo-1 + - match: { acknowledged: true } + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO" + } + } + ], + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": null + } + } + } + } + }, + "index_template_substitutions": { + "foo_index_template": { + "index_patterns":[ + "foo*" + ], + "composed_of": ["settings_template", "mappings_template"] + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: [] } + - not_exists: docs.0.doc.error + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO" + } + } + ], + "component_template_substitutions": { + "mappings_template": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "boolean" + } + } + } + } + } + }, + "index_template_substitutions": { + "foo_index_template": { + "index_patterns":[ + "foo*" + ], + "composed_of": ["settings_template", "mappings_template"] + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: true } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline"] } + - not_exists: docs.0.doc.error + +--- +"Test ingest simulate with index template substitutions for data streams": + # In this test, we make sure that when the index template is a data stream template, simulate ingest works the same whether the data + # stream has been created or not -- either way, we expect it to use the template rather than the data stream / index mappings and settings. + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.index.template.substitutions"] + reason: "ingest simulate component template substitutions added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: mappings_template + body: + template: + mappings: + dynamic: strict + properties: + foo: + type: boolean + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo-pipeline" + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - mappings_template + - settings_template + + - do: + allowed_warnings: + - "index template [my-template-1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template-1] will take precedence during new index creation" + indices.put_index_template: + name: my-template-1 + body: + index_patterns: [simple-data-stream1] + composed_of: + - mappings_template + - settings_template + data_stream: {} + + # Here we replace my-template-1 with a substitute version that uses the settings_template_2 and mappings_template_2 templates defined in + # this request, and foo-pipeline-2 defined in this request. + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": false + } + } + ], + "pipeline_substitutions": { + "foo-pipeline-2": { + "processors": [ + { + "set": { + "field": "foo", + "value": "FOO" + } + } + ] + } + }, + "component_template_substitutions": { + "settings_template_2": { + "template": { + "settings": { + "index": { + "default_pipeline": "foo-pipeline-2" + } + } + } + }, + "mappings_template_2": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + } + }, + "index_template_substitutions": { + "my-template-1": { + "index_patterns": ["simple-data-stream1"], + "composed_of": ["settings_template_2", "mappings_template_2"], + "data_stream": {} + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } + - not_exists: docs.0.doc.error + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + cluster.health: + wait_for_status: yellow + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": false + } + } + ], + "pipeline_substitutions": { + "foo-pipeline-2": { + "processors": [ + { + "set": { + "field": "foo", + "value": "FOO" + } + } + ] + } + }, + "component_template_substitutions": { + "settings_template_2": { + "template": { + "settings": { + "index": { + "default_pipeline": "foo-pipeline-2" + } + } + } + }, + "mappings_template_2": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + } + }, + "index_template_substitutions": { + "my-template-1": { + "index_patterns": ["simple-data-stream1"], + "composed_of": ["settings_template_2", "mappings_template_2"], + "data_stream": {} + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } + - not_exists: docs.0.doc.error diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java index 91674b7ce9050..af99a0344e030 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java @@ -59,7 +59,7 @@ public void testMappingValidationIndexExists() { } """; indicesAdmin().create(new CreateIndexRequest(indexName).mapping(mapping)).actionGet(); - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -90,7 +90,7 @@ public void testMappingValidationIndexExists() { } @SuppressWarnings("unchecked") - public void testMappingValidationIndexExistsWithComponentTemplate() throws IOException { + public void testMappingValidationIndexExistsTemplateSubstitutions() throws IOException { /* * This test simulates a BulkRequest of two documents into an existing index. Then we make sure the index contains no documents, and * that the index's mapping in the cluster state has not been updated with the two new field. With the mapping from the template @@ -122,16 +122,19 @@ public void testMappingValidationIndexExistsWithComponentTemplate() throws IOExc .indexPatterns(List.of("my-index-*")) .componentTemplates(List.of("test-component-template")) .build(); - TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test"); + final String indexTemplateName = "test-index-template"; + TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request( + indexTemplateName + ); request.indexTemplate(composableIndexTemplate); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); String indexName = "my-index-1"; // First, run before the index is created: - assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName); + assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName, indexTemplateName); // Now, create the index and make sure the component template substitutions work the same: indicesAdmin().create(new CreateIndexRequest(indexName)).actionGet(); - assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName); + assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName, indexTemplateName); // Now make sure nothing was actually changed: indicesAdmin().refresh(new RefreshRequest(indexName)).actionGet(); SearchResponse searchResponse = client().search(new SearchRequest(indexName)).actionGet(); @@ -143,7 +146,7 @@ public void testMappingValidationIndexExistsWithComponentTemplate() throws IOExc assertThat(fields.size(), equalTo(1)); } - private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String indexName) { + private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String indexName, String indexTemplateName) { IndexRequest indexRequest1 = new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -156,7 +159,7 @@ private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String inde """, XContentType.JSON).id(randomUUID()); { // First we use the original component template, and expect a failure in the second document: - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(indexRequest1); bulkRequest.add(indexRequest2); BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); @@ -188,7 +191,42 @@ private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String inde ) ) ) - ) + ), + Map.of() + ); + bulkRequest.add(indexRequest1); + bulkRequest.add(indexRequest2); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[1].getResponse()).getException()); + } + + { + /* + * Now we substitute a "test-component-template-2" that defines both fields, and an index template that pulls it in, so we + * expect no exception: + */ + BulkRequest bulkRequest = new SimulateBulkRequest( + Map.of(), + Map.of( + "test-component-template-2", + Map.of( + "template", + Map.of( + "mappings", + Map.of( + "dynamic", + "strict", + "properties", + Map.of("foo1", Map.of("type", "text"), "foo3", Map.of("type", "text")) + ) + ) + ) + ), + Map.of(indexTemplateName, Map.of("index_patterns", List.of(indexName), "composed_of", List.of("test-component-template-2"))) ); bulkRequest.add(indexRequest1); bulkRequest.add(indexRequest2); @@ -207,7 +245,7 @@ public void testMappingValidationIndexDoesNotExistsNoTemplate() { * mapping-less "random-index-template" created by the parent class), so we expect no mapping validation failure. */ String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -254,7 +292,7 @@ public void testMappingValidationIndexDoesNotExistsV2Template() throws IOExcepti request.indexTemplate(composableIndexTemplate); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -286,7 +324,7 @@ public void testMappingValidationIndexDoesNotExistsV1Template() { indicesAdmin().putTemplate( new PutIndexTemplateRequest("test-template").patterns(List.of("my-index-*")).mapping("foo1", "type=integer") ).actionGet(); - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -340,7 +378,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); { // First, try with no @timestamp to make sure we're picking up data-stream-specific templates - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -366,7 +404,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti } { // Now with @timestamp - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "@timestamp": "2024-08-27", diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 2095ba47ee377..78fddad603cab 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -237,6 +237,7 @@ static TransportVersion def(int id) { public static final TransportVersion DATE_TIME_DOC_VALUES_LOCALES = def(8_761_00_0); public static final TransportVersion FAST_REFRESH_RCO = def(8_762_00_0); public static final TransportVersion TEXT_SIMILARITY_RERANKER_QUERY_REWRITE = def(8_763_00_0); + public static final TransportVersion SIMULATE_INDEX_TEMPLATES_SUBSTITUTIONS = def(8_764_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java index af1782ac1ade3..78e603fba9be0 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java @@ -15,11 +15,17 @@ import java.util.Set; import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS; +import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS; import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION; import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION_TEMPLATES; public class BulkFeatures implements FeatureSpecification { public Set getFeatures() { - return Set.of(SIMULATE_MAPPING_VALIDATION, SIMULATE_MAPPING_VALIDATION_TEMPLATES, SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS); + return Set.of( + SIMULATE_MAPPING_VALIDATION, + SIMULATE_MAPPING_VALIDATION_TEMPLATES, + SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS, + SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS + ); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index 558901f102299..f62b2f48fa2fd 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -506,6 +507,10 @@ public Map getComponentTemplateSubstitutions() throws return Map.of(); } + public Map getIndexTemplateSubstitutions() throws IOException { + return Map.of(); + } + record IncrementalState(Map shardLevelFailures, boolean indexingPressureAccounted) implements Writeable { static final IncrementalState EMPTY = new IncrementalState(Collections.emptyMap(), false); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java index 3cc7fa12733bf..6fa22151396df 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java @@ -11,6 +11,7 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; @@ -22,8 +23,8 @@ import java.util.Map; /** - * This extends BulkRequest with support for providing substitute pipeline definitions and component template definitions. In a user - * request, the substitutions will look something like this: + * This extends BulkRequest with support for providing substitute pipeline definitions, component template definitions, and index template + * substitutions. In a user request, the substitutions will look something like this: * * "pipeline_substitutions": { * "my-pipeline-1": { @@ -72,6 +73,16 @@ * } * } * } + * }, + * "index_template_substitutions": { + * "my-index-template-1": { + * "template": { + * "index_patterns": ["foo*", "bar*"] + * "composed_of": [ + * "component-template-1", + * "component-template-2" + * ] + * } * } * } * @@ -82,6 +93,7 @@ public class SimulateBulkRequest extends BulkRequest { private final Map> pipelineSubstitutions; private final Map> componentTemplateSubstitutions; + private final Map> indexTemplateSubstitutions; /** * @param pipelineSubstitutions The pipeline definitions that are to be used in place of any pre-existing pipeline definitions with @@ -89,14 +101,18 @@ public class SimulateBulkRequest extends BulkRequest { * parsed by XContentHelper.convertToMap(). * @param componentTemplateSubstitutions The component template definitions that are to be used in place of any pre-existing * component template definitions with the same name. + * @param indexTemplateSubstitutions The index template definitions that are to be used in place of any pre-existing + * index template definitions with the same name. */ public SimulateBulkRequest( @Nullable Map> pipelineSubstitutions, - @Nullable Map> componentTemplateSubstitutions + @Nullable Map> componentTemplateSubstitutions, + @Nullable Map> indexTemplateSubstitutions ) { super(); this.pipelineSubstitutions = pipelineSubstitutions; this.componentTemplateSubstitutions = componentTemplateSubstitutions; + this.indexTemplateSubstitutions = indexTemplateSubstitutions; } @SuppressWarnings("unchecked") @@ -108,6 +124,11 @@ public SimulateBulkRequest(StreamInput in) throws IOException { } else { componentTemplateSubstitutions = Map.of(); } + if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_INDEX_TEMPLATES_SUBSTITUTIONS)) { + this.indexTemplateSubstitutions = (Map>) in.readGenericValue(); + } else { + indexTemplateSubstitutions = Map.of(); + } } @Override @@ -117,6 +138,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_COMPONENT_TEMPLATES_SUBSTITUTIONS)) { out.writeGenericValue(componentTemplateSubstitutions); } + if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_INDEX_TEMPLATES_SUBSTITUTIONS)) { + out.writeGenericValue(indexTemplateSubstitutions); + } } public Map> getPipelineSubstitutions() { @@ -140,6 +164,18 @@ public Map getComponentTemplateSubstitutions() throws return result; } + @Override + public Map getIndexTemplateSubstitutions() throws IOException { + if (indexTemplateSubstitutions == null) { + return Map.of(); + } + Map result = new HashMap<>(indexTemplateSubstitutions.size()); + for (Map.Entry> rawEntry : indexTemplateSubstitutions.entrySet()) { + result.put(rawEntry.getKey(), convertRawTemplateToIndexTemplate(rawEntry.getValue())); + } + return result; + } + private static ComponentTemplate convertRawTemplateToComponentTemplate(Map rawTemplate) throws IOException { ComponentTemplate componentTemplate; try (var parser = XContentHelper.mapToXContentParser(XContentParserConfiguration.EMPTY, rawTemplate)) { @@ -148,9 +184,21 @@ private static ComponentTemplate convertRawTemplateToComponentTemplate(Map rawTemplate) throws IOException { + ComposableIndexTemplate indexTemplate; + try (var parser = XContentHelper.mapToXContentParser(XContentParserConfiguration.EMPTY, rawTemplate)) { + indexTemplate = ComposableIndexTemplate.parse(parser); + } + return indexTemplate; + } + @Override public BulkRequest shallowClone() { - BulkRequest bulkRequest = new SimulateBulkRequest(pipelineSubstitutions, componentTemplateSubstitutions); + BulkRequest bulkRequest = new SimulateBulkRequest( + pipelineSubstitutions, + componentTemplateSubstitutions, + indexTemplateSubstitutions + ); bulkRequest.setRefreshPolicy(getRefreshPolicy()); bulkRequest.waitForActiveShards(waitForActiveShards()); bulkRequest.timeout(timeout()); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java index 8c6565e52daa7..111e4d72c57c6 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; @@ -183,7 +184,9 @@ private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor exec boolean hasIndexRequestsWithPipelines = false; final Metadata metadata; Map componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); - if (bulkRequest.isSimulated() && componentTemplateSubstitutions.isEmpty() == false) { + Map indexTemplateSubstitutions = bulkRequest.getIndexTemplateSubstitutions(); + if (bulkRequest.isSimulated() + && (componentTemplateSubstitutions.isEmpty() == false || indexTemplateSubstitutions.isEmpty() == false)) { /* * If this is a simulated request, and there are template substitutions, then we want to create and use a new metadata that has * those templates. That is, we want to add the new templates (which will replace any that already existed with the same name), @@ -197,6 +200,12 @@ private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor exec updatedComponentTemplates.putAll(componentTemplateSubstitutions); simulatedMetadataBuilder.componentTemplates(updatedComponentTemplates); } + if (indexTemplateSubstitutions.isEmpty() == false) { + Map updatedIndexTemplates = new HashMap<>(); + updatedIndexTemplates.putAll(clusterService.state().metadata().templatesV2()); + updatedIndexTemplates.putAll(indexTemplateSubstitutions); + simulatedMetadataBuilder.indexTemplates(updatedIndexTemplates); + } /* * We now remove the index from the simulated metadata to force the templates to be used. Note that simulated requests are * always index requests -- no other type of request is supported. diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 713116c4cf98e..d7c555879c00f 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; @@ -73,6 +74,7 @@ public class TransportSimulateBulkAction extends TransportAbstractBulkAction { public static final NodeFeature SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS = new NodeFeature( "simulate.component.template.substitutions" ); + public static final NodeFeature SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS = new NodeFeature("simulate.index.template.substitutions"); private final IndicesService indicesService; private final NamedXContentRegistry xContentRegistry; private final Set indexSettingProviders; @@ -119,11 +121,12 @@ protected void doInternalExecute( : "TransportSimulateBulkAction should only ever be called with a SimulateBulkRequest but got a " + bulkRequest.getClass(); final AtomicArray responses = new AtomicArray<>(bulkRequest.requests.size()); Map componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); + Map indexTemplateSubstitutions = bulkRequest.getIndexTemplateSubstitutions(); for (int i = 0; i < bulkRequest.requests.size(); i++) { DocWriteRequest docRequest = bulkRequest.requests.get(i); assert docRequest instanceof IndexRequest : "TransportSimulateBulkAction should only ever be called with IndexRequests"; IndexRequest request = (IndexRequest) docRequest; - Exception mappingValidationException = validateMappings(componentTemplateSubstitutions, request); + Exception mappingValidationException = validateMappings(componentTemplateSubstitutions, indexTemplateSubstitutions, request); responses.set( i, BulkItemResponse.success( @@ -153,7 +156,11 @@ protected void doInternalExecute( * @param request The IndexRequest whose source will be validated against the mapping (if it exists) of its index * @return a mapping exception if the source does not match the mappings, otherwise null */ - private Exception validateMappings(Map componentTemplateSubstitutions, IndexRequest request) { + private Exception validateMappings( + Map componentTemplateSubstitutions, + Map indexTemplateSubstitutions, + IndexRequest request + ) { final SourceToParse sourceToParse = new SourceToParse( request.id(), request.source(), @@ -167,7 +174,7 @@ private Exception validateMappings(Map componentTempl Exception mappingValidationException = null; IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(request.index()); try { - if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty()) { + if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty() && indexTemplateSubstitutions.isEmpty()) { /* * In this case the index exists and we don't have any component template overrides. So we can just use withTempIndexService * to do the mapping validation, using all the existing logic for validation. @@ -222,6 +229,12 @@ private Exception validateMappings(Map componentTempl updatedComponentTemplates.putAll(componentTemplateSubstitutions); simulatedMetadata.componentTemplates(updatedComponentTemplates); } + if (indexTemplateSubstitutions.isEmpty() == false) { + Map updatedIndexTemplates = new HashMap<>(); + updatedIndexTemplates.putAll(state.metadata().templatesV2()); + updatedIndexTemplates.putAll(indexTemplateSubstitutions); + simulatedMetadata.indexTemplates(updatedIndexTemplates); + } ClusterState simulatedState = simulatedClusterStateBuilder.metadata(simulatedMetadata).build(); String matchingTemplate = findV2Template(simulatedState.metadata(), request.index(), false); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index abeb3279b7b50..2a2cf6743a877 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1421,44 +1421,24 @@ public static Settings resolveSettings(final List templat * Resolve the given v2 template into a collected {@link Settings} object */ public static Settings resolveSettings(final Metadata metadata, final String templateName) { - return resolveSettings(metadata, templateName, Map.of()); - } - - public static Settings resolveSettings( - final Metadata metadata, - final String templateName, - Map templateSubstitutions - ) { final ComposableIndexTemplate template = metadata.templatesV2().get(templateName); assert template != null : "attempted to resolve settings for a template [" + templateName + "] that did not exist in the cluster state"; if (template == null) { return Settings.EMPTY; } - return resolveSettings(template, metadata.componentTemplates(), templateSubstitutions); + return resolveSettings(template, metadata.componentTemplates()); } /** * Resolve the provided v2 template and component templates into a collected {@link Settings} object */ public static Settings resolveSettings(ComposableIndexTemplate template, Map componentTemplates) { - return resolveSettings(template, componentTemplates, Map.of()); - } - - public static Settings resolveSettings( - ComposableIndexTemplate template, - Map componentTemplates, - Map templateSubstitutions - ) { Objects.requireNonNull(template, "attempted to resolve settings for a null template"); Objects.requireNonNull(componentTemplates, "attempted to resolve settings with null component templates"); - Map combinedComponentTemplates = new HashMap<>(); - combinedComponentTemplates.putAll(componentTemplates); - // We want any substitutions to take precedence: - combinedComponentTemplates.putAll(templateSubstitutions); List componentSettings = template.composedOf() .stream() - .map(combinedComponentTemplates::get) + .map(componentTemplates::get) .filter(Objects::nonNull) .map(ComponentTemplate::template) .map(Template::settings) diff --git a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java index 6de15b0046f1b..680860332fe74 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java @@ -76,7 +76,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC Map sourceMap = XContentHelper.convertToMap(sourceTuple.v2(), false, sourceTuple.v1()).v2(); SimulateBulkRequest bulkRequest = new SimulateBulkRequest( (Map>) sourceMap.remove("pipeline_substitutions"), - (Map>) sourceMap.remove("component_template_substitutions") + (Map>) sourceMap.remove("component_template_substitutions"), + (Map>) sourceMap.remove("index_template_substitutions") ); BytesReference transformedData = convertToBulkRequestXContentBytes(sourceMap); bulkRequest.add( diff --git a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java index b6b1770e2ed5c..c94e4e46c9ee3 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.ESTestCase; @@ -27,18 +28,27 @@ public class SimulateBulkRequestTests extends ESTestCase { public void testSerialization() throws Exception { - testSerialization(getTestPipelineSubstitutions(), getTestTemplateSubstitutions()); - testSerialization(getTestPipelineSubstitutions(), null); - testSerialization(null, getTestTemplateSubstitutions()); - testSerialization(null, null); - testSerialization(Map.of(), Map.of()); + testSerialization(getTestPipelineSubstitutions(), getTestComponentTemplateSubstitutions(), getTestIndexTemplateSubstitutions()); + testSerialization(getTestPipelineSubstitutions(), null, null); + testSerialization(getTestPipelineSubstitutions(), getTestComponentTemplateSubstitutions(), null); + testSerialization(getTestPipelineSubstitutions(), null, getTestIndexTemplateSubstitutions()); + testSerialization(null, getTestComponentTemplateSubstitutions(), getTestIndexTemplateSubstitutions()); + testSerialization(null, getTestComponentTemplateSubstitutions(), null); + testSerialization(null, null, getTestIndexTemplateSubstitutions()); + testSerialization(null, null, null); + testSerialization(Map.of(), Map.of(), Map.of()); } private void testSerialization( Map> pipelineSubstitutions, - Map> templateSubstitutions + Map> componentTemplateSubstitutions, + Map> indexTemplateSubstitutions ) throws IOException { - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, templateSubstitutions); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest( + pipelineSubstitutions, + componentTemplateSubstitutions, + indexTemplateSubstitutions + ); /* * Note: SimulateBulkRequest does not implement equals or hashCode, so we can't test serialization in the usual way for a * Writable @@ -49,7 +59,7 @@ private void testSerialization( @SuppressWarnings({ "unchecked", "rawtypes" }) public void testGetComponentTemplateSubstitutions() throws IOException { - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); assertThat(simulateBulkRequest.getComponentTemplateSubstitutions(), equalTo(Map.of())); String substituteComponentTemplatesString = """ { @@ -83,7 +93,7 @@ public void testGetComponentTemplateSubstitutions() throws IOException { XContentType.JSON ).v2(); Map> substituteComponentTemplates = (Map>) tempMap; - simulateBulkRequest = new SimulateBulkRequest(Map.of(), substituteComponentTemplates); + simulateBulkRequest = new SimulateBulkRequest(Map.of(), substituteComponentTemplates, Map.of()); Map componentTemplateSubstitutions = simulateBulkRequest.getComponentTemplateSubstitutions(); assertThat(componentTemplateSubstitutions.size(), equalTo(2)); assertThat( @@ -107,8 +117,70 @@ public void testGetComponentTemplateSubstitutions() throws IOException { ); } + public void testGetIndexTemplateSubstitutions() throws IOException { + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); + assertThat(simulateBulkRequest.getIndexTemplateSubstitutions(), equalTo(Map.of())); + String substituteIndexTemplatesString = """ + { + "foo_template": { + "index_patterns": ["foo*"], + "composed_of": ["foo_mapping_template", "foo_settings_template"], + "template": { + "mappings": { + "dynamic": "true", + "properties": { + "foo": { + "type": "keyword" + } + } + }, + "settings": { + "index": { + "default_pipeline": "foo-pipeline" + } + } + } + }, + "bar_template": { + "index_patterns": ["bar*"], + "composed_of": ["bar_mapping_template", "bar_settings_template"] + } + } + """; + + @SuppressWarnings("unchecked") + Map> substituteIndexTemplates = (Map>) (Map) XContentHelper.convertToMap( + new BytesArray(substituteIndexTemplatesString.getBytes(StandardCharsets.UTF_8)), + randomBoolean(), + XContentType.JSON + ).v2(); + simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), substituteIndexTemplates); + Map indexTemplateSubstitutions = simulateBulkRequest.getIndexTemplateSubstitutions(); + assertThat(indexTemplateSubstitutions.size(), equalTo(2)); + assertThat( + XContentHelper.convertToMap( + XContentHelper.toXContent(indexTemplateSubstitutions.get("foo_template").template(), XContentType.JSON, randomBoolean()), + randomBoolean(), + XContentType.JSON + ).v2(), + equalTo(substituteIndexTemplates.get("foo_template").get("template")) + ); + + assertThat(indexTemplateSubstitutions.get("foo_template").template().settings().size(), equalTo(1)); + assertThat( + indexTemplateSubstitutions.get("foo_template").template().settings().get("index.default_pipeline"), + equalTo("foo-pipeline") + ); + assertNull(indexTemplateSubstitutions.get("bar_template").template()); + assertNull(indexTemplateSubstitutions.get("bar_template").template()); + } + public void testShallowClone() throws IOException { - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(getTestPipelineSubstitutions(), getTestTemplateSubstitutions()); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest( + getTestPipelineSubstitutions(), + getTestComponentTemplateSubstitutions(), + getTestIndexTemplateSubstitutions() + ); simulateBulkRequest.setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values())); simulateBulkRequest.waitForActiveShards(randomIntBetween(1, 10)); simulateBulkRequest.timeout(randomTimeValue()); @@ -144,7 +216,7 @@ private static Map> getTestPipelineSubstitutions() { ); } - private static Map> getTestTemplateSubstitutions() { + private static Map> getTestComponentTemplateSubstitutions() { return Map.of( "template1", Map.of( @@ -155,4 +227,25 @@ private static Map> getTestTemplateSubstitutions() { Map.of("template", Map.of("mappings", Map.of(), "settings", Map.of())) ); } + + private static Map> getTestIndexTemplateSubstitutions() { + return Map.of( + "template1", + Map.of( + "template", + Map.of( + "index_patterns", + List.of("foo*", "bar*"), + "composed_of", + List.of("template_1", "template_2"), + "mappings", + Map.of("_source", Map.of("enabled", false), "properties", Map.of()), + "settings", + Map.of() + ) + ), + "template2", + Map.of("template", Map.of("index_patterns", List.of("foo*", "bar*"), "mappings", Map.of(), "settings", Map.of())) + ); + } } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java index f4e53912d09a7..71bc31334920e 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java @@ -135,7 +135,7 @@ public void tearDown() throws Exception { public void testIndexData() throws IOException { Task task = mock(Task.class); // unused - BulkRequest bulkRequest = new SimulateBulkRequest(null, null); + BulkRequest bulkRequest = new SimulateBulkRequest(null, null, null); int bulkItemCount = randomIntBetween(0, 200); for (int i = 0; i < bulkItemCount; i++) { Map source = Map.of(randomAlphaOfLength(10), randomAlphaOfLength(5)); @@ -218,7 +218,7 @@ public void testIndexDataWithValidation() throws IOException { * (7) An indexing request to a nonexistent index that matches no templates */ Task task = mock(Task.class); // unused - BulkRequest bulkRequest = new SimulateBulkRequest(null, null); + BulkRequest bulkRequest = new SimulateBulkRequest(null, null, null); int bulkItemCount = randomIntBetween(0, 200); Map indicesMap = new HashMap<>(); Map v1Templates = new HashMap<>(); diff --git a/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java index 332a04e40e43d..3b3f5bdc747b5 100644 --- a/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java @@ -65,7 +65,7 @@ public void testGetPipeline() { ingestService.innerUpdatePipelines(ingestMetadata); { // First we make sure that if there are no substitutions that we get our original pipeline back: - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(null, null); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(null, null, null); SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest); Pipeline pipeline = simulateIngestService.getPipeline("pipeline1"); assertThat(pipeline.getProcessors(), contains(transformedMatch(Processor::getType, equalTo("processor1")))); @@ -83,7 +83,7 @@ public void testGetPipeline() { ); pipelineSubstitutions.put("pipeline2", newHashMap("processors", List.of(newHashMap("processor3", Collections.emptyMap())))); - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, null); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, null, null); SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest); Pipeline pipeline1 = simulateIngestService.getPipeline("pipeline1"); assertThat( @@ -103,7 +103,7 @@ public void testGetPipeline() { */ Map> pipelineSubstitutions = new HashMap<>(); pipelineSubstitutions.put("pipeline2", newHashMap("processors", List.of(newHashMap("processor3", Collections.emptyMap())))); - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, null); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, null, null); SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest); Pipeline pipeline1 = simulateIngestService.getPipeline("pipeline1"); assertThat(pipeline1.getProcessors(), contains(transformedMatch(Processor::getType, equalTo("processor1")))); From e676f6bb588c84291a5ee59c2d7147dceabc1642 Mon Sep 17 00:00:00 2001 From: Panagiotis Bailis Date: Wed, 9 Oct 2024 08:48:13 +0300 Subject: [PATCH 06/10] Updating RRF license to Enterprise (#113100) --- .../java/org/elasticsearch/xpack/rank/rrf/RRFRankPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankPlugin.java b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankPlugin.java index ece08d1a3d558..9404d863f1d28 100644 --- a/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankPlugin.java +++ b/x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/rrf/RRFRankPlugin.java @@ -25,7 +25,7 @@ public class RRFRankPlugin extends Plugin implements SearchPlugin { public static final LicensedFeature.Momentary RANK_RRF_FEATURE = LicensedFeature.momentary( null, "rank-rrf", - License.OperationMode.PLATINUM + License.OperationMode.ENTERPRISE ); public static final String NAME = "rrf"; From 276f3b8836c4b82bdc174d68e9fabcf83abc4f0a Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 9 Oct 2024 06:59:38 +0100 Subject: [PATCH 07/10] Avoid leaking blackholed register ops in tests (#114287) Today when we reboot a node in a test case derived from `AbstractCoordinatorTestCase` we lose the contents of `blackholedRegisterOperations`, but it's important that these operations _eventually_ run. With this commit we copy these operations over into the new node. --- .../cluster/coordination/AbstractCoordinatorTestCase.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java index 262dba80caa24..ddfa61b53a0af 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/AbstractCoordinatorTestCase.java @@ -1250,7 +1250,7 @@ ClusterNode restartedNode( .roles(localNode.isMasterNode() && DiscoveryNode.isMasterNode(settings) ? ALL_ROLES_EXCEPT_VOTING_ONLY : emptySet()) .build(); try { - return new ClusterNode( + final var restartedNode = new ClusterNode( nodeIndex, newLocalNode, (node, threadPool) -> createPersistedStateFromExistingState( @@ -1263,6 +1263,8 @@ ClusterNode restartedNode( settings, nodeHealthService ); + restartedNode.blackholedRegisterOperations.addAll(blackholedRegisterOperations); + return restartedNode; } finally { clearableRecycler.clear(); } From c41f1e7bb5a10611646ec392f117776e6579c1b6 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 9 Oct 2024 17:05:11 +1100 Subject: [PATCH 08/10] Mute org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT testProfileOrdinalsGroupingOperator {SYNC} #114380 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 5ab607fab2090..0ee6b21cedca3 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -371,6 +371,9 @@ tests: - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT method: test {yaml=cluster.stats/30_ccs_stats/cross-cluster search stats search} issue: https://github.com/elastic/elasticsearch/issues/114371 +- class: org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT + method: testProfileOrdinalsGroupingOperator {SYNC} + issue: https://github.com/elastic/elasticsearch/issues/114380 # Examples: # From f79705d9b6c528a96cb5badaa251377525cbd628 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Wed, 9 Oct 2024 10:22:39 +0300 Subject: [PATCH 09/10] Skip storing ignored source for single-element leaf arrays (#113937) * Minimize storing array source * restrict to fields * revert changes for `addIgnoredFieldFromContext` * fix test * spotless * count nulls --- .../index/mapper/DocumentParser.java | 39 ++++++++++++++----- .../index/mapper/DocumentParserContext.java | 6 +++ .../mapper/IgnoredSourceFieldMapperTests.java | 32 +++++++++++++++ .../index/mapper/MapperTestCase.java | 6 ++- 4 files changed, 73 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 19bd4f9980baf..0ff754d953934 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -799,17 +799,30 @@ private static void parseNonDynamicArray( String fullPath = context.path().pathAsText(arrayFieldName); // Check if we need to record the array source. This only applies to synthetic source. + boolean canRemoveSingleLeafElement = false; if (context.canAddIgnoredField()) { - boolean objectRequiresStoringSource = mapper instanceof ObjectMapper objectMapper - && (getSourceKeepMode(context, objectMapper.sourceKeepMode()) == Mapper.SourceKeepMode.ALL - || (getSourceKeepMode(context, objectMapper.sourceKeepMode()) == Mapper.SourceKeepMode.ARRAYS - && objectMapper instanceof NestedObjectMapper == false)); - boolean fieldWithFallbackSyntheticSource = mapper instanceof FieldMapper fieldMapper - && fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK; - boolean fieldWithStoredArraySource = mapper instanceof FieldMapper fieldMapper - && getSourceKeepMode(context, fieldMapper.sourceKeepMode()) != Mapper.SourceKeepMode.NONE; + Mapper.SourceKeepMode mode = Mapper.SourceKeepMode.NONE; + boolean objectWithFallbackSyntheticSource = false; + if (mapper instanceof ObjectMapper objectMapper) { + mode = getSourceKeepMode(context, objectMapper.sourceKeepMode()); + objectWithFallbackSyntheticSource = (mode == Mapper.SourceKeepMode.ALL + || (mode == Mapper.SourceKeepMode.ARRAYS && objectMapper instanceof NestedObjectMapper == false)); + } + boolean fieldWithFallbackSyntheticSource = false; + boolean fieldWithStoredArraySource = false; + if (mapper instanceof FieldMapper fieldMapper) { + mode = getSourceKeepMode(context, fieldMapper.sourceKeepMode()); + fieldWithFallbackSyntheticSource = fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK; + fieldWithStoredArraySource = mode != Mapper.SourceKeepMode.NONE; + } boolean copyToFieldHasValuesInDocument = context.isWithinCopyTo() == false && context.isCopyToDestinationField(fullPath); - if (objectRequiresStoringSource + + canRemoveSingleLeafElement = mapper instanceof FieldMapper + && mode == Mapper.SourceKeepMode.ARRAYS + && fieldWithFallbackSyntheticSource == false + && copyToFieldHasValuesInDocument == false; + + if (objectWithFallbackSyntheticSource || fieldWithFallbackSyntheticSource || fieldWithStoredArraySource || copyToFieldHasValuesInDocument) { @@ -829,20 +842,28 @@ private static void parseNonDynamicArray( XContentParser parser = context.parser(); XContentParser.Token token; + int elements = 0; while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if (token == XContentParser.Token.START_OBJECT) { + elements = Integer.MAX_VALUE; parseObject(context, lastFieldName); } else if (token == XContentParser.Token.START_ARRAY) { + elements = Integer.MAX_VALUE; parseArray(context, lastFieldName); } else if (token == XContentParser.Token.VALUE_NULL) { + elements++; parseNullValue(context, lastFieldName); } else if (token == null) { throwEOFOnParseArray(arrayFieldName, context); } else { assert token.isValue(); + elements++; parseValue(context, lastFieldName); } } + if (elements <= 1 && canRemoveSingleLeafElement) { + context.removeLastIgnoredField(fullPath); + } postProcessDynamicArrayMapping(context, lastFieldName); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index eebe95e260dcf..ac236e5a7e5fd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -296,6 +296,12 @@ public final void addIgnoredField(IgnoredSourceFieldMapper.NameValue values) { } } + final void removeLastIgnoredField(String name) { + if (ignoredFieldValues.isEmpty() == false && ignoredFieldValues.getLast().name().equals(name)) { + ignoredFieldValues.removeLast(); + } + } + /** * Return the collection of values for fields that have been ignored so far. */ diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 8c65424fb8560..205ff08c397b2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -532,6 +532,38 @@ public void testIndexStoredArraySourceRootValueArrayDisabled() throws IOExceptio {"bool_value":true,"int_value":[10,20,30]}""", syntheticSource); } + public void testIndexStoredArraySourceSingleLeafElement() throws IOException { + DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { + b.startObject("int_value").field("type", "integer").endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> b.array("int_value", new int[] { 10 })); + assertEquals("{\"int_value\":10}", syntheticSource); + ParsedDocument doc = documentMapper.parse(source(syntheticSource)); + assertNull(doc.rootDoc().getField("_ignored_source")); + } + + public void testIndexStoredArraySourceSingleLeafElementAndNull() throws IOException { + DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { + b.startObject("value").field("type", "keyword").endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> b.array("value", new String[] { "foo", null })); + assertEquals("{\"value\":[\"foo\",null]}", syntheticSource); + } + + public void testIndexStoredArraySourceSingleObjectElement() throws IOException { + DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> { + b.startObject("path").startObject("properties"); + { + b.startObject("int_value").field("type", "integer").endObject(); + } + b.endObject().endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startArray("path").startObject().field("int_value", 10).endObject().endArray(); + }); + assertEquals("{\"path\":[{\"int_value\":10}]}", syntheticSource); + } + public void testFieldStoredArraySourceRootValueArray() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("int_value").field("type", "integer").field(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, "arrays").endObject(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index ca26779f3376d..eef7fc4e50080 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1580,7 +1580,11 @@ public void testSyntheticSourceKeepArrays() throws IOException { buildInput.accept(builder); builder.endObject(); String expected = Strings.toString(builder); - assertThat(syntheticSource(mapperAll, buildInput), equalTo(expected)); + String actual = syntheticSource(mapperAll, buildInput); + // Check for single-element array, the array source is not stored in this case. + if (expected.replace("[", "").replace("]", "").equals(actual) == false) { + assertThat(actual, equalTo(expected)); + } } @Override From b2903520d87e70c780f8d36348666da5bc5b35c5 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 9 Oct 2024 18:47:41 +1100 Subject: [PATCH 10/10] Mute org.elasticsearch.xpack.inference.services.cohere.CohereServiceTests testInfer_StreamRequest #114385 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 0ee6b21cedca3..ef27eeeffc14a 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -374,6 +374,9 @@ tests: - class: org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT method: testProfileOrdinalsGroupingOperator {SYNC} issue: https://github.com/elastic/elasticsearch/issues/114380 +- class: org.elasticsearch.xpack.inference.services.cohere.CohereServiceTests + method: testInfer_StreamRequest + issue: https://github.com/elastic/elasticsearch/issues/114385 # Examples: #