Merge remote-tracking branch 'origin/8.x' into backport/8.x/pr-113194

carlosdelest · Nov 14, 2024 · bf4d992 · bf4d992
2 parents 93ae064 + e51dc09
commit bf4d992
Show file tree

Hide file tree

Showing 79 changed files with 1,201 additions and 308 deletions.
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java
@@ -10,7 +10,6 @@
 package org.elasticsearch.benchmark.index.mapper;
 
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.util.Accountable;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -28,7 +27,6 @@
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
-import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.script.Script;
@@ -56,13 +54,7 @@ public static MapperService create(String mappings) {
         MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
 
         SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
-        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() {
-            @Override
-            public void onCache(ShardId shardId, Accountable accountable) {}
-
-            @Override
-            public void onRemoval(ShardId shardId, Accountable accountable) {}
-        });
+        BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP);
         MapperService mapperService = new MapperService(
             () -> TransportVersion.current(),
             indexSettings,

diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
@@ -14,7 +14,7 @@ log4j             = 2.19.0
 slf4j             = 2.0.6
 ecsLogging        = 1.2.0
 jna               = 5.12.1
-netty             = 4.1.109.Final
+netty             = 4.1.115.Final
 commons_lang3     = 3.9
 google_oauth_client = 1.34.1
 

diff --git a/docs/changelog/115687.yaml b/docs/changelog/115687.yaml
@@ -0,0 +1,5 @@
+pr: 115687
+summary: Add default ILM policies and switch to ILM for apm-data plugin
+area: Data streams
+type: feature
+issues: []
diff --git a/docs/changelog/115814.yaml b/docs/changelog/115814.yaml
@@ -0,0 +1,6 @@
+pr: 115814
+summary: "[ES|QL] Implicit casting string literal to intervals"
+area: ES|QL
+type: enhancement
+issues:
+ - 115352
diff --git a/docs/reference/esql/implicit-casting.asciidoc b/docs/reference/esql/implicit-casting.asciidoc
@@ -5,7 +5,7 @@
 <titleabbrev>Implicit casting</titleabbrev>
 ++++
 
-Often users will input `datetime`, `ip`, `version`, or geospatial objects as simple strings in their queries for use in predicates, functions, or expressions. {esql} provides <<esql-type-conversion-functions, type conversion functions>> to explicitly convert these strings into the desired data types.
+Often users will input `date`, `ip`, `version`, `date_period` or `time_duration` as simple strings in their queries for use in predicates, functions, or expressions. {esql} provides <<esql-type-conversion-functions, type conversion functions>> to explicitly convert these strings into the desired data types.
 
 Without implicit casting users must explicitly code these `to_X` functions in their queries, when string literals don't match the target data types they are assigned or compared to. Here is an example of using `to_datetime` to explicitly perform a data type conversion.
 
@@ -18,7 +18,7 @@ FROM employees
 | LIMIT 1
 ----
 
-Implicit casting improves usability, by automatically converting string literals to the target data type. This is most useful when the target data type is `datetime`, `ip`, `version` or a geo spatial. It is natural to specify these as a string in queries.
+Implicit casting improves usability, by automatically converting string literals to the target data type. This is most useful when the target data type is `date`, `ip`, `version`, `date_period` or `time_duration`. It is natural to specify these as a string in queries.
 
 The first query can be coded without calling the `to_datetime` function, as follows:
 
@@ -38,16 +38,28 @@ The following table details which {esql} operations support implicit casting for
 
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
-||ScalarFunction|BinaryComparison|ArithmeticOperation|InListPredicate|AggregateFunction
-|DATETIME|Y|Y|Y|Y|N
-|DOUBLE|Y|N|N|N|N
-|LONG|Y|N|N|N|N
-|INTEGER|Y|N|N|N|N
-|IP|Y|Y|Y|Y|N
-|VERSION|Y|Y|Y|Y|N
-|GEO_POINT|Y|N|N|N|N
-|GEO_SHAPE|Y|N|N|N|N
-|CARTESIAN_POINT|Y|N|N|N|N
-|CARTESIAN_SHAPE|Y|N|N|N|N
-|BOOLEAN|Y|Y|Y|Y|N
+||ScalarFunction*|Operator*|<<esql-group-functions, GroupingFunction>>|<<esql-agg-functions, AggregateFunction>>
+|DATE|Y|Y|Y|N
+|IP|Y|Y|Y|N
+|VERSION|Y|Y|Y|N
+|BOOLEAN|Y|Y|Y|N
+|DATE_PERIOD/TIME_DURATION|Y|N|Y|N
 |===
+
+ScalarFunction* includes:
+
+<<esql-conditional-functions-and-expressions, Conditional Functions and Expressions>>
+
+<<esql-date-time-functions, Date and Time Functions>>
+
+<<esql-ip-functions, IP Functions>>
+
+
+Operator* includes:
+
+<<esql-binary-operators, Binary Operators>>
+
+<<esql-unary-operators, Unary Operator>>
+
+<<esql-in-operator, IN>>
+
diff --git a/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc b/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc
@@ -106,6 +106,8 @@ As a long-term solution, we recommend you do one of the following best suited
 to your use case: 
 
 * add nodes to the affected <<data-tiers,data tiers>>
++
+TIP: You should enable <<xpack-autoscaling,autoscaling>> for clusters deployed using our {ess}, {ece}, and {eck} platforms.
 
 * upgrade existing nodes to increase disk space
 +

diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
@@ -1361,109 +1361,109 @@
             <sha256 value="a3ebec96768ee4a2d3db44597e84cea2d0bdd68ca04822397980ea9f67075a86" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-buffer" version="4.1.109.Final">
-         <artifact name="netty-buffer-4.1.109.Final.jar">
-            <sha256 value="ed49956c1e5d99534af2e9e9ecb80f4d3390c631c6cedd2a94d23aedcc911bb9" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-buffer" version="4.1.115.Final">
+         <artifact name="netty-buffer-4.1.115.Final.jar">
+            <sha256 value="4a7b331d3770c566ab70eb02a0d1feed63b95cf6e4d68c8fe778c4c9de2d116d" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-buffer" version="4.1.42.Final">
          <artifact name="netty-buffer-4.1.42.Final.jar">
             <sha256 value="7b0171a4e8bcd573e08d9f2bba053c67b557ab5012106a5982ccbae5743814c0" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-codec" version="4.1.109.Final">
-         <artifact name="netty-codec-4.1.109.Final.jar">
-            <sha256 value="2df943cd3ed24f86d61d442314e8a9a4c06b85a9a7aa99cec8c6520e77ff3f1c" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-codec" version="4.1.115.Final">
+         <artifact name="netty-codec-4.1.115.Final.jar">
+            <sha256 value="cd189afb70ec6eacfcdfdd3a5f472b4e705a5c91d5bd3ef0386421f2ae15ec77" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-codec" version="4.1.42.Final">
          <artifact name="netty-codec-4.1.42.Final.jar">
             <sha256 value="e96ced697fb7df589da7c20c995e01f75a9cb246be242bbc4cd3b4af424ff189" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-codec-dns" version="4.1.109.Final">
-         <artifact name="netty-codec-dns-4.1.109.Final.jar">
-            <sha256 value="a67a84c0a46094815e2044ac1314d8d81089d219ec8e8c732f9a12ba4cd451bd" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-codec-dns" version="4.1.115.Final">
+         <artifact name="netty-codec-dns-4.1.115.Final.jar">
+            <sha256 value="23dd6806bcc326855f13e69838c6411d0490e6b1aeb12e217a19a3dd6ad3f10d" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-codec-http" version="4.1.109.Final">
-         <artifact name="netty-codec-http-4.1.109.Final.jar">
-            <sha256 value="fdf056b08e156797d8f6caa727af0370dea8265b1eff8fc6f9e739c5c1f093d1" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-codec-http" version="4.1.115.Final">
+         <artifact name="netty-codec-http-4.1.115.Final.jar">
+            <sha256 value="e6dbe971c59373bbae9802021c63b9bc1d8800fead382863d67e79e79b023166" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-codec-http2" version="4.1.109.Final">
-         <artifact name="netty-codec-http2-4.1.109.Final.jar">
-            <sha256 value="a75498995087968b4582aefe42cfa8ca257759f32279edbec3c874e54acf633c" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-codec-http2" version="4.1.115.Final">
+         <artifact name="netty-codec-http2-4.1.115.Final.jar">
+            <sha256 value="cbed9829a5d582e91e314e209edce9a0c2eb369f23bb4fb74a5bc8b7990222c2" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-codec-socks" version="4.1.109.Final">
-         <artifact name="netty-codec-socks-4.1.109.Final.jar">
-            <sha256 value="a8b2c52807e4071f8fdd6239b8fd0c4d2cd282ebf08406f8d431d3229b69418e" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-codec-socks" version="4.1.115.Final">
+         <artifact name="netty-codec-socks-4.1.115.Final.jar">
+            <sha256 value="e9b1cc744dc6195894450b1fd4d271a821ab167fe21ae3c459b27cdadc70e81f" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-common" version="4.1.109.Final">
-         <artifact name="netty-common-4.1.109.Final.jar">
-            <sha256 value="33bf51077021de2b9d80b9ec64a4fae68f22e6ec56f52431441eaf6dd79cce71" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-common" version="4.1.115.Final">
+         <artifact name="netty-common-4.1.115.Final.jar">
+            <sha256 value="39f1b5a2aaa4eab5d036dfd0486e35a4276df412e092d36b2d88b494705a134d" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-common" version="4.1.42.Final">
          <artifact name="netty-common-4.1.42.Final.jar">
             <sha256 value="3d0a918d78292eeca02a7bb2188daa4e5053b6e29b71e6308309033e121242b5" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-handler" version="4.1.109.Final">
-         <artifact name="netty-handler-4.1.109.Final.jar">
-            <sha256 value="1161e2ed8e1f037124a8ddfe3d4184ebaa7be17aea27e7fca87f510f17898ea5" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-handler" version="4.1.115.Final">
+         <artifact name="netty-handler-4.1.115.Final.jar">
+            <sha256 value="5972028cc863b74927ce0d11fb8d58f65da2560bef5602fe8ce8903bd306ca07" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-handler" version="4.1.42.Final">
          <artifact name="netty-handler-4.1.42.Final.jar">
             <sha256 value="11eda86500c33b9d386719b5419f513fd9c097d13894f25dd0c75b610d636e03" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-handler-proxy" version="4.1.109.Final">
-         <artifact name="netty-handler-proxy-4.1.109.Final.jar">
-            <sha256 value="42e34c84423f074a4db4576e6d1ddabf56eb26705213e5f9b6ef537ce36bcd0b" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-handler-proxy" version="4.1.115.Final">
+         <artifact name="netty-handler-proxy-4.1.115.Final.jar">
+            <sha256 value="807e67cfb17136927d11db42df62031169d1fa0883e13f254906994c84ffbe87" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-resolver" version="4.1.109.Final">
-         <artifact name="netty-resolver-4.1.109.Final.jar">
-            <sha256 value="e168920249ea8d65076b0847e64464b5132843f1f1b2ff514e45fa255b64300b" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-resolver" version="4.1.115.Final">
+         <artifact name="netty-resolver-4.1.115.Final.jar">
+            <sha256 value="7b3455d14f59828765a00573bc3967dc59379e874bd62a67eb1926d6512109d1" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-resolver" version="4.1.42.Final">
          <artifact name="netty-resolver-4.1.42.Final.jar">
             <sha256 value="89768242b6b7cce9bd9f5945ad21d1b4bae515c6b1bf03a8af5d1899779cebc9" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-resolver-dns" version="4.1.109.Final">
-         <artifact name="netty-resolver-dns-4.1.109.Final.jar">
-            <sha256 value="7e354a85f8147744aeb9d3e1e64ace18f6432a25c09f8bde57383959a842c967" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-resolver-dns" version="4.1.115.Final">
+         <artifact name="netty-resolver-dns-4.1.115.Final.jar">
+            <sha256 value="4aca31593e5896c64ab7e041bbc6c0d851bd9634ec3a4354208141a35576619f" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-transport" version="4.1.109.Final">
-         <artifact name="netty-transport-4.1.109.Final.jar">
-            <sha256 value="c1ac17ce337d945290be680d7be942b7b4ff4c989be8ec24c684b3e94438da52" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-transport" version="4.1.115.Final">
+         <artifact name="netty-transport-4.1.115.Final.jar">
+            <sha256 value="c3d71faaa736ffd2c9260ab0b498024b814c39c7d764bea8113fa98de6e2bdd2" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-transport" version="4.1.42.Final">
          <artifact name="netty-transport-4.1.42.Final.jar">
             <sha256 value="dfa817a156ea263aa9ad8364a2e226527665c9722aca40a7945f228c2c14f1da" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-transport-classes-epoll" version="4.1.109.Final">
-         <artifact name="netty-transport-classes-epoll-4.1.109.Final.jar">
-            <sha256 value="02ce34ba9ab6ecc62de667ed35a8680202a902ec6e90bed587bea2e574fae38d" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-transport-classes-epoll" version="4.1.115.Final">
+         <artifact name="netty-transport-classes-epoll-4.1.115.Final.jar">
+            <sha256 value="40aa67b4463cca0ab346e393c87f6c37e8954d18ec8b78567d95b55aa1f2b3aa" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-transport-native-epoll" version="4.1.42.Final">
          <artifact name="netty-transport-native-epoll-4.1.42.Final.jar">
             <sha256 value="3c7d659b3bd773e0ea9b7517d2d6baffa275a3d2ae8eb4c10cb8f0a7724b11d5" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-transport-native-unix-common" version="4.1.109.Final">
-         <artifact name="netty-transport-native-unix-common-4.1.109.Final.jar">
-            <sha256 value="61e66832820f257bbee68722ec088b39da383c213eafbd1767097a7fff5d029f" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-transport-native-unix-common" version="4.1.115.Final">
+         <artifact name="netty-transport-native-unix-common-4.1.115.Final.jar">
+            <sha256 value="4b03e716272657c296b0204b57c140b2b2ca96b1a746c92da41f595892ec6d88" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-transport-native-unix-common" version="4.1.42.Final">

diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle
@@ -177,9 +177,8 @@ tasks.named("thirdPartyAudit").configure {
     'com.google.protobuf.nano.CodedOutputByteBufferNano',
     'com.google.protobuf.nano.MessageNano',
     'com.github.luben.zstd.Zstd',
-    'com.github.luben.zstd.BaseZstdBufferDecompressingStreamNoFinalizer',
-    'com.github.luben.zstd.ZstdBufferDecompressingStreamNoFinalizer',
-    'com.github.luben.zstd.ZstdDirectBufferDecompressingStreamNoFinalizer',
+    'com.github.luben.zstd.ZstdInputStreamNoFinalizer',
+    'com.github.luben.zstd.util.Native',
     'com.jcraft.jzlib.Deflater',
     'com.jcraft.jzlib.Inflater',
     'com.jcraft.jzlib.JZlib$WrapperType',
@@ -231,8 +230,14 @@ tasks.named("thirdPartyAudit").configure {
     'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField',
     'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField',
     'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField',
+    'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerIndexField',
+    'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerIndexField',
+    'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField',
+    'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField',
+    'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField',
     'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess',
     'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess',
+    'io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess',
     'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator',
     'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$1',
     'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$2',

diff --git a/muted-tests.yml b/muted-tests.yml
@@ -330,6 +330,9 @@ tests:
 - class: org.elasticsearch.xpack.spatial.search.GeoGridAggAndQueryConsistencyIT
   method: testGeoShapeGeoTile
   issue: https://github.com/elastic/elasticsearch/issues/115717
+- class: org.elasticsearch.search.StressSearchServiceReaperIT
+  method: testStressReaper
+  issue: https://github.com/elastic/elasticsearch/issues/115816
 
 # Examples:
 #

diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -176,6 +176,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY = def(8_772_00_0);
     public static final TransportVersion INFERENCE_DONT_PERSIST_ON_READ_BACKPORT_8_16 = def(8_772_00_1);
     public static final TransportVersion ADD_COMPATIBILITY_VERSIONS_TO_NODE_INFO_BACKPORT_8_16 = def(8_772_00_2);
+    public static final TransportVersion SKIP_INNER_HITS_SEARCH_SOURCE_BACKPORT_8_16 = def(8_772_00_3);
     public static final TransportVersion REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(8_773_00_0);
     public static final TransportVersion REVERT_REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(8_774_00_0);
     public static final TransportVersion ESQL_FIELD_ATTRIBUTE_PARENT_SIMPLIFIED = def(8_775_00_0);

diff --git a/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java b/server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
@@ -336,5 +336,13 @@ public interface Listener {
          * @param accountable the bitsets ram representation
          */
         void onRemoval(ShardId shardId, Accountable accountable);
+
+        Listener NOOP = new Listener() {
+            @Override
+            public void onCache(ShardId shardId, Accountable accountable) {}
+
+            @Override
+            public void onRemoval(ShardId shardId, Accountable accountable) {}
+        };
     }
 }