Merge branch 'main' into revert-116725

davidkyle · Nov 13, 2024 · f35b70e · f35b70e
2 parents 095836b + adf7328
commit f35b70e
Show file tree

Hide file tree

Showing 54 changed files with 1,281 additions and 713 deletions.
diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
@@ -14,7 +14,7 @@ log4j             = 2.19.0
 slf4j             = 2.0.6
 ecsLogging        = 1.2.0
 jna               = 5.12.1
-netty             = 4.1.109.Final
+netty             = 4.1.115.Final
 commons_lang3     = 3.9
 google_oauth_client = 1.34.1
 

diff --git a/docs/changelog/113194.yaml b/docs/changelog/113194.yaml
@@ -0,0 +1,5 @@
+pr: 113194
+summary: Add Search Phase APM metrics
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/116077.yaml b/docs/changelog/116077.yaml
@@ -0,0 +1,14 @@
+pr: 116077
+summary: Remove `ecs` option on `user_agent` processor
+area: Ingest Node
+type: breaking
+issues: []
+breaking:
+  title: Remove `ecs` option on `user_agent` processor
+  area: Ingest
+  details: >-
+    The `user_agent` ingest processor no longer accepts the `ecs` option. (It was previously deprecated and ignored.)
+  impact: >-
+    Users should stop using the `ecs` option when creating instances of the `user_agent` ingest processor.
+    The option will be removed from existing processors stored in the cluster state on upgrade.
+  notable: false
diff --git a/docs/changelog/116357.yaml b/docs/changelog/116357.yaml
@@ -0,0 +1,5 @@
+pr: 116357
+summary: Add tracking for query rule types
+area: Relevance
+type: enhancement
+issues: []
diff --git a/docs/reference/query-rules/apis/list-query-rulesets.asciidoc b/docs/reference/query-rules/apis/list-query-rulesets.asciidoc
@@ -124,7 +124,7 @@ PUT _query_rules/ruleset-3
         },
         {
             "rule_id": "rule-3",
-            "type": "pinned",
+            "type": "exclude",
             "criteria": [
                 {
                     "type": "fuzzy",
@@ -178,6 +178,9 @@ A sample response:
             "rule_total_count": 1,
             "rule_criteria_types_counts": {
                 "exact": 1
+            },
+            "rule_type_counts": {
+                "pinned": 1
             }
         },
         {
@@ -186,6 +189,9 @@ A sample response:
             "rule_criteria_types_counts": {
                 "exact": 1,
                 "fuzzy": 1
+            },
+            "rule_type_counts": {
+                "pinned": 2
             }
         },
         {
@@ -194,6 +200,10 @@ A sample response:
             "rule_criteria_types_counts": {
                 "exact": 1,
                 "fuzzy": 2
+            },
+            "rule_type_counts": {
+                "pinned": 2,
+                "exclude": 1
             }
         }
     ]

diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
@@ -1366,109 +1366,109 @@
             <sha256 value="a3ebec96768ee4a2d3db44597e84cea2d0bdd68ca04822397980ea9f67075a86" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-buffer" version="4.1.109.Final">
-         <artifact name="netty-buffer-4.1.109.Final.jar">
-            <sha256 value="ed49956c1e5d99534af2e9e9ecb80f4d3390c631c6cedd2a94d23aedcc911bb9" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-buffer" version="4.1.115.Final">
+         <artifact name="netty-buffer-4.1.115.Final.jar">
+            <sha256 value="4a7b331d3770c566ab70eb02a0d1feed63b95cf6e4d68c8fe778c4c9de2d116d" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-buffer" version="4.1.42.Final">
          <artifact name="netty-buffer-4.1.42.Final.jar">
             <sha256 value="7b0171a4e8bcd573e08d9f2bba053c67b557ab5012106a5982ccbae5743814c0" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-codec" version="4.1.109.Final">
-         <artifact name="netty-codec-4.1.109.Final.jar">
-            <sha256 value="2df943cd3ed24f86d61d442314e8a9a4c06b85a9a7aa99cec8c6520e77ff3f1c" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-codec" version="4.1.115.Final">
+         <artifact name="netty-codec-4.1.115.Final.jar">
+            <sha256 value="cd189afb70ec6eacfcdfdd3a5f472b4e705a5c91d5bd3ef0386421f2ae15ec77" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-codec" version="4.1.42.Final">
          <artifact name="netty-codec-4.1.42.Final.jar">
             <sha256 value="e96ced697fb7df589da7c20c995e01f75a9cb246be242bbc4cd3b4af424ff189" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-codec-dns" version="4.1.109.Final">
-         <artifact name="netty-codec-dns-4.1.109.Final.jar">
-            <sha256 value="a67a84c0a46094815e2044ac1314d8d81089d219ec8e8c732f9a12ba4cd451bd" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-codec-dns" version="4.1.115.Final">
+         <artifact name="netty-codec-dns-4.1.115.Final.jar">
+            <sha256 value="23dd6806bcc326855f13e69838c6411d0490e6b1aeb12e217a19a3dd6ad3f10d" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-codec-http" version="4.1.109.Final">
-         <artifact name="netty-codec-http-4.1.109.Final.jar">
-            <sha256 value="fdf056b08e156797d8f6caa727af0370dea8265b1eff8fc6f9e739c5c1f093d1" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-codec-http" version="4.1.115.Final">
+         <artifact name="netty-codec-http-4.1.115.Final.jar">
+            <sha256 value="e6dbe971c59373bbae9802021c63b9bc1d8800fead382863d67e79e79b023166" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-codec-http2" version="4.1.109.Final">
-         <artifact name="netty-codec-http2-4.1.109.Final.jar">
-            <sha256 value="a75498995087968b4582aefe42cfa8ca257759f32279edbec3c874e54acf633c" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-codec-http2" version="4.1.115.Final">
+         <artifact name="netty-codec-http2-4.1.115.Final.jar">
+            <sha256 value="cbed9829a5d582e91e314e209edce9a0c2eb369f23bb4fb74a5bc8b7990222c2" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-codec-socks" version="4.1.109.Final">
-         <artifact name="netty-codec-socks-4.1.109.Final.jar">
-            <sha256 value="a8b2c52807e4071f8fdd6239b8fd0c4d2cd282ebf08406f8d431d3229b69418e" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-codec-socks" version="4.1.115.Final">
+         <artifact name="netty-codec-socks-4.1.115.Final.jar">
+            <sha256 value="e9b1cc744dc6195894450b1fd4d271a821ab167fe21ae3c459b27cdadc70e81f" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-common" version="4.1.109.Final">
-         <artifact name="netty-common-4.1.109.Final.jar">
-            <sha256 value="33bf51077021de2b9d80b9ec64a4fae68f22e6ec56f52431441eaf6dd79cce71" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-common" version="4.1.115.Final">
+         <artifact name="netty-common-4.1.115.Final.jar">
+            <sha256 value="39f1b5a2aaa4eab5d036dfd0486e35a4276df412e092d36b2d88b494705a134d" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-common" version="4.1.42.Final">
          <artifact name="netty-common-4.1.42.Final.jar">
             <sha256 value="3d0a918d78292eeca02a7bb2188daa4e5053b6e29b71e6308309033e121242b5" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-handler" version="4.1.109.Final">
-         <artifact name="netty-handler-4.1.109.Final.jar">
-            <sha256 value="1161e2ed8e1f037124a8ddfe3d4184ebaa7be17aea27e7fca87f510f17898ea5" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-handler" version="4.1.115.Final">
+         <artifact name="netty-handler-4.1.115.Final.jar">
+            <sha256 value="5972028cc863b74927ce0d11fb8d58f65da2560bef5602fe8ce8903bd306ca07" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-handler" version="4.1.42.Final">
          <artifact name="netty-handler-4.1.42.Final.jar">
             <sha256 value="11eda86500c33b9d386719b5419f513fd9c097d13894f25dd0c75b610d636e03" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-handler-proxy" version="4.1.109.Final">
-         <artifact name="netty-handler-proxy-4.1.109.Final.jar">
-            <sha256 value="42e34c84423f074a4db4576e6d1ddabf56eb26705213e5f9b6ef537ce36bcd0b" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-handler-proxy" version="4.1.115.Final">
+         <artifact name="netty-handler-proxy-4.1.115.Final.jar">
+            <sha256 value="807e67cfb17136927d11db42df62031169d1fa0883e13f254906994c84ffbe87" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-resolver" version="4.1.109.Final">
-         <artifact name="netty-resolver-4.1.109.Final.jar">
-            <sha256 value="e168920249ea8d65076b0847e64464b5132843f1f1b2ff514e45fa255b64300b" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-resolver" version="4.1.115.Final">
+         <artifact name="netty-resolver-4.1.115.Final.jar">
+            <sha256 value="7b3455d14f59828765a00573bc3967dc59379e874bd62a67eb1926d6512109d1" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-resolver" version="4.1.42.Final">
          <artifact name="netty-resolver-4.1.42.Final.jar">
             <sha256 value="89768242b6b7cce9bd9f5945ad21d1b4bae515c6b1bf03a8af5d1899779cebc9" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-resolver-dns" version="4.1.109.Final">
-         <artifact name="netty-resolver-dns-4.1.109.Final.jar">
-            <sha256 value="7e354a85f8147744aeb9d3e1e64ace18f6432a25c09f8bde57383959a842c967" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-resolver-dns" version="4.1.115.Final">
+         <artifact name="netty-resolver-dns-4.1.115.Final.jar">
+            <sha256 value="4aca31593e5896c64ab7e041bbc6c0d851bd9634ec3a4354208141a35576619f" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-transport" version="4.1.109.Final">
-         <artifact name="netty-transport-4.1.109.Final.jar">
-            <sha256 value="c1ac17ce337d945290be680d7be942b7b4ff4c989be8ec24c684b3e94438da52" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-transport" version="4.1.115.Final">
+         <artifact name="netty-transport-4.1.115.Final.jar">
+            <sha256 value="c3d71faaa736ffd2c9260ab0b498024b814c39c7d764bea8113fa98de6e2bdd2" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-transport" version="4.1.42.Final">
          <artifact name="netty-transport-4.1.42.Final.jar">
             <sha256 value="dfa817a156ea263aa9ad8364a2e226527665c9722aca40a7945f228c2c14f1da" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-transport-classes-epoll" version="4.1.109.Final">
-         <artifact name="netty-transport-classes-epoll-4.1.109.Final.jar">
-            <sha256 value="02ce34ba9ab6ecc62de667ed35a8680202a902ec6e90bed587bea2e574fae38d" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-transport-classes-epoll" version="4.1.115.Final">
+         <artifact name="netty-transport-classes-epoll-4.1.115.Final.jar">
+            <sha256 value="40aa67b4463cca0ab346e393c87f6c37e8954d18ec8b78567d95b55aa1f2b3aa" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-transport-native-epoll" version="4.1.42.Final">
          <artifact name="netty-transport-native-epoll-4.1.42.Final.jar">
             <sha256 value="3c7d659b3bd773e0ea9b7517d2d6baffa275a3d2ae8eb4c10cb8f0a7724b11d5" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.netty" name="netty-transport-native-unix-common" version="4.1.109.Final">
-         <artifact name="netty-transport-native-unix-common-4.1.109.Final.jar">
-            <sha256 value="61e66832820f257bbee68722ec088b39da383c213eafbd1767097a7fff5d029f" origin="Generated by Gradle"/>
+      <component group="io.netty" name="netty-transport-native-unix-common" version="4.1.115.Final">
+         <artifact name="netty-transport-native-unix-common-4.1.115.Final.jar">
+            <sha256 value="4b03e716272657c296b0204b57c140b2b2ca96b1a746c92da41f595892ec6d88" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.netty" name="netty-transport-native-unix-common" version="4.1.42.Final">

diff --git a/modules/ingest-user-agent/src/main/java/module-info.java b/modules/ingest-user-agent/src/main/java/module-info.java
@@ -10,4 +10,5 @@
 module org.elasticsearch.ingest.useragent {
     requires org.elasticsearch.server;
     requires org.elasticsearch.xcontent;
+    requires org.elasticsearch.base;
 }
diff --git a/...st-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/...st-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java
@@ -9,7 +9,9 @@
 
 package org.elasticsearch.ingest.useragent;
 
+import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.ingest.IngestMetadata;
 import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -23,6 +25,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.UnaryOperator;
 import java.util.stream.Stream;
 
 public class IngestUserAgentPlugin extends Plugin implements IngestPlugin {
@@ -97,4 +100,15 @@ static Map<String, UserAgentParser> createUserAgentParsers(Path userAgentConfigD
     public List<Setting<?>> getSettings() {
         return List.of(CACHE_SIZE_SETTING);
     }
+
+    @Override
+    public Map<String, UnaryOperator<Metadata.Custom>> getCustomMetadataUpgraders() {
+        return Map.of(
+            IngestMetadata.TYPE,
+            ingestMetadata -> ((IngestMetadata) ingestMetadata).maybeUpgradeProcessors(
+                UserAgentProcessor.TYPE,
+                UserAgentProcessor::maybeUpgradeConfig
+            )
+        );
+    }
 }
diff --git a/...ngest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/...ngest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
@@ -9,9 +9,8 @@
 
 package org.elasticsearch.ingest.useragent;
 
-import org.elasticsearch.common.logging.DeprecationCategory;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
@@ -32,8 +31,6 @@
 
 public class UserAgentProcessor extends AbstractProcessor {
 
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(UserAgentProcessor.class);
-
     public static final String TYPE = "user_agent";
 
     private final String field;
@@ -198,21 +195,13 @@ public UserAgentProcessor create(
             String processorTag,
             String description,
             Map<String, Object> config
-        ) throws Exception {
+        ) {
             String field = readStringProperty(TYPE, processorTag, config, "field");
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "user_agent");
             String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);
             List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
             boolean extractDeviceType = readBooleanProperty(TYPE, processorTag, config, "extract_device_type", false);
             boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
-            Object ecsValue = config.remove("ecs");
-            if (ecsValue != null) {
-                deprecationLogger.warn(
-                    DeprecationCategory.SETTINGS,
-                    "ingest_useragent_ecs_settings",
-                    "setting [ecs] is deprecated as ECS format is the default and only option"
-                );
-            }
 
             UserAgentParser parser = userAgentParsers.get(regexFilename);
             if (parser == null) {
@@ -272,4 +261,14 @@ public static Property parseProperty(String propertyName) {
             }
         }
     }
+
+    @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
+    // This can be removed in V10. It's not possible to create an instance with the ecs property in V9, and all instances created by V8 or
+    // earlier will have been fixed when upgraded to V9.
+    static boolean maybeUpgradeConfig(Map<String, Object> config) {
+        // Instances created using ES 8.x (or earlier) may have the 'ecs' config entry.
+        // This was ignored in 8.x and is unsupported in 9.0.
+        // In 9.x, we should remove it from any existing processors on startup.
+        return config.remove("ecs") != null;
+    }
 }
diff --git a/...-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/...-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java
@@ -331,4 +331,18 @@ public void testExtractDeviceTypeDisabled() {
         device.put("name", "Other");
         assertThat(target.get("device"), is(device));
     }
+
+    public void testMaybeUpgradeConfig_removesEcsIfPresent() {
+        Map<String, Object> config = new HashMap<>(Map.of("field", "user-agent", "ecs", "whatever"));
+        boolean changed = UserAgentProcessor.maybeUpgradeConfig(config);
+        assertThat(changed, is(true));
+        assertThat(config, is(Map.of("field", "user-agent")));
+    }
+
+    public void testMaybeUpgradeConfig_doesNothingIfEcsAbsent() {
+        Map<String, Object> config = new HashMap<>(Map.of("field", "user-agent"));
+        boolean changed = UserAgentProcessor.maybeUpgradeConfig(config);
+        assertThat(changed, is(false));
+        assertThat(config, is(Map.of("field", "user-agent")));
+    }
 }
diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle
@@ -177,9 +177,8 @@ tasks.named("thirdPartyAudit").configure {
     'com.google.protobuf.nano.CodedOutputByteBufferNano',
     'com.google.protobuf.nano.MessageNano',
     'com.github.luben.zstd.Zstd',
-    'com.github.luben.zstd.BaseZstdBufferDecompressingStreamNoFinalizer',
-    'com.github.luben.zstd.ZstdBufferDecompressingStreamNoFinalizer',
-    'com.github.luben.zstd.ZstdDirectBufferDecompressingStreamNoFinalizer',
+    'com.github.luben.zstd.ZstdInputStreamNoFinalizer',
+    'com.github.luben.zstd.util.Native',
     'com.jcraft.jzlib.Deflater',
     'com.jcraft.jzlib.Inflater',
     'com.jcraft.jzlib.JZlib$WrapperType',
@@ -231,8 +230,14 @@ tasks.named("thirdPartyAudit").configure {
     'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField',
     'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField',
     'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField',
+    'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerIndexField',
+    'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerIndexField',
+    'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField',
+    'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField',
+    'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField',
     'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess',
     'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess',
+    'io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess',
     'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator',
     'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$1',
     'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$2',