Merge branch 'main' into index-metadata-system

Signed-off-by: Craig Perkins <[email protected]>

CHANGELOG.md

@@ -38,6 +38,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.xerial.snappy:snappy-java` from 1.1.10.6 to 1.1.10.7 ([#16665](https://github.com/opensearch-project/OpenSearch/pull/16665))
 - Bump `codecov/codecov-action` from 4 to 5 ([#16667](https://github.com/opensearch-project/OpenSearch/pull/16667))
 - Bump `org.apache.logging.log4j:log4j-core` from 2.24.1 to 2.24.2 ([#16718](https://github.com/opensearch-project/OpenSearch/pull/16718))
+- Bump `jackson` from 2.17.2 to 2.18.2 ([#16733](https://github.com/opensearch-project/OpenSearch/pull/16733))
+- Bump `ch.qos.logback:logback-classic` from 1.2.13 to 1.5.12 ([#16716](https://github.com/opensearch-project/OpenSearch/pull/16716))
 
 ### Changed
 - Indexed IP field supports `terms_query` with more than 1025 IP masks [#16391](https://github.com/opensearch-project/OpenSearch/pull/16391)
@@ -58,6 +60,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix `doc_values` only (`index:false`) IP field searching for masks ([#16628](https://github.com/opensearch-project/OpenSearch/pull/16628))
 - Fix stale cluster state custom file deletion ([#16670](https://github.com/opensearch-project/OpenSearch/pull/16670))
 - [Tiered Caching] Fix bug in cache stats API ([#16560](https://github.com/opensearch-project/OpenSearch/pull/16560))
+- Bound the size of cache in deprecation logger ([16702](https://github.com/opensearch-project/OpenSearch/issues/16702))
 - Ensure consistency of system flag on IndexMetadata after diff is applied ([#16644](https://github.com/opensearch-project/OpenSearch/pull/16644))
 
 ### Security

client/sniffer/licenses/jackson-core-2.18.2.jar.sha1

@@ -0,0 +1 @@
+fb64ccac5c27dca8819418eb4e443a9f496d9ee7

distribution/tools/upgrade-cli/licenses/jackson-annotations-2.18.2.jar.sha1

@@ -0,0 +1 @@
+985d77751ebc7fce5db115a986bc9aa82f973f4a

distribution/tools/upgrade-cli/licenses/jackson-databind-2.18.2.jar.sha1

@@ -0,0 +1 @@
+deef8697b92141fb6caf7aa86966cff4eec9b04f

gradle/libs.versions.toml

@@ -8,8 +8,8 @@ bundled_jdk = "23.0.1+11"
 # optional dependencies
 spatial4j         = "0.7"
 jts               = "1.15.0"
-jackson           = "2.17.2"
-jackson_databind  = "2.17.2"
+jackson           = "2.18.2"
+jackson_databind  = "2.18.2"
 snakeyaml         = "2.1"
 icu4j             = "75.1"
 supercsv          = "2.4.0"

libs/arrow-spi/licenses/jackson-annotations-2.18.2.jar.sha1

@@ -0,0 +1 @@
+985d77751ebc7fce5db115a986bc9aa82f973f4a

libs/arrow-spi/licenses/jackson-databind-2.18.2.jar.sha1

@@ -0,0 +1 @@
+deef8697b92141fb6caf7aa86966cff4eec9b04f

libs/core/licenses/jackson-core-2.18.2.jar.sha1

@@ -0,0 +1 @@
+fb64ccac5c27dca8819418eb4e443a9f496d9ee7

libs/x-content/licenses/jackson-core-2.18.2.jar.sha1

@@ -0,0 +1 @@
+fb64ccac5c27dca8819418eb4e443a9f496d9ee7

libs/x-content/licenses/jackson-dataformat-cbor-2.18.2.jar.sha1

@@ -0,0 +1 @@
+d4870757eff0344130f60e3ddb882b2336640f73

libs/x-content/licenses/jackson-dataformat-smile-2.18.2.jar.sha1

@@ -0,0 +1 @@
+44caf62d743bb5e5876e95efba5a55a1cab1b0db

libs/x-content/licenses/jackson-dataformat-yaml-2.18.2.jar.sha1

@@ -0,0 +1 @@
+d000e13505d1cf564371516fa3d5b8769a779dc9

modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/75_update.yml

@@ -42,9 +42,9 @@ teardown:
 ---
 "update operation with predefined default or final pipeline returns warning header":
   - skip:
-      version:  " - 2.99.99"
-      reason:   "this change is added in 3.0.0"
-      features: allowed_warnings
+      version:  " - 2.18.99"
+      reason:   "this change is added in 2.19.0"
+      features: [warnings, headers]
   - do:
       index:
         index: test_1
@@ -57,7 +57,8 @@ teardown:
   - match: { result:        created }
 
   - do:
-      allowed_warnings:
+      headers: { "X-Opaque-Id": "default_pipeline_request" }
+      warnings:
         - "the index [test_1] has a default ingest pipeline or a final ingest pipeline, the support of the ingest pipelines for update operation causes unexpected result and will be removed in 3.0.0"
       update:
         index:            test_1
@@ -85,7 +86,8 @@ teardown:
   - match: { result:        created }
 
   - do:
-      allowed_warnings:
+      headers: { "X-Opaque-Id": "final_pipeline_request" }
+      warnings:
         - "the index [test_2] has a default ingest pipeline or a final ingest pipeline, the support of the ingest pipelines for update operation causes unexpected result and will be removed in 3.0.0"
       update:
         index:            test_2

modules/ingest-geoip/licenses/jackson-annotations-2.18.2.jar.sha1

@@ -0,0 +1 @@
+985d77751ebc7fce5db115a986bc9aa82f973f4a

modules/ingest-geoip/licenses/jackson-databind-2.18.2.jar.sha1

@@ -0,0 +1 @@
+deef8697b92141fb6caf7aa86966cff4eec9b04f

plugins/crypto-kms/licenses/jackson-annotations-2.18.2.jar.sha1

@@ -0,0 +1 @@
+985d77751ebc7fce5db115a986bc9aa82f973f4a

plugins/crypto-kms/licenses/jackson-databind-2.18.2.jar.sha1

@@ -0,0 +1 @@
+deef8697b92141fb6caf7aa86966cff4eec9b04f

plugins/discovery-ec2/licenses/jackson-annotations-2.18.2.jar.sha1

@@ -0,0 +1 @@
+985d77751ebc7fce5db115a986bc9aa82f973f4a

plugins/discovery-ec2/licenses/jackson-databind-2.18.2.jar.sha1

@@ -0,0 +1 @@
+deef8697b92141fb6caf7aa86966cff4eec9b04f

plugins/repository-azure/licenses/jackson-annotations-2.18.2.jar.sha1

@@ -0,0 +1 @@
+985d77751ebc7fce5db115a986bc9aa82f973f4a

plugins/repository-azure/licenses/jackson-databind-2.18.2.jar.sha1

@@ -0,0 +1 @@
+deef8697b92141fb6caf7aa86966cff4eec9b04f

plugins/repository-azure/licenses/jackson-dataformat-xml-2.18.2.jar.sha1

@@ -0,0 +1 @@
+069cb3b7bd34b3f7842cc4a6fd717981433bf73e

plugins/repository-azure/licenses/jackson-datatype-jsr310-2.18.2.jar.sha1

@@ -0,0 +1 @@
+7b6ff96adf421f4c6edbd694e797dd8fe434510a

plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.18.2.jar.sha1

@@ -0,0 +1 @@
+048c96032e5a428739e28ff04973717c032df598

plugins/repository-s3/licenses/jackson-annotations-2.18.2.jar.sha1

@@ -0,0 +1 @@
+985d77751ebc7fce5db115a986bc9aa82f973f4a

plugins/repository-s3/licenses/jackson-databind-2.18.2.jar.sha1

@@ -0,0 +1 @@
+deef8697b92141fb6caf7aa86966cff4eec9b04f

server/licenses/jackson-core-2.18.2.jar.sha1

@@ -0,0 +1 @@
+fb64ccac5c27dca8819418eb4e443a9f496d9ee7

server/licenses/jackson-dataformat-cbor-2.18.2.jar.sha1

@@ -0,0 +1 @@
+d4870757eff0344130f60e3ddb882b2336640f73

server/licenses/jackson-dataformat-smile-2.18.2.jar.sha1

@@ -0,0 +1 @@
+44caf62d743bb5e5876e95efba5a55a1cab1b0db

server/licenses/jackson-dataformat-yaml-2.18.2.jar.sha1

@@ -0,0 +1 @@
+d000e13505d1cf564371516fa3d5b8769a779dc9

server/src/main/java/org/opensearch/common/logging/DeprecatedMessage.java

@@ -47,12 +47,17 @@
  */
 public class DeprecatedMessage extends OpenSearchLogMessage {
     public static final String X_OPAQUE_ID_FIELD_NAME = "x-opaque-id";
-    private static final Set<String> keys = ConcurrentHashMap.newKeySet();
+
+    // Arbitrary maximum size, should be much larger than unique number of
+    // loggers, but small relative to heap size.
+    static final int MAX_DEDUPE_CACHE_ENTRIES = 16_384;
+
+    private static final Set<String> keyDedupeCache = ConcurrentHashMap.newKeySet();
     private final String keyWithXOpaqueId;
 
     public DeprecatedMessage(String key, String xOpaqueId, String messagePattern, Object... args) {
         super(fieldMap(key, xOpaqueId), messagePattern, args);
-        this.keyWithXOpaqueId = new StringBuilder().append(key).append(xOpaqueId).toString();
+        this.keyWithXOpaqueId = key + xOpaqueId;
     }
 
     /**
@@ -62,7 +67,7 @@ public DeprecatedMessage(String key, String xOpaqueId, String messagePattern, Ob
      * Otherwise, a warning can be logged by some test and the upcoming test can be impacted by it.
      */
     public static void resetDeprecatedMessageForTests() {
-        keys.clear();
+        keyDedupeCache.clear();
     }
 
     private static Map<String, Object> fieldMap(String key, String xOpaqueId) {
@@ -77,6 +82,15 @@ private static Map<String, Object> fieldMap(String key, String xOpaqueId) {
     }
 
     public boolean isAlreadyLogged() {
-        return !keys.add(keyWithXOpaqueId);
+        if (keyDedupeCache.contains(keyWithXOpaqueId)) {
+            return true;
+        }
+        if (keyDedupeCache.size() >= MAX_DEDUPE_CACHE_ENTRIES) {
+            // Stop logging if max size is breached to avoid performance problems from
+            // excessive logging. The historical logs will be full of deprecation warnings
+            // at this point anyway.
+            return true;
+        }
+        return !keyDedupeCache.add(keyWithXOpaqueId);
     }
 }

server/src/main/java/org/opensearch/common/logging/DeprecationLogger.java

@@ -116,9 +116,12 @@ public DeprecationLoggerBuilder deprecate(final String key, final String msg, fi
     public class DeprecationLoggerBuilder {
 
         public DeprecationLoggerBuilder withDeprecation(String key, String msg, Object[] params) {
-            DeprecatedMessage deprecationMessage = new DeprecatedMessage(key, HeaderWarning.getXOpaqueId(), msg, params);
-            if (!deprecationMessage.isAlreadyLogged()) {
-                logger.log(DEPRECATION, deprecationMessage);
+            // Check if the logger is enabled to skip the overhead of deduplicating messages if the logger is disabled
+            if (logger.isEnabled(DEPRECATION)) {
+                DeprecatedMessage deprecationMessage = new DeprecatedMessage(key, HeaderWarning.getXOpaqueId(), msg, params);
+                if (!deprecationMessage.isAlreadyLogged()) {
+                    logger.log(DEPRECATION, deprecationMessage);
+                }
             }
             return this;
         }

server/src/test/java/org/opensearch/common/logging/DeprecationLoggerTests.java

@@ -69,4 +69,22 @@ public void testDuplicateLogMessages() {
         // assert that only unique warnings are logged
         assertWarnings("Deprecated message 1", "Deprecated message 2", "Deprecated message 3");
     }
+
+    public void testMaximumSizeOfCache() {
+        final int maxEntries = DeprecatedMessage.MAX_DEDUPE_CACHE_ENTRIES;
+        // Fill up the cache, asserting every message is new
+        for (int i = 0; i < maxEntries; i++) {
+            DeprecatedMessage message = new DeprecatedMessage("key-" + i, "message-" + i, "");
+            assertFalse(message.toString(), message.isAlreadyLogged());
+        }
+        // Do the same thing except assert every message has been seen
+        for (int i = 0; i < maxEntries; i++) {
+            DeprecatedMessage message = new DeprecatedMessage("key-" + i, "message-" + i, "");
+            assertTrue(message.toString(), message.isAlreadyLogged());
+        }
+        // Add one more new entry, asserting it will forever been seen as already logged (cache is full)
+        DeprecatedMessage message = new DeprecatedMessage("key-new", "message-new", "");
+        assertTrue(message.toString(), message.isAlreadyLogged());
+        assertTrue(message.toString(), message.isAlreadyLogged());
+    }
 }

test/fixtures/hdfs-fixture/build.gradle

@@ -75,7 +75,7 @@ dependencies {
   api "org.apache.commons:commons-text:1.12.0"
   api "commons-net:commons-net:3.11.1"
   api "ch.qos.logback:logback-core:1.5.12"
-  api "ch.qos.logback:logback-classic:1.2.13"
+  api "ch.qos.logback:logback-classic:1.5.12"
   api "org.jboss.xnio:xnio-nio:3.8.16.Final"
   api 'org.jline:jline:3.27.1'
   api 'org.apache.commons:commons-configuration2:2.11.0'