Update jackson to 2.9.10

core/src/main/java/org/apache/druid/timeline/SegmentWithOvershadowedStatus.java

@@ -34,17 +34,25 @@ public class SegmentWithOvershadowedStatus implements Comparable<SegmentWithOver
   private final boolean overshadowed;
   /**
    * dataSegment is serialized "unwrapped", i.e. it's properties are included as properties of
-   * enclosing class. If in future, if {@Code SegmentWithOvershadowedStatus} were to extend {@link DataSegment},
+   * enclosing class. If in future, if {@code SegmentWithOvershadowedStatus} were to extend {@link DataSegment},
    * there will be no change in the serialized format.
    */
   @JsonUnwrapped
   private final DataSegment dataSegment;
 
   @JsonCreator
   public SegmentWithOvershadowedStatus(
-      @JsonProperty("dataSegment") DataSegment dataSegment,
       @JsonProperty("overshadowed") boolean overshadowed
   )
+  {
+    // Jackson will overwrite dataSegment if needed (even though the field is final)
+    this(null, overshadowed);
+  }
+
+  public SegmentWithOvershadowedStatus(
+      DataSegment dataSegment,
+      boolean overshadowed
+  )
   {
     this.dataSegment = dataSegment;
     this.overshadowed = overshadowed;
@@ -94,4 +102,13 @@ public int compareTo(SegmentWithOvershadowedStatus o)
   {
     return dataSegment.getId().compareTo(o.dataSegment.getId());
   }
+
+  @Override
+  public String toString()
+  {
+    return "SegmentWithOvershadowedStatus{" +
+           "overshadowed=" + overshadowed +
+           ", dataSegment=" + dataSegment +
+           '}';
+  }
 }

core/src/test/java/org/apache/druid/data/input/impl/ParseSpecTest.java

@@ -206,7 +206,7 @@ public void testBadTypeSerde() throws IOException
 
     expectedException.expect(IllegalArgumentException.class);
     expectedException.expectCause(CoreMatchers.instanceOf(JsonMappingException.class));
-    expectedException.expectMessage("Could not resolve type id 'foo' into a subtype");
+    expectedException.expectMessage("Could not resolve type id 'foo' as a subtype");
     mapper.convertValue(mapValue, ParseSpec.class);
   }
 }

core/src/test/java/org/apache/druid/timeline/SegmentWithOvershadowedStatusTest.java

@@ -34,7 +34,6 @@
 import org.apache.druid.timeline.partition.ShardSpec;
 import org.joda.time.Interval;
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.Test;
 
 import javax.annotation.Nullable;
@@ -44,28 +43,29 @@
 
 public class SegmentWithOvershadowedStatusTest
 {
-  private static final ObjectMapper MAPPER = new TestObjectMapper();
+  private static final ObjectMapper MAPPER = createObjectMapper();
+  private static final Interval INTERVAL = Intervals.of("2011-10-01/2011-10-02");
+  private static final ImmutableMap<String, Object> LOAD_SPEC = ImmutableMap.of("something", "or_other");
+  private static final boolean OVERSHADOWED = true;
   private static final int TEST_VERSION = 0x9;
+  private static final SegmentWithOvershadowedStatus SEGMENT = createSegmentWithOvershadowedStatus();
 
-  @Before
-  public void setUp()
+  private static ObjectMapper createObjectMapper()
   {
+    ObjectMapper objectMapper = new TestObjectMapper();
     InjectableValues.Std injectableValues = new InjectableValues.Std();
     injectableValues.addValue(PruneSpecsHolder.class, PruneSpecsHolder.DEFAULT);
-    MAPPER.setInjectableValues(injectableValues);
+    objectMapper.setInjectableValues(injectableValues);
+    return objectMapper;
   }
 
-  @Test
-  public void testUnwrappedSegmentWithOvershadowedStatusDeserialization() throws Exception
+  private static SegmentWithOvershadowedStatus createSegmentWithOvershadowedStatus()
   {
-    final Interval interval = Intervals.of("2011-10-01/2011-10-02");
-    final ImmutableMap<String, Object> loadSpec = ImmutableMap.of("something", "or_other");
-
-    final DataSegment dataSegment = new DataSegment(
+    DataSegment dataSegment = new DataSegment(
         "something",
-        interval,
+        INTERVAL,
         "1",
-        loadSpec,
+        LOAD_SPEC,
         Arrays.asList("dim1", "dim2"),
         Arrays.asList("met1", "met2"),
         NoneShardSpec.instance(),
@@ -74,42 +74,58 @@ public void testUnwrappedSegmentWithOvershadowedStatusDeserialization() throws E
         1
     );
 
-    final SegmentWithOvershadowedStatus segment = new SegmentWithOvershadowedStatus(dataSegment, false);
+    return new SegmentWithOvershadowedStatus(dataSegment, OVERSHADOWED);
+  }
 
+  @Test
+  public void testUnwrappedSegmentWithOvershadowedStatusDeserialization() throws Exception
+  {
     final Map<String, Object> objectMap = MAPPER.readValue(
-        MAPPER.writeValueAsString(segment),
+        MAPPER.writeValueAsString(SEGMENT),
         JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT
     );
 
     Assert.assertEquals(11, objectMap.size());
     Assert.assertEquals("something", objectMap.get("dataSource"));
-    Assert.assertEquals(interval.toString(), objectMap.get("interval"));
+    Assert.assertEquals(INTERVAL.toString(), objectMap.get("interval"));
     Assert.assertEquals("1", objectMap.get("version"));
-    Assert.assertEquals(loadSpec, objectMap.get("loadSpec"));
+    Assert.assertEquals(LOAD_SPEC, objectMap.get("loadSpec"));
     Assert.assertEquals("dim1,dim2", objectMap.get("dimensions"));
     Assert.assertEquals("met1,met2", objectMap.get("metrics"));
     Assert.assertEquals(ImmutableMap.of("type", "none"), objectMap.get("shardSpec"));
     Assert.assertEquals(TEST_VERSION, objectMap.get("binaryVersion"));
     Assert.assertEquals(1, objectMap.get("size"));
-    Assert.assertEquals(false, objectMap.get("overshadowed"));
+    Assert.assertEquals(OVERSHADOWED, objectMap.get("overshadowed"));
 
-    final String json = MAPPER.writeValueAsString(segment);
+    final String json = MAPPER.writeValueAsString(SEGMENT);
 
     final TestSegmentWithOvershadowedStatus deserializedSegment = MAPPER.readValue(
         json,
         TestSegmentWithOvershadowedStatus.class
     );
 
-    Assert.assertEquals(segment.getDataSegment().getDataSource(), deserializedSegment.getDataSource());
-    Assert.assertEquals(segment.getDataSegment().getInterval(), deserializedSegment.getInterval());
-    Assert.assertEquals(segment.getDataSegment().getVersion(), deserializedSegment.getVersion());
-    Assert.assertEquals(segment.getDataSegment().getLoadSpec(), deserializedSegment.getLoadSpec());
-    Assert.assertEquals(segment.getDataSegment().getDimensions(), deserializedSegment.getDimensions());
-    Assert.assertEquals(segment.getDataSegment().getMetrics(), deserializedSegment.getMetrics());
-    Assert.assertEquals(segment.getDataSegment().getShardSpec(), deserializedSegment.getShardSpec());
-    Assert.assertEquals(segment.getDataSegment().getSize(), deserializedSegment.getSize());
-    Assert.assertEquals(segment.getDataSegment().getId(), deserializedSegment.getId());
+    DataSegment dataSegment = SEGMENT.getDataSegment();
+    Assert.assertEquals(dataSegment.getDataSource(), deserializedSegment.getDataSource());
+    Assert.assertEquals(dataSegment.getInterval(), deserializedSegment.getInterval());
+    Assert.assertEquals(dataSegment.getVersion(), deserializedSegment.getVersion());
+    Assert.assertEquals(dataSegment.getLoadSpec(), deserializedSegment.getLoadSpec());
+    Assert.assertEquals(dataSegment.getDimensions(), deserializedSegment.getDimensions());
+    Assert.assertEquals(dataSegment.getMetrics(), deserializedSegment.getMetrics());
+    Assert.assertEquals(dataSegment.getShardSpec(), deserializedSegment.getShardSpec());
+    Assert.assertEquals(dataSegment.getSize(), deserializedSegment.getSize());
+    Assert.assertEquals(dataSegment.getId(), deserializedSegment.getId());
+  }
 
+  // Previously, the implementation of SegmentWithOvershadowedStatus had @JsonCreator/@JsonProperty and @JsonUnwrapped
+  // on the same field (dataSegment), which used to work in Jackson 2.6, but does not work with Jackson 2.9:
+  // https://github.com/FasterXML/jackson-databind/issues/265#issuecomment-264344051
+  @Test
+  public void testJsonCreatorAndJsonUnwrappedAnnotationsAreCompatible() throws Exception
+  {
+    String json = MAPPER.writeValueAsString(SEGMENT);
+    SegmentWithOvershadowedStatus segment = MAPPER.readValue(json, SegmentWithOvershadowedStatus.class);
+    Assert.assertEquals(SEGMENT, segment);
+    Assert.assertEquals(json, MAPPER.writeValueAsString(segment));
   }
 }
 

indexing-hadoop/src/main/java/org/apache/druid/indexer/IndexGeneratorJob.java

@@ -77,6 +77,7 @@
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Iterator;
@@ -117,7 +118,7 @@ public static List<DataSegment> getPublishedSegments(HadoopDruidIndexerConfig co
       FileSystem fs = descriptorInfoDir.getFileSystem(conf);
 
       for (FileStatus status : fs.listStatus(descriptorInfoDir)) {
-        final DataSegment segment = jsonMapper.readValue(fs.open(status.getPath()), DataSegment.class);
+        final DataSegment segment = jsonMapper.readValue((InputStream) fs.open(status.getPath()), DataSegment.class);
         publishedSegmentsBuilder.add(segment);
         log.info("Adding segment %s to the list of published segments", segment.getId());
       }

integration-tests/src/test/resources/results/auth_test_sys_schema_segments.json

@@ -13,6 +13,6 @@
     "is_available": 1,
     "is_realtime": 0,
     "is_overshadowed": 0,
-    "payload": "{\"dataSource\":\"auth_test\",\"interval\":\"2012-12-29T00:00:00.000Z/2013-01-10T08:00:00.000Z\",\"version\":\"2013-01-10T08:13:47.830Z_v9\",\"loadSpec\":{\"load spec is pruned, because it's not needed on Brokers, but eats a lot of heap space\":\"\"},\"dimensions\":\"anonymous,area_code,city,continent_code,country_name,dma_code,geo,language,namespace,network,newpage,page,postal_code,region_lookup,robot,unpatrolled,user\",\"metrics\":\"added,count,deleted,delta,delta_hist,unique_users,variation\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":446027801,\"identifier\":\"auth_test_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9\",\"overshadowed\":false}"
+    "payload": "{\"overshadowed\":false,\"dataSource\":\"auth_test\",\"interval\":\"2012-12-29T00:00:00.000Z/2013-01-10T08:00:00.000Z\",\"version\":\"2013-01-10T08:13:47.830Z_v9\",\"loadSpec\":{\"load spec is pruned, because it's not needed on Brokers, but eats a lot of heap space\":\"\"},\"dimensions\":\"anonymous,area_code,city,continent_code,country_name,dma_code,geo,language,namespace,network,newpage,page,postal_code,region_lookup,robot,unpatrolled,user\",\"metrics\":\"added,count,deleted,delta,delta_hist,unique_users,variation\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":446027801,\"identifier\":\"auth_test_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9\"}"
   }
 ]

licenses.yaml

@@ -196,10 +196,11 @@ name: Jackson
 license_category: binary
 module: java-core
 license_name: Apache License version 2.0
-version: 2.6.7
+version: 2.9.10
 libraries:
   - com.fasterxml.jackson.core: jackson-annotations
   - com.fasterxml.jackson.core: jackson-core
+  - com.fasterxml.jackson.core: jackson-databind
   - com.fasterxml.jackson.dataformat: jackson-dataformat-cbor
   - com.fasterxml.jackson.dataformat: jackson-dataformat-smile
   - com.fasterxml.jackson.datatype: jackson-datatype-guava
@@ -232,37 +233,6 @@ notice: |
 
 ---
 
-name: Jackson
-license_category: binary
-module: java-core
-license_name: Apache License version 2.0
-version: 2.6.7.3
-libraries:
-  - com.fasterxml.jackson.core: jackson-databind
-notice: |
-  # Jackson JSON processor
-
-  Jackson is a high-performance, Free/Open Source JSON processing library.
-  It was originally written by Tatu Saloranta ([email protected]), and has
-  been in development since 2007.
-  It is currently developed by a community of developers, as well as supported
-  commercially by FasterXML.com.
-
-  ## Licensing
-
-  Jackson core and extension components may licensed under different licenses.
-  To find the details that apply to this artifact see the accompanying LICENSE file.
-  For more information, including possible other licensing options, contact
-  FasterXML.com (http://fasterxml.com).
-
-  ## Credits
-
-  A list of contributors may be found from CREDITS file, which is included
-  in some artifacts (usually source distributions); but is always available
-  from the source code management (SCM) system project uses.
-
----
-
 name: Caffeine
 license_category: binary
 module: java-core
@@ -1165,7 +1135,7 @@ name: Apache Calcite Avatica
 license_category: binary
 module: java-core
 license_name: Apache License version 2.0
-version: 1.12.0
+version: 1.15.0
 libraries:
   - org.apache.calcite.avatica: avatica-core
   - org.apache.calcite.avatica: avatica-metrics

pom.xml

@@ -78,7 +78,7 @@
         <aether.version>0.9.0.M2</aether.version>
         <apache.curator.version>4.1.0</apache.curator.version>
         <apache.curator.test.version>2.12.0</apache.curator.test.version>
-        <avatica.version>1.12.0</avatica.version>
+        <avatica.version>1.15.0</avatica.version>
         <avro.version>1.9.1</avro.version>
         <calcite.version>1.21.0</calcite.version>
         <derby.version>10.14.2.0</derby.version>
@@ -88,8 +88,7 @@
         <hamcrest.version>1.3</hamcrest.version>
         <jetty.version>9.4.12.v20180830</jetty.version>
         <jersey.version>1.19.3</jersey.version>
-        <!-- jackson 2.7.x causes injection error and 2.8.x can't be used because avatica is using 2.6.3 -->
-        <jackson.version>2.6.7</jackson.version>
+        <jackson.version>2.9.10</jackson.version>
         <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
         <log4j.version>2.8.2</log4j.version>
         <netty3.version>3.10.6.Final</netty3.version>
@@ -429,7 +428,7 @@
             <dependency>
                 <groupId>com.fasterxml.jackson.core</groupId>
                 <artifactId>jackson-databind</artifactId>
-                <version>${jackson.version}.3</version>
+                <version>${jackson.version}</version>
             </dependency>
             <dependency>
                 <groupId>com.fasterxml.jackson.datatype</groupId>
@@ -441,6 +440,15 @@
                 <artifactId>jackson-datatype-joda</artifactId>
                 <version>${jackson.version}</version>
             </dependency>
+            <dependency>
+                <!--
+                  ~ This is a transitive dependency of com.amazonaws:aws-java-sdk-core. Override the version here so
+                  ~ that it is the same as the other jackson dependencies.
+                  -->
+                <groupId>com.fasterxml.jackson.dataformat</groupId>
+                <artifactId>jackson-dataformat-cbor</artifactId>
+                <version>${jackson.version}</version>
+            </dependency>
             <dependency>
                 <groupId>com.fasterxml.jackson.dataformat</groupId>
                 <artifactId>jackson-dataformat-smile</artifactId>

processing/src/test/java/org/apache/druid/query/select/SelectQueryTest.java

@@ -53,7 +53,7 @@ public void testSerde() throws Exception
   {
     final String exceptionMessage =
         StringUtils.format(
-            "Instantiation of [simple type, class org.apache.druid.query.select.SelectQuery] value failed: %s",
+            "Cannot construct instance of `org.apache.druid.query.select.SelectQuery`, problem: %s",
             SelectQuery.REMOVED_ERROR_MESSAGE
         );
     expectedException.expect(JsonMappingException.class);

server/src/test/java/org/apache/druid/guice/FirehoseModuleTest.java

@@ -19,7 +19,6 @@
 
 package org.apache.druid.guice;
 
-import com.fasterxml.jackson.databind.AnnotationIntrospector;
 import com.fasterxml.jackson.databind.Module;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.cfg.MapperConfig;
@@ -69,8 +68,7 @@ private static Set<Class> getFirehoseFactorySubtypeClasses(ObjectMapper objectMa
   {
     Class parentClass = FirehoseFactory.class;
     MapperConfig config = objectMapper.getDeserializationConfig();
-    AnnotationIntrospector annotationIntrospector = config.getAnnotationIntrospector();
-    AnnotatedClass ac = AnnotatedClass.constructWithoutSuperTypes(parentClass, annotationIntrospector, config);
+    AnnotatedClass ac = AnnotatedClass.constructWithoutSuperTypes(parentClass, config);
     Collection<NamedType> subtypes = objectMapper.getSubtypeResolver().collectAndResolveSubtypesByClass(config, ac);
     Assert.assertNotNull(subtypes);
     return subtypes.stream()

server/src/test/java/org/apache/druid/segment/indexing/DataSchemaTest.java

@@ -282,7 +282,7 @@ public void testSerdeWithInvalidParserMap() throws Exception
     expectedException.expect(CoreMatchers.instanceOf(IllegalArgumentException.class));
     expectedException.expectCause(CoreMatchers.instanceOf(JsonMappingException.class));
     expectedException.expectMessage(
-        "Instantiation of [simple type, class org.apache.druid.data.input.impl.StringInputRowParser] value failed: parseSpec"
+        "Cannot construct instance of `org.apache.druid.data.input.impl.StringInputRowParser`, problem: parseSpec"
     );
 
     // Jackson creates a default type parser (StringInputRowParser) for an invalid type.

server/src/test/java/org/apache/druid/server/log/RequestLoggerProviderTest.java

@@ -85,7 +85,7 @@ public void testLoggerPropertiesWithNoType()
     );
 
     expectedException.expect(ProvisionException.class);
-    expectedException.expectMessage("missing property 'type'");
+    expectedException.expectMessage("missing type id property 'type'");
 
     configurator.configurate(
         properties,