add missing jsonPropNameIgnore feature to AvroSchemaComparator (#480)

linkedin · Apr 21, 2023 · 304c3f2 · 304c3f2
1 parent 9651633
commit 304c3f2
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 18 deletions.
diff --git a/avro-codegen/build.gradle b/avro-codegen/build.gradle
@@ -19,6 +19,8 @@ dependencies {
     implementation "org.apache.logging.log4j:log4j-api:2.17.1"
     implementation "com.beust:jcommander:1.78"
     implementation "org.apache.ant:ant:1.10.7"
+    testImplementation "com.fasterxml.jackson.core:jackson-core:2.10.2"
+    testImplementation "com.fasterxml.jackson.core:jackson-databind:2.10.2"
 
     compileOnly ("org.apache.avro:avro:1.4.1") {
         exclude group: "org.mortbay.jetty"

diff --git a/parser/src/main/java/com/linkedin/avroutil1/util/ConfigurableAvroSchemaComparator.java b/parser/src/main/java/com/linkedin/avroutil1/util/ConfigurableAvroSchemaComparator.java
@@ -113,13 +113,14 @@ private static void equals(AvroSchema a, AvroSchema b, SchemaComparisonConfigura
     boolean considerJsonStringProps = config.isCompareStringJsonProps();
     boolean considerJsonNonStringProps = config.isCompareNonStringJsonProps();
     boolean considerJsonProps = considerJsonStringProps || considerJsonNonStringProps;
+    Set<String> jsonPropNamesToIgnore = config.getJsonPropNamesToIgnore();
 
-    if (considerJsonProps && !hasSameObjectProps(a, b, considerJsonStringProps,
-        considerJsonNonStringProps)) {
+    if (considerJsonProps && !hasSameObjectProps(a, b, considerJsonStringProps, considerJsonNonStringProps,
+        jsonPropNamesToIgnore)) {
       AvroSchemaDifference difference = new AvroSchemaDifference(a.getCodeLocation(), b.getCodeLocation(),
           AvroSchemaDifferenceType.JSON_PROPERTY_MISMATCH,
-          String.format("Json properties of %s in schemaA does not match with the json properties of %s in schemaB",
-              a, b));
+          String.format("Json properties of %s in schemaA does not match with the json properties of %s in schemaB", a,
+              b));
       differences.add(difference);
       if (fastFail) {
         return;
@@ -301,6 +302,7 @@ private static void recordSchemaEquals(AvroRecordSchema a, AvroRecordSchema b, S
     boolean considerJsonNonStringProps = config.isCompareNonStringJsonProps();
     boolean considerAliases = config.isCompareAliases();
     boolean considerJsonProps = considerJsonStringProps || considerJsonNonStringProps;
+    Set<String> jsonPropNamesToIgnore = config.getJsonPropNamesToIgnore();
 
     try {
       if (considerAliases && !hasSameAliases(a, b)) {
@@ -367,11 +369,11 @@ private static void recordSchemaEquals(AvroRecordSchema a, AvroRecordSchema b, S
           }
         }
         if (considerJsonProps && !hasSameObjectProps(aField, bField, considerJsonStringProps,
-            considerJsonNonStringProps)) {
+            considerJsonNonStringProps, jsonPropNamesToIgnore)) {
           AvroSchemaDifference difference = new AvroSchemaDifference(aField.getCodeLocation(), bField.getCodeLocation(),
-              AvroSchemaDifferenceType.JSON_PROPERTY_MISMATCH,
-              String.format("Json properties of field \"%s\" in schemaA does not match with the json properties in schemaB",
-                  aField.getName(), bField.getName()));
+              AvroSchemaDifferenceType.JSON_PROPERTY_MISMATCH, String.format(
+              "Json properties of field \"%s\" in schemaA does not match with the json properties in schemaB",
+              aField.getName(), bField.getName()));
           differences.add(difference);
           if (fastFail) {
             return;
@@ -425,20 +427,22 @@ private static boolean hasSameAliases(AvroSchemaField a, AvroSchemaField b) {
   }
 
   private static boolean hasSameObjectProps(JsonPropertiesContainer aPropContainer,
-      JsonPropertiesContainer bPropContainer, boolean compareStringProps, boolean compareNonStringProps)
-      throws IOException {
-    if (aPropContainer.hasProperties()) {
-      Map<String, JsonNode> aProperties = getObjectProps(aPropContainer, aPropContainer.propertyNames());
-      Map<String, JsonNode> bProperties = getObjectProps(bPropContainer, bPropContainer.propertyNames());
-      return Jackson2Utils.compareJsonProperties(aProperties, bProperties, compareStringProps,
-          compareNonStringProps);
-    } return !bPropContainer.hasProperties();
+      JsonPropertiesContainer bPropContainer, boolean compareStringProps, boolean compareNonStringProps,
+      Set<String> jsonPropNamesToIgnore) {
+      Map<String, JsonNode> aProperties =
+          getObjectProps(aPropContainer, aPropContainer.propertyNames(), jsonPropNamesToIgnore);
+      Map<String, JsonNode> bProperties =
+          getObjectProps(bPropContainer, bPropContainer.propertyNames(), jsonPropNamesToIgnore);
+      return Jackson2Utils.compareJsonProperties(aProperties, bProperties, compareStringProps, compareNonStringProps);
   }
 
   private static Map<String, JsonNode> getObjectProps(JsonPropertiesContainer jsonPropertiesContainer,
-      Set<String> propNames) throws IOException {
+      Set<String> propNames, Set<String> jsonPropNamesToIgnore) {
     Map<String, JsonNode> propMap = new HashMap<>(propNames.size());
     for (String propName : propNames) {
+      if (jsonPropNamesToIgnore.contains(propName)) {
+        continue;
+      }
       propMap.put(propName,
           Jackson2Utils.toJsonNode(jsonPropertiesContainer.getPropertyAsJsonLiteral(propName), false));
     }

diff --git a/parser/src/test/java/com/linkedin/avroutil1/util/ConfigurableAvroSchemaComparatorTest.java b/parser/src/test/java/com/linkedin/avroutil1/util/ConfigurableAvroSchemaComparatorTest.java
@@ -14,7 +14,9 @@
 import com.linkedin.avroutil1.parser.avsc.AvscParser;
 import com.linkedin.avroutil1.testcommon.TestUtil;
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
 import org.testng.Assert;
 import org.testng.annotations.DataProvider;
@@ -73,7 +75,7 @@ public void testJsonProps() throws IOException {
     List<String> differences =
         ConfigurableAvroSchemaComparator.findDifference(schema1, schema2, SchemaComparisonConfiguration.STRICT)
             .stream()
-            .map(diff -> diff.toString())
+            .map(AvroSchemaDifference::toString)
             .collect(Collectors.toList());
     Assert.assertEquals(differences.size(), 6);
     Assert.assertTrue(differences.contains(
@@ -95,4 +97,16 @@ public void testJsonProps() throws IOException {
         "[JSON_PROPERTY_MISMATCH] Json properties of fixed vs14.FixedJsonPropMismatch in schemaA does not match with the json properties of fixed vs14.FixedJsonPropMismatch in schemaB\n"
             + "SchemaALocation: lines 35-39. SchemaBLocation: lines 41-46"));
   }
+
+  @Test
+  public void testJsonPropsWithIgnorePropsByName() throws IOException {
+    // test that json props are compared correctly in fields
+    AvroRecordSchema schema1 = (AvroRecordSchema) validateAndGetAvroRecordSchema("schemas/TestJsonPropsInFields1.avsc");
+    AvroRecordSchema schema2 = (AvroRecordSchema) validateAndGetAvroRecordSchema("schemas/TestJsonPropsInFields2.avsc");
+    Set<String> jsonPropNamesToIgnore = new HashSet<>();
+    jsonPropNamesToIgnore.add("specialAnnotation");
+    List<AvroSchemaDifference> differences = ConfigurableAvroSchemaComparator.findDifference(schema1, schema2,
+            SchemaComparisonConfiguration.STRICT.jsonPropNamesToIgnore(jsonPropNamesToIgnore));
+    Assert.assertEquals(differences.size(), 0);
+  }
 }