diff --git a/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvscGenerationConfig.java b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvscGenerationConfig.java index e5865603e..87a4d2a9a 100644 --- a/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvscGenerationConfig.java +++ b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvscGenerationConfig.java @@ -90,7 +90,7 @@ public class AvscGenerationConfig { */ public static final AvscGenerationConfig CANONICAL_ONELINE = new AvscGenerationConfig( false, false, false, Optional.of(Boolean.FALSE), false, false, - false, false, false, false, true, false); + false, false, false, false, true, false, false); /** * Broad Canonical form of avro, includes defaults and field / schema aliases @@ -98,7 +98,7 @@ public class AvscGenerationConfig { */ public static final AvscGenerationConfig CANONICAL_BROAD_ONELINE = new AvscGenerationConfig( false, false, false, Optional.of(Boolean.FALSE), false, true, - false, true, false, true, true, false); + false, true, false, true, true, false, false); /** * if this value is set to true, and the rest of the values on this config object @@ -162,6 +162,10 @@ public class AvscGenerationConfig { * true to write namespace relative to parent namespace. False writes full namespace. */ public final boolean writeRelativeNamespace; + /** + * true to write using legacy AvscWriter, order of fields may be non-deterministic + */ + public final boolean isLegacy; public AvscGenerationConfig( @@ -188,6 +192,7 @@ public AvscGenerationConfig( this.retainSchemaAliases = true; this.writeNamespaceExplicitly = false; this.writeRelativeNamespace = true; + this.isLegacy = true; } public AvscGenerationConfig( @@ -221,6 +226,42 @@ public AvscGenerationConfig( this.retainSchemaAliases = retainSchemaAliases; this.writeNamespaceExplicitly = writeNamespaceExplicitly; this.writeRelativeNamespace = writeRelativeNamespace; + this.isLegacy = true; + } + + public AvscGenerationConfig( + boolean preferUseOfRuntimeAvro, + boolean forceUseOfRuntimeAvro, + boolean prettyPrint, + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") + Optional retainPreAvro702Logic, + boolean addAvro702Aliases, + boolean retainDefaults, + boolean retainDocs, + boolean retainFieldAliases, + boolean retainNonClaimedProps, + boolean retainSchemaAliases, + boolean writeNamespaceExplicitly, + boolean writeRelativeNamespace, + boolean isLegacy + ) { + //noinspection OptionalAssignedToNull + if (retainPreAvro702Logic == null) { + throw new IllegalArgumentException("retainPreAvro702Logic cannot be null"); + } + this.preferUseOfRuntimeAvro = preferUseOfRuntimeAvro; + this.forceUseOfRuntimeAvro = forceUseOfRuntimeAvro; + this.prettyPrint = prettyPrint; + this.retainPreAvro702Logic = retainPreAvro702Logic; + this.addAvro702Aliases = addAvro702Aliases; + this.retainDefaults = retainDefaults; + this.retainDocs = retainDocs; + this.retainFieldAliases = retainFieldAliases; + this.retainNonClaimedProps = retainNonClaimedProps; + this.retainSchemaAliases = retainSchemaAliases; + this.writeNamespaceExplicitly = writeNamespaceExplicitly; + this.writeRelativeNamespace = writeRelativeNamespace; + this.isLegacy = isLegacy; } public boolean isPreferUseOfRuntimeAvro() { diff --git a/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvscWriter.java b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvscWriter.java index 9a98a2588..8158eb29a 100644 --- a/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvscWriter.java +++ b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvscWriter.java @@ -16,6 +16,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -81,6 +82,10 @@ public abstract class AvscWriter> { * true to write namespace relative to parent namespace. False writes full namespace. */ public final boolean writeRelativeNamespace; + /** + * true to write using legacy AvscWriter, order of fields may be non-deterministic + */ + public final boolean isLegacy; protected AvscWriter(boolean pretty, boolean preAvro702, boolean addAliasesForAvro702) { this.pretty = pretty; @@ -93,6 +98,7 @@ protected AvscWriter(boolean pretty, boolean preAvro702, boolean addAliasesForAv this.retainDefaults = true; this.writeNamespaceExplicitly = false; this.writeRelativeNamespace = true; + this.isLegacy = true; _plugins = new ArrayList(0); } @@ -109,6 +115,25 @@ protected AvscWriter(boolean pretty, boolean preAvro702, boolean addAliasesForAv this.retainDefaults = retainDefaults; this.writeNamespaceExplicitly = writeNamespaceExplicitly; this.writeRelativeNamespace = writeRelativeNamespace; + this.isLegacy = true; + this._plugins = plugins == null ? Collections.emptyList() : plugins; + } + + protected AvscWriter(boolean pretty, boolean preAvro702, boolean addAliasesForAvro702, boolean retainDefaults, + boolean retainDocs, boolean retainFieldAliases, boolean retainNonClaimedProps, boolean retainSchemaAliases, + boolean writeNamespaceExplicitly, boolean writeRelativeNamespace, boolean isLegacy, + List plugins) { + this.pretty = pretty; + this.preAvro702 = preAvro702; + this.addAliasesForAvro702 = addAliasesForAvro702; + this.retainDocs = retainDocs; + this.retainSchemaAliases = retainSchemaAliases; + this.retainFieldAliases = retainFieldAliases; + this.retainNonClaimedProps = retainNonClaimedProps; + this.retainDefaults = retainDefaults; + this.writeNamespaceExplicitly = writeNamespaceExplicitly; + this.writeRelativeNamespace = writeRelativeNamespace; + this.isLegacy = isLegacy; this._plugins = plugins == null ? Collections.emptyList() : plugins; } @@ -117,7 +142,12 @@ public String toAvsc(Schema schema) { AvroNames names = new AvroNames(); StringWriter writer = new StringWriter(); G gen = createJsonGenerator(writer); - toJson(schema, names, "", "", gen); + if(isLegacy) { + toJsonLegacy(schema, names, "", "", gen); + } else { + toJson(schema, names, "", "", gen); + } + gen.flush(); return writer.toString(); } catch (IOException e) { @@ -125,6 +155,129 @@ public String toAvsc(Schema schema) { } } + protected void toJsonLegacy( + Schema schema, + AvroNames names, + String contextNamespaceWhenParsed, + String contextNamespaceWhenParsedUnder702, + G gen + ) throws IOException { + Avro702Data avro702Data; + switch (schema.getType()) { + case ENUM: + //taken from EnumSchema.toJson() in avro 1.11 + if (writeNameRef(schema, names, gen)) { + return; + } + gen.writeStartObject(); + gen.writeStringField("type", "enum"); + avro702Data = writeName(schema, names, contextNamespaceWhenParsed, contextNamespaceWhenParsedUnder702, gen); + if (schema.getDoc() != null) { + gen.writeStringField("doc", schema.getDoc()); + } + gen.writeArrayFieldStart("symbols"); + for (String symbol : schema.getEnumSymbols()) { + gen.writeString(symbol); + } + gen.writeEndArray(); + writeEnumDefault(schema, gen); + writePropsLegacy(schema, gen); + aliasesToJsonLegacy(schema, avro702Data.getExtraAliases(), gen); + gen.writeEndObject(); + break; + case FIXED: + //taken from FixedSchema.toJson() in avro 1.11 + if (writeNameRef(schema, names, gen)) { + return; + } + gen.writeStartObject(); + gen.writeStringField("type", "fixed"); + avro702Data = writeName(schema, names, contextNamespaceWhenParsed, contextNamespaceWhenParsedUnder702, gen); + if (schema.getDoc() != null) { + gen.writeStringField("doc", schema.getDoc()); + } + gen.writeNumberField("size", schema.getFixedSize()); + writePropsLegacy(schema, gen); + aliasesToJsonLegacy(schema, avro702Data.getExtraAliases(), gen); + gen.writeEndObject(); + break; + case RECORD: + //taken from RecordSchema.toJson() in avro 1.11 + if (writeNameRef(schema, names, gen)) { + return; + } + gen.writeStartObject(); + gen.writeStringField("type", schema.isError() ? "error" : "record"); + avro702Data = writeName(schema, names, contextNamespaceWhenParsed, contextNamespaceWhenParsedUnder702, gen); + AvroName name = AvroName.of(schema); + + //save current namespaces - both 1.4 and correct one + String savedBadSpace = names.badSpace(); //save avro-702 mode namespace + String savedCorrectSpace = names.correctSpace(); //save correct namespace + //avro 1.4 only ever sets namespace if the current is null + if (savedBadSpace == null) { + names.badSpace(name.getSpace()); + } + names.correctSpace(name.getSpace()); //always update correct namespace + + if (schema.getDoc() != null) { + gen.writeStringField("doc", schema.getDoc()); + } + if (schema.getFields() != null) { + gen.writeFieldName("fields"); + fieldsToJsonLegacy( + schema, + names, + avro702Data.getNamespaceWhenParsing(), + avro702Data.getNamespaceWhenParsing702(), + gen + ); + } + writePropsLegacy(schema, gen); + aliasesToJsonLegacy(schema, avro702Data.getExtraAliases(), gen); + gen.writeEndObject(); + //avro 1.4 never restores namespace, so we never restore space + names.correctSpace(savedCorrectSpace); //always restore correct namespace + break; + case ARRAY: + //taken from ArraySchema.toJson() in avro 1.11 + gen.writeStartObject(); + gen.writeStringField("type", "array"); + gen.writeFieldName("items"); + toJsonLegacy(schema.getElementType(), names, contextNamespaceWhenParsed, contextNamespaceWhenParsedUnder702, gen); + writePropsLegacy(schema, gen); + gen.writeEndObject(); + break; + case MAP: + //taken from MapSchema.toJson() in avro 1.11 + gen.writeStartObject(); + gen.writeStringField("type", "map"); + gen.writeFieldName("values"); + toJsonLegacy(schema.getValueType(), names, contextNamespaceWhenParsed, contextNamespaceWhenParsedUnder702, gen); + writePropsLegacy(schema, gen); + gen.writeEndObject(); + break; + case UNION: + //taken from UnionSchema.toJson() in avro 1.11 + gen.writeStartArray(); + for (Schema type : schema.getTypes()) { + toJsonLegacy(type, names, contextNamespaceWhenParsed, contextNamespaceWhenParsedUnder702, gen); + } + gen.writeEndArray(); + break; + default: + //all other schema types (taken from Schema.toJson() in avro 1.11) + if (!hasProps(schema)) { // no props defined + gen.writeString(schema.getName()); // just write name + } else { + gen.writeStartObject(); + gen.writeStringField("type", schema.getName()); + writePropsLegacy(schema, gen); + gen.writeEndObject(); + } + } + } + protected void toJson( Schema schema, AvroNames names, @@ -152,7 +305,7 @@ protected void toJson( if(retainDefaults) { writeEnumDefault(schema, gen); } - aliasesToJson(schema, avro702Data.getExtraAliases(), gen, retainSchemaAliases); + aliasesToJsonLegacy(schema, avro702Data.getExtraAliases(), gen, retainSchemaAliases); if (retainDocs && schema.getDoc() != null) { gen.writeStringField("doc", schema.getDoc()); } @@ -173,7 +326,7 @@ protected void toJson( gen.writeNumberField("size", schema.getFixedSize()); - aliasesToJson(schema, avro702Data.getExtraAliases(), gen, retainSchemaAliases); + aliasesToJsonLegacy(schema, avro702Data.getExtraAliases(), gen, retainSchemaAliases); if (retainDocs && schema.getDoc() != null) { gen.writeStringField("doc", schema.getDoc()); } @@ -211,7 +364,7 @@ protected void toJson( gen ); } - aliasesToJson(schema, avro702Data.getExtraAliases(), gen, retainSchemaAliases); + aliasesToJsonLegacy(schema, avro702Data.getExtraAliases(), gen, retainSchemaAliases); if (retainDocs && schema.getDoc() != null) { gen.writeStringField("doc", schema.getDoc()); } @@ -350,7 +503,45 @@ private List executeAvscWriterPluginsForField(Schema.Field field, G gen) return claimedProps; } - protected void aliasesToJson(Schema schema, List extraAliases, G gen, boolean retainSchemaAliases) throws IOException { + protected void aliasesToJsonLegacy(Schema schema, List extraAliases, G gen) throws IOException { + Set userDefinedAliases = schema.getAliases(); + Set allAliases = userDefinedAliases; //could be null + if (addAliasesForAvro702 && extraAliases != null) { + allAliases = new HashSet<>(); + if (userDefinedAliases != null) { + allAliases.addAll(userDefinedAliases); + } + for (AvroName extraAlias : extraAliases) { + allAliases.add(extraAlias.getFull()); + } + } + if (allAliases == null || allAliases.isEmpty()) { + return; + } + AvroName name = AvroName.of(schema); + //"context" namespace for aliases is the fullname of the names type on which they are defined + //except for the extra alias. scenarios where extraAlias is used are those where the "effective" + //name of this schema (or its correct form) is different to its full name, so we want + //to very explicitly use the fullname of the alias for those. + String referenceNamespace = name.getSpace(); + gen.writeFieldName("aliases"); + gen.writeStartArray(); + //TODO - avro702 may have an impact on (regular) aliases, meaning we may need to + // add yet more aliases to account for those! + for (String s : allAliases) { + AvroName alias = new AvroName(s, null); + if (addAliasesForAvro702 && extraAliases != null && extraAliases.contains(alias)) { + //always emit any avro-702 related aliases as fullnames + gen.writeString(alias.getFull()); + } else { + String relative = alias.getQualified(referenceNamespace); + gen.writeString(relative); + } + } + gen.writeEndArray(); + } + + protected void aliasesToJsonLegacy(Schema schema, List extraAliases, G gen, boolean retainSchemaAliases) throws IOException { Set userDefinedAliases = retainSchemaAliases ? getSortedFullyQualifiedSchemaAliases(schema.getAliases(), schema.getNamespace()) : null; Set allAliases = userDefinedAliases == null ? new TreeSet<>() : userDefinedAliases; @@ -410,6 +601,42 @@ private Set getSortedFullyQualifiedSchemaAliases(Set aliases, St return sortedAliases; } + protected void fieldsToJsonLegacy( + Schema schema, + AvroNames names, + String contextNamespaceWhenParsed, + String contextNamespaceWhenParsedUnder702, + G gen + ) throws IOException { + gen.writeStartArray(); + List fields = schema.getFields(); + for (Schema.Field f : fields) { + gen.writeStartObject(); + gen.writeStringField("name", f.name()); + gen.writeFieldName("type"); + toJsonLegacy(f.schema(), names, contextNamespaceWhenParsed, contextNamespaceWhenParsedUnder702, gen); + if (f.doc() != null) { + gen.writeStringField("doc", f.doc()); + } + writeDefaultValue(f, gen); + if (f.order() != Schema.Field.Order.ASCENDING) { + gen.writeStringField("order", f.order().name()); + } + Set aliases = getAliases(f); + if (aliases != null && aliases.size() != 0) { + gen.writeFieldName("aliases"); + gen.writeStartArray(); + for (String alias : aliases) { + gen.writeString(alias); + } + gen.writeEndArray(); + } + writePropsLegacy(f, gen); + gen.writeEndObject(); + } + gen.writeEndArray(); + } + protected void fieldsToJson( Schema schema, AvroNames names, @@ -502,6 +729,9 @@ protected Avro702Data writeName( */ protected abstract void writeProps(Schema schema, G gen, Set propNames) throws IOException; + protected abstract void writePropsLegacy(Schema schema, G gen) throws IOException; + protected abstract void writePropsLegacy(Schema.Field field, G gen) throws IOException; + /*** * Write json props from field, for the keys provided in propNames * @param field diff --git a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/Avro110Adapter.java b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/Avro110Adapter.java index 9fd3be38d..bffc9d501 100644 --- a/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/Avro110Adapter.java +++ b/helper/impls/helper-impl-110/src/main/java/com/linkedin/avroutil1/compatibility/avro110/Avro110Adapter.java @@ -508,7 +508,7 @@ public AvscWriter getAvscWriter(AvscGenerationConfig config, List schemaPlugins) { + super(pretty, preAvro702, addAliasesForAvro702, retainDefaults, retainDocs, retainFieldAliases, + retainNonClaimedProps, retainSchemaAliases, writeNamespaceExplicitly, writeRelativeNamespace, isLegacy, + schemaPlugins); + } + @Override protected Jackson2JsonGeneratorWrapper createJsonGenerator(StringWriter writer) throws IOException { JsonGenerator gen = FACTORY.createGenerator(writer); @@ -118,4 +127,20 @@ private void writeProps(Map props, Jackson2JsonGeneratorWrapper delegate.writeObjectField(entry.getKey(), JacksonUtils.toJsonNode(o)); } } + + @Override + protected void writePropsLegacy(Schema schema, Jackson2JsonGeneratorWrapper gen) throws IOException { + Map props = schema.getObjectProps(); + if (props != null && !props.isEmpty()) { + writeProps(props, gen); + } + } + + @Override + protected void writePropsLegacy(Schema.Field field, Jackson2JsonGeneratorWrapper gen) throws IOException { + Map props = field.getObjectProps(); + if (props != null && !props.isEmpty()) { + writeProps(props, gen); + } + } } diff --git a/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/Avro111Adapter.java b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/Avro111Adapter.java index 043a4b8a7..d5947c792 100644 --- a/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/Avro111Adapter.java +++ b/helper/impls/helper-impl-111/src/main/java/com/linkedin/avroutil1/compatibility/avro111/Avro111Adapter.java @@ -506,7 +506,7 @@ public AvscWriter getAvscWriter(AvscGenerationConfig config, List schemaPlugins) { + super(pretty, preAvro702, addAliasesForAvro702, retainDefaults, retainDocs, retainFieldAliases, + retainNonClaimedProps, retainSchemaAliases, writeNamespaceExplicitly, writeRelativeNamespace, isLegacy, + schemaPlugins); + } + @Override protected Jackson2JsonGeneratorWrapper createJsonGenerator(StringWriter writer) throws IOException { JsonGenerator gen = FACTORY.createGenerator(writer); @@ -117,4 +126,20 @@ private void writeProps(Map props, Jackson2JsonGeneratorWrapper delegate.writeObjectField(entry.getKey(), JacksonUtils.toJsonNode(o)); } } + + @Override + protected void writePropsLegacy(Schema schema, Jackson2JsonGeneratorWrapper gen) throws IOException { + Map props = schema.getObjectProps(); + if (props != null && !props.isEmpty()) { + writeProps(props, gen); + } + } + + @Override + protected void writePropsLegacy(Schema.Field field, Jackson2JsonGeneratorWrapper gen) throws IOException { + Map props = field.getObjectProps(); + if (props != null && !props.isEmpty()) { + writeProps(props, gen); + } + } } diff --git a/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/Avro14Adapter.java b/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/Avro14Adapter.java index 40e00bfb6..69d296837 100644 --- a/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/Avro14Adapter.java +++ b/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/Avro14Adapter.java @@ -482,7 +482,7 @@ public AvscWriter getAvscWriter(AvscGenerationConfig config, List schemaPlugins) { + super(pretty, preAvro702, addAliasesForAvro702, retainDefaults, retainDocs, retainFieldAliases, + retainNonClaimedProps, retainSchemaAliases, writeNamespaceExplicitly, writeRelativeNamespace, isLegacy, + schemaPlugins); + } + @Override protected Jackson1JsonGeneratorWrapper createJsonGenerator(StringWriter writer) throws IOException { JsonGenerator gen = FACTORY.createJsonGenerator(writer); @@ -176,4 +185,20 @@ private void writeProps( } } } + @Override + protected void writePropsLegacy(Schema schema, Jackson1JsonGeneratorWrapper gen) throws IOException { + Map props = getProps(schema); + //write all props except "default" for enums + if (schema.getType() == Schema.Type.ENUM) { + writeProps(props, gen, s -> !"default".equals(s)); + } else { + writeProps(props, gen); + } + } + + @Override + protected void writePropsLegacy(Schema.Field field, Jackson1JsonGeneratorWrapper gen) throws IOException { + Map props = getProps(field); + writeProps(props, gen); + } } diff --git a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/Avro15Adapter.java b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/Avro15Adapter.java index 569240f02..551f152fd 100644 --- a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/Avro15Adapter.java +++ b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/Avro15Adapter.java @@ -508,7 +508,7 @@ public AvscWriter getAvscWriter(AvscGenerationConfig config, List schemaPlugins) { + super(pretty, preAvro702, addAliasesForAvro702, retainDefaults, retainDocs, retainFieldAliases, + retainNonClaimedProps, retainSchemaAliases, writeNamespaceExplicitly, writeRelativeNamespace, isLegacy, + schemaPlugins); + } @Override protected Jackson1JsonGeneratorWrapper createJsonGenerator(StringWriter writer) throws IOException { JsonGenerator gen = FACTORY.createJsonGenerator(writer); @@ -176,4 +184,21 @@ private void writeProps( } } } + + @Override + protected void writePropsLegacy(Schema schema, Jackson1JsonGeneratorWrapper gen) throws IOException { + Map props = getProps(schema); + //write all props except "default" for enums + if (schema.getType() == Schema.Type.ENUM) { + writeProps(props, gen, s -> !"default".equals(s)); + } else { + writeProps(props, gen); + } + } + + @Override + protected void writePropsLegacy(Schema.Field field, Jackson1JsonGeneratorWrapper gen) throws IOException { + Map props = getProps(field); + writeProps(props, gen); + } } diff --git a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/Avro16Adapter.java b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/Avro16Adapter.java index eba8bbe99..9b08a643d 100644 --- a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/Avro16Adapter.java +++ b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/Avro16Adapter.java @@ -478,7 +478,7 @@ public AvscWriter getAvscWriter(AvscGenerationConfig config, List schemaPlugins) { + super(pretty, preAvro702, addAliasesForAvro702, retainDefaults, retainDocs, retainFieldAliases, + retainNonClaimedProps, retainSchemaAliases, writeNamespaceExplicitly, writeRelativeNamespace, isLegacy, + schemaPlugins); + } + @Override protected Jackson1JsonGeneratorWrapper createJsonGenerator(StringWriter writer) throws IOException { JsonGenerator gen = FACTORY.createJsonGenerator(writer); @@ -133,4 +142,21 @@ private void writeProps( } } } + + @Override + protected void writePropsLegacy(Schema schema, Jackson1JsonGeneratorWrapper gen) throws IOException { + Map props = schema.getProps(); + //write all props except "default" for enums + if (schema.getType() == Schema.Type.ENUM) { + writeProps(props, gen, s -> !"default".equals(s)); + } else { + writeProps(props, gen); + } + } + + @Override + protected void writePropsLegacy(Schema.Field field, Jackson1JsonGeneratorWrapper gen) throws IOException { + Map props = field.props(); + writeProps(props, gen); + } } diff --git a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Adapter.java b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Adapter.java index 9407886c0..fca7de289 100644 --- a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Adapter.java +++ b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Adapter.java @@ -475,7 +475,7 @@ public AvscWriter getAvscWriter(AvscGenerationConfig config, List schemaPlugins) { + super(pretty, preAvro702, addAliasesForAvro702, retainDefaults, retainDocs, retainFieldAliases, + retainNonClaimedProps, retainSchemaAliases, writeNamespaceExplicitly, writeRelativeNamespace, isLegacy, + schemaPlugins); + } + @Override protected Jackson1JsonGeneratorWrapper createJsonGenerator(StringWriter writer) throws IOException { JsonGenerator gen = FACTORY.createJsonGenerator(writer); @@ -137,4 +146,26 @@ private void writeProps( } } } + + @Override + protected void writePropsLegacy(Schema schema, Jackson1JsonGeneratorWrapper gen) throws IOException { + Map props = Avro17Utils.getProps(schema); + if (props == null || props.isEmpty()) { + return; + } + //write all props except "default" for enums + if (schema.getType() == Schema.Type.ENUM) { + writeProps(props, gen, s -> !"default".equals(s)); + } else { + writeProps(props, gen); + } + } + + @Override + protected void writePropsLegacy(Schema.Field field, Jackson1JsonGeneratorWrapper gen) throws IOException { + Map props = Avro17Utils.getProps(field); + if (props != null && !props.isEmpty()) { + writeProps(props, gen); + } + } } diff --git a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/Avro18Adapter.java b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/Avro18Adapter.java index e16ac86fa..b87668522 100644 --- a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/Avro18Adapter.java +++ b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/Avro18Adapter.java @@ -473,7 +473,7 @@ public AvscWriter getAvscWriter(AvscGenerationConfig config, List schemaPlugins) { + super(pretty, preAvro702, addAliasesForAvro702, retainDefaults, retainDocs, retainFieldAliases, + retainNonClaimedProps, retainSchemaAliases, writeNamespaceExplicitly, writeRelativeNamespace, isLegacy, + schemaPlugins); + } + @Override protected Jackson1JsonGeneratorWrapper createJsonGenerator(StringWriter writer) throws IOException { JsonGenerator gen = FACTORY.createJsonGenerator(writer); @@ -137,4 +146,26 @@ private void writeProps( } } } + + @Override + protected void writePropsLegacy(Schema schema, Jackson1JsonGeneratorWrapper gen) throws IOException { + Map props = schema.getJsonProps(); + if (props == null || props.isEmpty()) { + return; + } + //write all props except "default" for enums + if (schema.getType() == Schema.Type.ENUM) { + writeProps(props, gen, s -> !"default".equals(s)); + } else { + writeProps(props, gen); + } + } + + @Override + protected void writePropsLegacy(Schema.Field field, Jackson1JsonGeneratorWrapper gen) throws IOException { + Map props = field.getJsonProps(); + if (props != null && !props.isEmpty()) { + writeProps(props, gen); + } + } } diff --git a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java index fcd4a2025..d3d39de87 100644 --- a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java +++ b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java @@ -503,7 +503,7 @@ public AvscWriter getAvscWriter(AvscGenerationConfig config, List schemaPlugins) { + super(pretty, preAvro702, addAliasesForAvro702, retainDefaults, retainDocs, retainFieldAliases, + retainNonClaimedProps, retainSchemaAliases, writeNamespaceExplicitly, writeRelativeNamespace, isLegacy, + schemaPlugins); + } + @Override protected Jackson2JsonGeneratorWrapper createJsonGenerator(StringWriter writer) throws IOException { JsonGenerator gen = FACTORY.createGenerator(writer); @@ -118,6 +127,20 @@ private void writeProps(Map props, Jackson2JsonGeneratorWrapper delegate.writeObjectField(entry.getKey(), JacksonUtils.toJsonNode(o)); } } + @Override + protected void writePropsLegacy(Schema schema, Jackson2JsonGeneratorWrapper gen) throws IOException { + Map props = schema.getObjectProps(); + if (props != null && !props.isEmpty()) { + writeProps(props, gen); + } + } + @Override + protected void writePropsLegacy(Schema.Field field, Jackson2JsonGeneratorWrapper gen) throws IOException { + Map props = field.getObjectProps(); + if (props != null && !props.isEmpty()) { + writeProps(props, gen); + } + } } diff --git a/helper/tests/helper-tests-17/src/test/java/com/linkedin/avroutil1/compatibility/avro17/AvroUtilSchemaNormalizationTest17.java b/helper/tests/helper-tests-17/src/test/java/com/linkedin/avroutil1/compatibility/avro17/AvroUtilSchemaNormalizationTest17.java index 9902618ae..24eb366f2 100644 --- a/helper/tests/helper-tests-17/src/test/java/com/linkedin/avroutil1/compatibility/avro17/AvroUtilSchemaNormalizationTest17.java +++ b/helper/tests/helper-tests-17/src/test/java/com/linkedin/avroutil1/compatibility/avro17/AvroUtilSchemaNormalizationTest17.java @@ -123,7 +123,7 @@ public void testCanonicalBroadNoPlugin() throws IOException { public void testCanonicalStrictWithNonSpecificJsonIncluded() throws IOException { AvscGenerationConfig config = new AvscGenerationConfig( false, false, false, Optional.of(Boolean.TRUE), false, false, - false, false, true, false, true, false); + false, false, true, false, true, false, false); Schema schema = Schema.parse(TestUtil.load("Record1.avsc")); String str = AvroUtilSchemaNormalization.getCanonicalForm(schema, config, null); @@ -580,7 +580,7 @@ public void testOrder() throws IOException { AvscGenerationConfig config = new AvscGenerationConfig( false, false, false, Optional.of(Boolean.TRUE), false, true, - true, true, true, true, true, false); + true, true, true, true, true, false, false); String expectedForm = "{" diff --git a/helper/tests/helper-tests-19/src/test/java/com/linkedin/avroutil1/compatibility/avro19/AvroUtilSchemaNormalizationTest19.java b/helper/tests/helper-tests-19/src/test/java/com/linkedin/avroutil1/compatibility/avro19/AvroUtilSchemaNormalizationTest19.java index 7ae24725d..fc6871b8b 100644 --- a/helper/tests/helper-tests-19/src/test/java/com/linkedin/avroutil1/compatibility/avro19/AvroUtilSchemaNormalizationTest19.java +++ b/helper/tests/helper-tests-19/src/test/java/com/linkedin/avroutil1/compatibility/avro19/AvroUtilSchemaNormalizationTest19.java @@ -120,7 +120,7 @@ public void testCanonicalBroadNoPlugin() throws IOException { public void testCanonicalStrictWithNonSpecificJsonIncluded() throws IOException { AvscGenerationConfig config = new AvscGenerationConfig( false, false, false, Optional.of(Boolean.TRUE), false, false, - false, false, true, false, true, false); + false, false, true, false, true, false, false); Schema schema = Schema.parse(TestUtil.load("Record1.avsc")); String str = AvroUtilSchemaNormalization.getCanonicalForm(schema, config, null); @@ -169,7 +169,7 @@ public void testOrder() throws IOException { AvscGenerationConfig config = new AvscGenerationConfig( false, false, false, Optional.of(Boolean.TRUE), false, true, - true, true, true, true, true, false); + true, true, true, true, true, false, false); String expectedForm = "{" diff --git a/helper/tests/helper-tests-allavro/src/test/resources/allavro/ComplexRecordInherit.avsc b/helper/tests/helper-tests-allavro/src/test/resources/allavro/ComplexRecordInherit.avsc index 25a2c9521..488530a75 100644 --- a/helper/tests/helper-tests-allavro/src/test/resources/allavro/ComplexRecordInherit.avsc +++ b/helper/tests/helper-tests-allavro/src/test/resources/allavro/ComplexRecordInherit.avsc @@ -1,7 +1,7 @@ { + "type" : "record", "name" : "ComplexRecord", "namespace" : "TopNamespace", - "type" : "record", "fields" : [ { "name" : "longField", "type" : "long", @@ -9,8 +9,8 @@ }, { "name" : "fixedField", "type" : { - "name" : "FixedSchema", "type" : "fixed", + "name" : "FixedSchema", "size" : 100 }, "doc" : "fixed type" @@ -19,16 +19,16 @@ "type" : [ "null", { "type" : "array", "items" : { - "name" : "innerRecord", "type" : "record", + "name" : "innerRecord", "fields" : [ { "name" : "stringField", "type" : "string" }, { "name" : "unionWithEnum", "type" : [ "null", "int", { - "name" : "enumField", "type" : "enum", + "name" : "enumField", "symbols" : [ "A", "B" ] } ] }, { @@ -36,8 +36,8 @@ "type" : { "type" : "map", "values" : { - "name" : "mapUnionRecord", "type" : "record", + "name" : "mapUnionRecord", "fields" : [ { "name" : "stringField", "type" : [ "null", "string" ], diff --git a/helper/tests/helper-tests-allavro/src/test/resources/allavro/ComplexRecordInnerNamespaceInherit.avsc b/helper/tests/helper-tests-allavro/src/test/resources/allavro/ComplexRecordInnerNamespaceInherit.avsc index d62e1f4bf..876563666 100644 --- a/helper/tests/helper-tests-allavro/src/test/resources/allavro/ComplexRecordInnerNamespaceInherit.avsc +++ b/helper/tests/helper-tests-allavro/src/test/resources/allavro/ComplexRecordInnerNamespaceInherit.avsc @@ -1,7 +1,7 @@ { + "type" : "record", "name" : "ComplexRecord", "namespace" : "TopNamespace", - "type" : "record", "fields" : [ { "name" : "longField", "type" : "long", @@ -9,8 +9,8 @@ }, { "name" : "fixedField", "type" : { - "name" : "FixedSchema", "type" : "fixed", + "name" : "FixedSchema", "size" : 100 }, "doc" : "fixed type" @@ -19,17 +19,17 @@ "type" : [ "null", { "type" : "array", "items" : { + "type" : "record", "name" : "innerRecord", "namespace" : "InnerNamespace", - "type" : "record", "fields" : [ { "name" : "stringField", "type" : "string" }, { "name" : "unionWithEnum", "type" : [ "null", "int", { - "name" : "enumField", "type" : "enum", + "name" : "enumField", "symbols" : [ "A", "B" ] } ] }, { @@ -37,8 +37,8 @@ "type" : { "type" : "map", "values" : { - "name" : "mapUnionRecord", "type" : "record", + "name" : "mapUnionRecord", "fields" : [ { "name" : "stringField", "type" : [ "null", "string" ], diff --git a/helper/tests/helper-tests-allavro/src/test/resources/allavro/ComplexRecordInnerNamespaceNoInherit.avsc b/helper/tests/helper-tests-allavro/src/test/resources/allavro/ComplexRecordInnerNamespaceNoInherit.avsc index 7f2be4874..511892c49 100644 --- a/helper/tests/helper-tests-allavro/src/test/resources/allavro/ComplexRecordInnerNamespaceNoInherit.avsc +++ b/helper/tests/helper-tests-allavro/src/test/resources/allavro/ComplexRecordInnerNamespaceNoInherit.avsc @@ -1,7 +1,7 @@ { + "type" : "record", "name" : "ComplexRecord", "namespace" : "TopNamespace", - "type" : "record", "fields" : [ { "name" : "longField", "type" : "long", @@ -9,9 +9,9 @@ }, { "name" : "fixedField", "type" : { + "type" : "fixed", "name" : "FixedSchema", "namespace" : "", - "type" : "fixed", "size" : 100 }, "doc" : "fixed type" @@ -20,17 +20,17 @@ "type" : [ "null", { "type" : "array", "items" : { + "type" : "record", "name" : "innerRecord", "namespace" : "InnerNamespace", - "type" : "record", "fields" : [ { "name" : "stringField", "type" : "string" }, { "name" : "unionWithEnum", "type" : [ "null", "int", { - "name" : "enumField", "type" : "enum", + "name" : "enumField", "symbols" : [ "A", "B" ] } ] }, { @@ -38,8 +38,8 @@ "type" : { "type" : "map", "values" : { - "name" : "mapUnionRecord", "type" : "record", + "name" : "mapUnionRecord", "fields" : [ { "name" : "stringField", "type" : [ "null", "string" ], diff --git a/helper/tests/helper-tests-allavro/src/test/resources/allavro/ComplexRecordNoInherit.avsc b/helper/tests/helper-tests-allavro/src/test/resources/allavro/ComplexRecordNoInherit.avsc index 134c7afcb..0571d9847 100644 --- a/helper/tests/helper-tests-allavro/src/test/resources/allavro/ComplexRecordNoInherit.avsc +++ b/helper/tests/helper-tests-allavro/src/test/resources/allavro/ComplexRecordNoInherit.avsc @@ -1,7 +1,7 @@ { + "type" : "record", "name" : "ComplexRecord", "namespace" : "TopNamespace", - "type" : "record", "fields" : [ { "name" : "longField", "type" : "long", @@ -9,9 +9,9 @@ }, { "name" : "fixedField", "type" : { + "type" : "fixed", "name" : "FixedSchema", "namespace" : "", - "type" : "fixed", "size" : 100 }, "doc" : "fixed type" @@ -20,17 +20,17 @@ "type" : [ "null", { "type" : "array", "items" : { + "type" : "record", "name" : "innerRecord", "namespace" : "", - "type" : "record", "fields" : [ { "name" : "stringField", "type" : "string" }, { "name" : "unionWithEnum", "type" : [ "null", "int", { - "name" : "enumField", "type" : "enum", + "name" : "enumField", "symbols" : [ "A", "B" ] } ] }, { @@ -38,8 +38,8 @@ "type" : { "type" : "map", "values" : { - "name" : "mapUnionRecord", "type" : "record", + "name" : "mapUnionRecord", "fields" : [ { "name" : "stringField", "type" : [ "null", "string" ],