From 10bef2ff7f4d4a668e003e685a491141659945d1 Mon Sep 17 00:00:00 2001 From: xinxinzhenhuai <61451754+xinxinzhenhuai@users.noreply.github.com> Date: Wed, 27 May 2020 11:00:00 -0700 Subject: [PATCH] add newBinaryDecoder which re-configures a BinaryDecoder in AvroCompatibilityHelper (#52) Venice has a local optimization requires re-initialize a BinaryDecoder, but the method signature is different in avro 1.4 and versions after 1.4. In order to be compatible with all avro versions, we need to add this method in AvroCompatibilityHelper. Co-authored-by: xnma --- .../avroutil1/compatibility/AvroAdapter.java | 3 +++ .../AvroCompatibilityHelper.java | 16 ++++++++++++ .../compatibility/avro14/Avro14Adapter.java | 6 +++++ .../io/Avro14BinaryDecoderAccessUtil.java | 25 +++++++++++++++++++ .../compatibility/avro15/Avro15Adapter.java | 6 +++++ .../io/Avro15BinaryDecoderAccessUtil.java | 24 ++++++++++++++++++ .../compatibility/avro16/Avro16Adapter.java | 6 +++++ .../io/Avro16BinaryDecoderAccessUtil.java | 24 ++++++++++++++++++ .../compatibility/avro17/Avro17Adapter.java | 6 +++++ .../io/Avro17BinaryDecoderAccessUtil.java | 24 ++++++++++++++++++ .../compatibility/avro18/Avro18Adapter.java | 6 +++++ .../io/Avro18BinaryDecoderAccessUtil.java | 24 ++++++++++++++++++ .../compatibility/avro19/Avro19Adapter.java | 6 +++++ .../io/Avro19BinaryDecoderAccessUtil.java | 24 ++++++++++++++++++ 14 files changed, 200 insertions(+) create mode 100644 helper/impls/helper-impl-14/src/main/java/org/apache/avro/io/Avro14BinaryDecoderAccessUtil.java create mode 100644 helper/impls/helper-impl-15/src/main/java/org/apache/avro/io/Avro15BinaryDecoderAccessUtil.java create mode 100644 helper/impls/helper-impl-16/src/main/java/org/apache/avro/io/Avro16BinaryDecoderAccessUtil.java create mode 100644 helper/impls/helper-impl-17/src/main/java/org/apache/avro/io/Avro17BinaryDecoderAccessUtil.java create mode 100644 helper/impls/helper-impl-18/src/main/java/org/apache/avro/io/Avro18BinaryDecoderAccessUtil.java create mode 100644 helper/impls/helper-impl-19/src/main/java/org/apache/avro/io/Avro19BinaryDecoderAccessUtil.java diff --git a/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvroAdapter.java b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvroAdapter.java index 1e2f6b601..312d41246 100644 --- a/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvroAdapter.java +++ b/helper/helper-common/src/main/java/com/linkedin/avroutil1/compatibility/AvroAdapter.java @@ -37,6 +37,9 @@ public interface AvroAdapter { BinaryDecoder newBinaryDecoder(ObjectInput in); + BinaryDecoder newBinaryDecoder(byte[] bytes, int offset, + int length, BinaryDecoder reuse); + JsonEncoder newJsonEncoder(Schema schema, OutputStream out, boolean pretty) throws IOException; JsonDecoder newJsonDecoder(Schema schema, InputStream in) throws IOException; diff --git a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/AvroCompatibilityHelper.java b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/AvroCompatibilityHelper.java index cced6df2d..3fcd6537d 100644 --- a/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/AvroCompatibilityHelper.java +++ b/helper/helper/src/main/java/com/linkedin/avroutil1/compatibility/AvroCompatibilityHelper.java @@ -133,6 +133,22 @@ public static BinaryDecoder newBinaryDecoder(InputStream in, boolean buffered, B return ADAPTER.newBinaryDecoder(in, buffered, reuse); } + /** + * constructs or reinitializes a {@link BinaryDecoder} with the byte array + * provided as the source of data. + * @param bytes The byte array to initialize to + * @param offset The offset to start reading from + * @param length The maximum number of bytes to read from the byte array + * @param reuse The BinaryDecoder to attempt to reinitialize. + * @return A BinaryDecoder that uses bytes as its source of data. If + * reuse is null, this will be a new instance. + */ + public static BinaryDecoder newBinaryDecoder(byte[] bytes, int offset, + int length, BinaryDecoder reuse) { + assertAvroAvailable(); + return ADAPTER.newBinaryDecoder(bytes, offset, length, reuse); + } + /** * convenience method for getting a {@link BinaryDecoder} for a given byte[] * @param in byte array with data diff --git a/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/Avro14Adapter.java b/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/Avro14Adapter.java index 570525169..e5efb4dd0 100644 --- a/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/Avro14Adapter.java +++ b/helper/impls/helper-impl-14/src/main/java/com/linkedin/avroutil1/compatibility/avro14/Avro14Adapter.java @@ -36,6 +36,7 @@ import org.apache.avro.AvroRuntimeException; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; +import org.apache.avro.io.Avro14BinaryDecoderAccessUtil; import org.apache.avro.io.BinaryDecoder; import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.DecoderFactory; @@ -106,6 +107,11 @@ public BinaryDecoder newBinaryDecoder(ObjectInput in) { return newBinaryDecoder(new ObjectInputToInputStreamAdapter(in), false, null); } + @Override + public BinaryDecoder newBinaryDecoder(byte[] bytes, int offset, int length, BinaryDecoder reuse) { + return Avro14BinaryDecoderAccessUtil.newBinaryDecoder(bytes, offset, length, reuse); + } + @Override public JsonEncoder newJsonEncoder(Schema schema, OutputStream out, boolean pretty) throws IOException { return new JsonEncoder(schema, out); diff --git a/helper/impls/helper-impl-14/src/main/java/org/apache/avro/io/Avro14BinaryDecoderAccessUtil.java b/helper/impls/helper-impl-14/src/main/java/org/apache/avro/io/Avro14BinaryDecoderAccessUtil.java new file mode 100644 index 000000000..1c8e9d4ce --- /dev/null +++ b/helper/impls/helper-impl-14/src/main/java/org/apache/avro/io/Avro14BinaryDecoderAccessUtil.java @@ -0,0 +1,25 @@ +/* + * Copyright 2020 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package org.apache.avro.io; + +/** + * this class exists to allow us access to package-private classes and methods on class {@link BinaryDecoder} + * + * the difference between this method and {@link DecoderFactory#createBinaryDecoder(byte[], int, int, BinaryDecoder)} + * is that this method supports configuring custom BinaryDecoder since it does not check class type of BinaryDecoder. + */ +public class Avro14BinaryDecoderAccessUtil { + public static BinaryDecoder newBinaryDecoder(byte[] bytes, int offset, + int length, BinaryDecoder reuse) { + if (null != reuse) { + reuse.init(bytes, offset, length); + return reuse; + } else { + return new BinaryDecoder(bytes, offset, length); + } + } +} diff --git a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/Avro15Adapter.java b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/Avro15Adapter.java index 7bc0c31c0..39d72aea6 100644 --- a/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/Avro15Adapter.java +++ b/helper/impls/helper-impl-15/src/main/java/com/linkedin/avroutil1/compatibility/avro15/Avro15Adapter.java @@ -37,6 +37,7 @@ import java.util.stream.Collectors; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; +import org.apache.avro.io.Avro15BinaryDecoderAccessUtil; import org.apache.avro.io.BinaryDecoder; import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.DecoderFactory; @@ -109,6 +110,11 @@ public BinaryDecoder newBinaryDecoder(ObjectInput in) { return newBinaryDecoder(new ObjectInputToInputStreamAdapter(in), false, null); } + @Override + public BinaryDecoder newBinaryDecoder(byte[] bytes, int offset, int length, BinaryDecoder reuse) { + return Avro15BinaryDecoderAccessUtil.newBinaryDecoder(bytes, offset, length, reuse); + } + @Override public JsonEncoder newJsonEncoder(Schema schema, OutputStream out, boolean pretty) throws IOException { return EncoderFactory.get().jsonEncoder(schema, out); diff --git a/helper/impls/helper-impl-15/src/main/java/org/apache/avro/io/Avro15BinaryDecoderAccessUtil.java b/helper/impls/helper-impl-15/src/main/java/org/apache/avro/io/Avro15BinaryDecoderAccessUtil.java new file mode 100644 index 000000000..8adab2856 --- /dev/null +++ b/helper/impls/helper-impl-15/src/main/java/org/apache/avro/io/Avro15BinaryDecoderAccessUtil.java @@ -0,0 +1,24 @@ +/* + * Copyright 2020 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package org.apache.avro.io; + +/** + * this class exists to allow us access to package-private classes and methods on class {@link BinaryDecoder} + * + * the difference between this method and {@link DecoderFactory#binaryDecoder(byte[], int, int, BinaryDecoder)} + * is that this method supports configuring custom BinaryDecoder since it does not check class type of BinaryDecoder. + */ +public class Avro15BinaryDecoderAccessUtil { + public static BinaryDecoder newBinaryDecoder(byte[] bytes, int offset, + int length, BinaryDecoder reuse) { + if (null == reuse) { + return new BinaryDecoder(bytes, offset, length); + } else { + return reuse.configure(bytes, offset, length); + } + } +} diff --git a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/Avro16Adapter.java b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/Avro16Adapter.java index a434fcec5..fcac6eca3 100644 --- a/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/Avro16Adapter.java +++ b/helper/impls/helper-impl-16/src/main/java/com/linkedin/avroutil1/compatibility/avro16/Avro16Adapter.java @@ -36,6 +36,7 @@ import java.util.stream.Collectors; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; +import org.apache.avro.io.Avro16BinaryDecoderAccessUtil; import org.apache.avro.io.BinaryDecoder; import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.DecoderFactory; @@ -115,6 +116,11 @@ public BinaryDecoder newBinaryDecoder(ObjectInput in) { return newBinaryDecoder(new ObjectInputToInputStreamAdapter(in), false, null); } + @Override + public BinaryDecoder newBinaryDecoder(byte[] bytes, int offset, int length, BinaryDecoder reuse) { + return Avro16BinaryDecoderAccessUtil.newBinaryDecoder(bytes, offset, length, reuse); + } + @Override public JsonEncoder newJsonEncoder(Schema schema, OutputStream out, boolean pretty) throws IOException { return EncoderFactory.get().jsonEncoder(schema, out); diff --git a/helper/impls/helper-impl-16/src/main/java/org/apache/avro/io/Avro16BinaryDecoderAccessUtil.java b/helper/impls/helper-impl-16/src/main/java/org/apache/avro/io/Avro16BinaryDecoderAccessUtil.java new file mode 100644 index 000000000..48c0836f1 --- /dev/null +++ b/helper/impls/helper-impl-16/src/main/java/org/apache/avro/io/Avro16BinaryDecoderAccessUtil.java @@ -0,0 +1,24 @@ +/* + * Copyright 2020 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package org.apache.avro.io; + +/** + * this class exists to allow us access to package-private classes and methods on class {@link BinaryDecoder} + * + * the difference between this method and {@link DecoderFactory#binaryDecoder(byte[], int, int, BinaryDecoder)} + * is that this method supports configuring custom BinaryDecoder since it does not check class type of BinaryDecoder. + */ +public class Avro16BinaryDecoderAccessUtil { + public static BinaryDecoder newBinaryDecoder(byte[] bytes, int offset, + int length, BinaryDecoder reuse) { + if (null == reuse) { + return new BinaryDecoder(bytes, offset, length); + } else { + return reuse.configure(bytes, offset, length); + } + } +} diff --git a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Adapter.java b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Adapter.java index 6742dd987..e52f0bd69 100644 --- a/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Adapter.java +++ b/helper/impls/helper-impl-17/src/main/java/com/linkedin/avroutil1/compatibility/avro17/Avro17Adapter.java @@ -36,6 +36,7 @@ import org.apache.avro.Schema; import org.apache.avro.SchemaNormalization; import org.apache.avro.generic.GenericData; +import org.apache.avro.io.Avro17BinaryDecoderAccessUtil; import org.apache.avro.io.BinaryDecoder; import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.DecoderFactory; @@ -158,6 +159,11 @@ public BinaryDecoder newBinaryDecoder(ObjectInput in) { return newBinaryDecoder(new ObjectInputToInputStreamAdapter(in), false, null); } + @Override + public BinaryDecoder newBinaryDecoder(byte[] bytes, int offset, int length, BinaryDecoder reuse) { + return Avro17BinaryDecoderAccessUtil.newBinaryDecoder(bytes, offset, length, reuse); + } + @Override public JsonEncoder newJsonEncoder(Schema schema, OutputStream out, boolean pretty) throws IOException { return EncoderFactory.get().jsonEncoder(schema, out, pretty); diff --git a/helper/impls/helper-impl-17/src/main/java/org/apache/avro/io/Avro17BinaryDecoderAccessUtil.java b/helper/impls/helper-impl-17/src/main/java/org/apache/avro/io/Avro17BinaryDecoderAccessUtil.java new file mode 100644 index 000000000..eaa66c921 --- /dev/null +++ b/helper/impls/helper-impl-17/src/main/java/org/apache/avro/io/Avro17BinaryDecoderAccessUtil.java @@ -0,0 +1,24 @@ +/* + * Copyright 2020 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package org.apache.avro.io; + +/** + * this class exists to allow us access to package-private classes and methods on class {@link BinaryDecoder} + * + * the difference between this method and {@link DecoderFactory#binaryDecoder(byte[], int, int, BinaryDecoder)} + * is that this method supports configuring custom BinaryDecoder since it does not check class type of BinaryDecoder. + */ +public class Avro17BinaryDecoderAccessUtil { + public static BinaryDecoder newBinaryDecoder(byte[] bytes, int offset, + int length, BinaryDecoder reuse) { + if (null == reuse) { + return new BinaryDecoder(bytes, offset, length); + } else { + return reuse.configure(bytes, offset, length); + } + } +} diff --git a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/Avro18Adapter.java b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/Avro18Adapter.java index 5f7232dd4..414bbc592 100644 --- a/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/Avro18Adapter.java +++ b/helper/impls/helper-impl-18/src/main/java/com/linkedin/avroutil1/compatibility/avro18/Avro18Adapter.java @@ -32,6 +32,7 @@ import org.apache.avro.Schema; import org.apache.avro.SchemaNormalization; import org.apache.avro.generic.GenericData; +import org.apache.avro.io.Avro18BinaryDecoderAccessUtil; import org.apache.avro.io.BinaryDecoder; import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.DecoderFactory; @@ -117,6 +118,11 @@ public BinaryDecoder newBinaryDecoder(ObjectInput in) { return newBinaryDecoder(new ObjectInputToInputStreamAdapter(in), false, null); } + @Override + public BinaryDecoder newBinaryDecoder(byte[] bytes, int offset, int length, BinaryDecoder reuse) { + return Avro18BinaryDecoderAccessUtil.newBinaryDecoder(bytes, offset, length, reuse); + } + @Override public JsonEncoder newJsonEncoder(Schema schema, OutputStream out, boolean pretty) throws IOException { return EncoderFactory.get().jsonEncoder(schema, out, pretty); diff --git a/helper/impls/helper-impl-18/src/main/java/org/apache/avro/io/Avro18BinaryDecoderAccessUtil.java b/helper/impls/helper-impl-18/src/main/java/org/apache/avro/io/Avro18BinaryDecoderAccessUtil.java new file mode 100644 index 000000000..10896b309 --- /dev/null +++ b/helper/impls/helper-impl-18/src/main/java/org/apache/avro/io/Avro18BinaryDecoderAccessUtil.java @@ -0,0 +1,24 @@ +/* + * Copyright 2020 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package org.apache.avro.io; + +/** + * this class exists to allow us access to package-private classes and methods on class {@link BinaryDecoder} + * + * the difference between this method and {@link DecoderFactory#binaryDecoder(byte[], int, int, BinaryDecoder)} + * is that this method supports configuring custom BinaryDecoder since it does not check class type of BinaryDecoder. + */ +public class Avro18BinaryDecoderAccessUtil { + public static BinaryDecoder newBinaryDecoder(byte[] bytes, int offset, + int length, BinaryDecoder reuse) { + if (null == reuse) { + return new BinaryDecoder(bytes, offset, length); + } else { + return reuse.configure(bytes, offset, length); + } + } +} diff --git a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java index 2a3b37930..22abff87d 100644 --- a/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java +++ b/helper/impls/helper-impl-19/src/main/java/com/linkedin/avroutil1/compatibility/avro19/Avro19Adapter.java @@ -32,6 +32,7 @@ import org.apache.avro.Schema; import org.apache.avro.SchemaNormalization; import org.apache.avro.generic.GenericData; +import org.apache.avro.io.Avro19BinaryDecoderAccessUtil; import org.apache.avro.io.BinaryDecoder; import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.DecoderFactory; @@ -117,6 +118,11 @@ public BinaryDecoder newBinaryDecoder(ObjectInput in) { return newBinaryDecoder(new ObjectInputToInputStreamAdapter(in), false, null); } + @Override + public BinaryDecoder newBinaryDecoder(byte[] bytes, int offset, int length, BinaryDecoder reuse) { + return Avro19BinaryDecoderAccessUtil.newBinaryDecoder(bytes, offset, length, reuse); + } + @Override public JsonEncoder newJsonEncoder(Schema schema, OutputStream out, boolean pretty) throws IOException { return EncoderFactory.get().jsonEncoder(schema, out, pretty); diff --git a/helper/impls/helper-impl-19/src/main/java/org/apache/avro/io/Avro19BinaryDecoderAccessUtil.java b/helper/impls/helper-impl-19/src/main/java/org/apache/avro/io/Avro19BinaryDecoderAccessUtil.java new file mode 100644 index 000000000..0a1a9b8e6 --- /dev/null +++ b/helper/impls/helper-impl-19/src/main/java/org/apache/avro/io/Avro19BinaryDecoderAccessUtil.java @@ -0,0 +1,24 @@ +/* + * Copyright 2020 LinkedIn Corp. + * Licensed under the BSD 2-Clause License (the "License"). + * See License in the project root for license information. + */ + +package org.apache.avro.io; + +/** + * this class exists to allow us access to package-private classes and methods on class {@link BinaryDecoder} + * + * the difference between this method and {@link DecoderFactory#binaryDecoder(byte[], int, int, BinaryDecoder)} + * is that this method supports configuring custom BinaryDecoder since it does not check class type of BinaryDecoder. + */ +public class Avro19BinaryDecoderAccessUtil { + public static BinaryDecoder newBinaryDecoder(byte[] bytes, int offset, + int length, BinaryDecoder reuse) { + if (null == reuse) { + return new BinaryDecoder(bytes, offset, length); + } else { + return reuse.configure(bytes, offset, length); + } + } +} \ No newline at end of file