From 4cbb4cf0e5f841cb42f4fc36007d7397d534efc7 Mon Sep 17 00:00:00 2001 From: Vinoo Ganesh Date: Tue, 23 Apr 2024 16:15:33 -0400 Subject: [PATCH] [PARQUET-2460] Removing deprecated code (#237) --- .../parquet/format/InterningProtocol.java | 297 ------------------ .../apache/parquet/format/LogicalTypes.java | 58 ---- .../java/org/apache/parquet/format/Util.java | 239 -------------- .../parquet/format/event/Consumers.java | 201 ------------ .../format/event/EventBasedThriftReader.java | 131 -------- .../parquet/format/event/FieldConsumer.java | 45 --- .../parquet/format/event/TypedConsumer.java | 206 ------------ .../org/apache/parquet/format/TestUtil.java | 88 ------ 8 files changed, 1265 deletions(-) delete mode 100644 src/main/java/org/apache/parquet/format/InterningProtocol.java delete mode 100644 src/main/java/org/apache/parquet/format/LogicalTypes.java delete mode 100644 src/main/java/org/apache/parquet/format/Util.java delete mode 100644 src/main/java/org/apache/parquet/format/event/Consumers.java delete mode 100644 src/main/java/org/apache/parquet/format/event/EventBasedThriftReader.java delete mode 100644 src/main/java/org/apache/parquet/format/event/FieldConsumer.java delete mode 100644 src/main/java/org/apache/parquet/format/event/TypedConsumer.java delete mode 100644 src/test/java/org/apache/parquet/format/TestUtil.java diff --git a/src/main/java/org/apache/parquet/format/InterningProtocol.java b/src/main/java/org/apache/parquet/format/InterningProtocol.java deleted file mode 100644 index b6cbef4d8..000000000 --- a/src/main/java/org/apache/parquet/format/InterningProtocol.java +++ /dev/null @@ -1,297 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.parquet.format; - -import java.nio.ByteBuffer; -import java.util.UUID; - -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TField; -import org.apache.thrift.protocol.TList; -import org.apache.thrift.protocol.TMap; -import org.apache.thrift.protocol.TMessage; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.protocol.TSet; -import org.apache.thrift.protocol.TStruct; -import org.apache.thrift.transport.TTransport; - -/** - * TProtocol that interns the strings. - * - * @deprecated java code moved to the parquet-mr project: See org.apache.parquet:parquet-format-structures; Will be - * removed from here - */ -@Deprecated -public class InterningProtocol extends TProtocol { - - private final TProtocol delegate; - - public InterningProtocol(TProtocol delegate) { - super(delegate.getTransport()); - this.delegate = delegate; - } - - @Override - public TTransport getTransport() { - return delegate.getTransport(); - } - - @Override - public void writeMessageBegin(TMessage message) throws TException { - delegate.writeMessageBegin(message); - } - - @Override - public void writeMessageEnd() throws TException { - delegate.writeMessageEnd(); - } - - @Override - public int hashCode() { - return delegate.hashCode(); - } - - @Override - public void writeStructBegin(TStruct struct) throws TException { - delegate.writeStructBegin(struct); - } - - @Override - public void writeStructEnd() throws TException { - delegate.writeStructEnd(); - } - - @Override - public void writeFieldBegin(TField field) throws TException { - delegate.writeFieldBegin(field); - } - - @Override - public void writeFieldEnd() throws TException { - delegate.writeFieldEnd(); - } - - @Override - public void writeFieldStop() throws TException { - delegate.writeFieldStop(); - } - - @Override - public void writeMapBegin(TMap map) throws TException { - delegate.writeMapBegin(map); - } - - @Override - public void writeMapEnd() throws TException { - delegate.writeMapEnd(); - } - - @Override - public void writeListBegin(TList list) throws TException { - delegate.writeListBegin(list); - } - - @Override - public void writeListEnd() throws TException { - delegate.writeListEnd(); - } - - @Override - public void writeSetBegin(TSet set) throws TException { - delegate.writeSetBegin(set); - } - - @Override - public void writeSetEnd() throws TException { - delegate.writeSetEnd(); - } - - @Override - public void writeBool(boolean b) throws TException { - delegate.writeBool(b); - } - - @Override - public void writeByte(byte b) throws TException { - delegate.writeByte(b); - } - - @Override - public void writeI16(short i16) throws TException { - delegate.writeI16(i16); - } - - @Override - public void writeI32(int i32) throws TException { - delegate.writeI32(i32); - } - - @Override - public void writeI64(long i64) throws TException { - delegate.writeI64(i64); - } - - @Override - public void writeUuid(UUID uuid) throws TException { - delegate.writeUuid(uuid); - } - - @Override - public void writeDouble(double dub) throws TException { - delegate.writeDouble(dub); - } - - @Override - public void writeString(String str) throws TException { - delegate.writeString(str); - } - - @Override - public void writeBinary(ByteBuffer buf) throws TException { - delegate.writeBinary(buf); - } - - @Override - public TMessage readMessageBegin() throws TException { - return delegate.readMessageBegin(); - } - - @Override - public void readMessageEnd() throws TException { - delegate.readMessageEnd(); - } - - @Override - public TStruct readStructBegin() throws TException { - return delegate.readStructBegin(); - } - - @Override - public void readStructEnd() throws TException { - delegate.readStructEnd(); - } - - @Override - public TField readFieldBegin() throws TException { - return delegate.readFieldBegin(); - } - - @Override - public void readFieldEnd() throws TException { - delegate.readFieldEnd(); - } - - @Override - public TMap readMapBegin() throws TException { - return delegate.readMapBegin(); - } - - @Override - public void readMapEnd() throws TException { - delegate.readMapEnd(); - } - - @Override - public TList readListBegin() throws TException { - return delegate.readListBegin(); - } - - @Override - public void readListEnd() throws TException { - delegate.readListEnd(); - } - - @Override - public TSet readSetBegin() throws TException { - return delegate.readSetBegin(); - } - - @Override - public void readSetEnd() throws TException { - delegate.readSetEnd(); - } - - @Override - public boolean equals(Object obj) { - return delegate.equals(obj); - } - - @Override - public boolean readBool() throws TException { - return delegate.readBool(); - } - - @Override - public byte readByte() throws TException { - return delegate.readByte(); - } - - @Override - public short readI16() throws TException { - return delegate.readI16(); - } - - @Override - public int readI32() throws TException { - return delegate.readI32(); - } - - @Override - public long readI64() throws TException { - return delegate.readI64(); - } - - @Override - public UUID readUuid() throws TException { - return delegate.readUuid(); - } - - @Override - public double readDouble() throws TException { - return delegate.readDouble(); - } - - @Override - public String readString() throws TException { - // this is where we intern the strings - return delegate.readString().intern(); - } - - @Override - public ByteBuffer readBinary() throws TException { - return delegate.readBinary(); - } - - @Override - public int getMinSerializedSize(byte b) throws TException { - return delegate.getMinSerializedSize(b); - } - - @Override - public void reset() { - delegate.reset(); - } - - @Override - public String toString() { - return delegate.toString(); - } - -} diff --git a/src/main/java/org/apache/parquet/format/LogicalTypes.java b/src/main/java/org/apache/parquet/format/LogicalTypes.java deleted file mode 100644 index 13012181a..000000000 --- a/src/main/java/org/apache/parquet/format/LogicalTypes.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.parquet.format; - -/** - * Convenience instances of logical type classes. - * @deprecated java code moved to the parquet-mr project: See org.apache.parquet:parquet-format-structures; Will be - * removed from here - */ -@Deprecated -public class LogicalTypes { - public static class TimeUnits { - public static final TimeUnit MILLIS = TimeUnit.MILLIS(new MilliSeconds()); - public static final TimeUnit MICROS = TimeUnit.MICROS(new MicroSeconds()); - } - - public static LogicalType DECIMAL(int scale, int precision) { - return LogicalType.DECIMAL(new DecimalType(scale, precision)); - } - - public static final LogicalType UTF8 = LogicalType.STRING(new StringType()); - public static final LogicalType MAP = LogicalType.MAP(new MapType()); - public static final LogicalType LIST = LogicalType.LIST(new ListType()); - public static final LogicalType ENUM = LogicalType.ENUM(new EnumType()); - public static final LogicalType DATE = LogicalType.DATE(new DateType()); - public static final LogicalType TIME_MILLIS = LogicalType.TIME(new TimeType(true, TimeUnits.MILLIS)); - public static final LogicalType TIME_MICROS = LogicalType.TIME(new TimeType(true, TimeUnits.MICROS)); - public static final LogicalType TIMESTAMP_MILLIS = LogicalType.TIMESTAMP(new TimestampType(true, TimeUnits.MILLIS)); - public static final LogicalType TIMESTAMP_MICROS = LogicalType.TIMESTAMP(new TimestampType(true, TimeUnits.MICROS)); - public static final LogicalType INT_8 = LogicalType.INTEGER(new IntType((byte) 8, true)); - public static final LogicalType INT_16 = LogicalType.INTEGER(new IntType((byte) 16, true)); - public static final LogicalType INT_32 = LogicalType.INTEGER(new IntType((byte) 32, true)); - public static final LogicalType INT_64 = LogicalType.INTEGER(new IntType((byte) 64, true)); - public static final LogicalType UINT_8 = LogicalType.INTEGER(new IntType((byte) 8, false)); - public static final LogicalType UINT_16 = LogicalType.INTEGER(new IntType((byte) 16, false)); - public static final LogicalType UINT_32 = LogicalType.INTEGER(new IntType((byte) 32, false)); - public static final LogicalType UINT_64 = LogicalType.INTEGER(new IntType((byte) 64, false)); - public static final LogicalType UNKNOWN = LogicalType.UNKNOWN(new NullType()); - public static final LogicalType JSON = LogicalType.JSON(new JsonType()); - public static final LogicalType BSON = LogicalType.BSON(new BsonType()); -} diff --git a/src/main/java/org/apache/parquet/format/Util.java b/src/main/java/org/apache/parquet/format/Util.java deleted file mode 100644 index 0c831b673..000000000 --- a/src/main/java/org/apache/parquet/format/Util.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.parquet.format; - -import static org.apache.parquet.format.FileMetaData._Fields.CREATED_BY; -import static org.apache.parquet.format.FileMetaData._Fields.KEY_VALUE_METADATA; -import static org.apache.parquet.format.FileMetaData._Fields.NUM_ROWS; -import static org.apache.parquet.format.FileMetaData._Fields.ROW_GROUPS; -import static org.apache.parquet.format.FileMetaData._Fields.SCHEMA; -import static org.apache.parquet.format.FileMetaData._Fields.VERSION; -import static org.apache.parquet.format.event.Consumers.fieldConsumer; -import static org.apache.parquet.format.event.Consumers.listElementsOf; -import static org.apache.parquet.format.event.Consumers.listOf; -import static org.apache.parquet.format.event.Consumers.struct; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.List; - -import org.apache.thrift.TBase; -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TCompactProtocol; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.transport.TIOStreamTransport; - -import org.apache.parquet.format.event.Consumers.Consumer; -import org.apache.parquet.format.event.Consumers.DelegatingFieldConsumer; -import org.apache.parquet.format.event.EventBasedThriftReader; -import org.apache.parquet.format.event.TypedConsumer.I32Consumer; -import org.apache.parquet.format.event.TypedConsumer.I64Consumer; -import org.apache.parquet.format.event.TypedConsumer.StringConsumer; -import org.apache.thrift.transport.TTransportException; - -/** - * Utility to read/write metadata - * We use the TCompactProtocol to serialize metadata - * - * @deprecated java code moved to the parquet-mr project: See org.apache.parquet:parquet-format-structures; Will be - * removed from here - */ -@Deprecated -public class Util { - - public static void writeColumnIndex(ColumnIndex columnIndex, OutputStream to) throws IOException { - write(columnIndex, to); - } - - public static ColumnIndex readColumnIndex(InputStream from) throws IOException { - return read(from, new ColumnIndex()); - } - - public static void writeOffsetIndex(OffsetIndex offsetIndex, OutputStream to) throws IOException { - write(offsetIndex, to); - } - - public static OffsetIndex readOffsetIndex(InputStream from) throws IOException { - return read(from, new OffsetIndex()); - } - - public static void writePageHeader(PageHeader pageHeader, OutputStream to) throws IOException { - write(pageHeader, to); - } - - public static PageHeader readPageHeader(InputStream from) throws IOException { - return read(from, new PageHeader()); - } - - public static void writeFileMetaData(org.apache.parquet.format.FileMetaData fileMetadata, OutputStream to) throws IOException { - write(fileMetadata, to); - } - - public static FileMetaData readFileMetaData(InputStream from) throws IOException { - return read(from, new FileMetaData()); - } - /** - * reads the meta data from the stream - * @param from the stream to read the metadata from - * @param skipRowGroups whether row groups should be skipped - * @return the resulting metadata - * @throws IOException - */ - public static FileMetaData readFileMetaData(InputStream from, boolean skipRowGroups) throws IOException { - FileMetaData md = new FileMetaData(); - if (skipRowGroups) { - readFileMetaData(from, new DefaultFileMetaDataConsumer(md), skipRowGroups); - } else { - read(from, md); - } - return md; - } - - /** - * To read metadata in a streaming fashion. - */ - public static abstract class FileMetaDataConsumer { - abstract public void setVersion(int version); - abstract public void setSchema(List schema); - abstract public void setNumRows(long numRows); - abstract public void addRowGroup(RowGroup rowGroup); - abstract public void addKeyValueMetaData(KeyValue kv); - abstract public void setCreatedBy(String createdBy); - } - - /** - * Simple default consumer that sets the fields - */ - public static final class DefaultFileMetaDataConsumer extends FileMetaDataConsumer { - private final FileMetaData md; - - public DefaultFileMetaDataConsumer(FileMetaData md) { - this.md = md; - } - - @Override - public void setVersion(int version) { - md.setVersion(version); - } - - @Override - public void setSchema(List schema) { - md.setSchema(schema); - } - - @Override - public void setNumRows(long numRows) { - md.setNum_rows(numRows); - } - - @Override - public void setCreatedBy(String createdBy) { - md.setCreated_by(createdBy); - } - - @Override - public void addRowGroup(RowGroup rowGroup) { - md.addToRow_groups(rowGroup); - } - - @Override - public void addKeyValueMetaData(KeyValue kv) { - md.addToKey_value_metadata(kv); - } - } - - public static void readFileMetaData(InputStream from, FileMetaDataConsumer consumer) throws IOException { - readFileMetaData(from, consumer, false); - } - - public static void readFileMetaData(InputStream from, final FileMetaDataConsumer consumer, boolean skipRowGroups) throws IOException { - try { - DelegatingFieldConsumer eventConsumer = fieldConsumer() - .onField(VERSION, new I32Consumer() { - @Override - public void consume(int value) { - consumer.setVersion(value); - } - }).onField(SCHEMA, listOf(SchemaElement.class, new Consumer>() { - @Override - public void consume(List schema) { - consumer.setSchema(schema); - } - })).onField(NUM_ROWS, new I64Consumer() { - @Override - public void consume(long value) { - consumer.setNumRows(value); - } - }).onField(KEY_VALUE_METADATA, listElementsOf(struct(KeyValue.class, new Consumer() { - @Override - public void consume(KeyValue kv) { - consumer.addKeyValueMetaData(kv); - } - }))).onField(CREATED_BY, new StringConsumer() { - @Override - public void consume(String value) { - consumer.setCreatedBy(value); - } - }); - if (!skipRowGroups) { - eventConsumer = eventConsumer.onField(ROW_GROUPS, listElementsOf(struct(RowGroup.class, new Consumer() { - @Override - public void consume(RowGroup rowGroup) { - consumer.addRowGroup(rowGroup); - } - }))); - } - new EventBasedThriftReader(protocol(from)).readStruct(eventConsumer); - - } catch (TException e) { - throw new IOException("can not read FileMetaData: " + e.getMessage(), e); - } - } - - private static TProtocol protocol(OutputStream to) throws TTransportException { - return protocol(new TIOStreamTransport(to)); - } - - private static TProtocol protocol(InputStream from) throws TTransportException { - return protocol(new TIOStreamTransport(from)); - } - - private static InterningProtocol protocol(TIOStreamTransport t) { - return new InterningProtocol(new TCompactProtocol(t)); - } - - private static > T read(InputStream from, T tbase) throws IOException { - try { - tbase.read(protocol(from)); - return tbase; - } catch (TException e) { - throw new IOException("can not read " + tbase.getClass() + ": " + e.getMessage(), e); - } - } - - private static void write(TBase tbase, OutputStream to) throws IOException { - try { - tbase.write(protocol(to)); - } catch (TException e) { - throw new IOException("can not write " + tbase, e); - } - } -} diff --git a/src/main/java/org/apache/parquet/format/event/Consumers.java b/src/main/java/org/apache/parquet/format/event/Consumers.java deleted file mode 100644 index e57aa07e3..000000000 --- a/src/main/java/org/apache/parquet/format/event/Consumers.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.parquet.format.event; - -import static java.util.Collections.unmodifiableMap; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.thrift.TBase; -import org.apache.thrift.TException; -import org.apache.thrift.TFieldIdEnum; -import org.apache.thrift.protocol.TList; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.protocol.TProtocolUtil; - -import org.apache.parquet.format.event.Consumers.Consumer; -import org.apache.parquet.format.event.TypedConsumer.BoolConsumer; -import org.apache.parquet.format.event.TypedConsumer.ListConsumer; -import org.apache.parquet.format.event.TypedConsumer.StructConsumer; - -/** - * Entry point for reading thrift in a streaming fashion - * - * @author Julien Le Dem - * @deprecated java code moved to the parquet-mr project: See org.apache.parquet:parquet-format-structures; Will be - * removed from here - */ -@Deprecated -public class Consumers { - - /** - * To consume objects coming from a DelegatingFieldConsumer - * @author Julien Le Dem - * - * @param the type of consumed objects - */ - public static interface Consumer { - void consume(T t); - } - - /** - * Delegates reading the field to TypedConsumers. - * There is one TypedConsumer per thrift type. - * use {@link DelegatingFieldConsumer#onField(TFieldIdEnum, BoolConsumer)} et al. to consume specific thrift fields. - * @see Consumers#fieldConsumer() - * @author Julien Le Dem - * - */ - public static class DelegatingFieldConsumer implements FieldConsumer { - - private final Map contexts; - private final FieldConsumer defaultFieldEventConsumer; - - private DelegatingFieldConsumer(FieldConsumer defaultFieldEventConsumer, Map contexts) { - this.defaultFieldEventConsumer = defaultFieldEventConsumer; - this.contexts = unmodifiableMap(contexts); - } - - private DelegatingFieldConsumer() { - this(new SkippingFieldConsumer()); - } - - private DelegatingFieldConsumer(FieldConsumer defaultFieldEventConsumer) { - this(defaultFieldEventConsumer, Collections.emptyMap()); - } - - public DelegatingFieldConsumer onField(TFieldIdEnum e, TypedConsumer typedConsumer) { - Map newContexts = new HashMap(contexts); - newContexts.put(e.getThriftFieldId(), typedConsumer); - return new DelegatingFieldConsumer(defaultFieldEventConsumer, newContexts); - } - - @Override - public void consumeField( - TProtocol protocol, EventBasedThriftReader reader, - short id, byte type) throws TException { - TypedConsumer delegate = contexts.get(id); - if (delegate != null) { - delegate.read(protocol, reader, type); - } else { - defaultFieldEventConsumer.consumeField(protocol, reader, id, type); - } - } - } - - /** - * call onField on the resulting DelegatingFieldConsumer to handle individual fields - * @return a new DelegatingFieldConsumer - */ - public static DelegatingFieldConsumer fieldConsumer() { - return new DelegatingFieldConsumer(); - } - - /** - * To consume a list of elements - * @param c the type of the list content - * @param consumer the consumer that will receive the list - * @return a ListConsumer that can be passed to the DelegatingFieldConsumer - */ - public static > ListConsumer listOf(Class c, final Consumer> consumer) { - class ListConsumer implements Consumer { - List list; - @Override - public void consume(T t) { - list.add(t); - } - } - final ListConsumer co = new ListConsumer(); - return new DelegatingListElementsConsumer(struct(c, co)) { - @Override - public void consumeList(TProtocol protocol, - EventBasedThriftReader reader, TList tList) throws TException { - co.list = new ArrayList(); - super.consumeList(protocol, reader, tList); - consumer.consume(co.list); - } - }; - } - - /** - * To consume list elements one by one - * @param consumer the consumer that will read the elements - * @return a ListConsumer that can be passed to the DelegatingFieldConsumer - */ - public static ListConsumer listElementsOf(TypedConsumer consumer) { - return new DelegatingListElementsConsumer(consumer); - } - - public static > StructConsumer struct(final Class c, final Consumer consumer) { - return new TBaseStructConsumer(c, consumer); - } -} - -class SkippingFieldConsumer implements FieldConsumer { - @Override - public void consumeField(TProtocol protocol, EventBasedThriftReader reader, short id, byte type) throws TException { - TProtocolUtil.skip(protocol, type); - } -} - -class DelegatingListElementsConsumer extends ListConsumer { - - private TypedConsumer elementConsumer; - - protected DelegatingListElementsConsumer(TypedConsumer consumer) { - this.elementConsumer = consumer; - } - - @Override - public void consumeElement(TProtocol protocol, EventBasedThriftReader reader, byte elemType) throws TException { - elementConsumer.read(protocol, reader, elemType); - } -} -class TBaseStructConsumer> extends StructConsumer { - - private final Class c; - private Consumer consumer; - - public TBaseStructConsumer(Class c, Consumer consumer) { - this.c = c; - this.consumer = consumer; - } - - @Override - public void consumeStruct(TProtocol protocol, EventBasedThriftReader reader) throws TException { - T o = newObject(); - o.read(protocol); - consumer.consume(o); - } - - protected T newObject() { - try { - return c.newInstance(); - } catch (InstantiationException e) { - throw new RuntimeException(c.getName(), e); - } catch (IllegalAccessException e) { - throw new RuntimeException(c.getName(), e); - } - } - -} \ No newline at end of file diff --git a/src/main/java/org/apache/parquet/format/event/EventBasedThriftReader.java b/src/main/java/org/apache/parquet/format/event/EventBasedThriftReader.java deleted file mode 100644 index 6d1f6b84f..000000000 --- a/src/main/java/org/apache/parquet/format/event/EventBasedThriftReader.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.parquet.format.event; - -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TField; -import org.apache.thrift.protocol.TList; -import org.apache.thrift.protocol.TMap; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.protocol.TSet; -import org.apache.thrift.protocol.TType; - -import org.apache.parquet.format.event.TypedConsumer.ListConsumer; -import org.apache.parquet.format.event.TypedConsumer.MapConsumer; -import org.apache.parquet.format.event.TypedConsumer.SetConsumer; - -/** - * Event based reader for Thrift - * - * @author Julien Le Dem - * @deprecated java code moved to the parquet-mr project: See org.apache.parquet:parquet-format-structures; Will be - * removed from here - */ -@Deprecated -public final class EventBasedThriftReader { - - private final TProtocol protocol; - - /** - * @param protocol the protocol to read from - */ - public EventBasedThriftReader(TProtocol protocol) { - this.protocol = protocol; - } - - /** - * reads a Struct from the underlying protocol and passes the field events to the FieldConsumer - * @param c the field consumer - * @throws TException - */ - public void readStruct(FieldConsumer c) throws TException { - protocol.readStructBegin(); - readStructContent(c); - protocol.readStructEnd(); - } - - /** - * reads the content of a struct (fields) from the underlying protocol and passes the events to c - * @param c the field consumer - * @throws TException - */ - public void readStructContent(FieldConsumer c) throws TException { - TField field; - while (true) { - field = protocol.readFieldBegin(); - if (field.type == TType.STOP) { - break; - } - c.consumeField(protocol, this, field.id, field.type); - } - } - - /** - * reads the set content (elements) from the underlying protocol and passes the events to the set event consumer - * @param eventConsumer the consumer - * @param tSet the set descriptor - * @throws TException - */ - public void readSetContent(SetConsumer eventConsumer, TSet tSet) - throws TException { - for (int i = 0; i < tSet.size; i++) { - eventConsumer.consumeElement(protocol, this, tSet.elemType); - } - } - - /** - * reads the map content (key values) from the underlying protocol and passes the events to the map event consumer - * @param eventConsumer the consumer - * @param tMap the map descriptor - * @throws TException - */ - public void readMapContent(MapConsumer eventConsumer, TMap tMap) - throws TException { - for (int i = 0; i < tMap.size; i++) { - eventConsumer.consumeEntry(protocol, this, tMap.keyType, tMap.valueType); - } - } - - /** - * reads a key-value pair - * @param keyType the type of the key - * @param keyConsumer the consumer for the key - * @param valueType the type of the value - * @param valueConsumer the consumer for the value - * @throws TException - */ - public void readMapEntry(byte keyType, TypedConsumer keyConsumer, byte valueType, TypedConsumer valueConsumer) - throws TException { - keyConsumer.read(protocol, this, keyType); - valueConsumer.read(protocol, this, valueType); - } - - /** - * reads the list content (elements) from the underlying protocol and passes the events to the list event consumer - * @param eventConsumer the consumer - * @param tList the list descriptor - * @throws TException - */ - public void readListContent(ListConsumer eventConsumer, TList tList) - throws TException { - for (int i = 0; i < tList.size; i++) { - eventConsumer.consumeElement(protocol, this, tList.elemType); - } - } -} \ No newline at end of file diff --git a/src/main/java/org/apache/parquet/format/event/FieldConsumer.java b/src/main/java/org/apache/parquet/format/event/FieldConsumer.java deleted file mode 100644 index c882dcc2d..000000000 --- a/src/main/java/org/apache/parquet/format/event/FieldConsumer.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.parquet.format.event; - -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TProtocol; - -/** - * To receive Thrift field events - * - * @author Julien Le Dem - * @deprecated java code moved to the parquet-mr project: See org.apache.parquet:parquet-format-structures; Will be - * removed from here - */ -@Deprecated -public interface FieldConsumer { - - /** - * called by the EventBasedThriftReader when reading a field from a Struct - * @param protocol the underlying protocol - * @param eventBasedThriftReader the reader to delegate to further calls. - * @param id the id of the field - * @param type the type of the field - * @return the typed consumer to pass the value to - * @throws TException - */ - public void consumeField(TProtocol protocol, EventBasedThriftReader eventBasedThriftReader, short id, byte type) throws TException; - -} \ No newline at end of file diff --git a/src/main/java/org/apache/parquet/format/event/TypedConsumer.java b/src/main/java/org/apache/parquet/format/event/TypedConsumer.java deleted file mode 100644 index 4838de9ee..000000000 --- a/src/main/java/org/apache/parquet/format/event/TypedConsumer.java +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.parquet.format.event; - -import static org.apache.thrift.protocol.TType.BOOL; -import static org.apache.thrift.protocol.TType.BYTE; -import static org.apache.thrift.protocol.TType.DOUBLE; -import static org.apache.thrift.protocol.TType.I16; -import static org.apache.thrift.protocol.TType.I32; -import static org.apache.thrift.protocol.TType.I64; -import static org.apache.thrift.protocol.TType.LIST; -import static org.apache.thrift.protocol.TType.MAP; -import static org.apache.thrift.protocol.TType.SET; -import static org.apache.thrift.protocol.TType.STRING; -import static org.apache.thrift.protocol.TType.STRUCT; - -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TList; -import org.apache.thrift.protocol.TMap; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.protocol.TSet; - -/** - * receive thrift events of a given type - * - * @author Julien Le Dem - * @deprecated java code moved to the parquet-mr project: See org.apache.parquet:parquet-format-structures; Will be - * removed from here - */ -@Deprecated -abstract public class TypedConsumer { - - abstract public static class DoubleConsumer extends TypedConsumer { - protected DoubleConsumer() { super(DOUBLE); } - @Override - final void read(TProtocol protocol, EventBasedThriftReader reader) throws TException { - this.consume(protocol.readDouble()); - } - abstract public void consume(double value); - } - - abstract public static class ByteConsumer extends TypedConsumer { - protected ByteConsumer() { super(BYTE); } - @Override - final void read(TProtocol protocol, EventBasedThriftReader reader) throws TException { - this.consume(protocol.readByte()); - } - abstract public void consume(byte value); - } - - abstract public static class BoolConsumer extends TypedConsumer { - protected BoolConsumer() { super(BOOL); } - @Override - final void read(TProtocol protocol, EventBasedThriftReader reader) throws TException { - this.consume(protocol.readBool()); - } - abstract public void consume(boolean value); - } - - abstract public static class I32Consumer extends TypedConsumer { - protected I32Consumer() { super(I32); } - @Override - final void read(TProtocol protocol, EventBasedThriftReader reader) throws TException { - this.consume(protocol.readI32()); - } - abstract public void consume(int value); - } - - abstract public static class I64Consumer extends TypedConsumer { - protected I64Consumer() { super(I64); } - final void read(TProtocol protocol, EventBasedThriftReader reader) throws TException { - this.consume(protocol.readI64()); - } - abstract public void consume(long value); - } - - abstract public static class I16Consumer extends TypedConsumer { - protected I16Consumer() { super(I16); } - @Override - final void read(TProtocol protocol, EventBasedThriftReader reader) throws TException { - this.consume(protocol.readI16()); - } - abstract public void consume(short value); - } - - abstract public static class StringConsumer extends TypedConsumer { - protected StringConsumer() { super(STRING); } - @Override - final void read(TProtocol protocol, EventBasedThriftReader reader) throws TException { - this.consume(protocol.readString()); - } - abstract public void consume(String value); - } - - abstract public static class StructConsumer extends TypedConsumer { - protected StructConsumer() { super(STRUCT); } - @Override - final void read(TProtocol protocol, EventBasedThriftReader reader) throws TException { - this.consumeStruct(protocol, reader); - } - /** - * can either delegate to the reader or read the struct from the protocol - * reader.readStruct(fieldConsumer); - * @param protocol the underlying protocol - * @param reader the reader to delegate to - * @throws TException - */ - abstract public void consumeStruct(TProtocol protocol, EventBasedThriftReader reader) throws TException; - } - - abstract public static class ListConsumer extends TypedConsumer { - protected ListConsumer() { super(LIST); } - @Override - final void read(TProtocol protocol, EventBasedThriftReader reader) throws TException { - this.consumeList(protocol, reader, protocol.readListBegin()); - protocol.readListEnd(); - } - public void consumeList(TProtocol protocol, EventBasedThriftReader reader, TList tList) throws TException { - reader.readListContent(this, tList); - } - /** - * can either delegate to the reader or read the element from the protocol - * @param protocol the underlying protocol - * @param reader the reader to delegate to - * @throws TException - */ - abstract public void consumeElement(TProtocol protocol, EventBasedThriftReader reader, byte elemType) throws TException; - } - - abstract public static class SetConsumer extends TypedConsumer { - protected SetConsumer() { super(SET); } - @Override - final void read(TProtocol protocol, EventBasedThriftReader reader) throws TException { - this.consumeSet(protocol, reader, protocol.readSetBegin()); - protocol.readSetEnd(); - } - public void consumeSet(TProtocol protocol, EventBasedThriftReader reader, TSet tSet) throws TException { - reader.readSetContent(this, tSet); - } - /** - * can either delegate to the reader or read the set from the protocol - * @param protocol the underlying protocol - * @param reader the reader to delegate to - * @throws TException - */ - abstract public void consumeElement( - TProtocol protocol, EventBasedThriftReader reader, - byte elemType) throws TException; - } - - abstract public static class MapConsumer extends TypedConsumer { - protected MapConsumer() { super(MAP); } - @Override - final void read(TProtocol protocol, EventBasedThriftReader reader) - throws TException { - this.consumeMap(protocol, reader , protocol.readMapBegin()); - protocol.readMapEnd(); - } - public void consumeMap(TProtocol protocol, EventBasedThriftReader reader, TMap tMap) throws TException { - reader.readMapContent(this, tMap); - } - /** - * can either delegate to the reader or read the map entry from the protocol - * @param protocol the underlying protocol - * @param reader the reader to delegate to - * @throws TException - */ - abstract public void consumeEntry( - TProtocol protocol, EventBasedThriftReader reader, - byte keyType, byte valueType) throws TException; - } - - public final byte type; - - private TypedConsumer(byte type) { - this.type = type; - } - - final public void read(TProtocol protocol, EventBasedThriftReader reader, byte type) throws TException { - if (this.type != type) { - throw new TException( - "Incorrect type in stream. " - + "Expected " + this.type - + " but got " + type); - } - this.read(protocol, reader); - } - - abstract void read(TProtocol protocol, EventBasedThriftReader reader) throws TException; -} \ No newline at end of file diff --git a/src/test/java/org/apache/parquet/format/TestUtil.java b/src/test/java/org/apache/parquet/format/TestUtil.java deleted file mode 100644 index c91f8e27e..000000000 --- a/src/test/java/org/apache/parquet/format/TestUtil.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.parquet.format; - -import static java.util.Arrays.asList; -import static junit.framework.Assert.assertEquals; -import static junit.framework.Assert.assertNull; -import static org.apache.parquet.format.Util.readFileMetaData; -import static org.apache.parquet.format.Util.writeFileMetaData; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; - -import org.junit.Test; - -import org.apache.parquet.format.Util.DefaultFileMetaDataConsumer; -/** - * @deprecated java code moved to the parquet-mr project: See org.apache.parquet:parquet-format-structures; Will be - * removed from here - */ -@Deprecated -public class TestUtil { - - @Test - public void testReadFileMetadata() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - FileMetaData md = new FileMetaData( - 1, - asList(new SchemaElement("foo")), - 10, - asList( - new RowGroup( - asList( - new ColumnChunk(0), - new ColumnChunk(1) - ), - 10, - 5), - new RowGroup( - asList( - new ColumnChunk(2), - new ColumnChunk(3) - ), - 11, - 5) - ) - ); - writeFileMetaData(md , baos); - FileMetaData md2 = readFileMetaData(in(baos)); - FileMetaData md3 = new FileMetaData(); - readFileMetaData(in(baos), new DefaultFileMetaDataConsumer(md3)); - FileMetaData md4 = new FileMetaData(); - readFileMetaData(in(baos), new DefaultFileMetaDataConsumer(md4), true); - FileMetaData md5 = readFileMetaData(in(baos), true); - FileMetaData md6 = readFileMetaData(in(baos), false); - assertEquals(md, md2); - assertEquals(md, md3); - assertNull(md4.getRow_groups()); - assertNull(md5.getRow_groups()); - assertEquals(md4, md5); - md4.setRow_groups(md.getRow_groups()); - md5.setRow_groups(md.getRow_groups()); - assertEquals(md, md4); - assertEquals(md, md5); - assertEquals(md4, md5); - assertEquals(md, md6); - } - - private ByteArrayInputStream in(ByteArrayOutputStream baos) { - return new ByteArrayInputStream(baos.toByteArray()); - } -}