Skip to content

Commit

Permalink
fix: initial commit
Browse files Browse the repository at this point in the history
  • Loading branch information
vibhatha committed Aug 1, 2024
1 parent d4d92e4 commit 823610b
Show file tree
Hide file tree
Showing 6 changed files with 245 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,12 @@
import org.apache.arrow.vector.NullVector;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.complex.BaseRepeatedValueVector;
import org.apache.arrow.vector.complex.BaseRepeatedValueViewVector;
import org.apache.arrow.vector.complex.DenseUnionVector;
import org.apache.arrow.vector.complex.FixedSizeListVector;
import org.apache.arrow.vector.complex.LargeListVector;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.ListViewVector;
import org.apache.arrow.vector.complex.NonNullableStructVector;
import org.apache.arrow.vector.complex.UnionVector;

Expand Down Expand Up @@ -234,6 +236,14 @@ public Boolean visit(ExtensionTypeVector<?> left, Range range) {
return underlyingVisitor.rangeEquals(range);
}

@Override
public Boolean visit(ListViewVector left, Range range) {
if (!validate(left)) {
return false;
}
return compareListViewVectors(range);
}

protected RangeEqualsVisitor createInnerVisitor(
ValueVector leftInner,
ValueVector rightInner,
Expand Down Expand Up @@ -702,4 +712,51 @@ protected boolean compareLargeListVectors(Range range) {
}
return true;
}

protected boolean compareListViewVectors(Range range) {
ListViewVector leftVector = (ListViewVector) left;
ListViewVector rightVector = (ListViewVector) right;

RangeEqualsVisitor innerVisitor =
createInnerVisitor(
leftVector.getDataVector(), rightVector.getDataVector(), /*type comparator*/ null);
Range innerRange = new Range();

for (int i = 0; i < range.getLength(); i++) {
int leftIndex = range.getLeftStart() + i;
int rightIndex = range.getRightStart() + i;

boolean isNull = leftVector.isNull(leftIndex);
if (isNull != rightVector.isNull(rightIndex)) {
return false;
}

int offsetWidth = BaseRepeatedValueViewVector.OFFSET_WIDTH;
int sizeWidth = BaseRepeatedValueViewVector.SIZE_WIDTH;

if (!isNull) {
final int startIndexLeft =
leftVector.getOffsetBuffer().getInt((long) leftIndex * offsetWidth);
final int leftSize = leftVector.getSizeBuffer().getInt((long) leftIndex * sizeWidth);

final int startIndexRight =
rightVector.getOffsetBuffer().getInt((long) rightIndex * offsetWidth);
final int rightSize = rightVector.getSizeBuffer().getInt((long) rightIndex * sizeWidth);

if (leftSize != rightSize) {
return false;
}

innerRange =
innerRange
.setRightStart(startIndexRight)
.setLeftStart(startIndexLeft)
.setLength(leftSize);
if (!innerVisitor.rangeEquals(innerRange)) {
return false;
}
}
}
return true;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.apache.arrow.vector.complex.FixedSizeListVector;
import org.apache.arrow.vector.complex.LargeListVector;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.ListViewVector;
import org.apache.arrow.vector.complex.NonNullableStructVector;
import org.apache.arrow.vector.complex.UnionVector;
import org.apache.arrow.vector.types.pojo.Field;
Expand Down Expand Up @@ -124,6 +125,11 @@ public Boolean visit(ExtensionTypeVector<?> left, Void value) {
return compareField(left.getField(), right.getField());
}

@Override
public Boolean visit(ListViewVector left, Void value) {
return compareField(left.getField(), right.getField());
}

private boolean compareField(Field leftField, Field rightField) {

if (leftField == rightField) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.arrow.vector.complex.FixedSizeListVector;
import org.apache.arrow.vector.complex.LargeListVector;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.ListViewVector;
import org.apache.arrow.vector.complex.NonNullableStructVector;
import org.apache.arrow.vector.complex.UnionVector;

Expand Down Expand Up @@ -60,4 +61,8 @@ public interface VectorVisitor<OUT, IN> {
OUT visit(NullVector left, IN value);

OUT visit(ExtensionTypeVector<?> left, IN value);

default OUT visit(ListViewVector left, IN value) {
throw new UnsupportedOperationException("VectorVisitor for ListViewVector is not supported.");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2882,6 +2882,34 @@ public void testListVectorEqualsWithNull() {
}
}

@Test
public void testListViewVectorEqualsWithNull() {
try (final ListViewVector vector1 = ListViewVector.empty("listview", allocator);
final ListViewVector vector2 = ListViewVector.empty("listview", allocator); ) {

UnionListViewWriter writer1 = vector1.getWriter();
writer1.allocate();

// set some values
writeListViewVector(writer1, new int[] {1, 2});
writeListViewVector(writer1, new int[] {3, 4});
writeListViewVector(writer1, new int[] {});
writer1.setValueCount(3);

UnionListViewWriter writer2 = vector2.getWriter();
writer2.allocate();

// set some values
writeListViewVector(writer2, new int[] {1, 2});
writeListViewVector(writer2, new int[] {3, 4});
writer2.setValueCount(3);

VectorEqualsVisitor visitor = new VectorEqualsVisitor();

assertFalse(visitor.vectorEquals(vector1, vector2));
}
}

@Test
public void testListVectorEquals() {
try (final ListVector vector1 = ListVector.empty("list", allocator);
Expand Down Expand Up @@ -2914,6 +2942,38 @@ public void testListVectorEquals() {
}
}

@Test
public void testListViewVectorEquals() {
try (final ListViewVector vector1 = ListViewVector.empty("listview", allocator);
final ListViewVector vector2 = ListViewVector.empty("listview", allocator); ) {

UnionListViewWriter writer1 = vector1.getWriter();
writer1.allocate();

// set some values
writeListViewVector(writer1, new int[] {1, 2});
writeListViewVector(writer1, new int[] {3, 4});
writeListViewVector(writer1, new int[] {5, 6});
writer1.setValueCount(3);

UnionListViewWriter writer2 = vector2.getWriter();
writer2.allocate();

// set some values
writeListViewVector(writer2, new int[] {1, 2});
writeListViewVector(writer2, new int[] {3, 4});
writer2.setValueCount(2);

VectorEqualsVisitor visitor = new VectorEqualsVisitor();
assertFalse(visitor.vectorEquals(vector1, vector2));

writeListViewVector(writer2, new int[] {5, 6});
writer2.setValueCount(3);

assertTrue(visitor.vectorEquals(vector1, vector2));
}
}

@Test
public void testListVectorSetNull() {
try (final ListVector vector = ListVector.empty("list", allocator)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,12 @@
import org.apache.arrow.vector.complex.DenseUnionVector;
import org.apache.arrow.vector.complex.FixedSizeListVector;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.ListViewVector;
import org.apache.arrow.vector.complex.StructVector;
import org.apache.arrow.vector.complex.UnionVector;
import org.apache.arrow.vector.complex.impl.NullableStructWriter;
import org.apache.arrow.vector.complex.impl.UnionFixedSizeListWriter;
import org.apache.arrow.vector.complex.impl.UnionListViewWriter;
import org.apache.arrow.vector.complex.impl.UnionListWriter;
import org.apache.arrow.vector.holders.NullableBigIntHolder;
import org.apache.arrow.vector.holders.NullableFloat4Holder;
Expand Down Expand Up @@ -201,6 +203,24 @@ public void testListVectorWithDifferentChild() {
}
}

@Test
public void testListViewVectorWithDifferentChild() {
try (final ListViewVector vector1 = ListViewVector.empty("listview", allocator);
final ListViewVector vector2 = ListViewVector.empty("listview", allocator); ) {

vector1.allocateNew();
vector1.initializeChildrenFromFields(
Arrays.asList(Field.nullable("child", new ArrowType.Int(32, true))));

vector2.allocateNew();
vector2.initializeChildrenFromFields(
Arrays.asList(Field.nullable("child", new ArrowType.Int(64, true))));

RangeEqualsVisitor visitor = new RangeEqualsVisitor(vector1, vector2);
assertFalse(visitor.rangeEquals(new Range(0, 0, 0)));
}
}

@Test
public void testListVectorRangeEquals() {
try (final ListVector vector1 = ListVector.empty("list", allocator);
Expand Down Expand Up @@ -233,6 +253,38 @@ public void testListVectorRangeEquals() {
}
}

@Test
public void testListViewVectorRangeEquals() {
try (final ListViewVector vector1 = ListViewVector.empty("listview", allocator);
final ListViewVector vector2 = ListViewVector.empty("listview", allocator); ) {

UnionListViewWriter writer1 = vector1.getWriter();
writer1.allocate();

// set some values
writeListViewVector(writer1, new int[] {1, 2});
writeListViewVector(writer1, new int[] {3, 4});
writeListViewVector(writer1, new int[] {5, 6});
writeListViewVector(writer1, new int[] {7, 8});
writeListViewVector(writer1, new int[] {9, 10});
writer1.setValueCount(5);

UnionListViewWriter writer2 = vector2.getWriter();
writer2.allocate();

// set some values
writeListViewVector(writer2, new int[] {0, 0});
writeListViewVector(writer2, new int[] {3, 4});
writeListViewVector(writer2, new int[] {5, 6});
writeListViewVector(writer2, new int[] {7, 8});
writeListViewVector(writer2, new int[] {0, 0});
writer2.setValueCount(5);

RangeEqualsVisitor visitor = new RangeEqualsVisitor(vector1, vector2);
assertTrue(visitor.rangeEquals(new Range(1, 1, 3)));
}
}

@Test
public void testBitVectorRangeEquals() {
try (final BitVector vector1 = new BitVector("v1", allocator);
Expand Down Expand Up @@ -819,6 +871,38 @@ public void testListVectorApproxEquals() {
}
}

@Test
public void testListViewVectorApproxEquals() {
try (final ListViewVector right = ListViewVector.empty("listview", allocator);
final ListViewVector left1 = ListViewVector.empty("listview", allocator);
final ListViewVector left2 = ListViewVector.empty("listview", allocator); ) {

final float epsilon = 1.0E-6f;

UnionListViewWriter rightWriter = right.getWriter();
rightWriter.allocate();
writeListViewVector(rightWriter, new double[] {1, 2});
writeListViewVector(rightWriter, new double[] {1.01, 2.02});
rightWriter.setValueCount(2);

UnionListViewWriter leftWriter1 = left1.getWriter();
leftWriter1.allocate();
writeListViewVector(leftWriter1, new double[] {1, 2});
writeListViewVector(leftWriter1, new double[] {1.01 + epsilon / 2, 2.02 - epsilon / 2});
leftWriter1.setValueCount(2);

UnionListViewWriter leftWriter2 = left2.getWriter();
leftWriter2.allocate();
writeListViewVector(leftWriter2, new double[] {1, 2});
writeListViewVector(leftWriter2, new double[] {1.01 + epsilon * 2, 2.02 - epsilon * 2});
leftWriter2.setValueCount(2);

Range range = new Range(0, 0, right.getValueCount());
assertTrue(new ApproxEqualsVisitor(left1, right, epsilon, epsilon).rangeEquals(range));
assertFalse(new ApproxEqualsVisitor(left2, right, epsilon, epsilon).rangeEquals(range));
}
}

private void writeStructVector(NullableStructWriter writer, int value1, long value2) {
writer.start();
writer.integer("f0").writeInt(value1);
Expand All @@ -841,6 +925,14 @@ private void writeListVector(UnionListWriter writer, int[] values) {
writer.endList();
}

private void writeListViewVector(UnionListViewWriter writer, int[] values) {
writer.startListView();
for (int v : values) {
writer.integer().writeInt(v);
}
writer.endListView();
}

private void writeFixedSizeListVector(UnionFixedSizeListWriter writer, int[] values) {
writer.startList();
for (int v : values) {
Expand All @@ -856,4 +948,12 @@ private void writeListVector(UnionListWriter writer, double[] values) {
}
writer.endList();
}

private void writeListViewVector(UnionListViewWriter writer, double[] values) {
writer.startListView();
for (double v : values) {
writer.float8().writeFloat8(v);
}
writer.endListView();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.apache.arrow.vector.ViewVarCharVector;
import org.apache.arrow.vector.complex.DenseUnionVector;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.ListViewVector;
import org.apache.arrow.vector.complex.StructVector;
import org.apache.arrow.vector.complex.UnionVector;
import org.apache.arrow.vector.types.Types;
Expand Down Expand Up @@ -104,6 +105,22 @@ public void testListTypeEquals() {
}
}

@Test
public void testListViewTypeEquals() {
try (final ListViewVector right = ListViewVector.empty("listview", allocator);
final ListViewVector left1 = ListViewVector.empty("listview", allocator);
final ListViewVector left2 = ListViewVector.empty("listview", allocator)) {

right.addOrGetVector(FieldType.nullable(new ArrowType.Utf8()));
left1.addOrGetVector(FieldType.nullable(new ArrowType.Utf8()));
left2.addOrGetVector(FieldType.nullable(new ArrowType.FixedSizeBinary(2)));

TypeEqualsVisitor visitor = new TypeEqualsVisitor(right);
assertTrue(visitor.equals(left1));
assertFalse(visitor.equals(left2));
}
}

@Test
public void testStructTypeEquals() {
try (final StructVector right = StructVector.empty("struct", allocator);
Expand Down

0 comments on commit 823610b

Please sign in to comment.