Skip to content

Commit

Permalink
Fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@CurtHagenlocher
CurtHagenlocher committed Dec 21, 2023
1 parent 535ecac commit 8b82c0d
Show file tree
Hide file tree
Showing 6 changed files with 66 additions and 51 deletions.
4 changes: 2 additions & 2 deletions csharp/src/Apache.Arrow/C/CArrowArrayExporter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,11 +123,11 @@ private unsafe static void ConvertArray(ExportedAllocationOwner sharedOwner, Arr
cArray->buffers = null;
if (cArray->n_buffers > 0)
{
int* lengths = null;
long* lengths = null;
int bufferCount = array.Buffers.Length;
if (array.DataType.TypeId == ArrowTypeId.BinaryView || array.DataType.TypeId == ArrowTypeId.StringView)
{
lengths = (int*)sharedOwner.Allocate(4 * bufferCount); // overallocation to avoid edge case
lengths = (long*)sharedOwner.Allocate(8 * bufferCount); // overallocation to avoid edge case
bufferCount++;
cArray->n_buffers++;
}
Expand Down
4 changes: 2 additions & 2 deletions csharp/src/Apache.Arrow/C/CArrowArrayImporter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,13 +286,13 @@ private ArrowBuffer[] ImportByteArrayViewBuffers(CArrowArray* cArray)
int length = checked((int)cArray->length);
int viewsLength = length * 16;

int* bufferLengths = (int*)cArray->buffers[cArray->n_buffers - 1];
long* bufferLengths = (long*)cArray->buffers[cArray->n_buffers - 1];
ArrowBuffer[] buffers = new ArrowBuffer[cArray->n_buffers - 1];
buffers[0] = ImportValidityBuffer(cArray);
buffers[1] = new ArrowBuffer(AddMemory((IntPtr)cArray->buffers[1], 0, viewsLength));
for (int i = 2; i < buffers.Length; i++)
{
buffers[i] = new ArrowBuffer(AddMemory((IntPtr)cArray->buffers[i], 0, bufferLengths[i - 2]));
buffers[i] = new ArrowBuffer(AddMemory((IntPtr)cArray->buffers[i], 0, checked((int)bufferLengths[i - 2])));
}

return buffers;
Expand Down
2 changes: 1 addition & 1 deletion csharp/test/Apache.Arrow.Benchmarks/ArrowWriterBenchmark.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public class ArrowWriterBenchmark
[GlobalSetup]
public void GlobalSetup()
{
_batch = TestData.CreateSampleRecordBatch(BatchLength, ColumnSetCount, false);
_batch = TestData.CreateSampleRecordBatch(BatchLength, ColumnSetCount);
_memoryStream = new MemoryStream();
}

Expand Down
4 changes: 3 additions & 1 deletion csharp/test/Apache.Arrow.Tests/CDataInterfacePythonTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -741,7 +741,9 @@ public unsafe void ExportBatch()
[SkippableFact]
public unsafe void RoundTripTestBatch()
{
RecordBatch batch1 = TestData.CreateSampleRecordBatch(4, createDictionaryArray: true);
// TODO: Enable these once this the version of pyarrow referenced during testing supports them
HashSet<ArrowTypeId> unsupported = new HashSet<ArrowTypeId> { ArrowTypeId.ListView, ArrowTypeId.BinaryView, ArrowTypeId.StringView };
RecordBatch batch1 = TestData.CreateSampleRecordBatch(4, excludedTypes: unsupported);
RecordBatch batch2 = batch1.Clone();

CArrowArray* cExportArray = CArrowArray.Create();
Expand Down
93 changes: 51 additions & 42 deletions csharp/test/Apache.Arrow.Tests/TestData.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,57 +24,66 @@ namespace Apache.Arrow.Tests
{
public static class TestData
{
public static RecordBatch CreateSampleRecordBatch(int length, bool createDictionaryArray = true)
public static RecordBatch CreateSampleRecordBatch(int length, bool createDictionaryArray)
{
return CreateSampleRecordBatch(length, columnSetCount: 1, createDictionaryArray);
HashSet<ArrowTypeId> excluded = createDictionaryArray ? null : new HashSet<ArrowTypeId> { ArrowTypeId.Dictionary };
return CreateSampleRecordBatch(length, columnSetCount: 1, excluded);
}

public static RecordBatch CreateSampleRecordBatch(int length, int columnSetCount, bool createAdvancedTypeArrays)
public static RecordBatch CreateSampleRecordBatch(
int length,
int columnSetCount = 1,
HashSet<ArrowTypeId> excludedTypes = null)
{
Schema.Builder builder = new Schema.Builder();

void AddField(Field field)
{
if (excludedTypes == null || !excludedTypes.Contains(field.DataType.TypeId))
{
builder.Field(field);
}
}

for (int i = 0; i < columnSetCount; i++)
{
builder.Field(CreateField(new ListType(Int64Type.Default), i));
builder.Field(CreateField(new ListViewType(Int64Type.Default), i));
builder.Field(CreateField(BooleanType.Default, i));
builder.Field(CreateField(UInt8Type.Default, i));
builder.Field(CreateField(Int8Type.Default, i));
builder.Field(CreateField(UInt16Type.Default, i));
builder.Field(CreateField(Int16Type.Default, i));
builder.Field(CreateField(UInt32Type.Default, i));
builder.Field(CreateField(Int32Type.Default, i));
builder.Field(CreateField(UInt64Type.Default, i));
builder.Field(CreateField(Int64Type.Default, i));
builder.Field(CreateField(FloatType.Default, i));
builder.Field(CreateField(DoubleType.Default, i));
builder.Field(CreateField(Date32Type.Default, i));
builder.Field(CreateField(Date64Type.Default, i));
builder.Field(CreateField(Time32Type.Default, i));
builder.Field(CreateField(Time64Type.Default, i));
builder.Field(CreateField(TimestampType.Default, i));
builder.Field(CreateField(StringType.Default, i));
builder.Field(CreateField(StringViewType.Default, i));
builder.Field(CreateField(new StructType(new List<Field> { CreateField(StringType.Default, i), CreateField(Int32Type.Default, i) }), i));
builder.Field(CreateField(new Decimal128Type(10, 6), i));
builder.Field(CreateField(new Decimal256Type(16, 8), i));
builder.Field(CreateField(new MapType(StringType.Default, Int32Type.Default), i));
builder.Field(CreateField(IntervalType.YearMonth, i));
builder.Field(CreateField(IntervalType.DayTime, i));
builder.Field(CreateField(IntervalType.MonthDayNanosecond, i));
builder.Field(CreateField(BinaryType.Default, i));
builder.Field(CreateField(BinaryViewType.Default, i));
AddField(CreateField(new ListType(Int64Type.Default), i));
AddField(CreateField(new ListViewType(Int64Type.Default), i));
AddField(CreateField(BooleanType.Default, i));
AddField(CreateField(UInt8Type.Default, i));
AddField(CreateField(Int8Type.Default, i));
AddField(CreateField(UInt16Type.Default, i));
AddField(CreateField(Int16Type.Default, i));
AddField(CreateField(UInt32Type.Default, i));
AddField(CreateField(Int32Type.Default, i));
AddField(CreateField(UInt64Type.Default, i));
AddField(CreateField(Int64Type.Default, i));
#if NET5_0_OR_GREATER
builder.Field(CreateField(HalfFloatType.Default, i));
AddField(CreateField(HalfFloatType.Default, i));
#endif

if (createAdvancedTypeArrays)
{
builder.Field(CreateField(new DictionaryType(Int32Type.Default, StringType.Default, false), i));
builder.Field(CreateField(new FixedSizeBinaryType(16), i));
builder.Field(CreateField(new FixedSizeListType(Int32Type.Default, 3), i));
builder.Field(CreateField(new UnionType(new[] { CreateField(StringType.Default, i), CreateField(Int32Type.Default, i) }, new[] { 0, 1 }, UnionMode.Sparse), i));
builder.Field(CreateField(new UnionType(new[] { CreateField(StringType.Default, i), CreateField(Int32Type.Default, i) }, new[] { 0, 1 }, UnionMode.Dense), -i));
}
AddField(CreateField(FloatType.Default, i));
AddField(CreateField(DoubleType.Default, i));
AddField(CreateField(Date32Type.Default, i));
AddField(CreateField(Date64Type.Default, i));
AddField(CreateField(Time32Type.Default, i));
AddField(CreateField(Time64Type.Default, i));
AddField(CreateField(TimestampType.Default, i));
AddField(CreateField(StringType.Default, i));
AddField(CreateField(StringViewType.Default, i));
AddField(CreateField(new StructType(new List<Field> { CreateField(StringType.Default, i), CreateField(Int32Type.Default, i) }), i));
AddField(CreateField(new Decimal128Type(10, 6), i));
AddField(CreateField(new Decimal256Type(16, 8), i));
AddField(CreateField(new MapType(StringType.Default, Int32Type.Default), i));
AddField(CreateField(IntervalType.YearMonth, i));
AddField(CreateField(IntervalType.DayTime, i));
AddField(CreateField(IntervalType.MonthDayNanosecond, i));
AddField(CreateField(BinaryType.Default, i));
AddField(CreateField(BinaryViewType.Default, i));
AddField(CreateField(new FixedSizeBinaryType(16), i));
AddField(CreateField(new FixedSizeListType(Int32Type.Default, 3), i));
AddField(CreateField(new UnionType(new[] { CreateField(StringType.Default, i), CreateField(Int32Type.Default, i) }, new[] { 0, 1 }, UnionMode.Sparse), i));
AddField(CreateField(new UnionType(new[] { CreateField(StringType.Default, i), CreateField(Int32Type.Default, i) }, new[] { 0, 1 }, UnionMode.Dense), -i));
AddField(CreateField(new DictionaryType(Int32Type.Default, StringType.Default, false), i));
}

Schema schema = builder.Build();
Expand Down
10 changes: 7 additions & 3 deletions docs/source/status.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,13 @@ Data Types
+-------------------+-------+-------+-------+------------+-------+-------+-------+-------+
| Large Utf8 | ✓ | ✓ | ✓ | ✓ | | ✓ | ✓ | |
+-------------------+-------+-------+-------+------------+-------+-------+-------+-------+
| Binary View | ✓ | | ✓ | | | | | |
| Binary View | ✓ | | ✓ | | ✓ | | | |
+-------------------+-------+-------+-------+------------+-------+-------+-------+-------+
| String View | ✓ | | ✓ | | | | | |
| Large Binary View | ✓ | | ✓ | | | | | |
+-------------------+-------+-------+-------+------------+-------+-------+-------+-------+
| Utf8 View | ✓ | | ✓ | | ✓ | | | |
+-------------------+-------+-------+-------+------------+-------+-------+-------+-------+
| Large Utf8 View | ✓ | | ✓ | | | | | |
+-------------------+-------+-------+-------+------------+-------+-------+-------+-------+

+-------------------+-------+-------+-------+------------+-------+-------+-------+-------+
Expand All @@ -83,7 +87,7 @@ Data Types
+-------------------+-------+-------+-------+------------+-------+-------+-------+-------+
| Large List | ✓ | ✓ | ✓ | | | ✓ | ✓ | |
+-------------------+-------+-------+-------+------------+-------+-------+-------+-------+
| List View | ✓ | | ✓ | | | | | |
| List View | ✓ | | ✓ | | ✓ | | | |
+-------------------+-------+-------+-------+------------+-------+-------+-------+-------+
| Large List View | ✓ | | ✓ | | | | | |
+-------------------+-------+-------+-------+------------+-------+-------+-------+-------+
Expand Down

0 comments on commit 8b82c0d

Please sign in to comment.