From d24c76b049534b342c4cc2ebdbfccfabc250a51c Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Tue, 26 Nov 2024 18:45:10 +0100 Subject: [PATCH] fix: support arrow view types (#533) I've added support for view type passthrough some time ago in delta-rs. Arrow-rs finally catched up and FFI support was added for view types as well. I wanted to add enable view types export in polars during delta writes but delta-kernel-rs unfortunately hasn't the support yet: `SchemaMismatchError: Invalid data type for Delta Lake: Utf8View` This adds the view types as part of the conversion Co-authored-by: Zach Schuermann --- kernel/src/engine/arrow_conversion.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kernel/src/engine/arrow_conversion.rs b/kernel/src/engine/arrow_conversion.rs index 0d7c2775a..fbfdb487a 100644 --- a/kernel/src/engine/arrow_conversion.rs +++ b/kernel/src/engine/arrow_conversion.rs @@ -179,6 +179,7 @@ impl TryFrom<&ArrowDataType> for DataType { match arrow_datatype { ArrowDataType::Utf8 => Ok(DataType::STRING), ArrowDataType::LargeUtf8 => Ok(DataType::STRING), + ArrowDataType::Utf8View => Ok(DataType::STRING), ArrowDataType::Int64 => Ok(DataType::LONG), // undocumented type ArrowDataType::Int32 => Ok(DataType::INTEGER), ArrowDataType::Int16 => Ok(DataType::SHORT), @@ -193,6 +194,7 @@ impl TryFrom<&ArrowDataType> for DataType { ArrowDataType::Binary => Ok(DataType::BINARY), ArrowDataType::FixedSizeBinary(_) => Ok(DataType::BINARY), ArrowDataType::LargeBinary => Ok(DataType::BINARY), + ArrowDataType::BinaryView => Ok(DataType::BINARY), ArrowDataType::Decimal128(p, s) => { if *s < 0 { return Err(ArrowError::from_external_error( @@ -216,9 +218,15 @@ impl TryFrom<&ArrowDataType> for DataType { ArrowDataType::List(field) => { Ok(ArrayType::new((*field).data_type().try_into()?, (*field).is_nullable()).into()) } + ArrowDataType::ListView(field) => { + Ok(ArrayType::new((*field).data_type().try_into()?, (*field).is_nullable()).into()) + } ArrowDataType::LargeList(field) => { Ok(ArrayType::new((*field).data_type().try_into()?, (*field).is_nullable()).into()) } + ArrowDataType::LargeListView(field) => { + Ok(ArrayType::new((*field).data_type().try_into()?, (*field).is_nullable()).into()) + } ArrowDataType::FixedSizeList(field, _) => { Ok(ArrayType::new((*field).data_type().try_into()?, (*field).is_nullable()).into()) }