fix: support arrow view types (delta-io#533)

I've added support for view type passthrough some time ago in delta-rs. Arrow-rs finally catched up and FFI support was added for view types as well. I wanted to add enable view types export in polars during delta writes but delta-kernel-rs unfortunately hasn't the support yet: `SchemaMismatchError: Invalid data type for Delta Lake: Utf8View` This adds the view types as part of the conversion Co-authored-by: Zach Schuermann <[email protected]>
OussamaSaoudi-db · Nov 26, 2024 · d24c76b · d24c76b
1 parent 485525b
commit d24c76b
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/kernel/src/engine/arrow_conversion.rs b/kernel/src/engine/arrow_conversion.rs
@@ -179,6 +179,7 @@ impl TryFrom<&ArrowDataType> for DataType {
         match arrow_datatype {
             ArrowDataType::Utf8 => Ok(DataType::STRING),
             ArrowDataType::LargeUtf8 => Ok(DataType::STRING),
+            ArrowDataType::Utf8View => Ok(DataType::STRING),
             ArrowDataType::Int64 => Ok(DataType::LONG), // undocumented type
             ArrowDataType::Int32 => Ok(DataType::INTEGER),
             ArrowDataType::Int16 => Ok(DataType::SHORT),
@@ -193,6 +194,7 @@ impl TryFrom<&ArrowDataType> for DataType {
             ArrowDataType::Binary => Ok(DataType::BINARY),
             ArrowDataType::FixedSizeBinary(_) => Ok(DataType::BINARY),
             ArrowDataType::LargeBinary => Ok(DataType::BINARY),
+            ArrowDataType::BinaryView => Ok(DataType::BINARY),
             ArrowDataType::Decimal128(p, s) => {
                 if *s < 0 {
                     return Err(ArrowError::from_external_error(
@@ -216,9 +218,15 @@ impl TryFrom<&ArrowDataType> for DataType {
             ArrowDataType::List(field) => {
                 Ok(ArrayType::new((*field).data_type().try_into()?, (*field).is_nullable()).into())
             }
+            ArrowDataType::ListView(field) => {
+                Ok(ArrayType::new((*field).data_type().try_into()?, (*field).is_nullable()).into())
+            }
             ArrowDataType::LargeList(field) => {
                 Ok(ArrayType::new((*field).data_type().try_into()?, (*field).is_nullable()).into())
             }
+            ArrowDataType::LargeListView(field) => {
+                Ok(ArrayType::new((*field).data_type().try_into()?, (*field).is_nullable()).into())
+            }
             ArrowDataType::FixedSizeList(field, _) => {
                 Ok(ArrayType::new((*field).data_type().try_into()?, (*field).is_nullable()).into())
             }