From fe8fbe4639f36e1f40b33193de8e09bafaef8e11 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Tue, 19 Mar 2024 11:47:02 +0000 Subject: [PATCH] fixes --- pyvortex/src/lib.rs | 2 +- .../src/array/primitive/compute/as_arrow.rs | 2 + vortex-array/src/array/primitive/stats.rs | 4 + vortex-array/src/array/varbin/compute.rs | 2 +- vortex-array/src/array/varbinview/compute.rs | 80 ++++++++++++++++++- vortex-array/src/compute/flatten.rs | 3 + vortex-array/src/encode.rs | 48 +++++++++-- vortex-array/src/ptype.rs | 15 ++++ vortex-array/src/scalar/mod.rs | 4 + vortex-array/src/scalar/primitive.rs | 14 ++++ vortex-array/src/scalar/serde.rs | 8 ++ vortex-array/src/serde/ptype.rs | 6 ++ vortex-schema/src/dtype.rs | 7 +- vortex-schema/src/serde.rs | 3 + 14 files changed, 188 insertions(+), 10 deletions(-) diff --git a/pyvortex/src/lib.rs b/pyvortex/src/lib.rs index c455c5127..79a30fae1 100644 --- a/pyvortex/src/lib.rs +++ b/pyvortex/src/lib.rs @@ -74,7 +74,7 @@ fn dtype_bool(py: Python<'_>, nullable: bool) -> PyResult> { #[pyo3(signature = (width = None, signed = true, nullable = false))] fn dtype_int( py: Python<'_>, - width: Option, + width: Option, signed: bool, nullable: bool, ) -> PyResult> { diff --git a/vortex-array/src/array/primitive/compute/as_arrow.rs b/vortex-array/src/array/primitive/compute/as_arrow.rs index 07bc06361..776993723 100644 --- a/vortex-array/src/array/primitive/compute/as_arrow.rs +++ b/vortex-array/src/array/primitive/compute/as_arrow.rs @@ -20,10 +20,12 @@ impl AsArrowArray for PrimitiveArray { PType::U16 => Arc::new(as_arrow_array_primitive::(self)?), PType::U32 => Arc::new(as_arrow_array_primitive::(self)?), PType::U64 => Arc::new(as_arrow_array_primitive::(self)?), + PType::U128 => Arc::new(as_arrow_array_primitive::(self)?), PType::I8 => Arc::new(as_arrow_array_primitive::(self)?), PType::I16 => Arc::new(as_arrow_array_primitive::(self)?), PType::I32 => Arc::new(as_arrow_array_primitive::(self)?), PType::I64 => Arc::new(as_arrow_array_primitive::(self)?), + PType::I128 => Arc::new(as_arrow_array_primitive::(self)?), PType::F16 => Arc::new(as_arrow_array_primitive::(self)?), PType::F32 => Arc::new(as_arrow_array_primitive::(self)?), PType::F64 => Arc::new(as_arrow_array_primitive::(self)?), diff --git a/vortex-array/src/array/primitive/stats.rs b/vortex-array/src/array/primitive/stats.rs index d231eebef..242d75635 100644 --- a/vortex-array/src/array/primitive/stats.rs +++ b/vortex-array/src/array/primitive/stats.rs @@ -98,10 +98,12 @@ impl> BitWidth for T { PScalar::U16(i) => bit_width - i.leading_zeros() as usize, PScalar::U32(i) => bit_width - i.leading_zeros() as usize, PScalar::U64(i) => bit_width - i.leading_zeros() as usize, + PScalar::U128(i) => bit_width - i.leading_zeros() as usize, PScalar::I8(i) => bit_width - i.leading_zeros() as usize, PScalar::I16(i) => bit_width - i.leading_zeros() as usize, PScalar::I32(i) => bit_width - i.leading_zeros() as usize, PScalar::I64(i) => bit_width - i.leading_zeros() as usize, + PScalar::I128(i) => bit_width - i.leading_zeros() as usize, PScalar::F16(_) => bit_width, PScalar::F32(_) => bit_width, PScalar::F64(_) => bit_width, @@ -115,10 +117,12 @@ impl> BitWidth for T { PScalar::U16(i) => i.trailing_zeros() as usize, PScalar::U32(i) => i.trailing_zeros() as usize, PScalar::U64(i) => i.trailing_zeros() as usize, + PScalar::U128(i) => i.trailing_zeros() as usize, PScalar::I8(i) => i.trailing_zeros() as usize, PScalar::I16(i) => i.trailing_zeros() as usize, PScalar::I32(i) => i.trailing_zeros() as usize, PScalar::I64(i) => i.trailing_zeros() as usize, + PScalar::I128(i) => i.trailing_zeros() as usize, PScalar::F16(_) => 0, PScalar::F32(_) => 0, PScalar::F64(_) => 0, diff --git a/vortex-array/src/array/varbin/compute.rs b/vortex-array/src/array/varbin/compute.rs index cdd621b05..b8f14e18e 100644 --- a/vortex-array/src/array/varbin/compute.rs +++ b/vortex-array/src/array/varbin/compute.rs @@ -92,7 +92,7 @@ impl AsArrowArray for VarBinArray { &PType::U64 => flatten_primitive(cast(offsets.as_ref(), &PType::I64.into())?.as_ref())?, _ => flatten_primitive(cast(offsets.as_ref(), &PType::I32.into())?.as_ref())?, }; - let nulls = as_nulls(offsets.validity())?; + let nulls = as_nulls(self.validity())?; let data = flatten_primitive(self.bytes())?; assert_eq!(data.ptype(), &PType::U8); diff --git a/vortex-array/src/array/varbinview/compute.rs b/vortex-array/src/array/varbinview/compute.rs index 185577923..b2f4e8cb5 100644 --- a/vortex-array/src/array/varbinview/compute.rs +++ b/vortex-array/src/array/varbinview/compute.rs @@ -1,12 +1,30 @@ +use std::sync::Arc; + +use arrow_array::{ArrayRef as ArrowArrayRef, BinaryViewArray, StringViewArray}; +use itertools::Itertools; + +use vortex_schema::DType; + use crate::array::varbinview::VarBinViewArray; use crate::array::Array; +use crate::arrow::wrappers::{as_nulls, as_scalar_buffer}; +use crate::compute::as_arrow::AsArrowArray; +use crate::compute::flatten::{flatten, flatten_primitive, FlattenFn, FlattenedArray}; use crate::compute::scalar_at::ScalarAtFn; use crate::compute::ArrayCompute; use crate::error::VortexResult; +use crate::ptype::PType; use crate::scalar::Scalar; -use vortex_schema::DType; impl ArrayCompute for VarBinViewArray { + fn as_arrow(&self) -> Option<&dyn AsArrowArray> { + Some(self) + } + + fn flatten(&self) -> Option<&dyn FlattenFn> { + Some(self) + } + fn scalar_at(&self) -> Option<&dyn ScalarAtFn> { Some(self) } @@ -27,3 +45,63 @@ impl ScalarAtFn for VarBinViewArray { } } } + +impl FlattenFn for VarBinViewArray { + fn flatten(&self) -> VortexResult { + let views = flatten(self.views())?.into_array(); + let data = self + .data() + .iter() + .map(|d| flatten(d.as_ref()).unwrap().into_array()) + .collect::>(); + let validity = self + .validity() + .map(|v| flatten(v).map(FlattenedArray::into_array)) + .transpose()?; + Ok(FlattenedArray::VarBinView(VarBinViewArray::new( + views, + data, + self.dtype.clone(), + validity, + ))) + } +} + +impl AsArrowArray for VarBinViewArray { + fn as_arrow(&self) -> VortexResult { + // Ensure the offsets are either i32 or i64 + let views = flatten_primitive(self.views())?; + assert_eq!(views.ptype(), &PType::U128); + let nulls = as_nulls(self.validity())?; + + let data = self + .data() + .iter() + .map(|d| flatten_primitive(d.as_ref()).unwrap()) + .collect::>(); + if !data.is_empty() { + assert_eq!(data[0].ptype(), &PType::U8); + assert!(data.iter().map(|d| d.ptype()).all_equal()); + } + + let data = data + .iter() + .map(|p| p.buffer().to_owned()) + .collect::>(); + + // Switch on Arrow DType. + Ok(match self.dtype() { + DType::Binary(_) => Arc::new(BinaryViewArray::new( + as_scalar_buffer::(views), + data, + nulls, + )), + DType::Utf8(_) => Arc::new(StringViewArray::new( + as_scalar_buffer::(views), + data, + nulls, + )), + _ => return Err(VortexError::InvalidDType(self.dtype().clone())), + }) + } +} diff --git a/vortex-array/src/compute/flatten.rs b/vortex-array/src/compute/flatten.rs index 15679f52f..99360d677 100644 --- a/vortex-array/src/compute/flatten.rs +++ b/vortex-array/src/compute/flatten.rs @@ -4,6 +4,7 @@ use crate::array::composite::CompositeArray; use crate::array::primitive::PrimitiveArray; use crate::array::struct_::StructArray; use crate::array::varbin::VarBinArray; +use crate::array::varbinview::VarBinViewArray; use crate::array::{Array, ArrayRef}; use crate::error::{VortexError, VortexResult}; @@ -19,6 +20,7 @@ pub enum FlattenedArray { Primitive(PrimitiveArray), Struct(StructArray), VarBin(VarBinArray), + VarBinView(VarBinViewArray), } impl FlattenedArray { @@ -30,6 +32,7 @@ impl FlattenedArray { FlattenedArray::Primitive(array) => array.boxed(), FlattenedArray::Struct(array) => array.boxed(), FlattenedArray::VarBin(array) => array.boxed(), + FlattenedArray::VarBinView(array) => array.boxed(), } } } diff --git a/vortex-array/src/encode.rs b/vortex-array/src/encode.rs index b71807a83..8c0ac56b4 100644 --- a/vortex-array/src/encode.rs +++ b/vortex-array/src/encode.rs @@ -8,17 +8,18 @@ use arrow_array::array::{ use arrow_array::array::{ArrowPrimitiveType, OffsetSizeTrait}; use arrow_array::cast::{as_null_array, AsArray}; use arrow_array::types::{ - ByteArrayType, Date32Type, Date64Type, DurationMicrosecondType, DurationMillisecondType, - DurationNanosecondType, DurationSecondType, Time32MillisecondType, Time32SecondType, - Time64MicrosecondType, Time64NanosecondType, TimestampMicrosecondType, + ByteArrayType, ByteViewType, Date32Type, Date64Type, DurationMicrosecondType, + DurationMillisecondType, DurationNanosecondType, DurationSecondType, Time32MillisecondType, + Time32SecondType, Time64MicrosecondType, Time64NanosecondType, TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, }; use arrow_array::types::{ Float16Type, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, UInt16Type, UInt32Type, UInt64Type, UInt8Type, }; +use arrow_array::{BinaryViewArray, GenericByteViewArray, StringViewArray}; use arrow_buffer::buffer::{NullBuffer, OffsetBuffer}; -use arrow_buffer::Buffer; +use arrow_buffer::{ArrowNativeType, Buffer, ScalarBuffer}; use arrow_schema::{DataType, TimeUnit}; use vortex_schema::DType; @@ -28,9 +29,10 @@ use crate::array::constant::ConstantArray; use crate::array::primitive::PrimitiveArray; use crate::array::struct_::StructArray; use crate::array::varbin::VarBinArray; +use crate::array::varbinview::VarBinViewArray; use crate::array::{Array, ArrayRef}; use crate::datetime::{LocalDateTime, LocalDateTimeArray}; -use crate::ptype::PType; +use crate::ptype::{NativePType, PType}; use crate::scalar::NullScalar; pub trait FromArrowArray { @@ -49,6 +51,12 @@ impl From<&NullBuffer> for ArrayRef { } } +impl From<&ScalarBuffer> for ArrayRef { + fn from(value: &ScalarBuffer) -> Self { + PrimitiveArray::new(T::PTYPE, value.inner().to_owned(), None).boxed() + } +} + impl From<&OffsetBuffer> for ArrayRef { fn from(value: &OffsetBuffer) -> Self { let ptype = if O::IS_LARGE { PType::I64 } else { PType::I32 }; @@ -107,6 +115,28 @@ impl FromArrowArray<&GenericByteArray> for ArrayRef { } } +impl FromArrowArray<&GenericByteViewArray> for ArrayRef { + fn from_arrow(value: &GenericByteViewArray, nullable: bool) -> Self { + let dtype = match T::DATA_TYPE { + DataType::BinaryView => DType::Binary(nullable.into()), + DataType::Utf8View => DType::Utf8(nullable.into()), + _ => panic!("Invalid data type for ByteViewArray"), + }; + + VarBinViewArray::new( + value.views().into(), + value + .data_buffers() + .iter() + .map(|b| b.into()) + .collect::>(), + dtype, + nulls(value.nulls(), nullable, value.len()), + ) + .boxed() + } +} + impl FromArrowArray<&ArrowBooleanArray> for ArrayRef { fn from_arrow(value: &ArrowBooleanArray, nullable: bool) -> Self { BoolArray::new( @@ -184,6 +214,14 @@ impl FromArrowArray for ArrayRef { DataType::LargeUtf8 => ArrayRef::from_arrow(array.as_string::(), nullable), DataType::Binary => ArrayRef::from_arrow(array.as_binary::(), nullable), DataType::LargeBinary => ArrayRef::from_arrow(array.as_binary::(), nullable), + DataType::BinaryView => ArrayRef::from_arrow( + array.as_any().downcast_ref::().unwrap(), + nullable, + ), + DataType::Utf8View => ArrayRef::from_arrow( + array.as_any().downcast_ref::().unwrap(), + nullable, + ), DataType::Struct(_) => ArrayRef::from_arrow(array.as_struct(), nullable), DataType::Null => ArrayRef::from_arrow(as_null_array(array.as_ref()), nullable), DataType::Timestamp(u, _) => match u { diff --git a/vortex-array/src/ptype.rs b/vortex-array/src/ptype.rs index b13dd680a..a906199ad 100644 --- a/vortex-array/src/ptype.rs +++ b/vortex-array/src/ptype.rs @@ -17,10 +17,12 @@ pub enum PType { U16, U32, U64, + U128, I8, I16, I32, I64, + I128, F16, F32, F64, @@ -58,10 +60,12 @@ native_ptype!(u8, U8); native_ptype!(u16, U16); native_ptype!(u32, U32); native_ptype!(u64, U64); +native_ptype!(u128, U128); native_ptype!(i8, I8); native_ptype!(i16, I16); native_ptype!(i32, I32); native_ptype!(i64, I64); +native_ptype!(i128, I128); native_ptype!(f16, F16); native_ptype!(f32, F32); native_ptype!(f64, F64); @@ -77,10 +81,12 @@ macro_rules! match_each_native_ptype { PType::I16 => __with__! { i16 }, PType::I32 => __with__! { i32 }, PType::I64 => __with__! { i64 }, + PType::I128 => __with__! { i128}, PType::U8 => __with__! { u8 }, PType::U16 => __with__! { u16 }, PType::U32 => __with__! { u32 }, PType::U64 => __with__! { u64 }, + PType::U128 => __with__! { u128}, PType::F16 => __with__! { f16 }, PType::F32 => __with__! { f32 }, PType::F64 => __with__! { f64 }, @@ -99,10 +105,12 @@ macro_rules! match_each_integer_ptype { PType::I16 => __with__! { i16 }, PType::I32 => __with__! { i32 }, PType::I64 => __with__! { i64 }, + PType::I128 => __with__! { i128 }, PType::U8 => __with__! { u8 }, PType::U16 => __with__! { u16 }, PType::U32 => __with__! { u32 }, PType::U64 => __with__! { u64 }, + PType::U128 => __with__! { u128}, _ => panic!("Unsupported ptype {:?}", $self), } }) @@ -142,10 +150,12 @@ impl Display for PType { PType::U16 => write!(f, "u16"), PType::U32 => write!(f, "u32"), PType::U64 => write!(f, "u64"), + PType::U128 => write!(f, "u128"), PType::I8 => write!(f, "i8"), PType::I16 => write!(f, "i16"), PType::I32 => write!(f, "i32"), PType::I64 => write!(f, "i64"), + PType::I128 => write!(f, "i128"), PType::F16 => write!(f, "f16"), PType::F32 => write!(f, "f32"), PType::F64 => write!(f, "f64"), @@ -185,6 +195,11 @@ impl TryFrom<&DType> for PType { Unsigned => Ok(PType::U64), Signed => Ok(PType::I64), }, + IntWidth::_128 => match s { + Unknown => Ok(PType::I128), + Unsigned => Ok(PType::U128), + Signed => Ok(PType::I128), + }, }, DType::Float(f, _) => match f { FloatWidth::Unknown => Ok(PType::F64), diff --git a/vortex-array/src/scalar/mod.rs b/vortex-array/src/scalar/mod.rs index 82191cddc..f1bac6738 100644 --- a/vortex-array/src/scalar/mod.rs +++ b/vortex-array/src/scalar/mod.rs @@ -104,6 +104,9 @@ impl Scalar { (IntWidth::_64, Signedness::Unknown | Signedness::Signed) => { PrimitiveScalar::none(PType::I64).into() } + (IntWidth::_128, Signedness::Unknown | Signedness::Signed) => { + PrimitiveScalar::none(PType::I128).into() + } (IntWidth::Unknown, Signedness::Unsigned) => { PrimitiveScalar::none(PType::U64).into() } @@ -111,6 +114,7 @@ impl Scalar { (IntWidth::_16, Signedness::Unsigned) => PrimitiveScalar::none(PType::U16).into(), (IntWidth::_32, Signedness::Unsigned) => PrimitiveScalar::none(PType::U32).into(), (IntWidth::_64, Signedness::Unsigned) => PrimitiveScalar::none(PType::U64).into(), + (IntWidth::_128, Signedness::Unsigned) => PrimitiveScalar::none(PType::U128).into(), }, DType::Decimal(_, _, _) => unimplemented!("DecimalScalar"), DType::Float(w, _) => match w { diff --git a/vortex-array/src/scalar/primitive.rs b/vortex-array/src/scalar/primitive.rs index 09fff7ae8..3c0a01b68 100644 --- a/vortex-array/src/scalar/primitive.rs +++ b/vortex-array/src/scalar/primitive.rs @@ -97,10 +97,12 @@ pub enum PScalar { U16(u16), U32(u32), U64(u64), + U128(u128), I8(i8), I16(i16), I32(i32), I64(i64), + I128(i128), F16(f16), F32(f32), F64(f64), @@ -113,10 +115,12 @@ impl PScalar { PScalar::U16(_) => PType::U16, PScalar::U32(_) => PType::U32, PScalar::U64(_) => PType::U64, + PScalar::U128(_) => PType::U128, PScalar::I8(_) => PType::I8, PScalar::I16(_) => PType::I16, PScalar::I32(_) => PType::I32, PScalar::I64(_) => PType::I64, + PScalar::I128(_) => PType::I128, PScalar::F16(_) => PType::F16, PScalar::F32(_) => PType::F32, PScalar::F64(_) => PType::F64, @@ -131,10 +135,12 @@ impl PScalar { PType::U16 => Ok((*$v as u16).into()), PType::U32 => Ok((*$v as u32).into()), PType::U64 => Ok((*$v as u64).into()), + PType::U128 => Ok((*$v as u128).into()), PType::I8 => Ok((*$v as i8).into()), PType::I16 => Ok((*$v as i16).into()), PType::I32 => Ok((*$v as i32).into()), PType::I64 => Ok((*$v as i64).into()), + PType::I128 => Ok((*$v as i128).into()), PType::F16 => Ok(f16::from_f32(*$v as f32).into()), PType::F32 => Ok((*$v as f32).into()), PType::F64 => Ok((*$v as f64).into()), @@ -158,10 +164,12 @@ impl PScalar { PScalar::U16(v) => from_int!(ptype, v), PScalar::U32(v) => from_int!(ptype, v), PScalar::U64(v) => from_int!(ptype, v), + PScalar::U128(v) => from_int!(ptype, v), PScalar::I8(v) => from_int!(ptype, v), PScalar::I16(v) => from_int!(ptype, v), PScalar::I32(v) => from_int!(ptype, v), PScalar::I64(v) => from_int!(ptype, v), + PScalar::I128(v) => from_int!(ptype, v), PScalar::F16(v) => match ptype { PType::F16 => Ok((*v).into()), PType::F32 => Ok(v.to_f32().into()), @@ -233,10 +241,12 @@ pscalar!(u8, U8); pscalar!(u16, U16); pscalar!(u32, U32); pscalar!(u64, U64); +pscalar!(u128, U128); pscalar!(i8, I8); pscalar!(i16, I16); pscalar!(i32, I32); pscalar!(i64, I64); +pscalar!(i128, I128); pscalar!(f16, F16); pscalar!(f32, F32); pscalar!(f64, F64); @@ -305,10 +315,12 @@ impl TryFrom<&Scalar> for usize { PScalar::U16(v) => __with_pscalar__! { v }, PScalar::U32(v) => __with_pscalar__! { v }, PScalar::U64(v) => __with_pscalar__! { v }, + PScalar::U128(v) => __with_pscalar__! { v }, PScalar::I8(v) => __with_pscalar__! { v }, PScalar::I16(v) => __with_pscalar__! { v }, PScalar::I32(v) => __with_pscalar__! { v }, PScalar::I64(v) => __with_pscalar__! { v }, + PScalar::I128(v) => __with_pscalar__! { v }, _ => Err(VortexError::InvalidDType($self.ptype().into())), } }) @@ -336,10 +348,12 @@ impl Display for PScalar { PScalar::U16(p) => Display::fmt(p, f), PScalar::U32(p) => Display::fmt(p, f), PScalar::U64(p) => Display::fmt(p, f), + PScalar::U128(p) => Display::fmt(p, f), PScalar::I8(p) => Display::fmt(p, f), PScalar::I16(p) => Display::fmt(p, f), PScalar::I32(p) => Display::fmt(p, f), PScalar::I64(p) => Display::fmt(p, f), + PScalar::I128(p) => Display::fmt(p, f), PScalar::F16(p) => Display::fmt(p, f), PScalar::F32(p) => Display::fmt(p, f), PScalar::F64(p) => Display::fmt(p, f), diff --git a/vortex-array/src/scalar/serde.rs b/vortex-array/src/scalar/serde.rs index 3b07f9be6..57158e8e5 100644 --- a/vortex-array/src/scalar/serde.rs +++ b/vortex-array/src/scalar/serde.rs @@ -103,6 +103,9 @@ impl<'a, 'b> ScalarReader<'a, 'b> { PType::U64 => PrimitiveScalar::some(PScalar::U64(u64::from_le_bytes( self.reader.read_nbytes()?, ))), + PType::U128 => PrimitiveScalar::some(PScalar::U128(u128::from_le_bytes( + self.reader.read_nbytes()?, + ))), PType::I8 => PrimitiveScalar::some(PScalar::I8(i8::from_le_bytes( self.reader.read_nbytes()?, ))), @@ -115,6 +118,9 @@ impl<'a, 'b> ScalarReader<'a, 'b> { PType::I64 => PrimitiveScalar::some(PScalar::I64(i64::from_le_bytes( self.reader.read_nbytes()?, ))), + PType::I128 => PrimitiveScalar::some(PScalar::I128(i128::from_le_bytes( + self.reader.read_nbytes()?, + ))), PType::F16 => PrimitiveScalar::some(PScalar::F16(f16::from_le_bytes( self.reader.read_nbytes()?, ))), @@ -199,10 +205,12 @@ impl<'a, 'b> ScalarWriter<'a, 'b> { PScalar::I16(i) => self.writer.write_fixed_slice(i.to_le_bytes())?, PScalar::I32(i) => self.writer.write_fixed_slice(i.to_le_bytes())?, PScalar::I64(i) => self.writer.write_fixed_slice(i.to_le_bytes())?, + PScalar::I128(i) => self.writer.write_fixed_slice(i.to_le_bytes())?, PScalar::I8(i) => self.writer.write_fixed_slice(i.to_le_bytes())?, PScalar::U16(u) => self.writer.write_fixed_slice(u.to_le_bytes())?, PScalar::U32(u) => self.writer.write_fixed_slice(u.to_le_bytes())?, PScalar::U64(u) => self.writer.write_fixed_slice(u.to_le_bytes())?, + PScalar::U128(u) => self.writer.write_fixed_slice(u.to_le_bytes())?, PScalar::U8(u) => self.writer.write_fixed_slice(u.to_le_bytes())?, } } diff --git a/vortex-array/src/serde/ptype.rs b/vortex-array/src/serde/ptype.rs index 4481a6b3c..adc238722 100644 --- a/vortex-array/src/serde/ptype.rs +++ b/vortex-array/src/serde/ptype.rs @@ -9,10 +9,12 @@ pub enum PTypeTag { U16, U32, U64, + U128, I8, I16, I32, I64, + I128, F16, F32, F64, @@ -25,10 +27,12 @@ impl From for PTypeTag { PType::U16 => PTypeTag::U16, PType::U32 => PTypeTag::U32, PType::U64 => PTypeTag::U64, + PType::U128 => PTypeTag::U128, PType::I8 => PTypeTag::I8, PType::I16 => PTypeTag::I16, PType::I32 => PTypeTag::I32, PType::I64 => PTypeTag::I64, + PType::I128 => PTypeTag::I128, PType::F16 => PTypeTag::F16, PType::F32 => PTypeTag::F32, PType::F64 => PTypeTag::F64, @@ -43,10 +47,12 @@ impl From for PType { PTypeTag::U16 => PType::U16, PTypeTag::U32 => PType::U32, PTypeTag::U64 => PType::U64, + PTypeTag::U128 => PType::U128, PTypeTag::I8 => PType::I8, PTypeTag::I16 => PType::I16, PTypeTag::I32 => PType::I32, PTypeTag::I64 => PType::I64, + PTypeTag::I128 => PType::I128, PTypeTag::F16 => PType::F16, PTypeTag::F32 => PType::F32, PTypeTag::F64 => PType::F64, diff --git a/vortex-schema/src/dtype.rs b/vortex-schema/src/dtype.rs index 6f7de6c8d..9eca9d857 100644 --- a/vortex-schema/src/dtype.rs +++ b/vortex-schema/src/dtype.rs @@ -67,15 +67,17 @@ pub enum IntWidth { _16, _32, _64, + _128, } -impl From for IntWidth { - fn from(item: i8) -> Self { +impl From for IntWidth { + fn from(item: i16) -> Self { match item { 8 => IntWidth::_8, 16 => IntWidth::_16, 32 => IntWidth::_32, 64 => IntWidth::_64, + 128 => IntWidth::_128, _ => IntWidth::Unknown, } } @@ -89,6 +91,7 @@ impl Display for IntWidth { IntWidth::_16 => write!(f, "16"), IntWidth::_32 => write!(f, "32"), IntWidth::_64 => write!(f, "64"), + IntWidth::_128 => write!(f, "128"), } } } diff --git a/vortex-schema/src/serde.rs b/vortex-schema/src/serde.rs index ea083d8b7..a1909b5e6 100644 --- a/vortex-schema/src/serde.rs +++ b/vortex-schema/src/serde.rs @@ -342,6 +342,7 @@ enum IntWidthTag { _16, _32, _64, + _128, } #[allow(clippy::just_underscores_and_digits)] @@ -354,6 +355,7 @@ impl From for IntWidthTag { _16 => IntWidthTag::_16, _32 => IntWidthTag::_32, _64 => IntWidthTag::_64, + _128 => IntWidthTag::_128, } } } @@ -367,6 +369,7 @@ impl From for IntWidth { IntWidthTag::_16 => _16, IntWidthTag::_32 => _32, IntWidthTag::_64 => _64, + IntWidthTag::_128 => _128, } } }