From 687a5cacb618f8b0b7c55d8003996ea3cdd0971b Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Wed, 1 May 2024 16:11:29 +0100 Subject: [PATCH] Remove composite and decimal (#285) --- pyvortex/src/array.rs | 8 - pyvortex/src/lib.rs | 1 - vortex-array/src/array/composite/array.rs | 179 ------------------- vortex-array/src/array/composite/compress.rs | 40 ----- vortex-array/src/array/composite/compute.rs | 95 ---------- vortex-array/src/array/composite/mod.rs | 22 --- vortex-array/src/array/composite/typed.rs | 131 -------------- vortex-array/src/array/mod.rs | 1 - vortex-array/src/arrow/dtype.rs | 1 - vortex-array/src/compress.rs | 2 - vortex-array/src/flatten.rs | 3 - vortex-dtype/src/deserialize.rs | 8 - vortex-dtype/src/dtype.rs | 11 +- vortex-dtype/src/lib.rs | 12 -- vortex-dtype/src/serialize.rs | 13 -- vortex-scalar/src/composite.rs | 47 ----- vortex-scalar/src/lib.rs | 8 - vortex-scalar/src/serde.rs | 1 - 18 files changed, 1 insertion(+), 582 deletions(-) delete mode 100644 vortex-array/src/array/composite/array.rs delete mode 100644 vortex-array/src/array/composite/compress.rs delete mode 100644 vortex-array/src/array/composite/compute.rs delete mode 100644 vortex-array/src/array/composite/mod.rs delete mode 100644 vortex-array/src/array/composite/typed.rs delete mode 100644 vortex-scalar/src/composite.rs diff --git a/pyvortex/src/array.rs b/pyvortex/src/array.rs index 2f67a8aff3..79797fe7fb 100644 --- a/pyvortex/src/array.rs +++ b/pyvortex/src/array.rs @@ -2,7 +2,6 @@ use paste::paste; use pyo3::prelude::*; use vortex::array::bool::{Bool, BoolArray, BoolEncoding, OwnedBoolArray}; use vortex::array::chunked::{Chunked, ChunkedArray, ChunkedEncoding, OwnedChunkedArray}; -use vortex::array::composite::{Composite, CompositeArray, CompositeEncoding, OwnedCompositeArray}; use vortex::array::constant::{Constant, ConstantArray, ConstantEncoding, OwnedConstantArray}; use vortex::array::primitive::{OwnedPrimitiveArray, Primitive, PrimitiveArray, PrimitiveEncoding}; use vortex::array::r#struct::{OwnedStructArray, Struct, StructArray, StructEncoding}; @@ -65,7 +64,6 @@ macro_rules! pyarray { pyarray!(BoolEncoding, BoolArray, "BoolArray"); pyarray!(ChunkedEncoding, ChunkedArray, "ChunkedArray"); -pyarray!(CompositeEncoding, CompositeArray, "CompositeArray"); pyarray!(ConstantEncoding, ConstantArray, "ConstantArray"); pyarray!(PrimitiveEncoding, PrimitiveArray, "PrimitiveArray"); pyarray!(SparseEncoding, SparseArray, "SparseArray"); @@ -97,12 +95,6 @@ impl PyArray { OwnedChunkedArray::try_from(inner.into_array()).map_err(PyVortexError::map_err)?, )? .extract(py), - Composite::ID => PyCompositeArray::wrap( - py, - OwnedCompositeArray::try_from(inner.into_array()) - .map_err(PyVortexError::map_err)?, - )? - .extract(py), Constant::ID => PyConstantArray::wrap( py, OwnedConstantArray::try_from(inner.into_array()).map_err(PyVortexError::map_err)?, diff --git a/pyvortex/src/lib.rs b/pyvortex/src/lib.rs index 439706fab3..bc8fa85728 100644 --- a/pyvortex/src/lib.rs +++ b/pyvortex/src/lib.rs @@ -33,7 +33,6 @@ fn _lib(_py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/vortex-array/src/array/composite/array.rs b/vortex-array/src/array/composite/array.rs deleted file mode 100644 index dfc4e99057..0000000000 --- a/vortex-array/src/array/composite/array.rs +++ /dev/null @@ -1,179 +0,0 @@ -use flatbuffers::root; -use vortex_dtype::flatbuffers as fb; -use vortex_dtype::CompositeID; -use vortex_error::vortex_err; -use vortex_flatbuffers::{FlatBufferToBytes, ReadFlatBuffer}; - -use crate::array::composite::{find_extension, CompositeExtensionRef, TypedCompositeArray}; -use crate::compute::ArrayCompute; -use crate::stats::ArrayStatisticsCompute; -use crate::validity::{ArrayValidity, LogicalValidity}; -use crate::visitor::{AcceptArrayVisitor, ArrayVisitor}; -use crate::{ - impl_encoding, ArrayDType, ArrayFlatten, IntoArrayData, TryDeserializeArrayMetadata, - TrySerializeArrayMetadata, -}; - -pub trait UnderlyingMetadata: - 'static + Send + Sync + Debug + TrySerializeArrayMetadata + for<'m> TryDeserializeArrayMetadata<'m> -{ - fn id(&self) -> CompositeID; -} - -impl_encoding!("vortex.composite", Composite); - -#[derive(Debug, Clone)] -pub struct CompositeMetadata { - ext: CompositeExtensionRef, - underlying_dtype: DType, - underlying_metadata: Arc<[u8]>, -} - -impl TrySerializeArrayMetadata for CompositeMetadata { - fn try_serialize_metadata(&self) -> VortexResult> { - let mut fb = flexbuffers::Builder::default(); - { - let mut elems = fb.start_vector(); - elems.push(self.ext.id().0); - self.underlying_dtype - .with_flatbuffer_bytes(|b| elems.push(flexbuffers::Blob(b))); - elems.push(flexbuffers::Blob(self.underlying_metadata.as_ref())); - } - Ok(fb.take_buffer().into()) - } -} - -impl TryDeserializeArrayMetadata<'_> for CompositeMetadata { - fn try_deserialize_metadata(metadata: Option<&[u8]>) -> VortexResult { - let reader = flexbuffers::Reader::get_root(metadata.expect("missing metadata"))?; - let elems = reader.as_vector(); - - let ext_id = elems.index(0).expect("missing composite id").as_str(); - let ext = find_extension(ext_id) - .ok_or_else(|| vortex_err!("Unrecognized composite extension: {}", ext_id))?; - - let dtype_blob = elems.index(1).expect("missing dtype").as_blob(); - let underlying_dtype = - DType::read_flatbuffer(&root::(dtype_blob.0).expect("invalid dtype"))?; - - let underlying_metadata: Arc<[u8]> = elems - .index(2) - .expect("missing underlying metadata") - .as_blob() - .0 - .to_vec() - .into(); - - Ok(CompositeMetadata { - ext, - underlying_dtype, - underlying_metadata, - }) - } -} - -impl<'a> CompositeArray<'a> { - pub fn new(id: CompositeID, metadata: Arc<[u8]>, underlying: Array<'a>) -> Self { - let dtype = DType::Composite(id, underlying.dtype().is_nullable().into()); - let ext = find_extension(id.0).expect("Unrecognized composite extension"); - Self::try_from_parts( - dtype, - CompositeMetadata { - ext, - underlying_dtype: underlying.dtype().clone(), - underlying_metadata: metadata, - }, - [underlying.into_array_data()].into(), - StatsSet::new(), - ) - .unwrap() - } -} - -impl CompositeArray<'_> { - #[inline] - pub fn id(&self) -> CompositeID { - self.metadata().ext.id() - } - - #[inline] - pub fn extension(&self) -> CompositeExtensionRef { - find_extension(self.id().0).expect("Unrecognized composite extension") - } - - pub fn underlying_metadata(&self) -> &Arc<[u8]> { - &self.metadata().underlying_metadata - } - - pub fn underlying_dtype(&self) -> &DType { - &self.metadata().underlying_dtype - } - - #[inline] - pub fn underlying(&self) -> Array { - self.array() - .child(0, self.underlying_dtype()) - .expect("CompositeArray must have an underlying array") - } - - pub fn with_compute(&self, mut f: F) -> R - where - F: FnMut(&dyn ArrayCompute) -> R, - { - let mut result = None; - - self.extension() - .with_compute(self, &mut |c| { - result = Some(f(c)); - Ok(()) - }) - .unwrap(); - - // Now we unwrap the optional, which we know to be populated by the closure. - result.unwrap() - } -} - -impl<'a> CompositeArray<'a> { - pub fn as_typed(&'a self) -> VortexResult> { - Ok(TypedCompositeArray::new( - M::try_deserialize_metadata(Some(self.underlying_metadata()))?, - self.underlying().clone(), - )) - } -} - -impl ArrayTrait for CompositeArray<'_> { - fn len(&self) -> usize { - self.underlying().len() - } -} - -impl ArrayFlatten for CompositeArray<'_> { - fn flatten<'a>(self) -> VortexResult> - where - Self: 'a, - { - Ok(Flattened::Composite(self)) - } -} - -impl ArrayValidity for CompositeArray<'_> { - fn is_valid(&self, index: usize) -> bool { - self.underlying().with_dyn(|a| a.is_valid(index)) - } - - fn logical_validity(&self) -> LogicalValidity { - self.underlying().with_dyn(|a| a.logical_validity()) - } -} - -impl AcceptArrayVisitor for CompositeArray<'_> { - fn accept(&self, visitor: &mut dyn ArrayVisitor) -> VortexResult<()> { - visitor.visit_child("underlying", &self.underlying()) - } -} - -impl ArrayStatisticsCompute for CompositeArray<'_> {} - -impl EncodingCompression for CompositeEncoding {} diff --git a/vortex-array/src/array/composite/compress.rs b/vortex-array/src/array/composite/compress.rs deleted file mode 100644 index cca56880ff..0000000000 --- a/vortex-array/src/array/composite/compress.rs +++ /dev/null @@ -1,40 +0,0 @@ -use vortex_error::VortexResult; - -use crate::array::composite::{CompositeArray, CompositeEncoding}; -use crate::array::downcast::DowncastArrayBuiltin; -use crate::array::{Array, ArrayRef}; -use crate::compress::{CompressConfig, CompressCtx, EncodingCompression}; - -impl EncodingCompression for CompositeEncoding { - fn cost(&self) -> u8 { - 0 - } - - fn can_compress( - &self, - array: &dyn Array, - _config: &CompressConfig, - ) -> Option<&dyn EncodingCompression> { - (array.encoding().id() == Self::ID).then_some(self) - } - - fn compress( - &self, - array: &dyn Array, - like: Option<&dyn Array>, - ctx: CompressCtx, - ) -> VortexResult { - let composite_array = array.as_composite(); - let composite_like = like.map(|like_array| like_array.as_composite()); - - Ok(CompositeArray::new( - composite_array.id(), - composite_array.metadata().clone(), - ctx.compress( - composite_array.underlying(), - composite_like.map(|c| c.underlying()), - )?, - ) - .into_array()) - } -} diff --git a/vortex-array/src/array/composite/compute.rs b/vortex-array/src/array/composite/compute.rs deleted file mode 100644 index 1c9ca39ccf..0000000000 --- a/vortex-array/src/array/composite/compute.rs +++ /dev/null @@ -1,95 +0,0 @@ -use arrow_array::ArrayRef as ArrowArrayRef; -use itertools::Itertools; -use vortex_error::{vortex_err, VortexResult}; -use vortex_scalar::Scalar; - -use crate::array::composite::array::CompositeArray; -use crate::compute::as_arrow::AsArrowArray; -use crate::compute::as_contiguous::{as_contiguous, AsContiguousFn}; -use crate::compute::scalar_at::{scalar_at, ScalarAtFn}; -use crate::compute::slice::{slice, SliceFn}; -use crate::compute::take::{take, TakeFn}; -use crate::compute::ArrayCompute; -use crate::{Array, ArrayDType, IntoArray, OwnedArray}; - -impl ArrayCompute for CompositeArray<'_> { - fn as_arrow(&self) -> Option<&dyn AsArrowArray> { - Some(self) - } - - fn as_contiguous(&self) -> Option<&dyn AsContiguousFn> { - Some(self) - } - - fn scalar_at(&self) -> Option<&dyn ScalarAtFn> { - Some(self) - } - - fn slice(&self) -> Option<&dyn SliceFn> { - Some(self) - } - - fn take(&self) -> Option<&dyn TakeFn> { - Some(self) - } -} - -impl AsArrowArray for CompositeArray<'_> { - fn as_arrow(&self) -> VortexResult { - self.with_compute(|c| { - c.as_arrow().map(|a| a.as_arrow()).unwrap_or_else(|| { - Err(vortex_err!( - NotImplemented: "as_arrow", - format!("composite extension {}", self.id()) - )) - }) - }) - } -} - -impl AsContiguousFn for CompositeArray<'_> { - fn as_contiguous(&self, arrays: &[Array]) -> VortexResult { - let composites = arrays - .iter() - .map(|array| CompositeArray::try_from(array).unwrap()) - .collect_vec(); - let underlyings = composites.iter().map(|c| c.underlying()).collect_vec(); - Ok(CompositeArray::new( - self.id(), - self.underlying_metadata().clone(), - as_contiguous(&underlyings)?, - ) - .into_array()) - } -} - -impl ScalarAtFn for CompositeArray<'_> { - fn scalar_at(&self, index: usize) -> VortexResult { - // TODO(ngates): this seems wrong... I don't think we just cast scalars like this. - // e.g. how do we know what a datetime is in? - let underlying = scalar_at(&self.underlying(), index)?; - underlying.cast(self.dtype()) - } -} - -impl TakeFn for CompositeArray<'_> { - fn take(&self, indices: &Array) -> VortexResult { - Ok(CompositeArray::new( - self.id(), - self.underlying_metadata().clone(), - take(&self.underlying(), indices)?, - ) - .into_array()) - } -} - -impl SliceFn for CompositeArray<'_> { - fn slice(&self, start: usize, stop: usize) -> VortexResult { - Ok(CompositeArray::new( - self.id(), - self.underlying_metadata().clone(), - slice(&self.underlying(), start, stop)?, - ) - .into_array()) - } -} diff --git a/vortex-array/src/array/composite/mod.rs b/vortex-array/src/array/composite/mod.rs deleted file mode 100644 index 09f6bb5a5f..0000000000 --- a/vortex-array/src/array/composite/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -pub use array::*; -use linkme::distributed_slice; -pub use typed::*; -use vortex_dtype::CompositeID; - -mod array; -mod compute; -mod typed; - -#[distributed_slice] -pub static VORTEX_COMPOSITE_EXTENSIONS: [&'static dyn CompositeExtension] = [..]; - -pub fn find_extension(id: &str) -> Option<&'static dyn CompositeExtension> { - VORTEX_COMPOSITE_EXTENSIONS - .iter() - .find(|ext| ext.id().0 == id) - .copied() -} - -pub fn find_extension_id(id: &str) -> Option { - find_extension(id).map(|e| e.id()) -} diff --git a/vortex-array/src/array/composite/typed.rs b/vortex-array/src/array/composite/typed.rs deleted file mode 100644 index 7b4a158e5e..0000000000 --- a/vortex-array/src/array/composite/typed.rs +++ /dev/null @@ -1,131 +0,0 @@ -use std::fmt::Debug; - -use vortex_dtype::CompositeID; -use vortex_dtype::DType; -use vortex_error::{VortexError, VortexResult}; - -use crate::array::composite::array::CompositeArray; -use crate::array::composite::UnderlyingMetadata; -use crate::compute::ArrayCompute; -use crate::{Array, ArrayDType}; - -pub trait CompositeExtension: Debug + Send + Sync + 'static { - fn id(&self) -> CompositeID; - - fn with_compute<'a>( - &self, - array: &'a CompositeArray<'a>, - f: &mut dyn for<'b> FnMut(&'b (dyn ArrayCompute + 'a)) -> VortexResult<()>, - ) -> VortexResult<()>; -} - -pub type CompositeExtensionRef = &'static dyn CompositeExtension; - -#[derive(Debug, Clone)] -pub struct TypedCompositeArray<'a, M: UnderlyingMetadata> { - metadata: M, - underlying: Array<'a>, - dtype: DType, -} - -impl<'a, M: UnderlyingMetadata> TypedCompositeArray<'a, M> { - pub fn new(metadata: M, underlying: Array<'a>) -> Self { - let dtype = DType::Composite(metadata.id(), underlying.dtype().is_nullable().into()); - Self { - metadata, - underlying, - dtype, - } - } - - #[inline] - pub fn underlying_metadata(&self) -> &M { - &self.metadata - } - - #[inline] - pub fn underlying(&self) -> &Array<'a> { - &self.underlying - } - - #[inline] - pub fn dtype(&self) -> &DType { - &self.dtype - } - - pub fn as_composite(&self) -> VortexResult> { - Ok(CompositeArray::new( - self.underlying_metadata().id(), - self.underlying_metadata().try_serialize_metadata()?, - self.underlying().clone(), - )) - } -} - -impl<'a, M: UnderlyingMetadata> TryFrom<&'a CompositeArray<'a>> for TypedCompositeArray<'a, M> { - type Error = VortexError; - - fn try_from(value: &'a CompositeArray<'a>) -> Result { - value.as_typed::() - } -} - -#[macro_export] -macro_rules! impl_composite { - ($id:expr, $T:ty) => { - use linkme::distributed_slice; - use paste::paste; - use vortex_dtype::{CompositeID, DType, Nullability}; - use $crate::array::composite::{ - CompositeArray, CompositeExtension, TypedCompositeArray, UnderlyingMetadata, - VORTEX_COMPOSITE_EXTENSIONS, - }; - use $crate::compute::ArrayCompute; - use $crate::TryDeserializeArrayMetadata; - - paste! { - #[derive(Debug)] - pub struct [<$T Extension>]; - - impl [<$T Extension>] { - pub const ID: CompositeID = CompositeID($id); - - pub fn dtype(nullability: Nullability) -> DType { - DType::Composite(Self::ID, nullability) - } - } - - impl CompositeExtension for [<$T Extension>] { - fn id(&self) -> CompositeID { - Self::ID - } - - fn with_compute<'a>( - &self, - array: &'a CompositeArray<'a>, - f: &mut dyn for<'b> FnMut(&'b (dyn ArrayCompute + 'a)) -> VortexResult<()>, - ) -> VortexResult<()> { - if array.id() != Self::ID { - panic!("Incorrect CompositeID"); - } - let typed = TypedCompositeArray::new( - $T::try_deserialize_metadata(Some(array.underlying_metadata().as_ref()))?, - array.underlying().clone(), - ); - f(&typed) - } - } - - impl UnderlyingMetadata for $T { - fn id(&self) -> CompositeID { - [<$T Extension>]::ID - } - } - - #[distributed_slice(VORTEX_COMPOSITE_EXTENSIONS)] - static ENCODINGS_COMPOSITE_EXT: &'static dyn CompositeExtension = &[<$T Extension>]; - } - }; -} - -pub use impl_composite; diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs index 4ba6ea80f2..6c5836001a 100644 --- a/vortex-array/src/array/mod.rs +++ b/vortex-array/src/array/mod.rs @@ -1,6 +1,5 @@ pub mod bool; pub mod chunked; -pub mod composite; pub mod constant; pub mod datetime; pub mod primitive; diff --git a/vortex-array/src/arrow/dtype.rs b/vortex-array/src/arrow/dtype.rs index 7d6ce60413..554755c489 100644 --- a/vortex-array/src/arrow/dtype.rs +++ b/vortex-array/src/arrow/dtype.rs @@ -87,7 +87,6 @@ impl FromArrowType<&Field> for DType { .map(|f| DType::from_arrow(f.as_ref())) .collect_vec(), ), - DataType::Decimal128(p, s) | DataType::Decimal256(p, s) => Decimal(*p, *s, nullability), _ => unimplemented!("Arrow data type not yet supported: {:?}", field.data_type()), } } diff --git a/vortex-array/src/compress.rs b/vortex-array/src/compress.rs index 3e9560f69f..3fb44ff686 100644 --- a/vortex-array/src/compress.rs +++ b/vortex-array/src/compress.rs @@ -6,7 +6,6 @@ use log::{debug, info, warn}; use vortex_error::{vortex_bail, VortexResult}; use crate::array::chunked::{Chunked, ChunkedArray, ChunkedEncoding}; -use crate::array::composite::CompositeEncoding; use crate::array::constant::{Constant, ConstantArray}; use crate::array::r#struct::{Struct, StructArray, StructEncoding}; use crate::array::sparse::SparseEncoding; @@ -72,7 +71,6 @@ impl Default for CompressConfig { ree_average_run_threshold: 2.0, encodings: HashSet::from([ &ChunkedEncoding as EncodingRef, - &CompositeEncoding, &SparseEncoding, &StructEncoding, &VarBinEncoding, diff --git a/vortex-array/src/flatten.rs b/vortex-array/src/flatten.rs index 865bc0a6bc..adefd9b8dd 100644 --- a/vortex-array/src/flatten.rs +++ b/vortex-array/src/flatten.rs @@ -3,7 +3,6 @@ use vortex_error::{vortex_bail, VortexResult}; use crate::array::bool::BoolArray; use crate::array::chunked::ChunkedArray; -use crate::array::composite::CompositeArray; use crate::array::primitive::PrimitiveArray; use crate::array::r#struct::StructArray; use crate::array::varbin::VarBinArray; @@ -15,7 +14,6 @@ use crate::{Array, ArrayDType, IntoArray}; pub enum Flattened<'a> { Bool(BoolArray<'a>), Chunked(ChunkedArray<'a>), - Composite(CompositeArray<'a>), Primitive(PrimitiveArray<'a>), Struct(StructArray<'a>), VarBin(VarBinArray<'a>), @@ -84,7 +82,6 @@ impl<'a> IntoArray<'a> for Flattened<'a> { Flattened::Struct(a) => a.into_array(), Flattened::Chunked(a) => a.into_array(), Flattened::VarBin(a) => a.into_array(), - Flattened::Composite(a) => a.into_array(), Flattened::Extension(a) => a.into_array(), Flattened::VarBinView(a) => a.into_array(), } diff --git a/vortex-dtype/src/deserialize.rs b/vortex-dtype/src/deserialize.rs index b5bc4d06f4..03aa2fbdfa 100644 --- a/vortex-dtype/src/deserialize.rs +++ b/vortex-dtype/src/deserialize.rs @@ -23,14 +23,6 @@ impl ReadFlatBuffer for DType { fb_primitive.nullability().try_into()?, )) } - fb::Type::Decimal => { - let fb_decimal = fb.type__as_decimal().unwrap(); - Ok(DType::Decimal( - fb_decimal.precision(), - fb_decimal.scale(), - fb_decimal.nullability().try_into()?, - )) - } fb::Type::Binary => Ok(DType::Binary( fb.type__as_binary().unwrap().nullability().try_into()?, )), diff --git a/vortex-dtype/src/dtype.rs b/vortex-dtype/src/dtype.rs index 55f48dbfda..4ee0e6eb6c 100644 --- a/vortex-dtype/src/dtype.rs +++ b/vortex-dtype/src/dtype.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use itertools::Itertools; use DType::*; -use crate::{CompositeID, ExtDType, PType}; +use crate::{ExtDType, PType}; #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Ord, PartialOrd)] #[cfg_attr(feature = "serde", derive(::serde::Serialize, ::serde::Deserialize))] @@ -53,14 +53,11 @@ pub enum DType { Null, Bool(Nullability), Primitive(PType, Nullability), - Decimal(u8, i8, Nullability), Utf8(Nullability), Binary(Nullability), Struct(FieldNames, Vec), List(Box, Nullability), Extension(ExtDType, Nullability), - #[serde(skip)] - Composite(CompositeID, Nullability), } impl DType { @@ -80,13 +77,11 @@ impl DType { Null => true, Bool(n) => matches!(n, Nullable), Primitive(_, n) => matches!(n, Nullable), - Decimal(_, _, n) => matches!(n, Nullable), Utf8(n) => matches!(n, Nullable), Binary(n) => matches!(n, Nullable), Struct(_, fs) => fs.iter().all(|f| f.is_nullable()), List(_, n) => matches!(n, Nullable), Extension(_, n) => matches!(n, Nullable), - Composite(_, n) => matches!(n, Nullable), } } @@ -103,7 +98,6 @@ impl DType { Null => Null, Bool(_) => Bool(nullability), Primitive(p, _) => Primitive(*p, nullability), - Decimal(s, p, _) => Decimal(*s, *p, nullability), Utf8(_) => Utf8(nullability), Binary(_) => Binary(nullability), Struct(n, fs) => Struct( @@ -112,7 +106,6 @@ impl DType { ), List(c, _) => List(c.clone(), nullability), Extension(ext, _) => Extension(ext.clone(), nullability), - Composite(id, _) => Composite(*id, nullability), } } @@ -127,7 +120,6 @@ impl Display for DType { Null => write!(f, "null"), Bool(n) => write!(f, "bool{}", n), Primitive(p, n) => write!(f, "{}{}", p, n), - Decimal(p, s, n) => write!(f, "decimal({}, {}){}", p, s, n), Utf8(n) => write!(f, "utf8{}", n), Binary(n) => write!(f, "binary{}", n), Struct(n, dt) => write!( @@ -148,7 +140,6 @@ impl Display for DType { .unwrap_or_else(|| "".to_string()), n ), - Composite(id, n) => write!(f, "<{}>{}", id, n), } } } diff --git a/vortex-dtype/src/lib.rs b/vortex-dtype/src/lib.rs index 3ed41be2b6..869eff844a 100644 --- a/vortex-dtype/src/lib.rs +++ b/vortex-dtype/src/lib.rs @@ -1,5 +1,3 @@ -use std::fmt::{Display, Formatter}; - pub use dtype::*; pub use extension::*; pub use half; @@ -11,16 +9,6 @@ mod ptype; mod serde; mod serialize; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Hash)] -#[cfg_attr(feature = "serde", derive(::serde::Serialize, ::serde::Deserialize))] -pub struct CompositeID(pub &'static str); - -impl Display for CompositeID { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.0) - } -} - pub mod flatbuffers { #[allow(unused_imports)] #[allow(dead_code)] diff --git a/vortex-dtype/src/serialize.rs b/vortex-dtype/src/serialize.rs index 3bb47e22d0..42533fcbe1 100644 --- a/vortex-dtype/src/serialize.rs +++ b/vortex-dtype/src/serialize.rs @@ -31,15 +31,6 @@ impl WriteFlatBuffer for DType { }, ) .as_union_value(), - DType::Decimal(p, s, n) => fb::Decimal::create( - fbb, - &fb::DecimalArgs { - precision: *p, - scale: *s, - nullability: n.into(), - }, - ) - .as_union_value(), DType::Utf8(n) => fb::Utf8::create( fbb, &fb::Utf8Args { @@ -93,20 +84,17 @@ impl WriteFlatBuffer for DType { ) .as_union_value() } - DType::Composite(..) => todo!(), }; let dtype_type = match self { DType::Null => fb::Type::Null, DType::Bool(_) => fb::Type::Bool, DType::Primitive(..) => fb::Type::Primitive, - DType::Decimal(..) => fb::Type::Decimal, DType::Utf8(_) => fb::Type::Utf8, DType::Binary(_) => fb::Type::Binary, DType::Struct(..) => fb::Type::Struct_, DType::List(..) => fb::Type::List, DType::Extension { .. } => fb::Type::Extension, - DType::Composite(..) => unreachable!(), }; fb::DType::create( @@ -197,7 +185,6 @@ mod test { roundtrip_dtype(DType::Null); roundtrip_dtype(DType::Bool(Nullability::NonNullable)); roundtrip_dtype(DType::Primitive(PType::U64, Nullability::NonNullable)); - roundtrip_dtype(DType::Decimal(18, 9, Nullability::NonNullable)); roundtrip_dtype(DType::Binary(Nullability::NonNullable)); roundtrip_dtype(DType::Utf8(Nullability::NonNullable)); roundtrip_dtype(DType::List( diff --git a/vortex-scalar/src/composite.rs b/vortex-scalar/src/composite.rs deleted file mode 100644 index 710d6a1893..0000000000 --- a/vortex-scalar/src/composite.rs +++ /dev/null @@ -1,47 +0,0 @@ -use std::fmt::{Display, Formatter}; - -use vortex_dtype::DType; -use vortex_error::VortexResult; - -use crate::Scalar; - -#[derive(Debug, Clone, PartialEq)] -pub struct CompositeScalar { - dtype: DType, - scalar: Box, -} - -impl CompositeScalar { - pub fn new(dtype: DType, scalar: Box) -> Self { - Self { dtype, scalar } - } - - #[inline] - pub fn dtype(&self) -> &DType { - &self.dtype - } - - pub fn scalar(&self) -> &Scalar { - self.scalar.as_ref() - } - - pub fn cast(&self, _dtype: &DType) -> VortexResult { - todo!() - } - - pub fn nbytes(&self) -> usize { - self.scalar.nbytes() - } -} - -impl PartialOrd for CompositeScalar { - fn partial_cmp(&self, other: &Self) -> Option { - self.scalar.as_ref().partial_cmp(other.scalar.as_ref()) - } -} - -impl Display for CompositeScalar { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{} ({})", self.scalar, self.dtype) - } -} diff --git a/vortex-scalar/src/lib.rs b/vortex-scalar/src/lib.rs index 91a4634a9f..02bcdd23b0 100644 --- a/vortex-scalar/src/lib.rs +++ b/vortex-scalar/src/lib.rs @@ -2,7 +2,6 @@ use std::fmt::{Debug, Display, Formatter}; pub use binary::*; pub use bool::*; -pub use composite::*; pub use list::*; pub use null::*; pub use primitive::*; @@ -16,7 +15,6 @@ use crate::extension::ExtScalar; mod binary; mod bool; -mod composite; mod extension; mod list; mod null; @@ -55,7 +53,6 @@ pub enum Scalar { Struct(StructScalar), Utf8(Utf8Scalar), Extension(ExtScalar), - Composite(CompositeScalar), } macro_rules! impls_for_scalars { @@ -76,7 +73,6 @@ impls_for_scalars!(Primitive, PrimitiveScalar); impls_for_scalars!(Struct, StructScalar); impls_for_scalars!(Utf8, Utf8Scalar); impls_for_scalars!(Extension, ExtScalar); -impls_for_scalars!(Composite, CompositeScalar); macro_rules! match_each_scalar { ($self:expr, | $_:tt $scalar:ident | $($body:tt)*) => ({ @@ -90,7 +86,6 @@ macro_rules! match_each_scalar { Scalar::Struct(s) => __with_scalar__! { s }, Scalar::Utf8(s) => __with_scalar__! { s }, Scalar::Extension(s) => __with_scalar__! { s }, - Scalar::Composite(s) => __with_scalar__! { s }, } }) } @@ -123,7 +118,6 @@ impl Scalar { Scalar::Struct(_) => false, Scalar::Utf8(u) => u.value().is_none(), Scalar::Extension(e) => e.value().is_none(), - Scalar::Composite(c) => c.scalar().is_null(), } } @@ -133,13 +127,11 @@ impl Scalar { DType::Null => NullScalar::new().into(), DType::Bool(_) => BoolScalar::none().into(), DType::Primitive(p, _) => PrimitiveScalar::none_from_ptype(*p).into(), - DType::Decimal(..) => unimplemented!("DecimalScalar"), DType::Utf8(_) => Utf8Scalar::none().into(), DType::Binary(_) => BinaryScalar::none().into(), DType::Struct(..) => StructScalar::new(dtype.clone(), vec![]).into(), DType::List(..) => ListScalar::new(dtype.clone(), None).into(), DType::Extension(ext, _) => ExtScalar::null(ext.clone()).into(), - DType::Composite(..) => unimplemented!(), } } } diff --git a/vortex-scalar/src/serde.rs b/vortex-scalar/src/serde.rs index 1327b19695..a03ab231dc 100644 --- a/vortex-scalar/src/serde.rs +++ b/vortex-scalar/src/serde.rs @@ -103,7 +103,6 @@ impl WriteFlatBuffer for Scalar { nullability: self.nullability().into(), } } - Scalar::Composite(_) => panic!(), }; fb::Scalar::create(fbb, &union)