diff --git a/vortex-alp/src/array.rs b/vortex-alp/src/array.rs index acc56c4f43..3636ef9dcb 100644 --- a/vortex-alp/src/array.rs +++ b/vortex-alp/src/array.rs @@ -1,5 +1,6 @@ use std::sync::{Arc, RwLock}; +use vortex::array::validity::Validity; use vortex::array::{Array, ArrayKind, ArrayRef}; use vortex::compress::EncodingCompression; use vortex::compute::ArrayCompute; @@ -7,7 +8,6 @@ use vortex::encoding::{Encoding, EncodingId, EncodingRef}; use vortex::formatter::{ArrayDisplay, ArrayFormatter}; use vortex::serde::{ArraySerde, EncodingSerde}; use vortex::stats::{Stats, StatsSet}; -use vortex::validity::{ArrayValidity, Validity}; use vortex::{impl_array, ArrayWalker}; use vortex_error::{vortex_bail, vortex_err, VortexResult}; use vortex_schema::{DType, IntWidth, Signedness}; @@ -114,6 +114,10 @@ impl Array for ALPArray { self.encoded().nbytes() + self.patches().map(|p| p.nbytes()).unwrap_or(0) } + fn validity(&self) -> Option { + self.encoded().validity() + } + fn serde(&self) -> Option<&dyn ArraySerde> { Some(self) } @@ -131,12 +135,6 @@ impl ArrayDisplay for ALPArray { } } -impl ArrayValidity for ALPArray { - fn validity(&self) -> Option { - self.encoded().validity() - } -} - #[derive(Debug)] pub struct ALPEncoding; diff --git a/vortex-alp/src/compress.rs b/vortex-alp/src/compress.rs index 51a0021716..70d146a6c8 100644 --- a/vortex-alp/src/compress.rs +++ b/vortex-alp/src/compress.rs @@ -7,7 +7,6 @@ use vortex::compress::{CompressConfig, CompressCtx, EncodingCompression}; use vortex::compute::flatten::flatten_primitive; use vortex::compute::patch::patch; use vortex::ptype::{NativePType, PType}; -use vortex::validity::ArrayValidity; use vortex_error::{vortex_bail, vortex_err, VortexResult}; use crate::alp::ALPFloat; diff --git a/vortex-array/src/array/bool/compute/mod.rs b/vortex-array/src/array/bool/compute/mod.rs index bf315be67d..51abb561a7 100644 --- a/vortex-array/src/array/bool/compute/mod.rs +++ b/vortex-array/src/array/bool/compute/mod.rs @@ -6,7 +6,8 @@ use vortex_error::VortexResult; use crate::array::bool::BoolArray; use crate::array::downcast::DowncastArrayBuiltin; -use crate::array::{Array, ArrayRef}; +use crate::array::validity::Validity; +use crate::array::{Array, ArrayRef, ArrayValidity}; use crate::arrow::wrappers::as_nulls; use crate::compute::as_arrow::AsArrowArray; use crate::compute::as_contiguous::AsContiguousFn; @@ -16,7 +17,6 @@ use crate::compute::scalar_at::ScalarAtFn; use crate::compute::take::TakeFn; use crate::compute::ArrayCompute; use crate::scalar::{BoolScalar, Scalar}; -use crate::validity::{ArrayValidity, Validity}; mod take; @@ -122,8 +122,8 @@ impl FillForwardFn for BoolArray { mod test { use crate::array::bool::BoolArray; use crate::array::downcast::DowncastArrayBuiltin; + use crate::array::Array; use crate::compute; - use crate::validity::ArrayValidity; #[test] fn fill_forward() { diff --git a/vortex-array/src/array/bool/compute/take.rs b/vortex-array/src/array/bool/compute/take.rs index 3622350316..49f3b942ed 100644 --- a/vortex-array/src/array/bool/compute/take.rs +++ b/vortex-array/src/array/bool/compute/take.rs @@ -7,7 +7,6 @@ use crate::array::{Array, ArrayRef}; use crate::compute::flatten::flatten_primitive; use crate::compute::take::TakeFn; use crate::match_each_integer_ptype; -use crate::validity::ArrayValidity; impl TakeFn for BoolArray { fn take(&self, indices: &dyn Array) -> VortexResult { diff --git a/vortex-array/src/array/bool/mod.rs b/vortex-array/src/array/bool/mod.rs index 8c7583f0c3..c91903994a 100644 --- a/vortex-array/src/array/bool/mod.rs +++ b/vortex-array/src/array/bool/mod.rs @@ -6,13 +6,13 @@ use vortex_error::VortexResult; use vortex_schema::{DType, Nullability}; use super::{check_slice_bounds, Array, ArrayRef}; +use crate::array::validity::Validity; use crate::array::IntoArray; use crate::compute::ArrayCompute; use crate::encoding::{Encoding, EncodingId, EncodingRef, ENCODINGS}; use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::serde::{ArraySerde, EncodingSerde}; use crate::stats::{Stat, Stats, StatsSet}; -use crate::validity::{ArrayValidity, Validity}; use crate::{impl_array, ArrayWalker}; mod compute; @@ -102,6 +102,10 @@ impl Array for BoolArray { .into_array()) } + fn validity(&self) -> Option { + self.validity.clone() + } + #[inline] fn encoding(&self) -> EncodingRef { &BoolEncoding @@ -125,12 +129,6 @@ impl Array for BoolArray { } } -impl ArrayValidity for BoolArray { - fn validity(&self) -> Option { - self.validity.clone() - } -} - #[derive(Debug)] pub struct BoolEncoding; diff --git a/vortex-array/src/array/bool/serde.rs b/vortex-array/src/array/bool/serde.rs index 433ac7d33d..5fdb33c6d3 100644 --- a/vortex-array/src/array/bool/serde.rs +++ b/vortex-array/src/array/bool/serde.rs @@ -4,7 +4,6 @@ use vortex_error::VortexResult; use crate::array::bool::{BoolArray, BoolEncoding}; use crate::array::{Array, ArrayRef}; use crate::serde::{ArraySerde, ArrayView, BytesSerde, EncodingSerde, ReadCtx, WriteCtx}; -use crate::validity::ArrayValidity; impl ArraySerde for BoolArray { fn write(&self, ctx: &mut WriteCtx) -> VortexResult<()> { @@ -33,8 +32,8 @@ impl EncodingSerde for BoolEncoding { mod test { use crate::array::bool::BoolArray; use crate::array::downcast::DowncastArrayBuiltin; + use crate::array::Array; use crate::serde::test::roundtrip_array; - use crate::validity::ArrayValidity; #[test] fn roundtrip() { diff --git a/vortex-array/src/array/chunked/mod.rs b/vortex-array/src/array/chunked/mod.rs index 6ddb178744..1921137ebb 100644 --- a/vortex-array/src/array/chunked/mod.rs +++ b/vortex-array/src/array/chunked/mod.rs @@ -5,13 +5,13 @@ use linkme::distributed_slice; use vortex_error::{vortex_bail, VortexResult}; use vortex_schema::DType; +use crate::array::validity::Validity; use crate::array::{check_slice_bounds, Array, ArrayRef}; use crate::compute::ArrayCompute; use crate::encoding::{Encoding, EncodingId, EncodingRef, ENCODINGS}; use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::serde::{ArraySerde, EncodingSerde}; use crate::stats::{Stats, StatsSet}; -use crate::validity::{ArrayValidity, Validity}; use crate::{impl_array, ArrayWalker}; mod compute; @@ -145,15 +145,6 @@ impl Array for ChunkedArray { Some(self) } - fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> { - for chunk in self.chunks() { - walker.visit_child(&chunk)?; - } - Ok(()) - } -} - -impl ArrayValidity for ChunkedArray { fn validity(&self) -> Option { if !self.dtype.is_nullable() { return None; @@ -165,6 +156,13 @@ impl ArrayValidity for ChunkedArray { .unwrap_or_else(|| Validity::Valid(chunk.len())) }))) } + + fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> { + for chunk in self.chunks() { + walker.visit_child(&chunk)?; + } + Ok(()) + } } impl FromIterator for ChunkedArray { diff --git a/vortex-array/src/array/composite/array.rs b/vortex-array/src/array/composite/array.rs index 3cb28a9697..827cd93d71 100644 --- a/vortex-array/src/array/composite/array.rs +++ b/vortex-array/src/array/composite/array.rs @@ -6,6 +6,7 @@ use vortex_error::VortexResult; use vortex_schema::{CompositeID, DType}; use crate::array::composite::{find_extension, CompositeExtensionRef, TypedCompositeArray}; +use crate::array::validity::Validity; use crate::array::{Array, ArrayRef}; use crate::compress::EncodingCompression; use crate::compute::ArrayCompute; @@ -13,7 +14,6 @@ use crate::encoding::{Encoding, EncodingId, EncodingRef, ENCODINGS}; use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::serde::{ArraySerde, BytesSerde, EncodingSerde}; use crate::stats::{Stats, StatsCompute, StatsSet}; -use crate::validity::{ArrayValidity, Validity}; use crate::{impl_array, ArrayWalker}; pub trait CompositeMetadata: @@ -121,6 +121,10 @@ impl Array for CompositeArray { Some(self) } + fn validity(&self) -> Option { + self.underlying().validity() + } + fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> { walker.visit_child(self.underlying()) } @@ -128,12 +132,6 @@ impl Array for CompositeArray { impl StatsCompute for CompositeArray {} -impl ArrayValidity for CompositeArray { - fn validity(&self) -> Option { - self.underlying().validity() - } -} - impl ArrayDisplay for CompositeArray { fn fmt(&self, f: &mut ArrayFormatter) -> std::fmt::Result { f.property("metadata", format!("{:#?}", self.metadata().as_slice()))?; diff --git a/vortex-array/src/array/constant/compute.rs b/vortex-array/src/array/constant/compute.rs index f33bd600ec..c38603ce86 100644 --- a/vortex-array/src/array/constant/compute.rs +++ b/vortex-array/src/array/constant/compute.rs @@ -6,7 +6,8 @@ use crate::array::bool::BoolArray; use crate::array::constant::ConstantArray; use crate::array::downcast::DowncastArrayBuiltin; use crate::array::primitive::PrimitiveArray; -use crate::array::{Array, ArrayRef}; +use crate::array::validity::Validity; +use crate::array::{Array, ArrayRef, ArrayValidity}; use crate::compute::as_contiguous::AsContiguousFn; use crate::compute::flatten::{FlattenFn, FlattenedArray}; use crate::compute::scalar_at::ScalarAtFn; @@ -14,7 +15,6 @@ use crate::compute::take::TakeFn; use crate::compute::ArrayCompute; use crate::match_each_native_ptype; use crate::scalar::Scalar; -use crate::validity::{ArrayValidity, Validity}; impl ArrayCompute for ConstantArray { fn as_contiguous(&self) -> Option<&dyn AsContiguousFn> { diff --git a/vortex-array/src/array/constant/mod.rs b/vortex-array/src/array/constant/mod.rs index 8d98740e75..3a3aa9dd84 100644 --- a/vortex-array/src/array/constant/mod.rs +++ b/vortex-array/src/array/constant/mod.rs @@ -4,6 +4,7 @@ use linkme::distributed_slice; use vortex_error::VortexResult; use vortex_schema::DType; +use crate::array::validity::Validity; use crate::array::{check_slice_bounds, Array, ArrayRef}; use crate::compute::ArrayCompute; use crate::encoding::{Encoding, EncodingId, EncodingRef, ENCODINGS}; @@ -11,7 +12,6 @@ use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::scalar::Scalar; use crate::serde::{ArraySerde, EncodingSerde}; use crate::stats::{Stat, Stats, StatsSet}; -use crate::validity::{ArrayValidity, Validity}; use crate::{impl_array, ArrayWalker}; mod compute; @@ -96,12 +96,6 @@ impl Array for ConstantArray { Some(self) } - fn walk(&self, _walker: &mut dyn ArrayWalker) -> VortexResult<()> { - Ok(()) - } -} - -impl ArrayValidity for ConstantArray { fn validity(&self) -> Option { match self.scalar.dtype().is_nullable() { true => match self.scalar().is_null() { @@ -111,6 +105,10 @@ impl ArrayValidity for ConstantArray { false => None, } } + + fn walk(&self, _walker: &mut dyn ArrayWalker) -> VortexResult<()> { + Ok(()) + } } impl ArrayDisplay for ConstantArray { diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs index a27732df9b..1f45d74e58 100644 --- a/vortex-array/src/array/mod.rs +++ b/vortex-array/src/array/mod.rs @@ -13,13 +13,13 @@ use crate::array::downcast::DowncastArrayBuiltin; use crate::array::primitive::{PrimitiveArray, PrimitiveEncoding}; use crate::array::sparse::{SparseArray, SparseEncoding}; use crate::array::struct_::{StructArray, StructEncoding}; +use crate::array::validity::Validity; use crate::array::varbin::{VarBinArray, VarBinEncoding}; use crate::array::varbinview::{VarBinViewArray, VarBinViewEncoding}; use crate::compute::ArrayCompute; use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::serde::ArraySerde; use crate::stats::Stats; -use crate::validity::{ArrayValidity, Validity}; pub mod bool; pub mod chunked; @@ -29,6 +29,7 @@ pub mod downcast; pub mod primitive; pub mod sparse; pub mod struct_; +pub mod validity; pub mod varbin; pub mod varbinview; @@ -42,7 +43,7 @@ pub type ArrayRef = Arc; /// /// This differs from Apache Arrow where logical and physical are combined in /// the data type, e.g. LargeString, RunEndEncoded. -pub trait Array: ArrayValidity + ArrayDisplay + Debug + Send + Sync { +pub trait Array: ArrayDisplay + Debug + Send + Sync { /// Converts itself to a reference of [`Any`], which enables downcasting to concrete types. fn as_any(&self) -> &dyn Any; fn into_any(self: Arc) -> Arc; @@ -61,6 +62,8 @@ pub trait Array: ArrayValidity + ArrayDisplay + Debug + Send + Sync { /// Maybe we actually need to model stats more like compute? fn stats(&self) -> Stats; + fn validity(&self) -> Option; + /// Limit array to start..stop range fn slice(&self, start: usize, stop: usize) -> VortexResult; /// Encoding kind of the array @@ -104,6 +107,28 @@ impl WithArrayCompute for dyn Array + '_ { } } +pub trait ArrayValidity { + fn nullability(&self) -> Nullability; + + fn logical_validity(&self) -> Option; + + fn is_valid(&self, index: usize) -> bool; +} + +impl ArrayValidity for A { + fn nullability(&self) -> Nullability { + self.validity().is_some().into() + } + + fn logical_validity(&self) -> Option { + self.validity().and_then(|v| v.logical_validity()) + } + + fn is_valid(&self, index: usize) -> bool { + self.validity().map(|v| v.is_valid(index)).unwrap_or(true) + } +} + pub trait IntoArray { fn into_array(self) -> ArrayRef; } @@ -146,24 +171,6 @@ pub use impl_array; use crate::encoding::EncodingRef; use crate::ArrayWalker; -impl ArrayValidity for ArrayRef { - fn nullability(&self) -> Nullability { - self.as_ref().nullability() - } - - fn validity(&self) -> Option { - self.as_ref().validity() - } - - fn logical_validity(&self) -> Option { - self.as_ref().logical_validity() - } - - fn is_valid(&self, index: usize) -> bool { - self.as_ref().is_valid(index) - } -} - impl Array for ArrayRef { fn as_any(&self) -> &dyn Any { self.as_ref().as_any() @@ -220,6 +227,10 @@ impl Array for ArrayRef { self.as_ref().serde() } + fn validity(&self) -> Option { + self.as_ref().validity() + } + #[allow(unused_variables)] fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> { self.as_ref().walk(walker) @@ -232,24 +243,6 @@ impl ArrayDisplay for ArrayRef { } } -impl<'a, T: ArrayValidity> ArrayValidity for &'a T { - fn nullability(&self) -> Nullability { - T::nullability(self) - } - - fn validity(&self) -> Option { - T::validity(self) - } - - fn logical_validity(&self) -> Option { - T::logical_validity(self) - } - - fn is_valid(&self, index: usize) -> bool { - T::is_valid(self, index) - } -} - impl<'a, T: Array + Clone> Array for &'a T { fn as_any(&self) -> &dyn Any { T::as_any(self) @@ -283,6 +276,10 @@ impl<'a, T: Array + Clone> Array for &'a T { T::stats(self) } + fn validity(&self) -> Option { + T::validity(self) + } + fn slice(&self, start: usize, stop: usize) -> VortexResult { T::slice(self, start, stop) } diff --git a/vortex-array/src/array/primitive/compute/as_arrow.rs b/vortex-array/src/array/primitive/compute/as_arrow.rs index 84ff40159a..832a3b47c5 100644 --- a/vortex-array/src/array/primitive/compute/as_arrow.rs +++ b/vortex-array/src/array/primitive/compute/as_arrow.rs @@ -11,7 +11,6 @@ use crate::array::Array; use crate::arrow::wrappers::as_nulls; use crate::compute::as_arrow::AsArrowArray; use crate::ptype::PType; -use crate::validity::ArrayValidity; impl AsArrowArray for PrimitiveArray { fn as_arrow(&self) -> VortexResult { diff --git a/vortex-array/src/array/primitive/compute/as_contiguous.rs b/vortex-array/src/array/primitive/compute/as_contiguous.rs index 35584c9ae3..3fd6d8a816 100644 --- a/vortex-array/src/array/primitive/compute/as_contiguous.rs +++ b/vortex-array/src/array/primitive/compute/as_contiguous.rs @@ -4,11 +4,11 @@ use vortex_error::{vortex_bail, VortexResult}; use crate::array::downcast::DowncastArrayBuiltin; use crate::array::primitive::PrimitiveArray; +use crate::array::validity::Validity; use crate::array::{Array, ArrayRef}; use crate::compute::as_contiguous::AsContiguousFn; use crate::match_each_native_ptype; use crate::ptype::NativePType; -use crate::validity::{ArrayValidity, Validity}; impl AsContiguousFn for PrimitiveArray { fn as_contiguous(&self, arrays: &[ArrayRef]) -> VortexResult { diff --git a/vortex-array/src/array/primitive/compute/cast.rs b/vortex-array/src/array/primitive/compute/cast.rs index 2776f44d36..2f3e13a23a 100644 --- a/vortex-array/src/array/primitive/compute/cast.rs +++ b/vortex-array/src/array/primitive/compute/cast.rs @@ -6,7 +6,6 @@ use crate::array::{Array, ArrayRef}; use crate::compute::cast::CastFn; use crate::match_each_native_ptype; use crate::ptype::{NativePType, PType}; -use crate::validity::ArrayValidity; impl CastFn for PrimitiveArray { fn cast(&self, dtype: &DType) -> VortexResult { diff --git a/vortex-array/src/array/primitive/compute/fill.rs b/vortex-array/src/array/primitive/compute/fill.rs index fdfbd52d2b..c8e0687d07 100644 --- a/vortex-array/src/array/primitive/compute/fill.rs +++ b/vortex-array/src/array/primitive/compute/fill.rs @@ -6,7 +6,6 @@ use crate::array::IntoArray; use crate::array::{Array, ArrayRef}; use crate::compute::fill::FillForwardFn; use crate::match_each_native_ptype; -use crate::validity::ArrayValidity; impl FillForwardFn for PrimitiveArray { fn fill_forward(&self) -> VortexResult { @@ -41,8 +40,8 @@ impl FillForwardFn for PrimitiveArray { mod test { use crate::array::downcast::DowncastArrayBuiltin; use crate::array::primitive::PrimitiveArray; + use crate::array::Array; use crate::compute; - use crate::validity::ArrayValidity; #[test] fn leading_none() { diff --git a/vortex-array/src/array/primitive/compute/patch.rs b/vortex-array/src/array/primitive/compute/patch.rs index 685039343e..ac8d0d22b9 100644 --- a/vortex-array/src/array/primitive/compute/patch.rs +++ b/vortex-array/src/array/primitive/compute/patch.rs @@ -6,7 +6,6 @@ use crate::array::primitive::PrimitiveArray; use crate::array::sparse::{SparseArray, SparseEncoding}; use crate::array::{Array, ArrayRef}; use crate::compute::patch::PatchFn; -use crate::validity::ArrayValidity; use crate::{compute, match_each_native_ptype}; impl PatchFn for PrimitiveArray { diff --git a/vortex-array/src/array/primitive/compute/scalar_at.rs b/vortex-array/src/array/primitive/compute/scalar_at.rs index 587deb7ec9..2a69d34ad9 100644 --- a/vortex-array/src/array/primitive/compute/scalar_at.rs +++ b/vortex-array/src/array/primitive/compute/scalar_at.rs @@ -1,10 +1,10 @@ use vortex_error::VortexResult; use crate::array::primitive::PrimitiveArray; +use crate::array::ArrayValidity; use crate::compute::scalar_at::ScalarAtFn; use crate::match_each_native_ptype; use crate::scalar::{PrimitiveScalar, Scalar}; -use crate::validity::ArrayValidity; impl ScalarAtFn for PrimitiveArray { fn scalar_at(&self, index: usize) -> VortexResult { diff --git a/vortex-array/src/array/primitive/compute/take.rs b/vortex-array/src/array/primitive/compute/take.rs index eea359bd26..2269d63411 100644 --- a/vortex-array/src/array/primitive/compute/take.rs +++ b/vortex-array/src/array/primitive/compute/take.rs @@ -6,7 +6,6 @@ use crate::array::{Array, ArrayRef}; use crate::compute::flatten::flatten_primitive; use crate::compute::take::TakeFn; use crate::ptype::NativePType; -use crate::validity::ArrayValidity; use crate::{match_each_integer_ptype, match_each_native_ptype}; impl TakeFn for PrimitiveArray { diff --git a/vortex-array/src/array/primitive/mod.rs b/vortex-array/src/array/primitive/mod.rs index 25220c29f6..fcc44d18f1 100644 --- a/vortex-array/src/array/primitive/mod.rs +++ b/vortex-array/src/array/primitive/mod.rs @@ -12,15 +12,15 @@ use vortex_error::{vortex_bail, VortexResult}; use vortex_schema::{DType, Nullability}; use crate::accessor::ArrayAccessor; -use crate::array::IntoArray; +use crate::array::validity::Validity; use crate::array::{check_slice_bounds, Array, ArrayRef}; +use crate::array::{ArrayValidity, IntoArray}; use crate::encoding::{Encoding, EncodingId, EncodingRef, ENCODINGS}; use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::iterator::ArrayIter; use crate::ptype::{match_each_native_ptype, NativePType, PType}; use crate::serde::{ArraySerde, EncodingSerde}; use crate::stats::{Stats, StatsSet}; -use crate::validity::{ArrayValidity, Validity}; use crate::{impl_array, ArrayWalker}; mod compute; @@ -204,6 +204,10 @@ impl Array for PrimitiveArray { Some(self) } + fn validity(&self) -> Option { + self.validity.clone() + } + fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> { if let Some(v) = self.validity() { // FIXME(ngates): should validity implement Array? @@ -213,12 +217,6 @@ impl Array for PrimitiveArray { } } -impl ArrayValidity for PrimitiveArray { - fn validity(&self) -> Option { - self.validity.clone() - } -} - impl ArrayAccessor for PrimitiveArray { fn value(&self, index: usize) -> Option { if self.is_valid(index) { diff --git a/vortex-array/src/array/primitive/serde.rs b/vortex-array/src/array/primitive/serde.rs index 038e3f285e..005a9119ed 100644 --- a/vortex-array/src/array/primitive/serde.rs +++ b/vortex-array/src/array/primitive/serde.rs @@ -5,7 +5,6 @@ use crate::array::{Array, ArrayRef}; use crate::compute::ArrayCompute; use crate::match_each_native_ptype; use crate::serde::{ArraySerde, ArrayView, EncodingSerde, ReadCtx, WriteCtx}; -use crate::validity::ArrayValidity; impl ArraySerde for PrimitiveArray { fn write(&self, ctx: &mut WriteCtx) -> VortexResult<()> { @@ -43,8 +42,8 @@ impl EncodingSerde for PrimitiveEncoding { mod test { use crate::array::downcast::DowncastArrayBuiltin; use crate::array::primitive::PrimitiveArray; + use crate::array::Array; use crate::serde::test::roundtrip_array; - use crate::validity::ArrayValidity; #[test] fn roundtrip() { diff --git a/vortex-array/src/array/primitive/stats.rs b/vortex-array/src/array/primitive/stats.rs index 39060f9502..6c711489f7 100644 --- a/vortex-array/src/array/primitive/stats.rs +++ b/vortex-array/src/array/primitive/stats.rs @@ -5,11 +5,11 @@ use arrow_buffer::buffer::BooleanBuffer; use vortex_error::VortexResult; use crate::array::primitive::PrimitiveArray; +use crate::array::ArrayValidity; use crate::match_each_native_ptype; use crate::ptype::NativePType; use crate::scalar::{ListScalarVec, PScalar}; use crate::stats::{Stat, StatsCompute, StatsSet}; -use crate::validity::ArrayValidity; impl StatsCompute for PrimitiveArray { fn compute(&self, stat: &Stat) -> VortexResult { diff --git a/vortex-array/src/array/primitive/view.rs b/vortex-array/src/array/primitive/view.rs index 3cb07c0f6a..b053c821dd 100644 --- a/vortex-array/src/array/primitive/view.rs +++ b/vortex-array/src/array/primitive/view.rs @@ -2,6 +2,7 @@ use arrow_buffer::Buffer; use num_traits::PrimInt; use vortex_error::{vortex_err, VortexResult}; +use crate::array::validity::Validity; use crate::array::PrimitiveArray; use crate::array::{Array, ArrayRef}; use crate::compute::flatten::{flatten_primitive, FlattenFn, FlattenedArray}; @@ -10,12 +11,11 @@ use crate::compute::ArrayCompute; use crate::match_each_integer_ptype; use crate::ptype::{NativePType, PType}; use crate::serde::ArrayView; -use crate::validity::Validity; pub struct PrimitiveView<'a> { ptype: PType, buffer: &'a Buffer, - // TODO(ngates): look at a ValidityView? + // TODO(ngates): switch to ValidityView validity: Option, } diff --git a/vortex-array/src/array/sparse/mod.rs b/vortex-array/src/array/sparse/mod.rs index 441d470d9a..fecbccc166 100644 --- a/vortex-array/src/array/sparse/mod.rs +++ b/vortex-array/src/array/sparse/mod.rs @@ -5,6 +5,7 @@ use linkme::distributed_slice; use vortex_error::{vortex_bail, VortexResult}; use vortex_schema::DType; +use crate::array::validity::Validity; use crate::array::{check_slice_bounds, Array, ArrayRef}; use crate::compress::EncodingCompression; use crate::compute::cast::cast; @@ -16,7 +17,6 @@ use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::ptype::PType; use crate::serde::{ArraySerde, EncodingSerde}; use crate::stats::{Stats, StatsCompute, StatsSet}; -use crate::validity::{ArrayValidity, Validity}; use crate::{impl_array, ArrayWalker}; mod compress; @@ -141,6 +141,10 @@ impl Array for SparseArray { Some(self) } + fn validity(&self) -> Option { + todo!() + } + fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> { walker.visit_child(self.indices())?; walker.visit_child(self.values()) @@ -157,12 +161,6 @@ impl ArrayDisplay for SparseArray { } } -impl ArrayValidity for SparseArray { - fn validity(&self) -> Option { - todo!() - } -} - #[derive(Debug)] pub struct SparseEncoding; diff --git a/vortex-array/src/array/struct_/mod.rs b/vortex-array/src/array/struct_/mod.rs index 1d988e5050..747ace3809 100644 --- a/vortex-array/src/array/struct_/mod.rs +++ b/vortex-array/src/array/struct_/mod.rs @@ -6,13 +6,13 @@ use vortex_error::VortexResult; use vortex_schema::{DType, FieldNames}; use super::{check_slice_bounds, Array, ArrayRef}; +use crate::array::validity::Validity; use crate::compress::EncodingCompression; use crate::compute::ArrayCompute; use crate::encoding::{Encoding, EncodingId, EncodingRef, ENCODINGS}; use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::serde::{ArraySerde, EncodingSerde}; use crate::stats::{Stats, StatsCompute, StatsSet}; -use crate::validity::{ArrayValidity, Validity}; use crate::{impl_array, ArrayWalker}; mod compress; @@ -116,6 +116,10 @@ impl Array for StructArray { Some(self) } + fn validity(&self) -> Option { + todo!() + } + fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> { for field in self.fields() { walker.visit_child(field)?; @@ -124,12 +128,6 @@ impl Array for StructArray { } } -impl ArrayValidity for StructArray { - fn validity(&self) -> Option { - todo!() - } -} - impl StatsCompute for StructArray {} #[derive(Debug)] diff --git a/vortex-array/src/validity.rs b/vortex-array/src/array/validity/mod.rs similarity index 74% rename from vortex-array/src/validity.rs rename to vortex-array/src/array/validity/mod.rs index 9976069a12..ce57e55b07 100644 --- a/vortex-array/src/validity.rs +++ b/vortex-array/src/array/validity/mod.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use arrow_buffer::{BooleanBuffer, NullBuffer}; use itertools::Itertools; use vortex_error::VortexResult; @@ -10,7 +12,15 @@ use crate::compute::as_contiguous::as_contiguous; use crate::compute::flatten::flatten_bool; use crate::compute::scalar_at::scalar_at; use crate::compute::take::take; -use crate::stats::Stat; +use crate::compute::ArrayCompute; +use crate::encoding::{Encoding, EncodingId, EncodingRef}; +use crate::formatter::{ArrayDisplay, ArrayFormatter}; +use crate::stats::{Stat, Stats}; +use crate::{impl_array, ArrayWalker}; +mod serde; +mod view; + +pub use view::*; #[derive(Debug, Clone)] pub enum Validity { @@ -201,18 +211,81 @@ impl FromIterator for Validity { } } -pub trait ArrayValidity { - fn nullability(&self) -> Nullability { - self.validity().is_some().into() +impl Array for Validity { + impl_array!(); + + fn len(&self) -> usize { + match self { + Validity::Valid(len) | Validity::Invalid(len) => *len, + Validity::Array(a) => a.len(), + } + } + + fn is_empty(&self) -> bool { + match self { + Validity::Valid(len) | Validity::Invalid(len) => *len == 0, + Validity::Array(a) => a.is_empty(), + } + } + + fn dtype(&self) -> &DType { + &Validity::DTYPE + } + + fn stats(&self) -> Stats { + todo!() + } + + fn slice(&self, start: usize, stop: usize) -> VortexResult { + Ok(match self { + Validity::Valid(_) => Validity::Valid(stop - start), + Validity::Invalid(_) => Validity::Invalid(stop - start), + Validity::Array(a) => Validity::Array(a.slice(start, stop)?), + } + .into_array()) + } + + fn encoding(&self) -> EncodingRef { + &ValidityEncoding } - fn validity(&self) -> Option; + fn nbytes(&self) -> usize { + match self { + Validity::Valid(_) | Validity::Invalid(_) => 8, + Validity::Array(a) => a.nbytes(), + } + } - fn logical_validity(&self) -> Option { - self.validity().and_then(|v| v.logical_validity()) + fn validity(&self) -> Option { + None } - fn is_valid(&self, index: usize) -> bool { - self.validity().map(|v| v.is_valid(index)).unwrap_or(true) + fn walk(&self, _walker: &mut dyn ArrayWalker) -> VortexResult<()> { + Ok(()) + } +} + +impl ArrayDisplay for Validity { + fn fmt(&self, fmt: &'_ mut ArrayFormatter) -> std::fmt::Result { + match self { + Validity::Valid(_) => fmt.property("all", "valid"), + Validity::Invalid(_) => fmt.property("all", "invalid"), + Validity::Array(a) => fmt.child("validity", a), + } + } +} + +impl ArrayCompute for Validity {} + +#[derive(Debug)] +struct ValidityEncoding; + +impl ValidityEncoding { + const ID: EncodingId = EncodingId::new("vortex.validity"); +} + +impl Encoding for ValidityEncoding { + fn id(&self) -> EncodingId { + ValidityEncoding::ID } } diff --git a/vortex-array/src/array/validity/serde.rs b/vortex-array/src/array/validity/serde.rs new file mode 100644 index 0000000000..c2c6e0d876 --- /dev/null +++ b/vortex-array/src/array/validity/serde.rs @@ -0,0 +1,21 @@ +use vortex_error::VortexResult; + +use crate::array::validity::Validity; +use crate::array::ArrayRef; +use crate::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; + +impl ArraySerde for Validity { + fn write(&self, _ctx: &mut WriteCtx) -> VortexResult<()> { + todo!() + } + + fn metadata(&self) -> VortexResult>> { + todo!() + } +} + +impl EncodingSerde for Validity { + fn read(&self, _ctx: &mut ReadCtx) -> VortexResult { + todo!() + } +} diff --git a/vortex-array/src/array/validity/view.rs b/vortex-array/src/array/validity/view.rs new file mode 100644 index 0000000000..cdab4a436d --- /dev/null +++ b/vortex-array/src/array/validity/view.rs @@ -0,0 +1,8 @@ +use crate::array::Array; + +#[derive(Debug, Clone)] +pub enum ValidityView<'a> { + Valid(usize), + Invalid(usize), + Array(&'a dyn Array), +} diff --git a/vortex-array/src/array/varbin/accessor.rs b/vortex-array/src/array/varbin/accessor.rs index 7ecb0c7a12..bf2ef5faaf 100644 --- a/vortex-array/src/array/varbin/accessor.rs +++ b/vortex-array/src/array/varbin/accessor.rs @@ -4,10 +4,10 @@ use crate::accessor::ArrayAccessor; use crate::array::downcast::DowncastArrayBuiltin; use crate::array::varbin::VarBinArray; use crate::array::Array; +use crate::array::ArrayValidity; use crate::compute::flatten::flatten_primitive; use crate::compute::scalar_at::scalar_at; use crate::match_each_native_ptype; -use crate::validity::ArrayValidity; fn offset_at(array: &dyn Array, index: usize) -> usize { if let Some(parray) = array.maybe_primitive() { diff --git a/vortex-array/src/array/varbin/builder.rs b/vortex-array/src/array/varbin/builder.rs index e92128b57a..e2ff46481e 100644 --- a/vortex-array/src/array/varbin/builder.rs +++ b/vortex-array/src/array/varbin/builder.rs @@ -3,10 +3,10 @@ use num_traits::PrimInt; use vortex_schema::DType; use crate::array::primitive::PrimitiveArray; +use crate::array::validity::Validity; use crate::array::varbin::VarBinArray; use crate::array::Array; use crate::ptype::NativePType; -use crate::validity::Validity; pub struct VarBinBuilder { offsets: Vec, @@ -68,10 +68,9 @@ mod test { use vortex_schema::Nullability::Nullable; use crate::array::varbin::builder::VarBinBuilder; - use crate::array::Array; + use crate::array::{Array, ArrayValidity}; use crate::compute::scalar_at::scalar_at; use crate::scalar::Scalar; - use crate::validity::ArrayValidity; #[test] fn test_builder() { diff --git a/vortex-array/src/array/varbin/compress.rs b/vortex-array/src/array/varbin/compress.rs index ffd6000101..7516319074 100644 --- a/vortex-array/src/array/varbin/compress.rs +++ b/vortex-array/src/array/varbin/compress.rs @@ -4,7 +4,6 @@ use crate::array::downcast::DowncastArrayBuiltin; use crate::array::varbin::{VarBinArray, VarBinEncoding}; use crate::array::{Array, ArrayRef}; use crate::compress::{CompressConfig, CompressCtx, EncodingCompression}; -use crate::validity::ArrayValidity; impl EncodingCompression for VarBinEncoding { fn cost(&self) -> u8 { diff --git a/vortex-array/src/array/varbin/compute/mod.rs b/vortex-array/src/array/varbin/compute/mod.rs index 9dd18cefbc..e960acaebf 100644 --- a/vortex-array/src/array/varbin/compute/mod.rs +++ b/vortex-array/src/array/varbin/compute/mod.rs @@ -9,7 +9,9 @@ use vortex_schema::DType; use crate::array::downcast::DowncastArrayBuiltin; use crate::array::primitive::PrimitiveArray; +use crate::array::validity::Validity; use crate::array::varbin::VarBinArray; +use crate::array::ArrayValidity; use crate::array::{Array, ArrayRef}; use crate::arrow::wrappers::{as_nulls, as_offset_buffer}; use crate::compute::as_arrow::AsArrowArray; @@ -21,7 +23,6 @@ use crate::compute::take::TakeFn; use crate::compute::ArrayCompute; use crate::ptype::PType; use crate::scalar::{BinaryScalar, Scalar, Utf8Scalar}; -use crate::validity::{ArrayValidity, Validity}; mod take; diff --git a/vortex-array/src/array/varbin/compute/take.rs b/vortex-array/src/array/varbin/compute/take.rs index 57b5af4cc8..360a6a21ef 100644 --- a/vortex-array/src/array/varbin/compute/take.rs +++ b/vortex-array/src/array/varbin/compute/take.rs @@ -2,6 +2,7 @@ use num_traits::PrimInt; use vortex_error::VortexResult; use vortex_schema::DType; +use crate::array::validity::Validity; use crate::array::varbin::builder::VarBinBuilder; use crate::array::varbin::VarBinArray; use crate::array::{Array, ArrayRef}; @@ -9,7 +10,6 @@ use crate::compute::flatten::flatten_primitive; use crate::compute::take::TakeFn; use crate::match_each_integer_ptype; use crate::ptype::NativePType; -use crate::validity::{ArrayValidity, Validity}; impl TakeFn for VarBinArray { fn take(&self, indices: &dyn Array) -> VortexResult { diff --git a/vortex-array/src/array/varbin/mod.rs b/vortex-array/src/array/varbin/mod.rs index eb4b2f902d..caf35a3ca8 100644 --- a/vortex-array/src/array/varbin/mod.rs +++ b/vortex-array/src/array/varbin/mod.rs @@ -7,6 +7,7 @@ use vortex_schema::{DType, IntWidth, Nullability, Signedness}; use crate::array::downcast::DowncastArrayBuiltin; use crate::array::primitive::PrimitiveArray; +use crate::array::validity::Validity; use crate::array::{check_slice_bounds, Array, ArrayRef}; use crate::compress::EncodingCompression; use crate::compute::flatten::flatten_primitive; @@ -18,7 +19,6 @@ use crate::iterator::ArrayIter; use crate::ptype::NativePType; use crate::serde::{ArraySerde, EncodingSerde}; use crate::stats::{Stats, StatsSet}; -use crate::validity::{ArrayValidity, Validity}; use crate::{impl_array, ArrayWalker}; mod accessor; @@ -244,18 +244,16 @@ impl Array for VarBinArray { Some(self) } + fn validity(&self) -> Option { + self.validity.clone() + } + fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> { walker.visit_child(self.offsets())?; walker.visit_child(self.bytes()) } } -impl ArrayValidity for VarBinArray { - fn validity(&self) -> Option { - self.validity.clone() - } -} - #[derive(Debug)] pub struct VarBinEncoding; diff --git a/vortex-array/src/array/varbin/serde.rs b/vortex-array/src/array/varbin/serde.rs index 7a9d278b4b..b7f8ef5e56 100644 --- a/vortex-array/src/array/varbin/serde.rs +++ b/vortex-array/src/array/varbin/serde.rs @@ -3,7 +3,6 @@ use vortex_error::VortexResult; use crate::array::varbin::{VarBinArray, VarBinEncoding}; use crate::array::{Array, ArrayRef}; use crate::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; -use crate::validity::ArrayValidity; impl ArraySerde for VarBinArray { fn write(&self, ctx: &mut WriteCtx) -> VortexResult<()> { diff --git a/vortex-array/src/array/varbinview/compute.rs b/vortex-array/src/array/varbinview/compute.rs index 151b81349a..a273dcacf5 100644 --- a/vortex-array/src/array/varbinview/compute.rs +++ b/vortex-array/src/array/varbinview/compute.rs @@ -7,7 +7,7 @@ use vortex_error::{vortex_bail, VortexResult}; use vortex_schema::DType; use crate::array::varbinview::VarBinViewArray; -use crate::array::Array; +use crate::array::{Array, ArrayValidity}; use crate::arrow::wrappers::as_nulls; use crate::compute::as_arrow::AsArrowArray; use crate::compute::flatten::{flatten, flatten_primitive, FlattenFn, FlattenedArray}; @@ -15,7 +15,6 @@ use crate::compute::scalar_at::ScalarAtFn; use crate::compute::ArrayCompute; use crate::ptype::PType; use crate::scalar::Scalar; -use crate::validity::ArrayValidity; impl ArrayCompute for VarBinViewArray { fn as_arrow(&self) -> Option<&dyn AsArrowArray> { diff --git a/vortex-array/src/array/varbinview/mod.rs b/vortex-array/src/array/varbinview/mod.rs index aa0f9dc7a6..3e78e98525 100644 --- a/vortex-array/src/array/varbinview/mod.rs +++ b/vortex-array/src/array/varbinview/mod.rs @@ -5,6 +5,7 @@ use linkme::distributed_slice; use vortex_error::{vortex_bail, VortexResult}; use vortex_schema::{DType, IntWidth, Nullability, Signedness}; +use crate::array::validity::Validity; use crate::array::{check_slice_bounds, Array, ArrayRef}; use crate::compute::flatten::flatten_primitive; use crate::compute::ArrayCompute; @@ -12,7 +13,6 @@ use crate::encoding::{Encoding, EncodingId, EncodingRef, ENCODINGS}; use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::serde::{ArraySerde, EncodingSerde}; use crate::stats::{Stats, StatsSet}; -use crate::validity::{ArrayValidity, Validity}; use crate::{impl_array, ArrayWalker}; mod compute; @@ -233,6 +233,10 @@ impl Array for VarBinViewArray { Some(self) } + fn validity(&self) -> Option { + self.validity.clone() + } + fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> { walker.visit_child(self.views())?; for data in self.data() { @@ -242,12 +246,6 @@ impl Array for VarBinViewArray { } } -impl ArrayValidity for VarBinViewArray { - fn validity(&self) -> Option { - self.validity.clone() - } -} - #[derive(Debug)] pub struct VarBinViewEncoding; diff --git a/vortex-array/src/array/varbinview/serde.rs b/vortex-array/src/array/varbinview/serde.rs index 3d0eae040b..14deb1df91 100644 --- a/vortex-array/src/array/varbinview/serde.rs +++ b/vortex-array/src/array/varbinview/serde.rs @@ -3,7 +3,6 @@ use vortex_error::VortexResult; use crate::array::varbinview::{VarBinViewArray, VarBinViewEncoding}; use crate::array::{Array, ArrayRef}; use crate::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; -use crate::validity::ArrayValidity; impl ArraySerde for VarBinViewArray { fn write(&self, ctx: &mut WriteCtx) -> VortexResult<()> { diff --git a/vortex-array/src/arrow/wrappers.rs b/vortex-array/src/arrow/wrappers.rs index 3dc41ec924..5de8c3f5b7 100644 --- a/vortex-array/src/arrow/wrappers.rs +++ b/vortex-array/src/arrow/wrappers.rs @@ -2,8 +2,8 @@ use arrow_buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; use vortex_error::VortexResult; use crate::array::primitive::PrimitiveArray; +use crate::array::validity::Validity; use crate::ptype::NativePType; -use crate::validity::Validity; pub fn as_scalar_buffer(array: PrimitiveArray) -> ScalarBuffer { assert_eq!(array.ptype(), T::PTYPE); diff --git a/vortex-array/src/compress.rs b/vortex-array/src/compress.rs index 85c4c7fad0..0ae7249644 100644 --- a/vortex-array/src/compress.rs +++ b/vortex-array/src/compress.rs @@ -10,6 +10,7 @@ use crate::array::composite::CompositeEncoding; use crate::array::constant::ConstantArray; use crate::array::sparse::SparseEncoding; use crate::array::struct_::{StructArray, StructEncoding}; +use crate::array::validity::Validity; use crate::array::varbin::VarBinEncoding; use crate::array::{Array, ArrayKind, ArrayRef}; use crate::compute; @@ -18,7 +19,6 @@ use crate::encoding::{Encoding, EncodingRef, ENCODINGS}; use crate::formatter::display_tree; use crate::sampling::stratified_slices; use crate::stats::Stat; -use crate::validity::Validity; pub trait EncodingCompression: Encoding { fn cost(&self) -> u8 { diff --git a/vortex-array/src/datetime/localdatetime.rs b/vortex-array/src/datetime/localdatetime.rs index e0ebaf584a..45cbffa7aa 100644 --- a/vortex-array/src/datetime/localdatetime.rs +++ b/vortex-array/src/datetime/localdatetime.rs @@ -9,6 +9,7 @@ use vortex_error::VortexResult; use vortex_schema::CompositeID; use crate::array::composite::{composite_impl, TypedCompositeArray}; +use crate::array::Array; use crate::arrow::wrappers::as_nulls; use crate::compute::as_arrow::AsArrowArray; use crate::compute::cast::cast; @@ -16,7 +17,6 @@ use crate::compute::flatten::flatten_primitive; use crate::datetime::TimeUnit; use crate::ptype::PType; use crate::serde::BytesSerde; -use crate::validity::ArrayValidity; #[derive(Debug, Clone)] pub struct LocalDateTime { diff --git a/vortex-array/src/encode.rs b/vortex-array/src/encode.rs index a848e26cbb..8fbeebba1a 100644 --- a/vortex-array/src/encode.rs +++ b/vortex-array/src/encode.rs @@ -27,6 +27,7 @@ use crate::array::bool::BoolArray; use crate::array::constant::ConstantArray; use crate::array::primitive::PrimitiveArray; use crate::array::struct_::StructArray; +use crate::array::validity::Validity; use crate::array::varbin::VarBinArray; use crate::array::varbinview::VarBinViewArray; use crate::array::IntoArray; @@ -35,7 +36,6 @@ use crate::datetime::{LocalDateTime, LocalDateTimeArray}; use crate::ptype::{NativePType, PType}; use crate::scalar::NullScalar; use crate::stats::Stat; -use crate::validity::Validity; pub trait FromArrowArray { fn from_arrow(array: A, nullable: bool) -> Self; diff --git a/vortex-array/src/formatter.rs b/vortex-array/src/formatter.rs index 7f35b8eafd..d7feedf4d5 100644 --- a/vortex-array/src/formatter.rs +++ b/vortex-array/src/formatter.rs @@ -3,8 +3,8 @@ use std::fmt::{Display, Write}; use humansize::{format_size, DECIMAL}; +use crate::array::validity::Validity; use crate::array::{Array, ArrayRef}; -use crate::validity::Validity; pub trait ArrayDisplay { fn fmt(&self, fmt: &'_ mut ArrayFormatter) -> fmt::Result; diff --git a/vortex-array/src/lib.rs b/vortex-array/src/lib.rs index dc428993e1..ec0873b96c 100644 --- a/vortex-array/src/lib.rs +++ b/vortex-array/src/lib.rs @@ -16,7 +16,6 @@ pub mod ptype; mod sampling; pub mod serde; pub mod stats; -pub mod validity; mod walk; diff --git a/vortex-array/src/serde/mod.rs b/vortex-array/src/serde/mod.rs index cef05342f9..106a3a2256 100644 --- a/vortex-array/src/serde/mod.rs +++ b/vortex-array/src/serde/mod.rs @@ -10,12 +10,12 @@ use vortex_schema::DTypeSerdeContext; use vortex_schema::{DType, IntWidth, Nullability, Signedness}; use crate::array::composite::COMPOSITE_EXTENSIONS; +use crate::array::validity::Validity; use crate::array::{Array, ArrayRef}; use crate::encoding::{find_encoding, EncodingId, ENCODINGS}; use crate::ptype::PType; use crate::scalar::{Scalar, ScalarReader, ScalarWriter}; use crate::serde::ptype::PTypeTag; -use crate::validity::Validity; pub mod context; pub mod data; diff --git a/vortex-array/src/serde/view.rs b/vortex-array/src/serde/view.rs index 8c48da009b..dc3dbc96e1 100644 --- a/vortex-array/src/serde/view.rs +++ b/vortex-array/src/serde/view.rs @@ -6,6 +6,7 @@ use arrow_buffer::Buffer; use vortex_error::{vortex_bail, vortex_err, VortexResult}; use vortex_schema::DType; +use crate::array::validity::Validity; use crate::array::{Array, ArrayRef}; use crate::compute::ArrayCompute; use crate::encoding::EncodingRef; @@ -14,7 +15,6 @@ use crate::formatter::{ArrayDisplay, ArrayFormatter}; use crate::serde::context::SerdeContext; use crate::serde::EncodingSerde; use crate::stats::Stats; -use crate::validity::{ArrayValidity, Validity}; use crate::ArrayWalker; #[derive(Clone)] @@ -202,13 +202,11 @@ impl<'a> Array for ArrayView<'a> { .with_view_compute(self, f) } - fn walk(&self, _walker: &mut dyn ArrayWalker) -> VortexResult<()> { + fn validity(&self) -> Option { todo!() } -} -impl<'a> ArrayValidity for ArrayView<'a> { - fn validity(&self) -> Option { + fn walk(&self, _walker: &mut dyn ArrayWalker) -> VortexResult<()> { todo!() } } diff --git a/vortex-datetime/src/compress.rs b/vortex-datetime/src/compress.rs index 2e6ce69b7c..725c7e4cb5 100644 --- a/vortex-datetime/src/compress.rs +++ b/vortex-datetime/src/compress.rs @@ -7,7 +7,6 @@ use vortex::compute::cast::cast; use vortex::compute::flatten::flatten_primitive; use vortex::datetime::{LocalDateTime, LocalDateTimeArray, LocalDateTimeExtension, TimeUnit}; use vortex::ptype::PType; -use vortex::validity::ArrayValidity; use vortex_error::VortexResult; use crate::{DateTimeArray, DateTimeEncoding}; diff --git a/vortex-datetime/src/compute.rs b/vortex-datetime/src/compute.rs index e442dc65f2..cfaac1ee70 100644 --- a/vortex-datetime/src/compute.rs +++ b/vortex-datetime/src/compute.rs @@ -1,7 +1,6 @@ use vortex::array::{Array, ArrayRef}; use vortex::compute::take::{take, TakeFn}; use vortex::compute::ArrayCompute; -use vortex::validity::ArrayValidity; use vortex_error::VortexResult; use crate::DateTimeArray; diff --git a/vortex-datetime/src/datetime.rs b/vortex-datetime/src/datetime.rs index e4603111f1..bc98b8fec7 100644 --- a/vortex-datetime/src/datetime.rs +++ b/vortex-datetime/src/datetime.rs @@ -1,5 +1,6 @@ use std::sync::{Arc, RwLock}; +use vortex::array::validity::Validity; use vortex::array::{Array, ArrayRef}; use vortex::compress::EncodingCompression; use vortex::compute::ArrayCompute; @@ -7,7 +8,6 @@ use vortex::encoding::{Encoding, EncodingId, EncodingRef}; use vortex::formatter::{ArrayDisplay, ArrayFormatter}; use vortex::serde::{ArraySerde, EncodingSerde}; use vortex::stats::{Stats, StatsCompute, StatsSet}; -use vortex::validity::{ArrayValidity, Validity}; use vortex::{impl_array, ArrayWalker}; use vortex_error::{vortex_bail, VortexResult}; use vortex_schema::DType; @@ -119,6 +119,10 @@ impl Array for DateTimeArray { Some(self) } + fn validity(&self) -> Option { + self.validity.clone() + } + fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> { walker.visit_child(self.days())?; walker.visit_child(self.seconds())?; @@ -136,12 +140,6 @@ impl ArrayDisplay for DateTimeArray { } } -impl ArrayValidity for DateTimeArray { - fn validity(&self) -> Option { - self.validity.clone() - } -} - #[derive(Debug)] pub struct DateTimeEncoding; diff --git a/vortex-datetime/src/serde.rs b/vortex-datetime/src/serde.rs index 472b66f7a1..8a7fe9bc43 100644 --- a/vortex-datetime/src/serde.rs +++ b/vortex-datetime/src/serde.rs @@ -1,6 +1,5 @@ use vortex::array::{Array, ArrayRef}; use vortex::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; -use vortex::validity::ArrayValidity; use vortex_error::VortexResult; use crate::{DateTimeArray, DateTimeEncoding}; diff --git a/vortex-dict/src/compress.rs b/vortex-dict/src/compress.rs index 9ed1eb1474..7ae10b796f 100644 --- a/vortex-dict/src/compress.rs +++ b/vortex-dict/src/compress.rs @@ -6,6 +6,7 @@ use hashbrown::HashMap; use num_traits::AsPrimitive; use vortex::array::bool::BoolArray; use vortex::array::primitive::{PrimitiveArray, PrimitiveEncoding}; +use vortex::array::validity::Validity; use vortex::array::varbin::{VarBinArray, VarBinEncoding}; use vortex::array::{Array, ArrayKind, ArrayRef}; use vortex::compress::{CompressConfig, CompressCtx, EncodingCompression}; @@ -13,7 +14,6 @@ use vortex::match_each_native_ptype; use vortex::ptype::NativePType; use vortex::scalar::AsBytes; use vortex::stats::Stat; -use vortex::validity::Validity; use vortex_error::VortexResult; use vortex_schema::DType; diff --git a/vortex-dict/src/dict.rs b/vortex-dict/src/dict.rs index 12e5155186..65b841ea45 100644 --- a/vortex-dict/src/dict.rs +++ b/vortex-dict/src/dict.rs @@ -1,5 +1,6 @@ use std::sync::{Arc, RwLock}; +use vortex::array::validity::Validity; use vortex::array::{check_slice_bounds, Array, ArrayRef}; use vortex::compress::EncodingCompression; use vortex::compute::ArrayCompute; @@ -7,7 +8,6 @@ use vortex::encoding::{Encoding, EncodingId, EncodingRef}; use vortex::formatter::{ArrayDisplay, ArrayFormatter}; use vortex::serde::{ArraySerde, EncodingSerde}; use vortex::stats::{Stats, StatsSet}; -use vortex::validity::{ArrayValidity, Validity}; use vortex::{impl_array, ArrayWalker}; use vortex_error::{vortex_bail, VortexResult}; use vortex_schema::{DType, Signedness}; @@ -83,6 +83,10 @@ impl Array for DictArray { Some(self) } + fn validity(&self) -> Option { + todo!() + } + fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> { walker.visit_child(self.values())?; walker.visit_child(self.codes()) @@ -96,12 +100,6 @@ impl ArrayDisplay for DictArray { } } -impl ArrayValidity for DictArray { - fn validity(&self) -> Option { - todo!() - } -} - #[derive(Debug)] pub struct DictEncoding; diff --git a/vortex-fastlanes/src/bitpacking/compress.rs b/vortex-fastlanes/src/bitpacking/compress.rs index c7706864df..fab21e15f4 100644 --- a/vortex-fastlanes/src/bitpacking/compress.rs +++ b/vortex-fastlanes/src/bitpacking/compress.rs @@ -14,7 +14,6 @@ use vortex::ptype::PType::{I16, I32, I64, I8, U16, U32, U64, U8}; use vortex::ptype::{NativePType, PType}; use vortex::scalar::ListScalarVec; use vortex::stats::Stat; -use vortex::validity::ArrayValidity; use vortex_error::VortexResult; use crate::downcast::DowncastFastlanes; diff --git a/vortex-fastlanes/src/bitpacking/mod.rs b/vortex-fastlanes/src/bitpacking/mod.rs index a6d2223fb0..dc5c19782d 100644 --- a/vortex-fastlanes/src/bitpacking/mod.rs +++ b/vortex-fastlanes/src/bitpacking/mod.rs @@ -1,6 +1,7 @@ use std::cmp::min; use std::sync::{Arc, RwLock}; +use vortex::array::validity::Validity; use vortex::array::{Array, ArrayRef}; use vortex::compress::EncodingCompression; use vortex::compute::flatten::flatten_primitive; @@ -9,7 +10,6 @@ use vortex::encoding::{Encoding, EncodingId, EncodingRef}; use vortex::formatter::{ArrayDisplay, ArrayFormatter}; use vortex::serde::{ArraySerde, EncodingSerde}; use vortex::stats::{Stat, Stats, StatsCompute, StatsSet}; -use vortex::validity::{ArrayValidity, Validity}; use vortex::{impl_array, ArrayWalker}; use vortex_error::{vortex_bail, VortexResult}; use vortex_schema::{DType, IntWidth, Nullability, Signedness}; @@ -145,6 +145,10 @@ impl Array for BitPackedArray { Some(self) } + fn validity(&self) -> Option { + self.validity.clone() + } + fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> { walker.visit_child(self.encoded()) } @@ -159,12 +163,6 @@ impl ArrayDisplay for BitPackedArray { } } -impl ArrayValidity for BitPackedArray { - fn validity(&self) -> Option { - self.validity.clone() - } -} - impl StatsCompute for BitPackedArray { fn compute(&self, _stat: &Stat) -> VortexResult { Ok(StatsSet::default()) diff --git a/vortex-fastlanes/src/bitpacking/serde.rs b/vortex-fastlanes/src/bitpacking/serde.rs index c3a7a914ec..68ca0fb5c3 100644 --- a/vortex-fastlanes/src/bitpacking/serde.rs +++ b/vortex-fastlanes/src/bitpacking/serde.rs @@ -1,6 +1,5 @@ use vortex::array::{Array, ArrayRef}; use vortex::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; -use vortex::validity::ArrayValidity; use vortex_error::VortexResult; use crate::{BitPackedArray, BitPackedEncoding}; diff --git a/vortex-fastlanes/src/delta/compress.rs b/vortex-fastlanes/src/delta/compress.rs index 708d90be82..b6346e2438 100644 --- a/vortex-fastlanes/src/delta/compress.rs +++ b/vortex-fastlanes/src/delta/compress.rs @@ -5,14 +5,13 @@ use fastlanez::{transpose, untranspose_into, Delta}; use num_traits::{WrappingAdd, WrappingSub}; use vortex::array::downcast::DowncastArrayBuiltin; use vortex::array::primitive::PrimitiveArray; +use vortex::array::validity::Validity; use vortex::array::{Array, ArrayRef}; use vortex::compress::{CompressConfig, CompressCtx, EncodingCompression}; use vortex::compute::fill::fill_forward; use vortex::compute::flatten::flatten_primitive; use vortex::match_each_integer_ptype; use vortex::ptype::NativePType; -use vortex::validity::ArrayValidity; -use vortex::validity::Validity; use vortex_error::VortexResult; use crate::downcast::DowncastFastlanes; diff --git a/vortex-fastlanes/src/delta/mod.rs b/vortex-fastlanes/src/delta/mod.rs index b64b6fc8f4..c7817bb85d 100644 --- a/vortex-fastlanes/src/delta/mod.rs +++ b/vortex-fastlanes/src/delta/mod.rs @@ -1,5 +1,6 @@ use std::sync::{Arc, RwLock}; +use vortex::array::validity::Validity; use vortex::array::{Array, ArrayRef}; use vortex::compress::EncodingCompression; use vortex::compute::ArrayCompute; @@ -7,7 +8,6 @@ use vortex::encoding::{Encoding, EncodingId, EncodingRef}; use vortex::formatter::{ArrayDisplay, ArrayFormatter}; use vortex::serde::{ArraySerde, EncodingSerde}; use vortex::stats::{Stat, Stats, StatsCompute, StatsSet}; -use vortex::validity::{ArrayValidity, Validity}; use vortex::{impl_array, match_each_integer_ptype, ArrayWalker}; use vortex_error::{vortex_bail, VortexResult}; use vortex_schema::DType; @@ -134,6 +134,10 @@ impl Array for DeltaArray { Some(self) } + fn validity(&self) -> Option { + self.validity.clone() + } + fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> { walker.visit_child(self.bases())?; walker.visit_child(self.deltas()) @@ -154,12 +158,6 @@ impl ArrayDisplay for DeltaArray { } } -impl ArrayValidity for DeltaArray { - fn validity(&self) -> Option { - self.validity.clone() - } -} - impl StatsCompute for DeltaArray { fn compute(&self, _stat: &Stat) -> VortexResult { Ok(StatsSet::default()) diff --git a/vortex-fastlanes/src/delta/serde.rs b/vortex-fastlanes/src/delta/serde.rs index 464677faca..fd7227cf9f 100644 --- a/vortex-fastlanes/src/delta/serde.rs +++ b/vortex-fastlanes/src/delta/serde.rs @@ -1,6 +1,5 @@ use vortex::array::{Array, ArrayRef}; use vortex::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; -use vortex::validity::ArrayValidity; use vortex_error::VortexResult; use crate::{DeltaArray, DeltaEncoding}; diff --git a/vortex-fastlanes/src/for/compress.rs b/vortex-fastlanes/src/for/compress.rs index 72c0561d7e..0e08c95c01 100644 --- a/vortex-fastlanes/src/for/compress.rs +++ b/vortex-fastlanes/src/for/compress.rs @@ -10,7 +10,6 @@ use vortex::match_each_integer_ptype; use vortex::ptype::{NativePType, PType}; use vortex::scalar::ListScalarVec; use vortex::stats::Stat; -use vortex::validity::ArrayValidity; use vortex_error::VortexResult; use crate::downcast::DowncastFastlanes; diff --git a/vortex-fastlanes/src/for/compute.rs b/vortex-fastlanes/src/for/compute.rs index aa5ba950a6..fdc0919c9e 100644 --- a/vortex-fastlanes/src/for/compute.rs +++ b/vortex-fastlanes/src/for/compute.rs @@ -18,6 +18,7 @@ impl ArrayCompute for FoRArray { fn scalar_at(&self) -> Option<&dyn ScalarAtFn> { Some(self) } + fn take(&self) -> Option<&dyn TakeFn> { Some(self) } diff --git a/vortex-fastlanes/src/for/mod.rs b/vortex-fastlanes/src/for/mod.rs index 12b44d6980..d028eb7db5 100644 --- a/vortex-fastlanes/src/for/mod.rs +++ b/vortex-fastlanes/src/for/mod.rs @@ -1,5 +1,6 @@ use std::sync::{Arc, RwLock}; +use vortex::array::validity::Validity; use vortex::array::{Array, ArrayRef}; use vortex::compress::EncodingCompression; use vortex::compute::ArrayCompute; @@ -8,7 +9,6 @@ use vortex::formatter::{ArrayDisplay, ArrayFormatter}; use vortex::scalar::Scalar; use vortex::serde::{ArraySerde, EncodingSerde}; use vortex::stats::{Stat, Stats, StatsCompute, StatsSet}; -use vortex::validity::{ArrayValidity, Validity}; use vortex::{impl_array, ArrayWalker}; use vortex_error::{vortex_bail, VortexResult}; use vortex_schema::DType; @@ -102,6 +102,10 @@ impl Array for FoRArray { Some(self) } + fn validity(&self) -> Option { + self.encoded().validity() + } + fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> { walker.visit_child(self.encoded()) } @@ -115,12 +119,6 @@ impl ArrayDisplay for FoRArray { } } -impl ArrayValidity for FoRArray { - fn validity(&self) -> Option { - self.encoded().validity() - } -} - impl StatsCompute for FoRArray { fn compute(&self, _stat: &Stat) -> VortexResult { Ok(StatsSet::default()) diff --git a/vortex-ree/src/compress.rs b/vortex-ree/src/compress.rs index 48aa469986..a028877077 100644 --- a/vortex-ree/src/compress.rs +++ b/vortex-ree/src/compress.rs @@ -4,13 +4,13 @@ use itertools::Itertools; use num_traits::AsPrimitive; use vortex::array::downcast::DowncastArrayBuiltin; use vortex::array::primitive::{PrimitiveArray, PrimitiveEncoding}; +use vortex::array::validity::Validity; use vortex::array::{Array, ArrayRef}; use vortex::compress::{CompressConfig, CompressCtx, EncodingCompression}; use vortex::encoding::Encoding; use vortex::match_each_integer_ptype; use vortex::ptype::{match_each_native_ptype, NativePType}; use vortex::stats::Stat; -use vortex::validity::{ArrayValidity, Validity}; use vortex_error::VortexResult; use crate::downcast::DowncastREE; @@ -156,8 +156,8 @@ pub fn ree_decode_primitive + Ord, T: Native mod test { use vortex::array::downcast::DowncastArrayBuiltin; use vortex::array::primitive::PrimitiveArray; + use vortex::array::validity::Validity; use vortex::array::{Array, IntoArray}; - use vortex::validity::{ArrayValidity, Validity}; use crate::compress::{ree_decode, ree_encode}; use crate::REEArray; diff --git a/vortex-ree/src/compute.rs b/vortex-ree/src/compute.rs index 139b6540a1..2678131eeb 100644 --- a/vortex-ree/src/compute.rs +++ b/vortex-ree/src/compute.rs @@ -6,7 +6,6 @@ use vortex::compute::take::{take, TakeFn}; use vortex::compute::ArrayCompute; use vortex::match_each_integer_ptype; use vortex::scalar::Scalar; -use vortex::validity::ArrayValidity; use vortex_error::{vortex_bail, vortex_err, VortexResult}; use crate::compress::ree_decode; diff --git a/vortex-ree/src/ree.rs b/vortex-ree/src/ree.rs index 13268167c2..6a06880162 100644 --- a/vortex-ree/src/ree.rs +++ b/vortex-ree/src/ree.rs @@ -1,5 +1,6 @@ use std::sync::{Arc, RwLock}; +use vortex::array::validity::Validity; use vortex::array::{check_slice_bounds, Array, ArrayKind, ArrayRef}; use vortex::compress::EncodingCompression; use vortex::compute::search_sorted::SearchSortedSide; @@ -8,7 +9,6 @@ use vortex::encoding::{Encoding, EncodingId, EncodingRef}; use vortex::formatter::{ArrayDisplay, ArrayFormatter}; use vortex::serde::{ArraySerde, EncodingSerde}; use vortex::stats::{Stat, Stats, StatsCompute, StatsSet}; -use vortex::validity::{ArrayValidity, Validity}; use vortex::{compute, impl_array, ArrayWalker}; use vortex_error::{vortex_bail, vortex_err, VortexResult}; use vortex_schema::DType; @@ -157,6 +157,10 @@ impl Array for REEArray { Some(self) } + fn validity(&self) -> Option { + self.validity.clone() + } + fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> { walker.visit_child(self.values())?; walker.visit_child(self.ends()) @@ -165,12 +169,6 @@ impl Array for REEArray { impl StatsCompute for REEArray {} -impl ArrayValidity for REEArray { - fn validity(&self) -> Option { - self.validity.clone() - } -} - #[derive(Debug)] pub struct REEEncoding; diff --git a/vortex-ree/src/serde.rs b/vortex-ree/src/serde.rs index d8166b6b5d..6b0b8886c8 100644 --- a/vortex-ree/src/serde.rs +++ b/vortex-ree/src/serde.rs @@ -1,6 +1,5 @@ use vortex::array::{Array, ArrayRef}; use vortex::serde::{ArraySerde, EncodingSerde, ReadCtx, WriteCtx}; -use vortex::validity::ArrayValidity; use vortex_error::VortexResult; use crate::{REEArray, REEEncoding}; diff --git a/vortex-roaring/src/boolean/compute.rs b/vortex-roaring/src/boolean/compute.rs index 749769dfd5..d6ca387b82 100644 --- a/vortex-roaring/src/boolean/compute.rs +++ b/vortex-roaring/src/boolean/compute.rs @@ -1,10 +1,10 @@ use arrow_buffer::{BooleanBuffer, Buffer}; use vortex::array::bool::BoolArray; +use vortex::array::Array; use vortex::compute::flatten::{FlattenFn, FlattenedArray}; use vortex::compute::scalar_at::ScalarAtFn; use vortex::compute::ArrayCompute; use vortex::scalar::{AsBytes, Scalar}; -use vortex::validity::ArrayValidity; use vortex_error::{vortex_err, VortexResult}; use crate::RoaringBoolArray; diff --git a/vortex-roaring/src/boolean/mod.rs b/vortex-roaring/src/boolean/mod.rs index e413fdca2d..eda6fd307a 100644 --- a/vortex-roaring/src/boolean/mod.rs +++ b/vortex-roaring/src/boolean/mod.rs @@ -2,6 +2,7 @@ use std::sync::{Arc, RwLock}; use compress::roaring_encode; use croaring::{Bitmap, Native}; +use vortex::array::validity::Validity; use vortex::array::{check_slice_bounds, Array, ArrayKind, ArrayRef}; use vortex::compress::EncodingCompression; use vortex::compute::ArrayCompute; @@ -9,7 +10,6 @@ use vortex::encoding::{Encoding, EncodingId, EncodingRef}; use vortex::formatter::{ArrayDisplay, ArrayFormatter}; use vortex::serde::{ArraySerde, EncodingSerde}; use vortex::stats::{Stats, StatsSet}; -use vortex::validity::{ArrayValidity, Validity}; use vortex::{impl_array, ArrayWalker}; use vortex_error::{vortex_err, VortexResult}; use vortex_schema::DType; @@ -99,6 +99,13 @@ impl Array for RoaringBoolArray { Some(self) } + fn validity(&self) -> Option { + match self.dtype().is_nullable() { + true => Some(Validity::Valid(self.length)), + false => None, + } + } + fn walk(&self, _walker: &mut dyn ArrayWalker) -> VortexResult<()> { // TODO(ngates): should we store a buffer in memory? Or delay serialization? // Or serialize into metadata? The only reason we support buffers is so we can write to @@ -114,15 +121,6 @@ impl ArrayDisplay for RoaringBoolArray { } } -impl ArrayValidity for RoaringBoolArray { - fn validity(&self) -> Option { - match self.dtype().is_nullable() { - true => Some(Validity::Valid(self.length)), - false => None, - } - } -} - #[derive(Debug)] pub struct RoaringBoolEncoding; diff --git a/vortex-roaring/src/integer/mod.rs b/vortex-roaring/src/integer/mod.rs index ccce431463..3576c63b37 100644 --- a/vortex-roaring/src/integer/mod.rs +++ b/vortex-roaring/src/integer/mod.rs @@ -2,6 +2,7 @@ use std::sync::{Arc, RwLock}; use compress::roaring_encode; use croaring::{Bitmap, Native}; +use vortex::array::validity::Validity; use vortex::array::{check_slice_bounds, Array, ArrayKind, ArrayRef}; use vortex::compress::EncodingCompression; use vortex::compute::ArrayCompute; @@ -10,7 +11,6 @@ use vortex::formatter::{ArrayDisplay, ArrayFormatter}; use vortex::ptype::PType; use vortex::serde::{ArraySerde, EncodingSerde}; use vortex::stats::{Stats, StatsSet}; -use vortex::validity::{ArrayValidity, Validity}; use vortex::{impl_array, ArrayWalker}; use vortex_error::{vortex_bail, vortex_err, VortexResult}; use vortex_schema::DType; @@ -101,6 +101,13 @@ impl Array for RoaringIntArray { Some(self) } + fn validity(&self) -> Option { + match self.dtype().is_nullable() { + true => Some(Validity::Valid(self.len())), + false => None, + } + } + fn walk(&self, _walker: &mut dyn ArrayWalker) -> VortexResult<()> { todo!() } @@ -112,15 +119,6 @@ impl ArrayDisplay for RoaringIntArray { } } -impl ArrayValidity for RoaringIntArray { - fn validity(&self) -> Option { - match self.dtype().is_nullable() { - true => Some(Validity::Valid(self.len())), - false => None, - } - } -} - #[derive(Debug)] pub struct RoaringIntEncoding; diff --git a/vortex-zigzag/src/compress.rs b/vortex-zigzag/src/compress.rs index 8c934d34ab..4163e9c4ed 100644 --- a/vortex-zigzag/src/compress.rs +++ b/vortex-zigzag/src/compress.rs @@ -1,10 +1,10 @@ use vortex::array::downcast::DowncastArrayBuiltin; use vortex::array::primitive::PrimitiveArray; +use vortex::array::validity::Validity; use vortex::array::{Array, ArrayKind, ArrayRef}; use vortex::compress::{CompressConfig, CompressCtx, EncodingCompression}; use vortex::ptype::{NativePType, PType}; use vortex::stats::Stat; -use vortex::validity::{ArrayValidity, Validity}; use vortex_alloc::{AlignedVec, ALIGNED_ALLOCATOR}; use vortex_error::VortexResult; use zigzag::ZigZag; diff --git a/vortex-zigzag/src/zigzag.rs b/vortex-zigzag/src/zigzag.rs index bd11e785c7..04cbb80795 100644 --- a/vortex-zigzag/src/zigzag.rs +++ b/vortex-zigzag/src/zigzag.rs @@ -1,5 +1,6 @@ use std::sync::{Arc, RwLock}; +use vortex::array::validity::Validity; use vortex::array::{Array, ArrayKind, ArrayRef}; use vortex::compress::EncodingCompression; use vortex::compute::ArrayCompute; @@ -7,7 +8,6 @@ use vortex::encoding::{Encoding, EncodingId, EncodingRef}; use vortex::formatter::{ArrayDisplay, ArrayFormatter}; use vortex::serde::{ArraySerde, EncodingSerde}; use vortex::stats::{Stats, StatsSet}; -use vortex::validity::{ArrayValidity, Validity}; use vortex::{impl_array, ArrayWalker}; use vortex_error::{vortex_bail, vortex_err, VortexResult}; use vortex_schema::{DType, Signedness}; @@ -93,6 +93,10 @@ impl Array for ZigZagArray { Some(self) } + fn validity(&self) -> Option { + self.encoded().validity() + } + fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> { walker.visit_child(self.encoded()) } @@ -104,12 +108,6 @@ impl ArrayDisplay for ZigZagArray { } } -impl ArrayValidity for ZigZagArray { - fn validity(&self) -> Option { - self.encoded().validity() - } -} - #[derive(Debug)] pub struct ZigZagEncoding;