Validity array (#185)

Fixes #173

vortex-alp/src/array.rs

@@ -1,13 +1,13 @@
 use std::sync::{Arc, RwLock};
 
+use vortex::array::validity::Validity;
 use vortex::array::{Array, ArrayKind, ArrayRef};
 use vortex::compress::EncodingCompression;
 use vortex::compute::ArrayCompute;
 use vortex::encoding::{Encoding, EncodingId, EncodingRef};
 use vortex::formatter::{ArrayDisplay, ArrayFormatter};
 use vortex::serde::{ArraySerde, EncodingSerde};
 use vortex::stats::{Stats, StatsSet};
-use vortex::validity::{ArrayValidity, Validity};
 use vortex::{impl_array, ArrayWalker};
 use vortex_error::{vortex_bail, vortex_err, VortexResult};
 use vortex_schema::{DType, IntWidth, Signedness};
@@ -114,6 +114,10 @@ impl Array for ALPArray {
         self.encoded().nbytes() + self.patches().map(|p| p.nbytes()).unwrap_or(0)
     }
 
+    fn validity(&self) -> Option<Validity> {
+        self.encoded().validity()
+    }
+
     fn serde(&self) -> Option<&dyn ArraySerde> {
         Some(self)
     }
@@ -131,12 +135,6 @@ impl ArrayDisplay for ALPArray {
     }
 }
 
-impl ArrayValidity for ALPArray {
-    fn validity(&self) -> Option<Validity> {
-        self.encoded().validity()
-    }
-}
-
 #[derive(Debug)]
 pub struct ALPEncoding;
 

vortex-alp/src/compress.rs

@@ -7,7 +7,6 @@ use vortex::compress::{CompressConfig, CompressCtx, EncodingCompression};
 use vortex::compute::flatten::flatten_primitive;
 use vortex::compute::patch::patch;
 use vortex::ptype::{NativePType, PType};
-use vortex::validity::ArrayValidity;
 use vortex_error::{vortex_bail, vortex_err, VortexResult};
 
 use crate::alp::ALPFloat;

vortex-array/src/array/bool/compute/mod.rs

@@ -6,7 +6,8 @@ use vortex_error::VortexResult;
 
 use crate::array::bool::BoolArray;
 use crate::array::downcast::DowncastArrayBuiltin;
-use crate::array::{Array, ArrayRef};
+use crate::array::validity::Validity;
+use crate::array::{Array, ArrayRef, ArrayValidity};
 use crate::arrow::wrappers::as_nulls;
 use crate::compute::as_arrow::AsArrowArray;
 use crate::compute::as_contiguous::AsContiguousFn;
@@ -16,7 +17,6 @@ use crate::compute::scalar_at::ScalarAtFn;
 use crate::compute::take::TakeFn;
 use crate::compute::ArrayCompute;
 use crate::scalar::{BoolScalar, Scalar};
-use crate::validity::{ArrayValidity, Validity};
 
 mod take;
 
@@ -122,8 +122,8 @@ impl FillForwardFn for BoolArray {
 mod test {
     use crate::array::bool::BoolArray;
     use crate::array::downcast::DowncastArrayBuiltin;
+    use crate::array::Array;
     use crate::compute;
-    use crate::validity::ArrayValidity;
 
     #[test]
     fn fill_forward() {

vortex-array/src/array/bool/compute/take.rs

@@ -7,7 +7,6 @@ use crate::array::{Array, ArrayRef};
 use crate::compute::flatten::flatten_primitive;
 use crate::compute::take::TakeFn;
 use crate::match_each_integer_ptype;
-use crate::validity::ArrayValidity;
 
 impl TakeFn for BoolArray {
     fn take(&self, indices: &dyn Array) -> VortexResult<ArrayRef> {

vortex-array/src/array/bool/mod.rs

@@ -6,13 +6,13 @@ use vortex_error::VortexResult;
 use vortex_schema::{DType, Nullability};
 
 use super::{check_slice_bounds, Array, ArrayRef};
+use crate::array::validity::Validity;
 use crate::array::IntoArray;
 use crate::compute::ArrayCompute;
 use crate::encoding::{Encoding, EncodingId, EncodingRef, ENCODINGS};
 use crate::formatter::{ArrayDisplay, ArrayFormatter};
 use crate::serde::{ArraySerde, EncodingSerde};
 use crate::stats::{Stat, Stats, StatsSet};
-use crate::validity::{ArrayValidity, Validity};
 use crate::{impl_array, ArrayWalker};
 
 mod compute;
@@ -102,6 +102,10 @@ impl Array for BoolArray {
         .into_array())
     }
 
+    fn validity(&self) -> Option<Validity> {
+        self.validity.clone()
+    }
+
     #[inline]
     fn encoding(&self) -> EncodingRef {
         &BoolEncoding
@@ -125,12 +129,6 @@ impl Array for BoolArray {
     }
 }
 
-impl ArrayValidity for BoolArray {
-    fn validity(&self) -> Option<Validity> {
-        self.validity.clone()
-    }
-}
-
 #[derive(Debug)]
 pub struct BoolEncoding;
 

vortex-array/src/array/bool/serde.rs

@@ -4,7 +4,6 @@ use vortex_error::VortexResult;
 use crate::array::bool::{BoolArray, BoolEncoding};
 use crate::array::{Array, ArrayRef};
 use crate::serde::{ArraySerde, ArrayView, BytesSerde, EncodingSerde, ReadCtx, WriteCtx};
-use crate::validity::ArrayValidity;
 
 impl ArraySerde for BoolArray {
     fn write(&self, ctx: &mut WriteCtx) -> VortexResult<()> {
@@ -33,8 +32,8 @@ impl EncodingSerde for BoolEncoding {
 mod test {
     use crate::array::bool::BoolArray;
     use crate::array::downcast::DowncastArrayBuiltin;
+    use crate::array::Array;
     use crate::serde::test::roundtrip_array;
-    use crate::validity::ArrayValidity;
 
     #[test]
     fn roundtrip() {

vortex-array/src/array/chunked/mod.rs

@@ -5,13 +5,13 @@ use linkme::distributed_slice;
 use vortex_error::{vortex_bail, VortexResult};
 use vortex_schema::DType;
 
+use crate::array::validity::Validity;
 use crate::array::{check_slice_bounds, Array, ArrayRef};
 use crate::compute::ArrayCompute;
 use crate::encoding::{Encoding, EncodingId, EncodingRef, ENCODINGS};
 use crate::formatter::{ArrayDisplay, ArrayFormatter};
 use crate::serde::{ArraySerde, EncodingSerde};
 use crate::stats::{Stats, StatsSet};
-use crate::validity::{ArrayValidity, Validity};
 use crate::{impl_array, ArrayWalker};
 
 mod compute;
@@ -145,15 +145,6 @@ impl Array for ChunkedArray {
         Some(self)
     }
 
-    fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> {
-        for chunk in self.chunks() {
-            walker.visit_child(&chunk)?;
-        }
-        Ok(())
-    }
-}
-
-impl ArrayValidity for ChunkedArray {
     fn validity(&self) -> Option<Validity> {
         if !self.dtype.is_nullable() {
             return None;
@@ -165,6 +156,13 @@ impl ArrayValidity for ChunkedArray {
                 .unwrap_or_else(|| Validity::Valid(chunk.len()))
         })))
     }
+
+    fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> {
+        for chunk in self.chunks() {
+            walker.visit_child(&chunk)?;
+        }
+        Ok(())
+    }
 }
 
 impl FromIterator<ArrayRef> for ChunkedArray {

vortex-array/src/array/composite/array.rs

@@ -6,14 +6,14 @@ use vortex_error::VortexResult;
 use vortex_schema::{CompositeID, DType};
 
 use crate::array::composite::{find_extension, CompositeExtensionRef, TypedCompositeArray};
+use crate::array::validity::Validity;
 use crate::array::{Array, ArrayRef};
 use crate::compress::EncodingCompression;
 use crate::compute::ArrayCompute;
 use crate::encoding::{Encoding, EncodingId, EncodingRef, ENCODINGS};
 use crate::formatter::{ArrayDisplay, ArrayFormatter};
 use crate::serde::{ArraySerde, BytesSerde, EncodingSerde};
 use crate::stats::{Stats, StatsCompute, StatsSet};
-use crate::validity::{ArrayValidity, Validity};
 use crate::{impl_array, ArrayWalker};
 
 pub trait CompositeMetadata:
@@ -121,19 +121,17 @@ impl Array for CompositeArray {
         Some(self)
     }
 
+    fn validity(&self) -> Option<Validity> {
+        self.underlying().validity()
+    }
+
     fn walk(&self, walker: &mut dyn ArrayWalker) -> VortexResult<()> {
         walker.visit_child(self.underlying())
     }
 }
 
 impl StatsCompute for CompositeArray {}
 
-impl ArrayValidity for CompositeArray {
-    fn validity(&self) -> Option<Validity> {
-        self.underlying().validity()
-    }
-}
-
 impl ArrayDisplay for CompositeArray {
     fn fmt(&self, f: &mut ArrayFormatter) -> std::fmt::Result {
         f.property("metadata", format!("{:#?}", self.metadata().as_slice()))?;

vortex-array/src/array/constant/compute.rs

@@ -6,15 +6,15 @@ use crate::array::bool::BoolArray;
 use crate::array::constant::ConstantArray;
 use crate::array::downcast::DowncastArrayBuiltin;
 use crate::array::primitive::PrimitiveArray;
-use crate::array::{Array, ArrayRef};
+use crate::array::validity::Validity;
+use crate::array::{Array, ArrayRef, ArrayValidity};
 use crate::compute::as_contiguous::AsContiguousFn;
 use crate::compute::flatten::{FlattenFn, FlattenedArray};
 use crate::compute::scalar_at::ScalarAtFn;
 use crate::compute::take::TakeFn;
 use crate::compute::ArrayCompute;
 use crate::match_each_native_ptype;
 use crate::scalar::Scalar;
-use crate::validity::{ArrayValidity, Validity};
 
 impl ArrayCompute for ConstantArray {
     fn as_contiguous(&self) -> Option<&dyn AsContiguousFn> {

vortex-array/src/array/constant/mod.rs

@@ -4,14 +4,14 @@ use linkme::distributed_slice;
 use vortex_error::VortexResult;
 use vortex_schema::DType;
 
+use crate::array::validity::Validity;
 use crate::array::{check_slice_bounds, Array, ArrayRef};
 use crate::compute::ArrayCompute;
 use crate::encoding::{Encoding, EncodingId, EncodingRef, ENCODINGS};
 use crate::formatter::{ArrayDisplay, ArrayFormatter};
 use crate::scalar::Scalar;
 use crate::serde::{ArraySerde, EncodingSerde};
 use crate::stats::{Stat, Stats, StatsSet};
-use crate::validity::{ArrayValidity, Validity};
 use crate::{impl_array, ArrayWalker};
 
 mod compute;
@@ -96,12 +96,6 @@ impl Array for ConstantArray {
         Some(self)
     }
 
-    fn walk(&self, _walker: &mut dyn ArrayWalker) -> VortexResult<()> {
-        Ok(())
-    }
-}
-
-impl ArrayValidity for ConstantArray {
     fn validity(&self) -> Option<Validity> {
         match self.scalar.dtype().is_nullable() {
             true => match self.scalar().is_null() {
@@ -111,6 +105,10 @@ impl ArrayValidity for ConstantArray {
             false => None,
         }
     }
+
+    fn walk(&self, _walker: &mut dyn ArrayWalker) -> VortexResult<()> {
+        Ok(())
+    }
 }
 
 impl ArrayDisplay for ConstantArray {