Add clone_optional for Option<&dyn Array>

vortex-alp/src/alp.rs

@@ -79,8 +79,8 @@ impl ALPArray {
         self.exponents
     }
 
-    pub fn patches(&self) -> Option<&ArrayRef> {
-        self.patches.as_ref()
+    pub fn patches(&self) -> Option<&dyn Array> {
+        self.patches.as_deref()
     }
 }
 

vortex-alp/src/compress.rs

@@ -19,7 +19,7 @@ use codecz::alp::{ALPEncoded, ALPExponents, SupportsALP};
 use vortex::array::downcast::DowncastArrayBuiltin;
 use vortex::array::primitive::PrimitiveArray;
 use vortex::array::sparse::SparseArray;
-use vortex::array::{Array, ArrayRef};
+use vortex::array::{Array, ArrayRef, CloneOptionalArray};
 use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression};
 use vortex::ptype::{NativePType, PType};
 
@@ -61,10 +61,8 @@ fn alp_compressor(array: &dyn Array, like: Option<&dyn Array>, ctx: CompressCtx)
             .compress(encoded.as_ref(), like_alp.map(|a| a.encoded())),
         exponents,
         patches.map(|p| {
-            ctx.next_level().compress(
-                p.as_ref(),
-                like_alp.and_then(|a| a.patches()).map(|p| p.as_ref()),
-            )
+            ctx.next_level()
+                .compress(p.as_ref(), like_alp.and_then(|a| a.patches()))
         }),
     )
     .boxed()
@@ -108,7 +106,7 @@ fn alp_encode_like_parts(
 
 fn alp_encode_primitive<T: SupportsALP + NativePType>(
     values: &[T],
-    validity: Option<&ArrayRef>,
+    validity: Option<&dyn Array>,
     exponents: Option<ALPExponents>,
 ) -> (ArrayRef, ALPExponents, Option<ArrayRef>)
 where
@@ -124,7 +122,7 @@ where
         .map(|exp| alp::encode_with(values, exp))
         .unwrap_or_else(|| alp::encode(values))
         .unwrap();
-    let values = PrimitiveArray::from_nullable_in(values, validity.cloned()); // move and re-alias
+    let values = PrimitiveArray::from_nullable_in(values, validity.clone_optional()); // move and re-alias
 
     let patches = if num_exceptions == 0 {
         None

vortex-dict/src/compress.rs

@@ -23,7 +23,7 @@ use num_traits::{AsPrimitive, FromPrimitive, Unsigned};
 use vortex::array::downcast::DowncastArrayBuiltin;
 use vortex::array::primitive::PrimitiveArray;
 use vortex::array::varbin::VarBinArray;
-use vortex::array::{Array, ArrayKind, ArrayRef};
+use vortex::array::{Array, ArrayKind, ArrayRef, CloneOptionalArray};
 use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression};
 use vortex::dtype::DType;
 use vortex::match_each_native_ptype;
@@ -138,7 +138,7 @@ fn dict_encode_typed_primitive<
     }
 
     (
-        PrimitiveArray::from_nullable(codes, array.validity().cloned()),
+        PrimitiveArray::from_nullable(codes, array.validity().clone_optional()),
         PrimitiveArray::from_vec(values),
     )
 }
@@ -230,7 +230,7 @@ fn dict_encode_typed_varbin<O, K, V, U>(
     dtype: DType,
     value_lookup: V,
     len: usize,
-    validity: Option<&ArrayRef>,
+    validity: Option<&dyn Array>,
 ) -> (PrimitiveArray, VarBinArray)
 where
     O: NativePType + Unsigned + FromPrimitive,
@@ -268,7 +268,7 @@ where
         codes.push(code)
     }
     (
-        PrimitiveArray::from_nullable(codes, validity.cloned()),
+        PrimitiveArray::from_nullable(codes, validity.clone_optional()),
         VarBinArray::new(
             PrimitiveArray::from_vec(offsets).boxed(),
             PrimitiveArray::from_vec(bytes).boxed(),

vortex-fastlanes/src/bitpacking/mod.rs

@@ -51,7 +51,7 @@ impl BitPackedArray {
         len: usize,
     ) -> VortexResult<Self> {
         let validity = validity.filter(|v| !v.is_empty());
-        check_validity_buffer(validity.as_ref())?;
+        check_validity_buffer(validity.as_deref())?;
 
         // TODO(ngates): check encoded has type u8
 

vortex-ffor/src/compress.rs

@@ -19,7 +19,7 @@ use codecz::ffor::{FforEncoded, SupportsFFoR};
 use vortex::array::downcast::DowncastArrayBuiltin;
 use vortex::array::primitive::PrimitiveArray;
 use vortex::array::sparse::SparseArray;
-use vortex::array::{Array, ArrayRef};
+use vortex::array::{Array, ArrayRef, CloneOptionalArray};
 use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression};
 use vortex::match_each_integer_ptype;
 use vortex::ptype::NativePType;
@@ -72,12 +72,10 @@ fn ffor_compressor(array: &dyn Array, like: Option<&dyn Array>, ctx: CompressCtx
 
     FFORArray::new(
         encoded,
-        parray.validity().cloned(),
+        parray.validity().clone_optional(),
         patches.map(|p| {
-            ctx.next_level().compress(
-                p.as_ref(),
-                like_ffor.and_then(|lf| lf.patches()).map(|p| p.as_ref()),
-            )
+            ctx.next_level()
+                .compress(p.as_ref(), like_ffor.and_then(|lf| lf.patches()))
         }),
         min_val,
         num_bits,
@@ -90,7 +88,7 @@ pub fn ffor_encode(parray: &PrimitiveArray) -> FFORArray {
     let (encoded, patches, min_val, num_bits) = ffor_encode_parts(parray);
     FFORArray::new(
         encoded,
-        parray.validity().cloned(),
+        parray.validity().clone_optional(),
         patches,
         min_val,
         num_bits,

vortex-ffor/src/ffor.rs

@@ -63,7 +63,7 @@ impl FFORArray {
         len: usize,
     ) -> VortexResult<Self> {
         let validity = validity.filter(|v| !v.is_empty());
-        check_validity_buffer(validity.as_ref())?;
+        check_validity_buffer(validity.as_deref())?;
 
         if !matches!(min_val.dtype(), DType::Int(_, _, _)) {
             return Err(VortexError::InvalidDType(min_val.dtype().clone()));
@@ -107,13 +107,13 @@ impl FFORArray {
     }
 
     #[inline]
-    pub fn validity(&self) -> Option<&ArrayRef> {
-        self.validity.as_ref()
+    pub fn validity(&self) -> Option<&dyn Array> {
+        self.validity.as_deref()
     }
 
     #[inline]
-    pub fn patches(&self) -> Option<&ArrayRef> {
-        self.patches.as_ref()
+    pub fn patches(&self) -> Option<&dyn Array> {
+        self.patches.as_deref()
     }
 
     pub fn is_valid(&self, index: usize) -> bool {

vortex-ree/src/compress.rs

@@ -15,7 +15,7 @@
 use codecz::AlignedAllocator;
 use vortex::array::downcast::DowncastArrayBuiltin;
 use vortex::array::primitive::{PrimitiveArray, PrimitiveEncoding};
-use vortex::array::{Array, ArrayRef};
+use vortex::array::{Array, ArrayRef, CloneOptionalArray};
 use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression};
 use vortex::dtype::{DType, IntWidth, Nullability};
 use vortex::ptype::match_each_native_ptype;
@@ -60,7 +60,7 @@ fn ree_compressor(array: &dyn Array, like: Option<&dyn Array>, ctx: CompressCtx)
     REEArray::new(
         compressed_ends,
         compressed_values,
-        primitive_array.validity().cloned(),
+        primitive_array.validity().clone_optional(),
         array.len(),
     )
     .boxed()
@@ -110,7 +110,7 @@ mod test {
 
     use vortex::array::bool::BoolArray;
     use vortex::array::downcast::DowncastArrayBuiltin;
-    use vortex::array::Array;
+    use vortex::array::{Array, CloneOptionalArray};
 
     use crate::compress::ree_decode;
     use crate::REEArray;
@@ -133,7 +133,7 @@ mod test {
         let decoded = ree_decode(
             arr.ends().as_primitive(),
             arr.values().as_primitive(),
-            arr.validity().cloned(),
+            arr.validity().clone_optional(),
         );
 
         assert_eq!(

vortex-ree/src/ree.rs

@@ -25,7 +25,7 @@ use codecz::ree::SupportsREE;
 use vortex::array::primitive::PrimitiveArray;
 use vortex::array::{
     check_index_bounds, check_slice_bounds, check_validity_buffer, Array, ArrayKind, ArrayRef,
-    ArrowIterator, Encoding, EncodingId, EncodingRef,
+    ArrowIterator, CloneOptionalArray, Encoding, EncodingId, EncodingRef,
 };
 use vortex::arrow::match_arrow_numeric_type;
 use vortex::compress::EncodingCompression;
@@ -67,7 +67,7 @@ impl REEArray {
         validity: Option<ArrayRef>,
         length: usize,
     ) -> VortexResult<Self> {
-        check_validity_buffer(validity.as_ref())?;
+        check_validity_buffer(validity.as_deref())?;
 
         if !matches!(
             ends.dtype(),
@@ -108,10 +108,13 @@ impl REEArray {
         match ArrayKind::from(array) {
             ArrayKind::Primitive(p) => {
                 let (ends, values) = ree_encode(p);
-                Ok(
-                    REEArray::new(ends.boxed(), values.boxed(), p.validity().cloned(), p.len())
-                        .boxed(),
+                Ok(REEArray::new(
+                    ends.boxed(),
+                    values.boxed(),
+                    p.validity().clone_optional(),
+                    p.len(),
                 )
+                .boxed())
             }
             _ => Err(VortexError::InvalidEncoding(array.encoding().id().clone())),
         }
@@ -128,8 +131,8 @@ impl REEArray {
     }
 
     #[inline]
-    pub fn validity(&self) -> Option<&ArrayRef> {
-        self.validity.as_ref()
+    pub fn validity(&self) -> Option<&dyn Array> {
+        self.validity.as_deref()
     }
 }
 

vortex-zigzag/src/compress.rs

@@ -17,7 +17,7 @@ use zigzag::ZigZag;
 use crate::downcast::DowncastZigzag;
 use vortex::array::downcast::DowncastArrayBuiltin;
 use vortex::array::primitive::PrimitiveArray;
-use vortex::array::{Array, ArrayKind, ArrayRef};
+use vortex::array::{Array, ArrayKind, ArrayRef, CloneOptionalArray};
 use vortex::compress::{CompressConfig, CompressCtx, Compressor, EncodingCompression};
 use vortex::error::VortexResult;
 use vortex::ptype::{NativePType, PType};
@@ -83,14 +83,14 @@ pub fn zigzag_encode(parray: &PrimitiveArray) -> VortexResult<ZigZagArray> {
 
 fn zigzag_encode_primitive<T: ZigZag + NativePType>(
     values: &[T],
-    validity: Option<&ArrayRef>,
+    validity: Option<&dyn Array>,
 ) -> PrimitiveArray
 where
     <T as ZigZag>::UInt: NativePType,
 {
     let mut encoded = AlignedVec::with_capacity_in(values.len(), ALIGNED_ALLOCATOR);
     encoded.extend(values.iter().map(|v| T::encode(*v)));
-    PrimitiveArray::from_nullable_in(encoded, validity.cloned())
+    PrimitiveArray::from_nullable_in(encoded, validity.clone_optional())
 }
 
 #[allow(dead_code)]
@@ -113,12 +113,12 @@ pub fn zigzag_decode(parray: &PrimitiveArray) -> PrimitiveArray {
 #[allow(dead_code)]
 fn zigzag_decode_primitive<T: ZigZag + NativePType>(
     values: &[T::UInt],
-    validity: Option<&ArrayRef>,
+    validity: Option<&dyn Array>,
 ) -> PrimitiveArray
 where
     <T as ZigZag>::UInt: NativePType,
 {
     let mut encoded: AlignedVec<T> = AlignedVec::with_capacity_in(values.len(), ALIGNED_ALLOCATOR);
     encoded.extend(values.iter().map(|v| T::decode(*v)));
-    PrimitiveArray::from_nullable_in(encoded, validity.cloned())
+    PrimitiveArray::from_nullable_in(encoded, validity.clone_optional())
 }

vortex/src/array/bool/mod.rs

@@ -52,7 +52,7 @@ impl BoolArray {
 
     pub fn try_new(buffer: BooleanBuffer, validity: Option<ArrayRef>) -> VortexResult<Self> {
         let validity = validity.filter(|v| !v.is_empty());
-        check_validity_buffer(validity.as_ref())?;
+        check_validity_buffer(validity.as_deref())?;
 
         Ok(Self {
             buffer,
@@ -74,8 +74,8 @@ impl BoolArray {
     }
 
     #[inline]
-    pub fn validity(&self) -> Option<&ArrayRef> {
-        self.validity.as_ref()
+    pub fn validity(&self) -> Option<&dyn Array> {
+        self.validity.as_deref()
     }
 }
 

vortex/src/array/mod.rs

@@ -94,6 +94,16 @@ pub trait Array: ArrayDisplay + Debug + Send + Sync + dyn_clone::DynClone + 'sta
 
 dyn_clone::clone_trait_object!(Array);
 
+pub trait CloneOptionalArray {
+    fn clone_optional(&self) -> Option<ArrayRef>;
+}
+
+impl CloneOptionalArray for Option<&dyn Array> {
+    fn clone_optional(&self) -> Option<ArrayRef> {
+        self.map(dyn_clone::clone_box)
+    }
+}
+
 pub fn check_slice_bounds(array: &dyn Array, start: usize, stop: usize) -> VortexResult<()> {
     if start > array.len() {
         return Err(VortexError::OutOfBounds(start, 0, array.len()));
@@ -111,7 +121,7 @@ pub fn check_index_bounds(array: &dyn Array, index: usize) -> VortexResult<()> {
     Ok(())
 }
 
-pub fn check_validity_buffer(validity: Option<&ArrayRef>) -> VortexResult<()> {
+pub fn check_validity_buffer(validity: Option<&dyn Array>) -> VortexResult<()> {
     // TODO(ngates): take a length parameter and check that the length of the validity buffer matches
     if validity
         .map(|v| !matches!(v.dtype(), DType::Bool(Nullability::NonNullable)))

vortex/src/array/primitive/mod.rs

@@ -62,7 +62,7 @@ impl PrimitiveArray {
 
     pub fn try_new(ptype: PType, buffer: Buffer, validity: Option<ArrayRef>) -> VortexResult<Self> {
         let validity = validity.filter(|v| !v.is_empty());
-        check_validity_buffer(validity.as_ref())?;
+        check_validity_buffer(validity.as_deref())?;
         let dtype = if validity.is_some() {
             DType::from(ptype).as_nullable()
         } else {
@@ -139,8 +139,8 @@ impl PrimitiveArray {
     }
 
     #[inline]
-    pub fn validity(&self) -> Option<&ArrayRef> {
-        self.validity.as_ref()
+    pub fn validity(&self) -> Option<&dyn Array> {
+        self.validity.as_deref()
     }
 }
 

vortex/src/array/varbin/compress.rs

@@ -45,14 +45,9 @@ fn varbin_compressor(array: &dyn Array, like: Option<&dyn Array>, ctx: CompressC
             varbin_like.map(|typed_arr| typed_arr.bytes()),
         ),
         array.dtype().clone(),
-        varbin_array.validity().map(|v| {
-            ctx.compress(
-                v.as_ref(),
-                varbin_like
-                    .and_then(|vblike| vblike.validity())
-                    .map(|v| v.as_ref()),
-            )
-        }),
+        varbin_array
+            .validity()
+            .map(|v| ctx.compress(v.as_ref(), varbin_like.and_then(|vblike| vblike.validity()))),
     )
     .boxed()
 }

vortex/src/array/varbin/mod.rs

@@ -87,7 +87,7 @@ impl VarBinArray {
         }
 
         let validity = validity.filter(|v| !v.is_empty());
-        check_validity_buffer(validity.as_ref())?;
+        check_validity_buffer(validity.as_deref())?;
 
         let dtype = if validity.is_some() && !dtype.is_nullable() {
             dtype.as_nullable()
@@ -122,8 +122,8 @@ impl VarBinArray {
     }
 
     #[inline]
-    pub fn validity(&self) -> Option<&ArrayRef> {
-        self.validity.as_ref()
+    pub fn validity(&self) -> Option<&dyn Array> {
+        self.validity.as_deref()
     }
 
     pub fn from_vec<T: AsRef<[u8]>>(vec: Vec<T>, dtype: DType) -> Self {

vortex/src/array/varbinview/compress.rs

@@ -63,9 +63,7 @@ fn varbinview_compressor(
         varbinview_array.validity().map(|v| {
             ctx.compress(
                 v.as_ref(),
-                varbinview_like
-                    .and_then(|vbvlike| vbvlike.validity())
-                    .map(|v| v.as_ref()),
+                varbinview_like.and_then(|vbvlike| vbvlike.validity()),
             )
         }),
     )