Array Length

encodings/alp/src/array.rs

@@ -17,6 +17,7 @@ pub struct ALPMetadata {
     exponents: Exponents,
     encoded_dtype: DType,
     patches_dtype: Option<DType>,
+    patches_len: usize,
 }
 
 impl ALPArray {
@@ -32,7 +33,10 @@ impl ALPArray {
             d => vortex_bail!(MismatchedTypes: "int32 or int64", d),
         };
 
+        let length = encoded.len();
+
         let patches_dtype = patches.as_ref().map(|a| a.dtype().as_nullable());
+        let patches_len = patches.as_ref().map(|a| a.len()).unwrap_or(0);
         let mut children = Vec::with_capacity(2);
         children.push(encoded);
         if let Some(patch) = patches {
@@ -41,10 +45,12 @@ impl ALPArray {
 
         Self::try_from_parts(
             dtype,
+            length,
             ALPMetadata {
                 exponents,
                 encoded_dtype,
                 patches_dtype,
+                patches_len,
             },
             children.into(),
             Default::default(),
@@ -61,7 +67,7 @@ impl ALPArray {
 
     pub fn encoded(&self) -> Array {
         self.array()
-            .child(0, &self.metadata().encoded_dtype)
+            .child(0, &self.metadata().encoded_dtype, self.len())
             .expect("Missing encoded array")
     }
 
@@ -73,7 +79,7 @@ impl ALPArray {
     pub fn patches(&self) -> Option<Array> {
         self.metadata().patches_dtype.as_ref().map(|dt| {
             self.array()
-                .child(1, dt)
+                .child(1, dt, self.metadata().patches_len)
                 .expect("Missing patches with present metadata flag")
         })
     }
@@ -84,6 +90,8 @@ impl ALPArray {
     }
 }
 
+impl ArrayTrait for ALPArray {}
+
 impl ArrayValidity for ALPArray {
     fn is_valid(&self, index: usize) -> bool {
         self.encoded().with_dyn(|a| a.is_valid(index))
@@ -114,9 +122,3 @@ impl AcceptArrayVisitor for ALPArray {
 }
 
 impl ArrayStatisticsCompute for ALPArray {}
-
-impl ArrayTrait for ALPArray {
-    fn len(&self) -> usize {
-        self.encoded().len()
-    }
-}

encodings/byte_bool/src/compute/mod.rs

@@ -13,7 +13,7 @@ use vortex::{
     encoding::ArrayEncodingRef,
     stats::StatsSet,
     validity::ArrayValidity,
-    ArrayDType, ArrayData, ArrayTrait, IntoArray,
+    ArrayDType, ArrayData, IntoArray,
 };
 use vortex::{Array, IntoCanonical};
 use vortex_dtype::{match_each_integer_ptype, Nullability};
@@ -73,6 +73,7 @@ impl SliceFn for ByteBoolArray {
         ArrayData::try_new(
             self.encoding(),
             self.dtype().clone(),
+            length,
             slice_metadata,
             Some(self.buffer().slice(start..stop)),
             validity.into_array().into_iter().collect::<Vec<_>>().into(),

encodings/byte_bool/src/lib.rs

@@ -25,14 +25,15 @@ impl ByteBoolArray {
     pub fn validity(&self) -> Validity {
         self.metadata()
             .validity
-            .to_validity(self.array().child(0, &Validity::DTYPE))
+            .to_validity(self.array().child(0, &Validity::DTYPE, self.len()))
     }
 
     pub fn try_new(buffer: Buffer, validity: Validity) -> VortexResult<Self> {
         let length = buffer.len();
 
         let typed = TypedArray::try_from_parts(
             DType::Bool(validity.nullability()),
+            length,
             ByteBoolMetadata {
                 validity: validity.to_metadata(length)?,
             },
@@ -70,6 +71,8 @@ impl ByteBoolArray {
     }
 }
 
+impl ArrayTrait for ByteBoolArray {}
+
 impl From<Vec<bool>> for ByteBoolArray {
     fn from(value: Vec<bool>) -> Self {
         Self::try_from_vec(value, Validity::AllValid).unwrap()
@@ -87,12 +90,6 @@ impl From<Vec<Option<bool>>> for ByteBoolArray {
     }
 }
 
-impl ArrayTrait for ByteBoolArray {
-    fn len(&self) -> usize {
-        self.buffer().len()
-    }
-}
-
 impl IntoCanonical for ByteBoolArray {
     fn into_canonical(self) -> VortexResult<Canonical> {
         let boolean_buffer = BooleanBuffer::from(self.maybe_null_slice());

encodings/byte_bool/src/stats.rs

@@ -1,6 +1,6 @@
 use vortex::{
     stats::{ArrayStatisticsCompute, Stat, StatsSet},
-    ArrayTrait, AsArray, IntoCanonical,
+    AsArray, IntoCanonical,
 };
 use vortex_error::VortexResult;
 

encodings/datetime-parts/src/array.rs

@@ -47,6 +47,7 @@ impl DateTimePartsArray {
 
         Self::try_from_parts(
             dtype,
+            length,
             DateTimePartsMetadata {
                 days_dtype: days.dtype().clone(),
                 seconds_dtype: seconds.dtype().clone(),
@@ -59,23 +60,25 @@ impl DateTimePartsArray {
 
     pub fn days(&self) -> Array {
         self.array()
-            .child(0, &self.metadata().days_dtype)
+            .child(0, &self.metadata().days_dtype, self.len())
             .expect("Missing days array")
     }
 
     pub fn seconds(&self) -> Array {
         self.array()
-            .child(1, &self.metadata().seconds_dtype)
+            .child(1, &self.metadata().seconds_dtype, self.len())
             .expect("Missing seconds array")
     }
 
     pub fn subsecond(&self) -> Array {
         self.array()
-            .child(2, &self.metadata().subseconds_dtype)
+            .child(2, &self.metadata().subseconds_dtype, self.len())
             .expect("Missing subsecond array")
     }
 }
 
+impl ArrayTrait for DateTimePartsArray {}
+
 impl IntoCanonical for DateTimePartsArray {
     fn into_canonical(self) -> VortexResult<Canonical> {
         Ok(Canonical::Extension(
@@ -103,9 +106,3 @@ impl AcceptArrayVisitor for DateTimePartsArray {
 }
 
 impl ArrayStatisticsCompute for DateTimePartsArray {}
-
-impl ArrayTrait for DateTimePartsArray {
-    fn len(&self) -> usize {
-        self.days().len()
-    }
-}

encodings/datetime-parts/src/compress.rs

@@ -1,7 +1,7 @@
 use vortex::array::datetime::{LocalDateTimeArray, TimeUnit};
 use vortex::array::primitive::PrimitiveArray;
 use vortex::compute::unary::cast::try_cast;
-use vortex::{Array, ArrayTrait, IntoArray, IntoCanonical};
+use vortex::{Array, IntoArray, IntoCanonical};
 use vortex_dtype::PType;
 use vortex_error::VortexResult;
 

encodings/dict/src/dict.rs

@@ -14,6 +14,7 @@ impl_encoding!("vortex.dict", 20u16, Dict);
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct DictMetadata {
     codes_dtype: DType,
+    values_len: usize,
 }
 
 impl DictArray {
@@ -23,8 +24,10 @@ impl DictArray {
         }
         Self::try_from_parts(
             values.dtype().clone(),
+            codes.len(),
             DictMetadata {
                 codes_dtype: codes.dtype().clone(),
+                values_len: values.len(),
             },
             [values, codes].into(),
             StatsSet::new(),
@@ -33,17 +36,21 @@ impl DictArray {
 
     #[inline]
     pub fn values(&self) -> Array {
-        self.array().child(0, self.dtype()).expect("Missing values")
+        self.array()
+            .child(0, self.dtype(), self.metadata().values_len)
+            .expect("Missing values")
     }
 
     #[inline]
     pub fn codes(&self) -> Array {
         self.array()
-            .child(1, &self.metadata().codes_dtype)
+            .child(1, &self.metadata().codes_dtype, self.len())
             .expect("Missing codes")
     }
 }
 
+impl ArrayTrait for DictArray {}
+
 impl IntoCanonical for DictArray {
     fn into_canonical(self) -> VortexResult<Canonical> {
         take(&self.values(), &self.codes())?.into_canonical()
@@ -89,9 +96,3 @@ impl AcceptArrayVisitor for DictArray {
         visitor.visit_child("codes", &self.codes())
     }
 }
-
-impl ArrayTrait for DictArray {
-    fn len(&self) -> usize {
-        self.codes().len()
-    }
-}

encodings/fastlanes/src/bitpacking/compress.rs

@@ -6,7 +6,7 @@ use vortex::array::sparse::{Sparse, SparseArray};
 use vortex::stats::ArrayStatistics;
 use vortex::validity::Validity;
 use vortex::IntoArrayVariant;
-use vortex::{Array, ArrayDType, ArrayDef, ArrayTrait, IntoArray};
+use vortex::{Array, ArrayDType, ArrayDef, IntoArray};
 use vortex_dtype::{
     match_each_integer_ptype, match_each_unsigned_integer_ptype, NativePType, PType,
 };

encodings/fastlanes/src/bitpacking/compute/mod.rs

@@ -3,7 +3,7 @@ use vortex::compute::slice::SliceFn;
 use vortex::compute::take::TakeFn;
 use vortex::compute::unary::scalar_at::{scalar_at, ScalarAtFn};
 use vortex::compute::ArrayCompute;
-use vortex::{ArrayDType, ArrayTrait};
+use vortex::ArrayDType;
 use vortex_error::{vortex_err, VortexResult};
 use vortex_scalar::Scalar;
 

encodings/fastlanes/src/bitpacking/compute/search_sorted.rs

@@ -7,7 +7,7 @@ use vortex::array::sparse::SparseArray;
 use vortex::compute::search_sorted::{
     search_sorted, IndexOrd, Len, SearchResult, SearchSorted, SearchSortedFn, SearchSortedSide,
 };
-use vortex::{ArrayDType, ArrayTrait, IntoArrayVariant};
+use vortex::{ArrayDType, IntoArrayVariant};
 use vortex_dtype::{match_each_unsigned_integer_ptype, NativePType};
 use vortex_error::VortexResult;
 use vortex_scalar::Scalar;

encodings/fastlanes/src/bitpacking/compute/slice.rs

@@ -31,7 +31,7 @@ mod test {
     use vortex::array::primitive::PrimitiveArray;
     use vortex::compute::slice::slice;
     use vortex::compute::unary::scalar_at::scalar_at;
-    use vortex::{ArrayTrait, IntoArray};
+    use vortex::IntoArray;
 
     use crate::BitPackedArray;
 

encodings/fastlanes/src/bitpacking/compute/take.rs

@@ -7,7 +7,7 @@ use vortex::array::primitive::PrimitiveArray;
 use vortex::array::sparse::SparseArray;
 use vortex::compute::slice::slice;
 use vortex::compute::take::{take, TakeFn};
-use vortex::{Array, ArrayDType, ArrayTrait, IntoArray, IntoArrayVariant};
+use vortex::{Array, ArrayDType, IntoArray, IntoArrayVariant};
 use vortex_dtype::{
     match_each_integer_ptype, match_each_unsigned_integer_ptype, NativePType, PType,
 };

encodings/fastlanes/src/bitpacking/mod.rs

@@ -17,7 +17,7 @@ impl_encoding!("fastlanes.bitpacked", 14u16, BitPacked);
 pub struct BitPackedMetadata {
     // TODO(ngates): serialize into compact form
     validity: ValidityMetadata,
-    patches: bool,
+    patches_len: usize,
     bit_width: usize,
     offset: usize, // Know to be <1024
     length: usize, // Store end padding instead <1024
@@ -70,7 +70,7 @@ impl BitPackedArray {
 
         let metadata = BitPackedMetadata {
             validity: validity.to_metadata(length)?,
-            patches: patches.is_some(),
+            patches_len: patches.as_ref().map(|a| a.len()).unwrap_or(0),
             offset,
             length,
             bit_width,
@@ -85,13 +85,22 @@ impl BitPackedArray {
             children.push(a)
         }
 
-        Self::try_from_parts(dtype, metadata, children.into(), StatsSet::new())
+        Self::try_from_parts(dtype, length, metadata, children.into(), StatsSet::new())
+    }
+
+    fn packed_len(&self) -> usize {
+        ((self.len() + self.offset() + 1023) / 1024)
+            * (128 * self.bit_width() / self.ptype().byte_width())
     }
 
     #[inline]
     pub fn packed(&self) -> Array {
         self.array()
-            .child(0, &self.dtype().with_nullability(Nullability::NonNullable))
+            .child(
+                0,
+                &self.dtype().with_nullability(Nullability::NonNullable),
+                self.packed_len(),
+            )
             .expect("Missing packed array")
     }
 
@@ -102,11 +111,19 @@ impl BitPackedArray {
 
     #[inline]
     pub fn patches(&self) -> Option<Array> {
-        self.metadata().patches.then(|| {
-            self.array()
-                .child(1, &self.dtype().with_nullability(Nullability::Nullable))
-                .expect("Missing patches array")
-        })
+        if self.metadata().patches_len > 0 {
+            Some(
+                self.array()
+                    .child(
+                        1,
+                        &self.dtype().with_nullability(Nullability::Nullable),
+                        self.metadata().patches_len,
+                    )
+                    .expect("Missing patches array"),
+            )
+        } else {
+            None
+        }
     }
 
     #[inline]
@@ -116,8 +133,13 @@ impl BitPackedArray {
 
     pub fn validity(&self) -> Validity {
         self.metadata().validity.to_validity(self.array().child(
-            if self.metadata().patches { 2 } else { 1 },
+            if self.metadata().patches_len > 0 {
+                2
+            } else {
+                1
+            },
             &Validity::DTYPE,
+            self.len(),
         ))
     }
 
@@ -159,7 +181,7 @@ impl ArrayValidity for BitPackedArray {
 impl AcceptArrayVisitor for BitPackedArray {
     fn accept(&self, visitor: &mut dyn ArrayVisitor) -> VortexResult<()> {
         visitor.visit_child("packed", &self.packed())?;
-        if self.metadata().patches {
+        if self.metadata().patches_len > 0 {
             visitor.visit_child(
                 "patches",
                 &self.patches().expect("Expected patches to be present "),
@@ -172,10 +194,6 @@ impl AcceptArrayVisitor for BitPackedArray {
 impl ArrayStatisticsCompute for BitPackedArray {}
 
 impl ArrayTrait for BitPackedArray {
-    fn len(&self) -> usize {
-        self.metadata().length
-    }
-
     fn nbytes(&self) -> usize {
         // Ignore any overheads like padding or the bit-width flag.
         let packed_size = ((self.bit_width() * self.len()) + 7) / 8;

encodings/fastlanes/src/delta/compress.rs

@@ -186,7 +186,6 @@ mod test {
         let (bases, deltas) = delta_compress(&PrimitiveArray::from(input.clone())).unwrap();
 
         let delta = DeltaArray::try_new(
-            input.len(),
             bases.into_array(),
             deltas.into_array(),
             Validity::NonNullable,