Compressors

bench-vortex/src/lib.rs

@@ -91,6 +91,7 @@ mod test {
         .unwrap();
     }
 
+    #[ignore]
     #[test]
     fn compression_ratio() {
         setup_logger();

pyvortex/src/compress.rs

@@ -4,6 +4,7 @@ use pyo3::{pyclass, pyfunction, pymethods, Py, PyResult, Python};
 use vortex::compress::{CompressConfig, CompressCtx};
 
 use crate::array::PyArray;
+use crate::error::PyVortexError;
 
 #[derive(Clone)]
 #[pyclass(name = "CompressConfig", module = "vortex")]
@@ -33,6 +34,8 @@ pub fn compress(
 ) -> PyResult<Py<PyArray>> {
     let compress_opts = opts.map(|o| o.inner).unwrap_or_default();
     let ctx = CompressCtx::new(&compress_opts);
-    let compressed = py.allow_threads(|| ctx.compress(arr.unwrap(), None));
+    let compressed = py
+        .allow_threads(|| ctx.compress(arr.unwrap(), None))
+        .map_err(PyVortexError::map_err)?;
     PyArray::wrap(py, compressed)
 }

vortex-array/src/array/sparse/mod.rs

@@ -21,12 +21,11 @@ use crate::error::{VortexError, VortexResult};
 use crate::formatter::{ArrayDisplay, ArrayFormatter};
 use crate::match_arrow_numeric_type;
 use crate::serde::{ArraySerde, EncodingSerde};
-use crate::stats::{Stats, StatsSet};
+use crate::stats::{Stats, StatsCompute, StatsSet};
 
 mod compress;
 mod compute;
 mod serde;
-mod stats;
 
 #[derive(Debug, Clone)]
 pub struct SparseArray {
@@ -190,6 +189,8 @@ impl Array for SparseArray {
     }
 }
 
+impl StatsCompute for SparseArray {}
+
 impl<'arr> AsRef<(dyn Array + 'arr)> for SparseArray {
     fn as_ref(&self) -> &(dyn Array + 'arr) {
         self

vortex-array/src/array/struct_/mod.rs

@@ -13,7 +13,7 @@ use crate::dtype::{DType, FieldNames};
 use crate::error::VortexResult;
 use crate::formatter::{ArrayDisplay, ArrayFormatter};
 use crate::serde::{ArraySerde, EncodingSerde};
-use crate::stats::{Stats, StatsSet};
+use crate::stats::{Stats, StatsCompute, StatsSet};
 
 use super::{
     check_slice_bounds, Array, ArrayRef, ArrowIterator, Encoding, EncodingId, EncodingRef,
@@ -23,7 +23,6 @@ use super::{
 mod compress;
 mod compute;
 mod serde;
-mod stats;
 
 #[derive(Debug, Clone)]
 pub struct StructArray {
@@ -167,6 +166,8 @@ impl<'arr> AsRef<(dyn Array + 'arr)> for StructArray {
     }
 }
 
+impl StatsCompute for StructArray {}
+
 #[derive(Debug)]
 pub struct StructEncoding;
 

vortex-array/src/compress.rs

@@ -131,7 +131,7 @@ impl Default for CompressCtx<'_> {
 
 pub fn sampled_compression(array: &dyn Array, ctx: CompressCtx) -> VortexResult<ArrayRef> {
     // First, we try constant compression and shortcut any sampling.
-    if array.len() > 0
+    if !array.is_empty()
         && array
             .stats()
             .get_or_compute_as::<bool>(&Stat::IsConstant)

vortex-array/src/stats.rs

@@ -209,7 +209,9 @@ impl StatsSet {
 }
 
 pub trait StatsCompute {
-    fn compute(&self, stat: &Stat) -> VortexResult<StatsSet>;
+    fn compute(&self, _stat: &Stat) -> VortexResult<StatsSet> {
+        Ok(StatsSet::new())
+    }
 }
 
 pub struct Stats<'a> {

vortex-fastlanes/src/bitpacking/compress.rs

@@ -224,10 +224,12 @@ mod test {
         );
         let ctx = CompressCtx::new(&cfg);
 
-        let compressed = ctx.compress(
-            &PrimitiveArray::from_vec(Vec::from_iter((0..10_000).map(|i| (i % 63) as u8))),
-            None,
-        );
+        let compressed = ctx
+            .compress(
+                &PrimitiveArray::from_vec(Vec::from_iter((0..10_000).map(|i| (i % 63) as u8))),
+                None,
+            )
+            .unwrap();
         assert_eq!(compressed.encoding().id(), BitPackedEncoding.id());
         let bp = compressed
             .as_any()

vortex-fastlanes/src/for/compress.rs

@@ -93,7 +93,7 @@ mod test {
         // Create a range offset by a million
         let array = PrimitiveArray::from_vec((0u32..10_000).map(|v| v + 1_000_000).collect_vec());
 
-        let compressed = ctx.compress(&array, None);
+        let compressed = ctx.compress(&array, None).unwrap();
         assert_eq!(compressed.encoding().id(), FoREncoding.id());
         let fa = compressed.as_any().downcast_ref::<FoRArray>().unwrap();
         assert_eq!(fa.reference().try_into(), Ok(1_000_000u32));

vortex-ffor/src/ffor.rs

@@ -13,7 +13,7 @@ use vortex::error::{VortexError, VortexResult};
 use vortex::formatter::{ArrayDisplay, ArrayFormatter};
 use vortex::scalar::{Scalar, ScalarRef};
 use vortex::serde::{ArraySerde, EncodingSerde};
-use vortex::stats::{Stats, StatsSet};
+use vortex::stats::{Stats, StatsCompute, StatsSet};
 
 use crate::compress::ffor_encode;
 
@@ -168,6 +168,8 @@ impl Array for FFORArray {
     }
 }
 
+impl StatsCompute for FFORArray {}
+
 impl ArrayCompute for FFORArray {}
 
 impl<'arr> AsRef<(dyn Array + 'arr)> for FFORArray {

vortex-ffor/src/lib.rs

@@ -7,7 +7,6 @@ mod compress;
 mod downcast;
 mod ffor;
 mod serde;
-mod stats;
 
 #[distributed_slice(ENCODINGS)]
 static ENCODINGS_FFOR: EncodingRef = &FFoREncoding;

vortex-ree/src/lib.rs

@@ -8,7 +8,6 @@ mod compute;
 mod downcast;
 mod ree;
 mod serde;
-mod stats;
 
 #[distributed_slice(ENCODINGS)]
 static ENCODINGS_REE: EncodingRef = &REEEncoding;

vortex-ree/src/ree.rs

@@ -23,7 +23,7 @@ use vortex::error::{VortexError, VortexResult};
 use vortex::formatter::{ArrayDisplay, ArrayFormatter};
 use vortex::ptype::NativePType;
 use vortex::serde::{ArraySerde, EncodingSerde};
-use vortex::stats::{Stat, Stats, StatsSet};
+use vortex::stats::{Stat, Stats, StatsCompute, StatsSet};
 
 use crate::compress::ree_encode;
 
@@ -218,6 +218,8 @@ impl Array for REEArray {
     }
 }
 
+impl StatsCompute for REEArray {}
+
 impl<'arr> AsRef<(dyn Array + 'arr)> for REEArray {
     fn as_ref(&self) -> &(dyn Array + 'arr) {
         self