Move expression filters out of datafusion (#638)

Cargo.toml

@@ -7,6 +7,7 @@ members = [
     "vortex-array",
     "vortex-buffer",
     "vortex-datafusion",
+    "vortex-datetime-dtype",
     "vortex-dtype",
     "vortex-error",
     "vortex-expr",
@@ -130,6 +131,7 @@ vortex-array = { version = "0.7.0", path = "./vortex-array" }
 vortex-buffer = { version = "0.7.0", path = "./vortex-buffer" }
 vortex-byte-bool = { version = "0.7.0", path = "./encodings/byte-bool" }
 vortex-datafusion = { version = "0.7.0", path = "./vortex-datafusion" }
+vortex-datetime-dtype = { version = "0.7.0", path = "./vortex-datetime-dtype" }
 vortex-datetime-parts = { version = "0.7.0", path = "./encodings/datetime-parts" }
 vortex-dict = { version = "0.7.0", path = "./encodings/dict" }
 vortex-dtype = { version = "0.7.0", path = "./vortex-dtype", default-features = false }

encodings/byte-bool/Cargo.toml

@@ -24,7 +24,6 @@ vortex-array = { workspace = true }
 vortex-buffer = { workspace = true }
 vortex-dtype = { workspace = true }
 vortex-error = { workspace = true }
-vortex-expr = { workspace = true }
 vortex-scalar = { workspace = true }
 
 [dev-dependencies]

encodings/byte-bool/src/compute/mod.rs

@@ -3,12 +3,11 @@ use std::ops::{BitAnd, BitOr, BitXor, Not};
 use arrow_buffer::BooleanBuffer;
 use num_traits::AsPrimitive;
 use vortex::compute::unary::{FillForwardFn, ScalarAtFn};
-use vortex::compute::{ArrayCompute, CompareFn, SliceFn, TakeFn};
+use vortex::compute::{ArrayCompute, CompareFn, Operator, SliceFn, TakeFn};
 use vortex::validity::{ArrayValidity, Validity};
 use vortex::{Array, ArrayDType, IntoArray, IntoArrayVariant};
 use vortex_dtype::{match_each_integer_ptype, Nullability};
 use vortex_error::{vortex_err, VortexResult};
-use vortex_expr::Operator;
 use vortex_scalar::{Scalar, ScalarValue};
 
 use super::ByteBoolArray;

encodings/datetime-parts/Cargo.toml

@@ -12,6 +12,7 @@ workspace = true
 log = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 vortex-array = { workspace = true }
+vortex-datetime-dtype = { workspace = true }
 vortex-dtype = { workspace = true }
 vortex-error = { workspace = true }
 vortex-scalar = { workspace = true }

encodings/datetime-parts/src/compress.rs

@@ -1,6 +1,7 @@
-use vortex::array::{PrimitiveArray, TemporalArray, TimeUnit};
+use vortex::array::{PrimitiveArray, TemporalArray};
 use vortex::compute::unary::try_cast;
 use vortex::{Array, IntoArray, IntoArrayVariant};
+use vortex_datetime_dtype::TimeUnit;
 use vortex_dtype::PType;
 use vortex_error::{vortex_bail, VortexResult};
 

encodings/datetime-parts/src/compute.rs

@@ -1,9 +1,9 @@
-use vortex::array::temporal::TemporalMetadata;
-use vortex::array::{PrimitiveArray, TemporalArray, TimeUnit};
+use vortex::array::{PrimitiveArray, TemporalArray};
 use vortex::compute::unary::{scalar_at, ScalarAtFn};
 use vortex::compute::{slice, take, ArrayCompute, SliceFn, TakeFn};
 use vortex::validity::ArrayValidity;
 use vortex::{Array, ArrayDType, IntoArray, IntoArrayVariant};
+use vortex_datetime_dtype::{TemporalMetadata, TimeUnit};
 use vortex_dtype::DType;
 use vortex_error::{vortex_bail, VortexResult};
 use vortex_scalar::Scalar;
@@ -120,8 +120,9 @@ pub fn decode_to_temporal(array: &DateTimePartsArray) -> VortexResult<TemporalAr
 
 #[cfg(test)]
 mod test {
-    use vortex::array::{PrimitiveArray, TemporalArray, TimeUnit};
+    use vortex::array::{PrimitiveArray, TemporalArray};
     use vortex::{IntoArray, IntoArrayVariant};
+    use vortex_datetime_dtype::TimeUnit;
     use vortex_dtype::{DType, Nullability};
 
     use crate::compute::decode_to_temporal;

vortex-array/Cargo.toml

@@ -44,24 +44,24 @@ pin-project = { workspace = true }
 rand = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 vortex-buffer = { workspace = true }
+vortex-datetime-dtype = { workspace = true }
 vortex-dtype = { workspace = true }
 vortex-error = { workspace = true }
-vortex-expr = { workspace = true }
 vortex-flatbuffers = { workspace = true, optional = true }
 vortex-scalar = { workspace = true }
 
 [features]
 default = ["flatbuffers", "serde"]
 arbitrary = ["dep:arbitrary"]
 flatbuffers = [
-  "dep:flatbuffers",
-  "dep:flexbuffers",
-  "dep:vortex-flatbuffers",
-  "vortex-flatbuffers/array",
-  "vortex-dtype/flatbuffers",
-  "vortex-error/flatbuffers",
-  "vortex-error/flexbuffers",
-  "vortex-scalar/flatbuffers",
+    "dep:flatbuffers",
+    "dep:flexbuffers",
+    "dep:vortex-flatbuffers",
+    "vortex-flatbuffers/array",
+    "vortex-dtype/flatbuffers",
+    "vortex-error/flatbuffers",
+    "vortex-error/flexbuffers",
+    "vortex-scalar/flatbuffers",
 ]
 serde = ["vortex-dtype/serde", "vortex-scalar/serde"]
 
@@ -82,10 +82,6 @@ harness = false
 name = "scalar_subtract"
 harness = false
 
-[[bench]]
-name = "filter_indices"
-harness = false
-
 [[bench]]
 name = "compare"
 harness = false

vortex-array/benches/compare.rs

@@ -3,11 +3,11 @@ use itertools::Itertools;
 use rand::distributions::Uniform;
 use rand::{thread_rng, Rng};
 use vortex::array::BoolArray;
+use vortex::compute::Operator;
 use vortex::IntoArray;
 use vortex_error::VortexError;
-use vortex_expr::Operator;
 
-fn filter_bool_indices(c: &mut Criterion) {
+fn compare_bool(c: &mut Criterion) {
     let mut group = c.benchmark_group("compare");
 
     let mut rng = thread_rng();
@@ -34,7 +34,7 @@ fn filter_bool_indices(c: &mut Criterion) {
     });
 }
 
-fn filter_indices(c: &mut Criterion) {
+fn compare_primitive(c: &mut Criterion) {
     let mut group = c.benchmark_group("compare");
 
     let mut rng = thread_rng();
@@ -58,5 +58,5 @@ fn filter_indices(c: &mut Criterion) {
     });
 }
 
-criterion_group!(benches, filter_indices, filter_bool_indices);
+criterion_group!(benches, compare_primitive, compare_bool);
 criterion_main!(benches);

vortex-array/src/array/bool/compute/compare.rs

@@ -1,10 +1,9 @@
 use std::ops::{BitAnd, BitOr, BitXor, Not};
 
 use vortex_error::VortexResult;
-use vortex_expr::Operator;
 
 use crate::array::BoolArray;
-use crate::compute::CompareFn;
+use crate::compute::{CompareFn, Operator};
 use crate::{Array, IntoArray, IntoArrayVariant};
 
 impl CompareFn for BoolArray {
@@ -34,7 +33,6 @@ impl CompareFn for BoolArray {
 }
 
 #[cfg(test)]
-#[allow(clippy::panic_in_result_fn)]
 mod test {
     use itertools::Itertools;
 
@@ -54,17 +52,23 @@ mod test {
     }
 
     #[test]
-    fn test_basic_comparisons() -> VortexResult<()> {
+    fn test_basic_comparisons() {
         let arr = BoolArray::from_vec(
             vec![true, true, false, true, false],
             Validity::Array(BoolArray::from(vec![false, true, true, true, true]).into_array()),
         )
         .into_array();
 
-        let matches = compare(&arr, &arr, Operator::Eq)?.into_bool()?;
+        let matches = compare(&arr, &arr, Operator::Eq)
+            .unwrap()
+            .into_bool()
+            .unwrap();
         assert_eq!(to_int_indices(matches), [1u64, 2, 3, 4]);
 
-        let matches = compare(&arr, &arr, Operator::NotEq)?.into_bool()?;
+        let matches = compare(&arr, &arr, Operator::NotEq)
+            .unwrap()
+            .into_bool()
+            .unwrap();
         let empty: [u64; 0] = [];
         assert_eq!(to_int_indices(matches), empty);
 
@@ -74,17 +78,28 @@ mod test {
         )
         .into_array();
 
-        let matches = compare(&arr, &other, Operator::Lte)?.into_bool()?;
+        let matches = compare(&arr, &other, Operator::Lte)
+            .unwrap()
+            .into_bool()
+            .unwrap();
         assert_eq!(to_int_indices(matches), [2u64, 3, 4]);
 
-        let matches = compare(&arr, &other, Operator::Lt)?.into_bool()?;
+        let matches = compare(&arr, &other, Operator::Lt)
+            .unwrap()
+            .into_bool()
+            .unwrap();
         assert_eq!(to_int_indices(matches), [4u64]);
 
-        let matches = compare(&other, &arr, Operator::Gte)?.into_bool()?;
+        let matches = compare(&other, &arr, Operator::Gte)
+            .unwrap()
+            .into_bool()
+            .unwrap();
         assert_eq!(to_int_indices(matches), [2u64, 3, 4]);
 
-        let matches = compare(&other, &arr, Operator::Gt)?.into_bool()?;
+        let matches = compare(&other, &arr, Operator::Gt)
+            .unwrap()
+            .into_bool()
+            .unwrap();
         assert_eq!(to_int_indices(matches), [4u64]);
-        Ok(())
     }
 }

vortex-array/src/array/constant/compute.rs

@@ -2,17 +2,17 @@ use std::cmp::Ordering;
 use std::sync::Arc;
 
 use arrow_array::Datum;
+use arrow_ord::cmp;
 use vortex_dtype::Nullability;
 use vortex_error::{vortex_bail, vortex_err, VortexResult};
-use vortex_expr::Operator;
 use vortex_scalar::Scalar;
 
 use crate::array::constant::ConstantArray;
 use crate::arrow::FromArrowArray;
 use crate::compute::unary::{scalar_at, ScalarAtFn};
 use crate::compute::{
-    scalar_cmp, AndFn, ArrayCompute, CompareFn, FilterFn, OrFn, SearchResult, SearchSortedFn,
-    SearchSortedSide, SliceFn, TakeFn,
+    scalar_cmp, AndFn, ArrayCompute, CompareFn, FilterFn, Operator, OrFn, SearchResult,
+    SearchSortedFn, SearchSortedSide, SliceFn, TakeFn,
 };
 use crate::stats::{ArrayStatistics, Stat};
 use crate::{Array, ArrayDType, AsArray, IntoArray, IntoCanonical};
@@ -110,12 +110,12 @@ impl CompareFn for ConstantArray {
             let rhs = rhs.as_ref();
 
             let boolean_array = match operator {
-                Operator::Eq => arrow_ord::cmp::eq(datum.as_ref(), &rhs)?,
-                Operator::NotEq => arrow_ord::cmp::neq(datum.as_ref(), &rhs)?,
-                Operator::Gt => arrow_ord::cmp::gt(datum.as_ref(), &rhs)?,
-                Operator::Gte => arrow_ord::cmp::gt_eq(datum.as_ref(), &rhs)?,
-                Operator::Lt => arrow_ord::cmp::lt(datum.as_ref(), &rhs)?,
-                Operator::Lte => arrow_ord::cmp::lt_eq(datum.as_ref(), &rhs)?,
+                Operator::Eq => cmp::eq(datum.as_ref(), &rhs)?,
+                Operator::NotEq => cmp::neq(datum.as_ref(), &rhs)?,
+                Operator::Gt => cmp::gt(datum.as_ref(), &rhs)?,
+                Operator::Gte => cmp::gt_eq(datum.as_ref(), &rhs)?,
+                Operator::Lt => cmp::lt(datum.as_ref(), &rhs)?,
+                Operator::Lte => cmp::lt_eq(datum.as_ref(), &rhs)?,
             };
 
             Ok(Array::from_arrow(&boolean_array, true))