Skip to content

Commit

Permalink
Fields clean up
Browse files Browse the repository at this point in the history
  • Loading branch information
gatesn committed Jun 12, 2024
1 parent ae0db20 commit e53d004
Show file tree
Hide file tree
Showing 7 changed files with 138 additions and 132 deletions.
6 changes: 1 addition & 5 deletions vortex-array/benches/filter_indices.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,7 @@ fn filter_indices(c: &mut Criterion) {
.collect_vec()
.into_array();

let predicate = Disjunction {
conjunctions: vec![Conjunction {
predicates: vec![FieldPath::builder().build().lt(lit(50_000_000i64))],
}],
};
let predicate = Disjunction::from(Conjunction::from(FieldPath::root().lt(lit(50_000_000i64))));

group.bench_function("vortex", |b| {
b.iter(|| {
Expand Down
122 changes: 43 additions & 79 deletions vortex-array/src/array/primitive/compute/filter_indices.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,9 @@ use crate::compute::filter_indices::FilterIndicesFn;
use crate::{Array, ArrayTrait, IntoArray};

impl FilterIndicesFn for PrimitiveArray {
fn filter_indices(&self, predicate: &Disjunction) -> VortexResult<Array> {
let conjunction_indices = predicate.conjunctions.iter().map(|conj| {
conj.predicates
.iter()
fn filter_indices(&self, disjunction: &Disjunction) -> VortexResult<Array> {
let conjunction_indices = disjunction.iter().map(|conj| {
conj.iter()
.map(|pred| indices_matching_predicate(self, pred))
.reduce(|a, b| Ok(a?.bitand(&b?)))
.unwrap()
Expand Down Expand Up @@ -70,17 +69,14 @@ fn apply_predicate<T: NativePType, F: Fn(&T, &T) -> bool>(
#[cfg(test)]
mod test {
use itertools::Itertools;
use vortex_dtype::field::FieldPathBuilder;
use vortex_expr::FieldPathOperations;
use vortex_expr::{lit, Conjunction};
use vortex_dtype::field::FieldPath;
use vortex_expr::{lit, Conjunction, FieldPathOperations};

use super::*;
use crate::validity::Validity;

fn apply_conjunctive_filter(arr: &PrimitiveArray, conj: Conjunction) -> VortexResult<Array> {
arr.filter_indices(&Disjunction {
conjunctions: vec![conj],
})
arr.filter_indices(&Disjunction::from_iter([conj]))
}

fn to_int_indices(filtered_primitive: BoolArray) -> Vec<u64> {
Expand Down Expand Up @@ -110,64 +106,44 @@ mod test {
None,
]);

let field = FieldPathBuilder::new().build();
let filtered_primitive = apply_conjunctive_filter(
&arr,
Conjunction {
predicates: vec![field.clone().lt(lit(5u32))],
},
)
.unwrap()
.flatten_bool()
.unwrap();
let field = FieldPath::root();
let filtered_primitive =
apply_conjunctive_filter(&arr, Conjunction::from(field.lt(lit(5u32))))
.unwrap()
.flatten_bool()
.unwrap();
let filtered = to_int_indices(filtered_primitive);
assert_eq!(filtered, [0u64, 1, 2, 3]);

let filtered_primitive = apply_conjunctive_filter(
&arr,
Conjunction {
predicates: vec![field.clone().gt(lit(5u32))],
},
)
.unwrap()
.flatten_bool()
.unwrap();
let filtered_primitive =
apply_conjunctive_filter(&arr, Conjunction::from(field.gt(lit(5u32))))
.unwrap()
.flatten_bool()
.unwrap();
let filtered = to_int_indices(filtered_primitive);
assert_eq!(filtered, [6u64, 7, 8, 10]);

let filtered_primitive = apply_conjunctive_filter(
&arr,
Conjunction {
predicates: vec![field.clone().eq(lit(5u32))],
},
)
.unwrap()
.flatten_bool()
.unwrap();
let filtered_primitive =
apply_conjunctive_filter(&arr, Conjunction::from(field.equal(lit(5u32))))
.unwrap()
.flatten_bool()
.unwrap();
let filtered = to_int_indices(filtered_primitive);
assert_eq!(filtered, [5u64]);

let filtered_primitive = apply_conjunctive_filter(
&arr,
Conjunction {
predicates: vec![field.clone().gte(lit(5u32))],
},
)
.unwrap()
.flatten_bool()
.unwrap();
let filtered_primitive =
apply_conjunctive_filter(&arr, Conjunction::from(field.gte(lit(5u32))))
.unwrap()
.flatten_bool()
.unwrap();
let filtered = to_int_indices(filtered_primitive);
assert_eq!(filtered, [5u64, 6, 7, 8, 10]);

let filtered_primitive = apply_conjunctive_filter(
&arr,
Conjunction {
predicates: vec![field.clone().lte(lit(5u32))],
},
)
.unwrap()
.flatten_bool()
.unwrap();
let filtered_primitive =
apply_conjunctive_filter(&arr, Conjunction::from(field.lte(lit(5u32))))
.unwrap()
.flatten_bool()
.unwrap();
let filtered = to_int_indices(filtered_primitive);
assert_eq!(filtered, [0u64, 1, 2, 3, 5]);
}
Expand All @@ -176,12 +152,10 @@ mod test {
fn test_multiple_predicates() {
let arr =
PrimitiveArray::from_vec(vec![1u32, 2, 3, 4, 5, 6, 7, 8, 9, 10], Validity::AllValid);
let field = FieldPathBuilder::new().build();
let field = FieldPath::root();
let filtered_primitive = apply_conjunctive_filter(
&arr,
Conjunction {
predicates: vec![field.clone().lt(lit(5u32)), field.clone().gt(lit(2u32))],
},
Conjunction::from_iter([field.lt(lit(5u32)), field.gt(lit(2u32))]),
)
.unwrap()
.flatten_bool()
Expand All @@ -194,12 +168,10 @@ mod test {
fn test_disjoint_predicates() {
let arr =
PrimitiveArray::from_vec(vec![1u32, 2, 3, 4, 5, 6, 7, 8, 9, 10], Validity::AllValid);
let field = FieldPathBuilder::new().build();
let field = FieldPath::root();
let filtered_primitive = apply_conjunctive_filter(
&arr,
Conjunction {
predicates: vec![field.clone().lt(lit(5u32)), field.clone().gt(lit(5u32))],
},
Conjunction::from_iter([field.lt(lit(5u32)), field.gt(lit(5u32))]),
)
.unwrap()
.flatten_bool()
Expand All @@ -213,17 +185,11 @@ mod test {
fn test_disjunctive_predicate() {
let arr =
PrimitiveArray::from_vec(vec![1u32, 2, 3, 4, 5, 6, 7, 8, 9, 10], Validity::AllValid);
let field = FieldPathBuilder::new().build();
let c1 = Conjunction {
predicates: vec![field.clone().lt(lit(5u32))],
};
let c2 = Conjunction {
predicates: vec![field.clone().gt(lit(5u32))],
};

let disj = Disjunction {
conjunctions: vec![c1, c2],
};
let field = FieldPath::root();
let c1 = Conjunction::from(field.lt(lit(5u32)));
let c2 = Conjunction::from(field.gt(lit(5u32)));

let disj = Disjunction::from_iter([c1, c2]);
let filtered_primitive = arr.filter_indices(&disj).unwrap().flatten_bool().unwrap();
let filtered = to_int_indices(filtered_primitive);
assert_eq!(filtered, [0u64, 1, 2, 3, 5, 6, 7, 8, 9])
Expand All @@ -233,12 +199,10 @@ mod test {
fn test_invalid_path_err() {
let arr =
PrimitiveArray::from_vec(vec![1u32, 2, 3, 4, 5, 6, 7, 8, 9, 10], Validity::AllValid);
let field = FieldPathBuilder::new().join("some_field").build();
let field = FieldPath::from_name("some_field");
apply_conjunctive_filter(
&arr,
Conjunction {
predicates: vec![field.clone().lt(lit(5u32)), field.clone().gt(lit(5u32))],
},
Conjunction::from_iter([field.lt(lit(5u32)), field.gt(lit(5u32))]),
)
.expect_err("Cannot apply field reference to primitive array");
}
Expand Down
10 changes: 10 additions & 0 deletions vortex-dtype/src/field.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ impl Display for Field {
pub struct FieldPath(Vec<Field>);

impl FieldPath {
pub fn root() -> Self {
Self(vec![])
}

pub fn from_name(name: &str) -> Self {
Self(vec![Field::from(name)])
}
Expand All @@ -57,6 +61,12 @@ impl FromIterator<Field> for FieldPath {
}
}

impl From<Field> for FieldPath {
fn from(value: Field) -> Self {
FieldPath(vec![value])
}
}

impl From<Vec<Field>> for FieldPath {
fn from(value: Vec<Field>) -> Self {
FieldPath(value)
Expand Down
42 changes: 18 additions & 24 deletions vortex-expr/src/display.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,39 +27,33 @@ mod tests {

#[test]
fn test_predicate_formatting() {
let f1 = Field::from("field");
assert_eq!(format!("{}", f1.clone().lt(lit(1u32))), "($field < 1)");
assert_eq!(format!("{}", f1.clone().gte(lit(1u32))), "($field >= 1)");
assert_eq!(format!("{}", !f1.clone().lte(lit(1u32))), "($field > 1)");
let f1 = FieldPath::from_name("field");
assert_eq!(format!("{}", f1.lt(lit(1u32))), "($field < 1)");
assert_eq!(format!("{}", f1.gte(lit(1u32))), "($field >= 1)");
assert_eq!(format!("{}", !f1.lte(lit(1u32))), "($field > 1)");
assert_eq!(format!("{}", !lit(1u32).lte(f1)), "($field <= 1)");

// nested field path
let f2 = FieldPath::builder().join("field").join(0).build();
let f2 = FieldPath::from_iter([Field::from("field"), Field::from(0)]);
assert_eq!(format!("{}", !f2.lte(lit(1u32))), "($field.[0] > 1)");
}

#[test]
fn test_dnf_formatting() {
let path = FieldPath::builder().join(2).join("col1").build();
let d1 = Conjunction {
predicates: vec![
lit(1u32).lt(path.clone()),
path.clone().gte(lit(1u32)),
!lit(1u32).lte(path),
],
};
let path2 = FieldPath::builder().join("col1").join(2).build();
let d2 = Conjunction {
predicates: vec![
lit(2u32).lt(path2),
lit(3u32).gte(field(2)),
!lit(5u32).lte(field("col2")),
],
};
let path = FieldPath::from_iter([Field::from(2), Field::from("col1")]);
let d1 = Conjunction::from_iter([
lit(1u32).lt(path.clone()),
path.clone().gte(lit(1u32)),
!lit(1u32).lte(path),
]);
let path2 = FieldPath::from_iter([Field::from("col1"), Field::from(2)]);
let d2 = Conjunction::from_iter([
lit(2u32).lt(path2),
lit(3u32).gte(Field::from(2)),
!lit(5u32).lte(Field::from("col2")),
]);

let dnf = Disjunction {
conjunctions: vec![d1, d2],
};
let dnf = Disjunction::from_iter([d1, d2]);

let string = format!("{}", dnf);
print!("{}", string);
Expand Down
51 changes: 48 additions & 3 deletions vortex-expr/src/expressions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,39 @@ pub struct Disjunction {
conjunctions: Vec<Conjunction>,
}

impl Disjunction {
pub fn iter(&self) -> impl Iterator<Item = &Conjunction> {
self.conjunctions.iter()
}
}

impl From<Conjunction> for Disjunction {
fn from(value: Conjunction) -> Self {
Self {
conjunctions: vec![value],
}
}
}

impl FromIterator<Predicate> for Disjunction {
fn from_iter<T: IntoIterator<Item = Predicate>>(iter: T) -> Self {
Self {
conjunctions: iter
.into_iter()
.map(|predicate| Conjunction::from_iter([predicate]))
.collect(),
}
}
}

impl FromIterator<Conjunction> for Disjunction {
fn from_iter<T: IntoIterator<Item = Conjunction>>(iter: T) -> Self {
Self {
conjunctions: iter.into_iter().collect(),
}
}
}

impl Display for Disjunction {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
self.conjunctions
Expand All @@ -37,11 +70,23 @@ pub struct Conjunction {
}

impl Conjunction {
pub fn new() -> Self {
Self { predicates: vec![] }
}

pub fn iter(&self) -> impl Iterator<Item = &Predicate> {
self.predicates.iter()
}
}

impl From<Predicate> for Conjunction {
fn from(value: Predicate) -> Self {
Self {
predicates: vec![value],
}
}
}

impl FromIterator<Predicate> for Conjunction {
fn from_iter<T: IntoIterator<Item = Predicate>>(iter: T) -> Self {
Self {
Expand Down Expand Up @@ -135,17 +180,17 @@ impl Value {

#[cfg(test)]
mod test {
use vortex_dtype::field::field;
use vortex_dtype::field::Field;

use super::*;

#[test]
fn test_lit() {
let scalar: Scalar = 1.into();
let value: Value = lit(scalar);
let field = field("id");
let field = Field::from("id");
let expr = Predicate {
lhs: field,
lhs: FieldPath::from_iter([field]),
op: Operator::Eq,
rhs: value,
};
Expand Down
Loading

0 comments on commit e53d004

Please sign in to comment.