Skip to content

Commit

Permalink
Use shorthand canonicalize methods (#460)
Browse files Browse the repository at this point in the history
  • Loading branch information
robert3005 authored Jul 16, 2024
1 parent a9ee529 commit d4558a1
Show file tree
Hide file tree
Showing 20 changed files with 54 additions and 201 deletions.
6 changes: 3 additions & 3 deletions encodings/byte_bool/src/compute/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use std::sync::Arc;
use arrow_buffer::BooleanBuffer;
use num_traits::AsPrimitive;
use vortex::validity::Validity;
use vortex::ToArrayData;
use vortex::Array;
use vortex::{
compute::{
compare::CompareFn, slice::SliceFn, take::TakeFn, unary::fill_forward::FillForwardFn,
Expand All @@ -15,7 +15,7 @@ use vortex::{
validity::ArrayValidity,
ArrayDType, ArrayData, IntoArray,
};
use vortex::{Array, IntoCanonical};
use vortex::{IntoArrayVariant, ToArrayData};
use vortex_dtype::{match_each_integer_ptype, Nullability};
use vortex_error::{vortex_bail, VortexResult};
use vortex_expr::Operator;
Expand Down Expand Up @@ -130,7 +130,7 @@ impl TakeFn for ByteBoolArray {

impl CompareFn for ByteBoolArray {
fn compare(&self, other: &Array, op: Operator) -> VortexResult<Array> {
let canonical = other.clone().into_canonical()?.into_bool()?;
let canonical = other.clone().into_bool()?;
let lhs = BooleanBuffer::from(self.maybe_null_slice());
let rhs = canonical.boolean_buffer();

Expand Down
4 changes: 2 additions & 2 deletions encodings/byte_bool/src/stats.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use vortex::{
stats::{ArrayStatisticsCompute, Stat, StatsSet},
AsArray, IntoCanonical,
AsArray, IntoArrayVariant,
};
use vortex_error::VortexResult;

Expand All @@ -13,7 +13,7 @@ impl ArrayStatisticsCompute for ByteBoolArray {
}

// TODO(adamgs): This is slightly wasteful and could be optimized in the future
let bools = self.as_array_ref().clone().into_canonical()?.into_bool()?;
let bools = self.as_array_ref().clone().into_bool()?;
bools.compute_statistics(stat)
}
}
Expand Down
6 changes: 2 additions & 4 deletions encodings/datetime-parts/src/compress.rs
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
use vortex::array::datetime::{LocalDateTimeArray, TimeUnit};
use vortex::array::primitive::PrimitiveArray;
use vortex::compute::unary::cast::try_cast;
use vortex::{Array, IntoArray, IntoCanonical};
use vortex::{Array, IntoArray, IntoArrayVariant};
use vortex_dtype::PType;
use vortex_error::VortexResult;

pub fn compress_localdatetime(array: LocalDateTimeArray) -> VortexResult<(Array, Array, Array)> {
let timestamps = try_cast(&array.timestamps(), PType::I64.into())?
.into_canonical()?
.into_primitive()?;
let timestamps = try_cast(&array.timestamps(), PType::I64.into())?.into_primitive()?;

let divisor = match array.time_unit() {
TimeUnit::Ns => 1_000_000_000,
Expand Down
48 changes: 7 additions & 41 deletions encodings/dict/src/compute.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ impl SliceFn for DictArray {
mod test {
use vortex::array::primitive::PrimitiveArray;
use vortex::array::varbin::VarBinArray;
use vortex::{IntoArray, IntoCanonical, ToArray};
use vortex::{IntoArray, IntoArrayVariant, ToArray};
use vortex_dtype::{DType, Nullability};

use crate::{dict_encode_typed_primitive, dict_encode_varbin, DictArray};
Expand All @@ -67,12 +67,7 @@ mod test {
]);
let (codes, values) = dict_encode_typed_primitive::<i32>(&reference);
let dict = DictArray::try_new(codes.into_array(), values.into_array()).unwrap();
let flattened_dict = dict
.to_array()
.into_canonical()
.unwrap()
.into_primitive()
.unwrap();
let flattened_dict = dict.to_array().into_primitive().unwrap();
assert_eq!(flattened_dict.buffer(), reference.buffer());
}

Expand All @@ -84,43 +79,14 @@ mod test {
);
let (codes, values) = dict_encode_varbin(&reference);
let dict = DictArray::try_new(codes.into_array(), values.into_array()).unwrap();
let flattened_dict = dict
.to_array()
.into_canonical()
.unwrap()
.into_varbin()
.unwrap();
let flattened_dict = dict.to_array().into_varbin().unwrap();
assert_eq!(
flattened_dict
.offsets()
.into_canonical()
.unwrap()
.into_primitive()
.unwrap()
.buffer(),
reference
.offsets()
.into_canonical()
.unwrap()
.into_primitive()
.unwrap()
.buffer()
flattened_dict.offsets().into_primitive().unwrap().buffer(),
reference.offsets().into_primitive().unwrap().buffer()
);
assert_eq!(
flattened_dict
.bytes()
.into_canonical()
.unwrap()
.into_primitive()
.unwrap()
.buffer(),
reference
.bytes()
.into_canonical()
.unwrap()
.into_primitive()
.unwrap()
.buffer()
flattened_dict.bytes().into_primitive().unwrap().buffer(),
reference.bytes().into_primitive().unwrap().buffer()
);
}
}
9 changes: 2 additions & 7 deletions encodings/dict/src/dict.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use vortex::compute::take::take;
use vortex::compute::unary::scalar_at::scalar_at;
use vortex::validity::{ArrayValidity, LogicalValidity};
use vortex::visitor::{AcceptArrayVisitor, ArrayVisitor};
use vortex::{impl_encoding, ArrayDType, Canonical, IntoCanonical};
use vortex::{impl_encoding, ArrayDType, Canonical, IntoArrayVariant, IntoCanonical};
use vortex_dtype::match_each_integer_ptype;
use vortex_error::vortex_bail;

Expand Down Expand Up @@ -69,12 +69,7 @@ impl ArrayValidity for DictArray {

fn logical_validity(&self) -> LogicalValidity {
if self.dtype().is_nullable() {
let primitive_codes = self
.codes()
.into_canonical()
.unwrap()
.into_primitive()
.unwrap();
let primitive_codes = self.codes().into_primitive().unwrap();
match_each_integer_ptype!(primitive_codes.ptype(), |$P| {
ArrayAccessor::<$P>::with_iterator(&primitive_codes, |iter| {
LogicalValidity::Array(
Expand Down
4 changes: 1 addition & 3 deletions encodings/fastlanes/src/bitpacking/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ impl ArrayTrait for BitPackedArray {
#[cfg(test)]
mod test {
use vortex::array::primitive::PrimitiveArray;
use vortex::{IntoArray, IntoCanonical};
use vortex::{IntoArray, IntoArrayVariant};

use crate::BitPackedArray;

Expand All @@ -216,8 +216,6 @@ mod test {
let expected = &[1, 0, 1, 0, 1, 0, u64::MAX];
let results = packed
.into_array()
.into_canonical()
.unwrap()
.into_primitive()
.unwrap()
.maybe_null_slice::<u64>()
Expand Down
9 changes: 2 additions & 7 deletions encodings/runend/src/compute.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ impl SliceFn for RunEndArray {
mod test {
use vortex::array::primitive::PrimitiveArray;
use vortex::compute::take::take;
use vortex::{IntoCanonical, ToArray};
use vortex::{IntoArrayVariant, ToArray};

use crate::RunEndArray;

Expand All @@ -81,12 +81,7 @@ mod test {
.unwrap();
let taken = take(ree.array(), PrimitiveArray::from(vec![9, 8, 1, 3]).array()).unwrap();
assert_eq!(
taken
.into_canonical()
.unwrap()
.into_primitive()
.unwrap()
.maybe_null_slice::<i32>(),
taken.into_primitive().unwrap().maybe_null_slice::<i32>(),
&[5, 5, 1, 4]
);
}
Expand Down
14 changes: 3 additions & 11 deletions encodings/runend/src/runend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ mod test {
use vortex::compute::slice::slice;
use vortex::compute::unary::scalar_at::scalar_at;
use vortex::validity::Validity;
use vortex::{ArrayDType, IntoArray, IntoCanonical};
use vortex::{ArrayDType, IntoArray, IntoArrayVariant};
use vortex_dtype::{DType, Nullability, PType};

use crate::RunEndArray;
Expand Down Expand Up @@ -190,11 +190,7 @@ mod test {
assert_eq!(arr.len(), 5);

assert_eq!(
arr.into_canonical()
.unwrap()
.into_primitive()
.unwrap()
.maybe_null_slice::<i32>(),
arr.into_primitive().unwrap().maybe_null_slice::<i32>(),
vec![2, 2, 3, 3, 3]
);
}
Expand All @@ -209,11 +205,7 @@ mod test {
.unwrap();

assert_eq!(
arr.into_canonical()
.unwrap()
.into_primitive()
.unwrap()
.maybe_null_slice::<i32>(),
arr.into_primitive().unwrap().maybe_null_slice::<i32>(),
vec![1, 1, 2, 2, 2, 3, 3, 3, 3, 3]
);
}
Expand Down
4 changes: 1 addition & 3 deletions vortex-array/src/array/chunked/compute/take.rs
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ fn take_strict_sorted(chunked: &ChunkedArray, indices: &Array) -> VortexResult<A
mod test {
use crate::array::chunked::ChunkedArray;
use crate::compute::take::take;
use crate::{ArrayDType, AsArray, IntoArray, IntoCanonical};
use crate::{ArrayDType, AsArray, IntoArray, IntoArrayVariant};

#[test]
fn test_take() {
Expand All @@ -129,8 +129,6 @@ mod test {
let result = &ChunkedArray::try_from(take(arr.as_array_ref(), &indices).unwrap())
.unwrap()
.into_array()
.into_canonical()
.unwrap()
.into_primitive()
.unwrap();
assert_eq!(result.maybe_null_slice::<i32>(), &[1, 1, 1, 2]);
Expand Down
8 changes: 1 addition & 7 deletions vortex-array/src/array/chunked/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ mod test {
use crate::array::chunked::ChunkedArray;
use crate::compute::slice::slice;
use crate::compute::unary::scalar_subtract::subtract_scalar;
use crate::{Array, IntoArray, IntoArrayVariant, IntoCanonical, ToArray};
use crate::{Array, IntoArray, IntoArrayVariant, ToArray};

fn chunked_array() -> ChunkedArray {
ChunkedArray::try_new(
Expand Down Expand Up @@ -231,8 +231,6 @@ mod test {
let results = chunks_out
.next()
.unwrap()
.into_canonical()
.unwrap()
.into_primitive()
.unwrap()
.maybe_null_slice::<u64>()
Expand All @@ -241,8 +239,6 @@ mod test {
let results = chunks_out
.next()
.unwrap()
.into_canonical()
.unwrap()
.into_primitive()
.unwrap()
.maybe_null_slice::<u64>()
Expand All @@ -251,8 +247,6 @@ mod test {
let results = chunks_out
.next()
.unwrap()
.into_canonical()
.unwrap()
.into_primitive()
.unwrap()
.maybe_null_slice::<u64>()
Expand Down
24 changes: 2 additions & 22 deletions vortex-array/src/array/primitive/compute/filter_indices.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ mod test {

use super::*;
use crate::validity::Validity;
use crate::IntoCanonical;
use crate::IntoArrayVariant;

fn apply_conjunctive_filter(arr: &PrimitiveArray, conj: Conjunction) -> VortexResult<Array> {
arr.filter_indices(&Disjunction::from_iter([conj]))
Expand Down Expand Up @@ -107,8 +107,6 @@ mod test {
let field = FieldPath::root();
let filtered_primitive =
apply_conjunctive_filter(&arr, Conjunction::from(field.lt(lit(5u32))))
.unwrap()
.into_canonical()
.unwrap()
.into_bool()
.unwrap();
Expand All @@ -117,8 +115,6 @@ mod test {

let filtered_primitive =
apply_conjunctive_filter(&arr, Conjunction::from(field.gt(lit(5u32))))
.unwrap()
.into_canonical()
.unwrap()
.into_bool()
.unwrap();
Expand All @@ -127,8 +123,6 @@ mod test {

let filtered_primitive =
apply_conjunctive_filter(&arr, Conjunction::from(field.equal(lit(5u32))))
.unwrap()
.into_canonical()
.unwrap()
.into_bool()
.unwrap();
Expand All @@ -137,8 +131,6 @@ mod test {

let filtered_primitive =
apply_conjunctive_filter(&arr, Conjunction::from(field.gte(lit(5u32))))
.unwrap()
.into_canonical()
.unwrap()
.into_bool()
.unwrap();
Expand All @@ -147,8 +139,6 @@ mod test {

let filtered_primitive =
apply_conjunctive_filter(&arr, Conjunction::from(field.lte(lit(5u32))))
.unwrap()
.into_canonical()
.unwrap()
.into_bool()
.unwrap();
Expand All @@ -166,8 +156,6 @@ mod test {
Conjunction::from_iter([field.lt(lit(5u32)), field.gt(lit(2u32))]),
)
.unwrap()
.into_canonical()
.unwrap()
.into_bool()
.unwrap();
let filtered = to_int_indices(filtered_primitive);
Expand All @@ -184,8 +172,6 @@ mod test {
Conjunction::from_iter([field.lt(lit(5u32)), field.gt(lit(5u32))]),
)
.unwrap()
.into_canonical()
.unwrap()
.into_bool()
.unwrap();
let filtered = to_int_indices(filtered_primitive);
Expand All @@ -202,13 +188,7 @@ mod test {
let c2 = Conjunction::from(field.gt(lit(5u32)));

let disj = Disjunction::from_iter([c1, c2]);
let filtered_primitive = arr
.filter_indices(&disj)
.unwrap()
.into_canonical()
.unwrap()
.into_bool()
.unwrap();
let filtered_primitive = arr.filter_indices(&disj).unwrap().into_bool().unwrap();
let filtered = to_int_indices(filtered_primitive);
assert_eq!(filtered, [0u64, 1, 2, 3, 5, 6, 7, 8, 9])
}
Expand Down
Loading

0 comments on commit d4558a1

Please sign in to comment.