Skip to content

Commit

Permalink
Random access benchmarks are runnable again (#330)
Browse files Browse the repository at this point in the history
robert3005 authored May 17, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent 5f2db66 commit 4381be6
Showing 6 changed files with 24 additions and 23 deletions.
4 changes: 1 addition & 3 deletions bench-vortex/src/reader.rs
Original file line number Diff line number Diff line change
@@ -33,9 +33,7 @@ use crate::CTX;
pub const BATCH_SIZE: usize = 65_536;

pub fn open_vortex(path: &Path) -> VortexResult<Array> {
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
Runtime::new()
.unwrap()
.block_on(async {
let file = tokio::fs::File::open(path).await.unwrap();
6 changes: 0 additions & 6 deletions vortex-array/src/arrow/dtype.rs
Original file line number Diff line number Diff line change
@@ -24,12 +24,6 @@ impl TryFromArrowType<&DataType> for PType {
DataType::Float16 => Ok(Self::F16),
DataType::Float32 => Ok(Self::F32),
DataType::Float64 => Ok(Self::F64),
DataType::Time32(_) => Ok(Self::I32),
DataType::Time64(_) => Ok(Self::I64),
DataType::Timestamp(..) => Ok(Self::I64),
DataType::Date32 => Ok(Self::I32),
DataType::Date64 => Ok(Self::I64),
DataType::Duration(_) => Ok(Self::I64),
_ => Err(vortex_err!(
"Arrow datatype {:?} cannot be converted to ptype",
value
12 changes: 9 additions & 3 deletions vortex-fastlanes/src/bitpacking/compute/mod.rs
Original file line number Diff line number Diff line change
@@ -10,7 +10,9 @@ use vortex::compute::slice::{slice, SliceFn};
use vortex::compute::take::{take, TakeFn};
use vortex::compute::ArrayCompute;
use vortex::{Array, ArrayDType, ArrayTrait, IntoArray};
use vortex_dtype::{match_each_integer_ptype, match_each_unsigned_integer_ptype, NativePType};
use vortex_dtype::{
match_each_integer_ptype, match_each_unsigned_integer_ptype, NativePType, PType,
};
use vortex_error::{vortex_err, VortexResult};
use vortex_scalar::Scalar;

@@ -43,14 +45,18 @@ impl ScalarAtFn for BitPackedArray {
if self.bit_width() == 0 || patches.with_dyn(|a| a.is_valid(index)) {
return scalar_at(&patches, index)?.cast(self.dtype());
}
} else if self.bit_width() == 0 {
match_each_unsigned_integer_ptype!(PType::try_from(self.dtype())?, |$P| {
return Ok(Scalar::zero::<$P>(self.dtype().nullability()));
});
}
unpack_single(self, index)?.cast(self.dtype())
}
}

impl TakeFn for BitPackedArray {
fn take(&self, indices: &Array) -> VortexResult<Array> {
let ptype = self.dtype().try_into()?;
let ptype: PType = self.dtype().try_into()?;
let validity = self.validity();
let taken_validity = validity.take(indices)?;
if self.bit_width() == 0 {
@@ -66,7 +72,7 @@ impl TakeFn for BitPackedArray {
}

let indices = indices.clone().flatten_primitive()?;
let taken = match_each_unsigned_integer_ptype!(ptype, |$T| {
let taken = match_each_unsigned_integer_ptype!(ptype.to_unsigned(), |$T| {
PrimitiveArray::from_vec(take_primitive::<$T>(self, &indices)?, taken_validity)
});
Ok(taken.reinterpret_cast(ptype).into_array())
9 changes: 4 additions & 5 deletions vortex-fastlanes/src/for/compute.rs
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@ use vortex::compute::ArrayCompute;
use vortex::{Array, IntoArray};
use vortex_dtype::match_each_integer_ptype;
use vortex_error::{vortex_bail, VortexResult};
use vortex_scalar::{PrimitiveScalar, Scalar};
use vortex_scalar::{PrimitiveScalar, Scalar, ScalarValue};

use crate::FoRArray;

@@ -46,10 +46,9 @@ impl ScalarAtFn for FoRArray {

match_each_integer_ptype!(encoded.ptype(), |$P| {
use num_traits::WrappingAdd;
Ok(Scalar::primitive::<$P>(
(encoded.typed_value::<$P>().unwrap() << self.shift()).wrapping_add(reference.typed_value::<$P>().unwrap()),
encoded.dtype().nullability()
))
Ok(encoded.typed_value::<$P>().map(|v| (v << self.shift()).wrapping_add(reference.typed_value::<$P>().unwrap()))
.map(|v| Scalar::primitive::<$P>(v, encoded.dtype().nullability()))
.unwrap_or_else(|| Scalar::new(encoded.dtype().clone(), ScalarValue::Null)))
})
}
}
8 changes: 2 additions & 6 deletions vortex-ree/src/ree.rs
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize};
use vortex::array::primitive::{Primitive, PrimitiveArray};
use vortex::compute::scalar_at::scalar_at;
use vortex::compute::search_sorted::{search_sorted, SearchSortedSide};
use vortex::stats::{ArrayStatistics, ArrayStatisticsCompute, Stat};
use vortex::stats::{ArrayStatistics, ArrayStatisticsCompute};
use vortex::validity::{ArrayValidity, LogicalValidity, Validity, ValidityMetadata};
use vortex::visitor::{AcceptArrayVisitor, ArrayVisitor};
use vortex::{impl_encoding, ArrayDType, ArrayFlatten, IntoArrayData};
@@ -37,11 +37,7 @@ impl REEArray {
vortex_bail!("incorrect validity {:?}", validity);
}

if !ends
.statistics()
.get_as(Stat::IsStrictSorted)
.unwrap_or(true)
{
if !ends.statistics().compute_is_strict_sorted().unwrap_or(true) {
vortex_bail!("Ends array must be strictly sorted",);
}
let dtype = values.dtype().clone();
8 changes: 8 additions & 0 deletions vortex-scalar/src/primitive.rs
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@ use crate::pvalue::PValue;
use crate::value::ScalarValue;
use crate::Scalar;

#[derive(Debug, Clone)]
pub struct PrimitiveScalar<'a> {
dtype: &'a DType,
ptype: PType,
@@ -82,6 +83,13 @@ impl Scalar {
value: ScalarValue::Primitive(value.into()),
}
}

pub fn zero<T: NativePType + Into<PValue>>(nullability: Nullability) -> Self {
Self {
dtype: DType::Primitive(T::PTYPE, nullability),
value: ScalarValue::Primitive(T::zero().into()),
}
}
}

macro_rules! primitive_scalar {

0 comments on commit 4381be6

Please sign in to comment.