From 7c45dbbc9322f52a9a61d85c78c5e2f3f578998e Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Tue, 30 Apr 2024 10:41:55 +0100 Subject: [PATCH] Extract a vortex-scalar crate (#275) Fixes #63 --- Cargo.lock | 22 +++++++++++++++ vortex-alp/Cargo.toml | 1 + vortex-alp/src/compress.rs | 2 +- vortex-alp/src/compute.rs | 2 +- vortex-array/Cargo.toml | 3 ++- .../src/array/bool/compute/scalar_at.rs | 2 +- vortex-array/src/array/bool/stats.rs | 2 +- vortex-array/src/array/chunked/compute/mod.rs | 2 +- vortex-array/src/array/chunked/stats.rs | 2 +- vortex-array/src/array/composite/compute.rs | 2 +- vortex-array/src/array/constant/compute.rs | 2 +- vortex-array/src/array/constant/flatten.rs | 2 +- vortex-array/src/array/constant/stats.rs | 2 +- .../src/array/primitive/compute/scalar_at.rs | 4 +-- .../array/primitive/compute/search_sorted.rs | 2 +- vortex-array/src/array/primitive/stats.rs | 7 ++--- vortex-array/src/array/sparse/compute/mod.rs | 4 +-- vortex-array/src/array/sparse/flatten.rs | 2 +- vortex-array/src/array/sparse/mod.rs | 2 +- vortex-array/src/array/struct/compute.rs | 2 +- vortex-array/src/array/varbin/builder.rs | 2 +- vortex-array/src/array/varbin/compute/mod.rs | 2 +- vortex-array/src/array/varbin/mod.rs | 2 +- vortex-array/src/array/varbin/stats.rs | 2 +- vortex-array/src/array/varbinview/compute.rs | 2 +- vortex-array/src/array/varbinview/mod.rs | 2 +- vortex-array/src/array/varbinview/stats.rs | 2 +- vortex-array/src/arrow/array.rs | 2 +- vortex-array/src/compute/scalar_at.rs | 2 +- vortex-array/src/compute/search_sorted.rs | 2 +- vortex-array/src/data.rs | 2 +- vortex-array/src/implementation.rs | 2 +- vortex-array/src/lib.rs | 14 +++------- vortex-array/src/stats.rs | 3 +-- vortex-array/src/typed.rs | 2 +- vortex-datetime-parts/Cargo.toml | 1 + vortex-dict/Cargo.toml | 1 + vortex-dict/src/compress.rs | 4 +-- vortex-dict/src/compute.rs | 2 +- vortex-dict/src/stats.rs | 2 +- vortex-fastlanes/Cargo.toml | 1 + vortex-fastlanes/src/bitpacking/compress.rs | 2 +- .../src/bitpacking/compute/mod.rs | 2 +- vortex-fastlanes/src/for/compress.rs | 2 +- vortex-fastlanes/src/for/compute.rs | 2 +- vortex-ree/Cargo.toml | 1 + vortex-ree/src/compute.rs | 2 +- vortex-roaring/Cargo.toml | 1 + vortex-roaring/src/boolean/compute.rs | 2 +- vortex-roaring/src/boolean/mod.rs | 2 +- vortex-roaring/src/integer/compute.rs | 2 +- vortex-scalar/Cargo.toml | 27 +++++++++++++++++++ vortex-scalar/build.rs | 1 + .../flatbuffers/scalar.fbs | 0 .../scalar => vortex-scalar/src}/binary.rs | 4 +-- .../src/scalar => vortex-scalar/src}/bool.rs | 4 +-- .../scalar => vortex-scalar/src}/composite.rs | 2 +- .../scalar/mod.rs => vortex-scalar/src/lib.rs | 21 ++++++++++++++- .../src/scalar => vortex-scalar/src}/list.rs | 2 +- .../src/scalar => vortex-scalar/src}/null.rs | 2 +- .../scalar => vortex-scalar/src}/primitive.rs | 4 +-- .../src/scalar => vortex-scalar/src}/serde.rs | 4 ++- .../scalar => vortex-scalar/src}/struct_.rs | 2 +- .../src/scalar => vortex-scalar/src}/utf8.rs | 4 +-- .../src/scalar => vortex-scalar/src}/value.rs | 0 vortex-zigzag/Cargo.toml | 1 + vortex-zigzag/src/compute.rs | 2 +- 67 files changed, 147 insertions(+), 74 deletions(-) create mode 100644 vortex-scalar/Cargo.toml create mode 120000 vortex-scalar/build.rs rename {vortex-array => vortex-scalar}/flatbuffers/scalar.fbs (100%) rename {vortex-array/src/scalar => vortex-scalar/src}/binary.rs (95%) rename {vortex-array/src/scalar => vortex-scalar/src}/bool.rs (96%) rename {vortex-array/src/scalar => vortex-scalar/src}/composite.rs (96%) rename vortex-array/src/scalar/mod.rs => vortex-scalar/src/lib.rs (92%) rename {vortex-array/src/scalar => vortex-scalar/src}/list.rs (99%) rename {vortex-array/src/scalar => vortex-scalar/src}/null.rs (96%) rename {vortex-array/src/scalar => vortex-scalar/src}/primitive.rs (99%) rename {vortex-array/src/scalar => vortex-scalar/src}/serde.rs (99%) rename {vortex-array/src/scalar => vortex-scalar/src}/struct_.rs (98%) rename {vortex-array/src/scalar => vortex-scalar/src}/utf8.rs (96%) rename {vortex-array/src/scalar => vortex-scalar/src}/value.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 9d10735412..82364b79c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5171,6 +5171,7 @@ dependencies = [ "vortex-array", "vortex-dtype", "vortex-error", + "vortex-scalar", ] [[package]] @@ -5200,6 +5201,7 @@ dependencies = [ "vortex-dtype", "vortex-error", "vortex-flatbuffers", + "vortex-scalar", "walkdir", ] @@ -5213,6 +5215,7 @@ dependencies = [ "vortex-array", "vortex-dtype", "vortex-error", + "vortex-scalar", ] [[package]] @@ -5232,6 +5235,7 @@ dependencies = [ "vortex-array", "vortex-dtype", "vortex-error", + "vortex-scalar", ] [[package]] @@ -5280,6 +5284,7 @@ dependencies = [ "vortex-array", "vortex-dtype", "vortex-error", + "vortex-scalar", ] [[package]] @@ -5332,6 +5337,7 @@ dependencies = [ "vortex-array", "vortex-dtype", "vortex-error", + "vortex-scalar", ] [[package]] @@ -5348,6 +5354,21 @@ dependencies = [ "vortex-array", "vortex-dtype", "vortex-error", + "vortex-scalar", +] + +[[package]] +name = "vortex-scalar" +version = "0.1.0" +dependencies = [ + "flatbuffers", + "flatc", + "itertools 0.12.1", + "serde", + "vortex-dtype", + "vortex-error", + "vortex-flatbuffers", + "walkdir", ] [[package]] @@ -5361,6 +5382,7 @@ dependencies = [ "vortex-dtype", "vortex-error", "vortex-fastlanes", + "vortex-scalar", "zigzag", ] diff --git a/vortex-alp/Cargo.toml b/vortex-alp/Cargo.toml index 4ad0072df4..9e57fd10da 100644 --- a/vortex-alp/Cargo.toml +++ b/vortex-alp/Cargo.toml @@ -24,6 +24,7 @@ serde = { workspace = true, features = ["derive"] } vortex-array = { path = "../vortex-array" } vortex-error = { path = "../vortex-error" } vortex-dtype = { path = "../vortex-dtype" } +vortex-scalar = { path = "../vortex-scalar" } [dev-dependencies] divan = { workspace = true } diff --git a/vortex-alp/src/compress.rs b/vortex-alp/src/compress.rs index c469fda60f..8b8d0ce0f2 100644 --- a/vortex-alp/src/compress.rs +++ b/vortex-alp/src/compress.rs @@ -2,11 +2,11 @@ use itertools::Itertools; use vortex::array::primitive::PrimitiveArray; use vortex::array::sparse::{Sparse, SparseArray}; use vortex::compress::{CompressConfig, CompressCtx, EncodingCompression}; -use vortex::scalar::Scalar; use vortex::validity::Validity; use vortex::{Array, ArrayDType, ArrayDef, AsArray, IntoArray, OwnedArray}; use vortex_dtype::{NativePType, PType}; use vortex_error::{vortex_bail, vortex_err, VortexResult}; +use vortex_scalar::Scalar; use crate::alp::ALPFloat; use crate::array::{ALPArray, ALPEncoding}; diff --git a/vortex-alp/src/compute.rs b/vortex-alp/src/compute.rs index b5710c03ea..169bb56a8d 100644 --- a/vortex-alp/src/compute.rs +++ b/vortex-alp/src/compute.rs @@ -2,9 +2,9 @@ use vortex::compute::scalar_at::{scalar_at, ScalarAtFn}; use vortex::compute::slice::{slice, SliceFn}; use vortex::compute::take::{take, TakeFn}; use vortex::compute::ArrayCompute; -use vortex::scalar::Scalar; use vortex::{Array, ArrayDType, IntoArray, OwnedArray}; use vortex_error::VortexResult; +use vortex_scalar::Scalar; use crate::{match_each_alp_float_ptype, ALPArray}; diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml index be54dc3056..9b5299fba6 100644 --- a/vortex-array/Cargo.toml +++ b/vortex-array/Cargo.toml @@ -36,9 +36,10 @@ paste = { workspace = true } rand = { workspace = true } thiserror = { workspace = true } vortex-alloc = { path = "../vortex-alloc" } +vortex-dtype = { path = "../vortex-dtype", features = ["serde"] } vortex-error = { path = "../vortex-error", features = ["flexbuffers"] } vortex-flatbuffers = { path = "../vortex-flatbuffers" } -vortex-dtype = { path = "../vortex-dtype", features = ["serde"] } +vortex-scalar = { path = "../vortex-scalar", features = ["serde"] } serde = { workspace = true, features = ["derive"] } [build-dependencies] diff --git a/vortex-array/src/array/bool/compute/scalar_at.rs b/vortex-array/src/array/bool/compute/scalar_at.rs index fd711acbd6..06bcd23e82 100644 --- a/vortex-array/src/array/bool/compute/scalar_at.rs +++ b/vortex-array/src/array/bool/compute/scalar_at.rs @@ -1,8 +1,8 @@ use vortex_error::VortexResult; +use vortex_scalar::{BoolScalar, Scalar}; use crate::array::bool::BoolArray; use crate::compute::scalar_at::ScalarAtFn; -use crate::scalar::{BoolScalar, Scalar}; use crate::validity::ArrayValidity; use crate::ArrayDType; diff --git a/vortex-array/src/array/bool/stats.rs b/vortex-array/src/array/bool/stats.rs index 35fe73275e..9ea907f3a8 100644 --- a/vortex-array/src/array/bool/stats.rs +++ b/vortex-array/src/array/bool/stats.rs @@ -1,9 +1,9 @@ use std::collections::HashMap; use vortex_error::VortexResult; +use vortex_scalar::Scalar; use crate::array::bool::BoolArray; -use crate::scalar::Scalar; use crate::stats::{ArrayStatisticsCompute, Stat}; use crate::ArrayTrait; diff --git a/vortex-array/src/array/chunked/compute/mod.rs b/vortex-array/src/array/chunked/compute/mod.rs index 13ecc1889e..9057f619de 100644 --- a/vortex-array/src/array/chunked/compute/mod.rs +++ b/vortex-array/src/array/chunked/compute/mod.rs @@ -1,11 +1,11 @@ use vortex_error::VortexResult; +use vortex_scalar::Scalar; use crate::array::chunked::ChunkedArray; use crate::compute::as_contiguous::{as_contiguous, AsContiguousFn}; use crate::compute::scalar_at::{scalar_at, ScalarAtFn}; use crate::compute::take::TakeFn; use crate::compute::ArrayCompute; -use crate::scalar::Scalar; use crate::{Array, OwnedArray, ToStatic}; mod take; diff --git a/vortex-array/src/array/chunked/stats.rs b/vortex-array/src/array/chunked/stats.rs index c1acf0701e..419c96812a 100644 --- a/vortex-array/src/array/chunked/stats.rs +++ b/vortex-array/src/array/chunked/stats.rs @@ -1,9 +1,9 @@ use std::collections::HashMap; use vortex_error::VortexResult; +use vortex_scalar::Scalar; use crate::array::chunked::ChunkedArray; -use crate::scalar::Scalar; use crate::stats::{ArrayStatisticsCompute, Stat}; impl ArrayStatisticsCompute for ChunkedArray<'_> { diff --git a/vortex-array/src/array/composite/compute.rs b/vortex-array/src/array/composite/compute.rs index 2b371c4740..1c9ca39ccf 100644 --- a/vortex-array/src/array/composite/compute.rs +++ b/vortex-array/src/array/composite/compute.rs @@ -1,6 +1,7 @@ use arrow_array::ArrayRef as ArrowArrayRef; use itertools::Itertools; use vortex_error::{vortex_err, VortexResult}; +use vortex_scalar::Scalar; use crate::array::composite::array::CompositeArray; use crate::compute::as_arrow::AsArrowArray; @@ -9,7 +10,6 @@ use crate::compute::scalar_at::{scalar_at, ScalarAtFn}; use crate::compute::slice::{slice, SliceFn}; use crate::compute::take::{take, TakeFn}; use crate::compute::ArrayCompute; -use crate::scalar::Scalar; use crate::{Array, ArrayDType, IntoArray, OwnedArray}; impl ArrayCompute for CompositeArray<'_> { diff --git a/vortex-array/src/array/constant/compute.rs b/vortex-array/src/array/constant/compute.rs index 09790b1bf1..1698745c35 100644 --- a/vortex-array/src/array/constant/compute.rs +++ b/vortex-array/src/array/constant/compute.rs @@ -1,12 +1,12 @@ use itertools::Itertools; use vortex_error::{vortex_err, VortexResult}; +use vortex_scalar::Scalar; use crate::array::constant::ConstantArray; use crate::compute::as_contiguous::AsContiguousFn; use crate::compute::scalar_at::ScalarAtFn; use crate::compute::take::TakeFn; use crate::compute::ArrayCompute; -use crate::scalar::Scalar; use crate::{Array, ArrayTrait, IntoArray, OwnedArray}; impl ArrayCompute for ConstantArray<'_> { diff --git a/vortex-array/src/array/constant/flatten.rs b/vortex-array/src/array/constant/flatten.rs index 05037a2175..86d2658819 100644 --- a/vortex-array/src/array/constant/flatten.rs +++ b/vortex-array/src/array/constant/flatten.rs @@ -1,10 +1,10 @@ use vortex_dtype::{match_each_native_ptype, Nullability}; use vortex_error::VortexResult; +use vortex_scalar::Scalar; use crate::array::bool::BoolArray; use crate::array::constant::ConstantArray; use crate::array::primitive::PrimitiveArray; -use crate::scalar::Scalar; use crate::validity::Validity; use crate::{ArrayDType, ArrayTrait}; use crate::{ArrayFlatten, Flattened}; diff --git a/vortex-array/src/array/constant/stats.rs b/vortex-array/src/array/constant/stats.rs index 34a4bbb563..16ede2d8fe 100644 --- a/vortex-array/src/array/constant/stats.rs +++ b/vortex-array/src/array/constant/stats.rs @@ -2,9 +2,9 @@ use std::collections::HashMap; use vortex_dtype::DType; use vortex_error::VortexResult; +use vortex_scalar::Scalar; use crate::array::constant::ConstantArray; -use crate::scalar::Scalar; use crate::stats::{ArrayStatisticsCompute, Stat}; use crate::{ArrayDType, ArrayTrait}; diff --git a/vortex-array/src/array/primitive/compute/scalar_at.rs b/vortex-array/src/array/primitive/compute/scalar_at.rs index 0987054460..0cfd8a95ab 100644 --- a/vortex-array/src/array/primitive/compute/scalar_at.rs +++ b/vortex-array/src/array/primitive/compute/scalar_at.rs @@ -1,10 +1,10 @@ use vortex_dtype::match_each_native_ptype; use vortex_error::VortexResult; +use vortex_scalar::PrimitiveScalar; +use vortex_scalar::Scalar; use crate::array::primitive::PrimitiveArray; use crate::compute::scalar_at::ScalarAtFn; -use crate::scalar::PrimitiveScalar; -use crate::scalar::Scalar; use crate::validity::ArrayValidity; use crate::ArrayDType; diff --git a/vortex-array/src/array/primitive/compute/search_sorted.rs b/vortex-array/src/array/primitive/compute/search_sorted.rs index 3251908780..ece561a71b 100644 --- a/vortex-array/src/array/primitive/compute/search_sorted.rs +++ b/vortex-array/src/array/primitive/compute/search_sorted.rs @@ -1,10 +1,10 @@ use vortex_dtype::match_each_native_ptype; use vortex_error::VortexResult; +use vortex_scalar::Scalar; use crate::array::primitive::PrimitiveArray; use crate::compute::search_sorted::{SearchResult, SearchSorted}; use crate::compute::search_sorted::{SearchSortedFn, SearchSortedSide}; -use crate::scalar::Scalar; impl SearchSortedFn for PrimitiveArray<'_> { fn search_sorted(&self, value: &Scalar, side: SearchSortedSide) -> VortexResult { diff --git a/vortex-array/src/array/primitive/stats.rs b/vortex-array/src/array/primitive/stats.rs index 80fb9dfa76..2101e3cbe4 100644 --- a/vortex-array/src/array/primitive/stats.rs +++ b/vortex-array/src/array/primitive/stats.rs @@ -4,10 +4,10 @@ use std::mem::size_of; use arrow_buffer::buffer::BooleanBuffer; use vortex_dtype::match_each_native_ptype; use vortex_error::VortexResult; +use vortex_scalar::{ListScalarVec, PScalar}; +use vortex_scalar::{PScalarType, Scalar}; use crate::array::primitive::PrimitiveArray; -use crate::scalar::{ListScalarVec, PScalar}; -use crate::scalar::{PScalarType, Scalar}; use crate::stats::{ArrayStatisticsCompute, Stat}; use crate::validity::ArrayValidity; use crate::validity::LogicalValidity; @@ -229,8 +229,9 @@ impl StatsAccumulator { #[cfg(test)] mod test { + use vortex_scalar::ListScalarVec; + use crate::array::primitive::PrimitiveArray; - use crate::scalar::ListScalarVec; use crate::stats::{ArrayStatistics, Stat}; #[test] diff --git a/vortex-array/src/array/sparse/compute/mod.rs b/vortex-array/src/array/sparse/compute/mod.rs index 1342475424..ab97fb7a9d 100644 --- a/vortex-array/src/array/sparse/compute/mod.rs +++ b/vortex-array/src/array/sparse/compute/mod.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use itertools::Itertools; use vortex_dtype::match_each_integer_ptype; use vortex_error::{vortex_bail, VortexResult}; +use vortex_scalar::Scalar; use crate::array::primitive::{OwnedPrimitiveArray, PrimitiveArray}; use crate::array::sparse::SparseArray; @@ -11,7 +12,6 @@ use crate::compute::scalar_at::{scalar_at, ScalarAtFn}; use crate::compute::slice::SliceFn; use crate::compute::take::{take, TakeFn}; use crate::compute::ArrayCompute; -use crate::scalar::Scalar; use crate::{Array, ArrayDType, ArrayTrait, IntoArray, OwnedArray}; mod slice; @@ -139,6 +139,7 @@ fn take_search_sorted( mod test { use itertools::Itertools; use vortex_dtype::{DType, FloatWidth, Nullability}; + use vortex_scalar::Scalar; use crate::array::primitive::PrimitiveArray; use crate::array::sparse::compute::take_map; @@ -146,7 +147,6 @@ mod test { use crate::compute::as_contiguous::as_contiguous; use crate::compute::slice::slice; use crate::compute::take::take; - use crate::scalar::Scalar; use crate::validity::Validity; use crate::{ArrayTrait, IntoArray, OwnedArray}; diff --git a/vortex-array/src/array/sparse/flatten.rs b/vortex-array/src/array/sparse/flatten.rs index 78af21a34c..1b785263ce 100644 --- a/vortex-array/src/array/sparse/flatten.rs +++ b/vortex-array/src/array/sparse/flatten.rs @@ -2,10 +2,10 @@ use arrow_buffer::BooleanBufferBuilder; use itertools::Itertools; use vortex_dtype::{match_each_native_ptype, NativePType}; use vortex_error::{VortexError, VortexResult}; +use vortex_scalar::Scalar; use crate::array::primitive::PrimitiveArray; use crate::array::sparse::SparseArray; -use crate::scalar::Scalar; use crate::validity::Validity; use crate::{ArrayFlatten, ArrayTrait, Flattened}; diff --git a/vortex-array/src/array/sparse/mod.rs b/vortex-array/src/array/sparse/mod.rs index 2b9e50ac4a..164bca4bba 100644 --- a/vortex-array/src/array/sparse/mod.rs +++ b/vortex-array/src/array/sparse/mod.rs @@ -177,13 +177,13 @@ mod test { use vortex_dtype::Signedness::Signed; use vortex_dtype::{DType, IntWidth}; use vortex_error::VortexError; + use vortex_scalar::Scalar; use crate::accessor::ArrayAccessor; use crate::array::sparse::SparseArray; use crate::compute::cast::cast; use crate::compute::scalar_at::scalar_at; use crate::compute::slice::slice; - use crate::scalar::Scalar; use crate::{Array, IntoArray, OwnedArray}; fn nullable_fill() -> Scalar { diff --git a/vortex-array/src/array/struct/compute.rs b/vortex-array/src/array/struct/compute.rs index e5233ada6a..94bb83ca63 100644 --- a/vortex-array/src/array/struct/compute.rs +++ b/vortex-array/src/array/struct/compute.rs @@ -6,6 +6,7 @@ use arrow_array::{ use arrow_schema::{Field, Fields}; use itertools::Itertools; use vortex_error::VortexResult; +use vortex_scalar::{Scalar, StructScalar}; use crate::array::r#struct::StructArray; use crate::compute::as_arrow::{as_arrow, AsArrowArray}; @@ -14,7 +15,6 @@ use crate::compute::scalar_at::{scalar_at, ScalarAtFn}; use crate::compute::slice::{slice, SliceFn}; use crate::compute::take::{take, TakeFn}; use crate::compute::ArrayCompute; -use crate::scalar::{Scalar, StructScalar}; use crate::ArrayTrait; use crate::{Array, ArrayDType, IntoArray, OwnedArray}; diff --git a/vortex-array/src/array/varbin/builder.rs b/vortex-array/src/array/varbin/builder.rs index c59309b911..7a73472959 100644 --- a/vortex-array/src/array/varbin/builder.rs +++ b/vortex-array/src/array/varbin/builder.rs @@ -69,10 +69,10 @@ impl VarBinBuilder { mod test { use vortex_dtype::DType; use vortex_dtype::Nullability::Nullable; + use vortex_scalar::Utf8Scalar; use crate::array::varbin::builder::VarBinBuilder; use crate::compute::scalar_at::scalar_at; - use crate::scalar::Utf8Scalar; use crate::{ArrayDType, IntoArray}; #[test] diff --git a/vortex-array/src/array/varbin/compute/mod.rs b/vortex-array/src/array/varbin/compute/mod.rs index 49558d9037..9ac0df46f5 100644 --- a/vortex-array/src/array/varbin/compute/mod.rs +++ b/vortex-array/src/array/varbin/compute/mod.rs @@ -7,6 +7,7 @@ use itertools::Itertools; use vortex_dtype::DType; use vortex_dtype::PType; use vortex_error::{vortex_bail, VortexResult}; +use vortex_scalar::Scalar; use crate::array::primitive::PrimitiveArray; use crate::array::varbin::{varbin_scalar, VarBinArray}; @@ -18,7 +19,6 @@ use crate::compute::scalar_at::ScalarAtFn; use crate::compute::slice::SliceFn; use crate::compute::take::TakeFn; use crate::compute::ArrayCompute; -use crate::scalar::Scalar; use crate::validity::{ArrayValidity, Validity}; use crate::{Array, ArrayDType, IntoArray, OwnedArray, ToArray}; diff --git a/vortex-array/src/array/varbin/mod.rs b/vortex-array/src/array/varbin/mod.rs index 4a1389db76..536eed04f3 100644 --- a/vortex-array/src/array/varbin/mod.rs +++ b/vortex-array/src/array/varbin/mod.rs @@ -3,11 +3,11 @@ use serde::{Deserialize, Serialize}; use vortex_dtype::{match_each_native_ptype, NativePType}; use vortex_dtype::{IntWidth, Nullability, Signedness}; use vortex_error::{vortex_bail, VortexResult}; +use vortex_scalar::{BinaryScalar, Utf8Scalar}; use crate::array::varbin::builder::VarBinBuilder; use crate::compute::scalar_at::scalar_at; use crate::compute::slice::slice; -use crate::scalar::{BinaryScalar, Utf8Scalar}; use crate::validity::{Validity, ValidityMetadata}; use crate::ArrayDType; use crate::{impl_encoding, OwnedArray, ToArrayData}; diff --git a/vortex-array/src/array/varbin/stats.rs b/vortex-array/src/array/varbin/stats.rs index 15c2718117..02981160ac 100644 --- a/vortex-array/src/array/varbin/stats.rs +++ b/vortex-array/src/array/varbin/stats.rs @@ -3,10 +3,10 @@ use std::collections::HashMap; use vortex_dtype::DType; use vortex_error::VortexResult; +use vortex_scalar::Scalar; use crate::accessor::ArrayAccessor; use crate::array::varbin::{varbin_scalar, VarBinArray}; -use crate::scalar::Scalar; use crate::stats::{ArrayStatisticsCompute, Stat}; use crate::{ArrayDType, ArrayTrait}; diff --git a/vortex-array/src/array/varbinview/compute.rs b/vortex-array/src/array/varbinview/compute.rs index ae5caa1758..30668059db 100644 --- a/vortex-array/src/array/varbinview/compute.rs +++ b/vortex-array/src/array/varbinview/compute.rs @@ -7,6 +7,7 @@ use itertools::Itertools; use vortex_dtype::DType; use vortex_dtype::PType; use vortex_error::{vortex_bail, VortexResult}; +use vortex_scalar::Scalar; use crate::array::varbin::varbin_scalar; use crate::array::varbinview::{VarBinViewArray, VIEW_SIZE}; @@ -14,7 +15,6 @@ use crate::compute::as_arrow::AsArrowArray; use crate::compute::scalar_at::ScalarAtFn; use crate::compute::slice::{slice, SliceFn}; use crate::compute::ArrayCompute; -use crate::scalar::Scalar; use crate::validity::ArrayValidity; use crate::{ArrayDType, IntoArray, IntoArrayData, OwnedArray}; diff --git a/vortex-array/src/array/varbinview/mod.rs b/vortex-array/src/array/varbinview/mod.rs index 468937f6af..55c993df07 100644 --- a/vortex-array/src/array/varbinview/mod.rs +++ b/vortex-array/src/array/varbinview/mod.rs @@ -311,12 +311,12 @@ impl EncodingCompression for VarBinViewEncoding {} #[cfg(test)] mod test { use arrow_array::array::StringViewArray as ArrowStringViewArray; + use vortex_scalar::Scalar; use crate::array::varbinview::VarBinViewArray; use crate::compute::as_arrow::as_arrow; use crate::compute::scalar_at::scalar_at; use crate::compute::slice::slice; - use crate::scalar::Scalar; use crate::{ArrayTrait, IntoArray}; #[test] diff --git a/vortex-array/src/array/varbinview/stats.rs b/vortex-array/src/array/varbinview/stats.rs index 5d7b8f6976..8c7047f7f9 100644 --- a/vortex-array/src/array/varbinview/stats.rs +++ b/vortex-array/src/array/varbinview/stats.rs @@ -1,11 +1,11 @@ use std::collections::HashMap; use vortex_error::VortexResult; +use vortex_scalar::Scalar; use crate::accessor::ArrayAccessor; use crate::array::varbin::compute_stats; use crate::array::varbinview::VarBinViewArray; -use crate::scalar::Scalar; use crate::stats::{ArrayStatisticsCompute, Stat}; use crate::{ArrayDType, ArrayTrait}; diff --git a/vortex-array/src/arrow/array.rs b/vortex-array/src/arrow/array.rs index a620e5c731..13a3c3724e 100644 --- a/vortex-array/src/arrow/array.rs +++ b/vortex-array/src/arrow/array.rs @@ -23,6 +23,7 @@ use arrow_buffer::{ArrowNativeType, Buffer, ScalarBuffer}; use arrow_schema::{DataType, TimeUnit}; use vortex_dtype::DType; use vortex_dtype::NativePType; +use vortex_scalar::NullScalar; use crate::array::bool::BoolArray; use crate::array::constant::ConstantArray; @@ -32,7 +33,6 @@ use crate::array::r#struct::StructArray; use crate::array::varbin::VarBinArray; use crate::array::varbinview::VarBinViewArray; use crate::arrow::FromArrowArray; -use crate::scalar::NullScalar; use crate::stats::{Stat, Statistics}; use crate::validity::Validity; use crate::{ArrayData, IntoArray, IntoArrayData}; diff --git a/vortex-array/src/compute/scalar_at.rs b/vortex-array/src/compute/scalar_at.rs index 86787aff59..131ac19d1e 100644 --- a/vortex-array/src/compute/scalar_at.rs +++ b/vortex-array/src/compute/scalar_at.rs @@ -1,6 +1,6 @@ use vortex_error::{vortex_bail, vortex_err, VortexResult}; +use vortex_scalar::Scalar; -use crate::scalar::Scalar; use crate::Array; pub trait ScalarAtFn { diff --git a/vortex-array/src/compute/search_sorted.rs b/vortex-array/src/compute/search_sorted.rs index 5f73e778a0..74acf1e499 100644 --- a/vortex-array/src/compute/search_sorted.rs +++ b/vortex-array/src/compute/search_sorted.rs @@ -2,9 +2,9 @@ use std::cmp::Ordering; use std::cmp::Ordering::{Equal, Greater, Less}; use vortex_error::{vortex_err, VortexResult}; +use vortex_scalar::Scalar; use crate::compute::scalar_at::scalar_at; -use crate::scalar::Scalar; use crate::{Array, ArrayDType}; #[derive(Debug, Copy, Clone)] diff --git a/vortex-array/src/data.rs b/vortex-array/src/data.rs index d6c6920ab6..6137a06545 100644 --- a/vortex-array/src/data.rs +++ b/vortex-array/src/data.rs @@ -3,10 +3,10 @@ use std::sync::{Arc, RwLock}; use vortex_dtype::DType; use vortex_error::VortexResult; +use vortex_scalar::Scalar; use crate::buffer::{Buffer, OwnedBuffer}; use crate::encoding::EncodingRef; -use crate::scalar::Scalar; use crate::stats::Stat; use crate::stats::Statistics; use crate::{Array, ArrayMetadata, IntoArray, OwnedArray, ToArray}; diff --git a/vortex-array/src/implementation.rs b/vortex-array/src/implementation.rs index 0064435a26..fff2fcee31 100644 --- a/vortex-array/src/implementation.rs +++ b/vortex-array/src/implementation.rs @@ -51,7 +51,6 @@ macro_rules! impl_encoding { VORTEX_ENCODINGS, }; use $crate::stats::Stat; - use $crate::scalar::Scalar; use std::any::Any; use std::collections::HashMap; use std::fmt::Debug; @@ -59,6 +58,7 @@ macro_rules! impl_encoding { use std::sync::Arc; use vortex_error::VortexError; use vortex_dtype::DType; + use vortex_scalar::Scalar; /// The array definition trait #[derive(Debug, Clone)] diff --git a/vortex-array/src/lib.rs b/vortex-array/src/lib.rs index fcd242de5a..a125258738 100644 --- a/vortex-array/src/lib.rs +++ b/vortex-array/src/lib.rs @@ -11,7 +11,6 @@ mod flatten; mod implementation; mod metadata; mod sampling; -pub mod scalar; pub mod stats; mod tree; mod typed; @@ -43,7 +42,6 @@ use crate::visitor::{AcceptArrayVisitor, ArrayVisitor}; pub mod flatbuffers { pub use gen_array::vortex::*; - pub use gen_scalar::vortex::*; #[allow(unused_imports)] #[allow(dead_code)] @@ -53,19 +51,15 @@ pub mod flatbuffers { include!(concat!(env!("OUT_DIR"), "/flatbuffers/array.rs")); } - #[allow(unused_imports)] - #[allow(dead_code)] - #[allow(non_camel_case_types)] - #[allow(clippy::all)] - mod gen_scalar { - include!(concat!(env!("OUT_DIR"), "/flatbuffers/scalar.rs")); - } - mod deps { pub mod dtype { #[allow(unused_imports)] pub use vortex_dtype::flatbuffers as dtype; } + pub mod scalar { + #[allow(unused_imports)] + pub use vortex_scalar::flatbuffers as scalar; + } } } diff --git a/vortex-array/src/stats.rs b/vortex-array/src/stats.rs index e4a3ffa306..d5cdcf1d1a 100644 --- a/vortex-array/src/stats.rs +++ b/vortex-array/src/stats.rs @@ -3,8 +3,7 @@ use std::collections::HashMap; use vortex_dtype::DType; use vortex_dtype::NativePType; use vortex_error::VortexResult; - -use crate::scalar::Scalar; +use vortex_scalar::Scalar; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Stat { diff --git a/vortex-array/src/typed.rs b/vortex-array/src/typed.rs index e116e18120..f2464dd2eb 100644 --- a/vortex-array/src/typed.rs +++ b/vortex-array/src/typed.rs @@ -3,9 +3,9 @@ use std::sync::Arc; use vortex_dtype::DType; use vortex_error::{vortex_err, VortexError, VortexResult}; +use vortex_scalar::Scalar; use crate::buffer::OwnedBuffer; -use crate::scalar::Scalar; use crate::stats::Stat; use crate::{Array, ArrayData, ArrayDef, AsArray, IntoArray, ToArray, TryDeserializeArrayMetadata}; diff --git a/vortex-datetime-parts/Cargo.toml b/vortex-datetime-parts/Cargo.toml index f5dd3239c6..cbcf57f4cd 100644 --- a/vortex-datetime-parts/Cargo.toml +++ b/vortex-datetime-parts/Cargo.toml @@ -10,6 +10,7 @@ workspace = true vortex-array = { "path" = "../vortex-array" } vortex-error = { path = "../vortex-error" } vortex-dtype = { "path" = "../vortex-dtype" } +vortex-scalar = { path = "../vortex-scalar" } linkme = { workspace = true } log = { workspace = true } serde = { workspace = true, features = ["derive"] } diff --git a/vortex-dict/Cargo.toml b/vortex-dict/Cargo.toml index e2f2dd1c2e..3db1d3b95e 100644 --- a/vortex-dict/Cargo.toml +++ b/vortex-dict/Cargo.toml @@ -22,6 +22,7 @@ serde = { workspace = true } vortex-array = { path = "../vortex-array" } vortex-error = { path = "../vortex-error" } vortex-dtype = { path = "../vortex-dtype" } +vortex-scalar = { path = "../vortex-scalar" } [lints] workspace = true diff --git a/vortex-dict/src/compress.rs b/vortex-dict/src/compress.rs index b4ed6d5033..e24c61771f 100644 --- a/vortex-dict/src/compress.rs +++ b/vortex-dict/src/compress.rs @@ -8,13 +8,13 @@ use vortex::accessor::ArrayAccessor; use vortex::array::primitive::{Primitive, PrimitiveArray}; use vortex::array::varbin::{VarBin, VarBinArray}; use vortex::compress::{CompressConfig, CompressCtx, EncodingCompression}; -use vortex::scalar::AsBytes; use vortex::stats::{ArrayStatistics, Stat}; use vortex::validity::Validity; use vortex::{Array, ArrayDType, ArrayDef, IntoArray, OwnedArray, ToArray}; use vortex_dtype::NativePType; use vortex_dtype::{match_each_native_ptype, DType}; use vortex_error::VortexResult; +use vortex_scalar::AsBytes; use crate::dict::{DictArray, DictEncoding}; @@ -254,7 +254,7 @@ mod test { use vortex::array::primitive::PrimitiveArray; use vortex::array::varbin::VarBinArray; use vortex::compute::scalar_at::scalar_at; - use vortex::scalar::PrimitiveScalar; + use vortex_scalar::PrimitiveScalar; use crate::compress::{dict_encode_typed_primitive, dict_encode_varbin}; diff --git a/vortex-dict/src/compute.rs b/vortex-dict/src/compute.rs index c4dc88b290..ef26882dd9 100644 --- a/vortex-dict/src/compute.rs +++ b/vortex-dict/src/compute.rs @@ -2,9 +2,9 @@ use vortex::compute::scalar_at::{scalar_at, ScalarAtFn}; use vortex::compute::slice::{slice, SliceFn}; use vortex::compute::take::{take, TakeFn}; use vortex::compute::ArrayCompute; -use vortex::scalar::Scalar; use vortex::{Array, IntoArray, OwnedArray}; use vortex_error::VortexResult; +use vortex_scalar::Scalar; use crate::DictArray; diff --git a/vortex-dict/src/stats.rs b/vortex-dict/src/stats.rs index 7002c0d3f2..1b0ebe40c8 100644 --- a/vortex-dict/src/stats.rs +++ b/vortex-dict/src/stats.rs @@ -1,8 +1,8 @@ use std::collections::HashMap; -use vortex::scalar::Scalar; use vortex::stats::{ArrayStatistics, ArrayStatisticsCompute, Stat}; use vortex_error::VortexResult; +use vortex_scalar::Scalar; use crate::dict::DictArray; diff --git a/vortex-fastlanes/Cargo.toml b/vortex-fastlanes/Cargo.toml index da4620b906..6b286967ef 100644 --- a/vortex-fastlanes/Cargo.toml +++ b/vortex-fastlanes/Cargo.toml @@ -26,6 +26,7 @@ serde = { workspace = true } vortex-array = { path = "../vortex-array" } vortex-error = { path = "../vortex-error" } vortex-dtype = { path = "../vortex-dtype" } +vortex-scalar = { path = "../vortex-scalar" } [dev-dependencies] criterion = { workspace = true } diff --git a/vortex-fastlanes/src/bitpacking/compress.rs b/vortex-fastlanes/src/bitpacking/compress.rs index 59136a127f..8ee3791c6e 100644 --- a/vortex-fastlanes/src/bitpacking/compress.rs +++ b/vortex-fastlanes/src/bitpacking/compress.rs @@ -4,13 +4,13 @@ use vortex::array::primitive::PrimitiveArray; use vortex::array::sparse::{Sparse, SparseArray}; use vortex::compress::{CompressConfig, CompressCtx, EncodingCompression}; use vortex::compute::cast::cast; -use vortex::scalar::{ListScalarVec, Scalar}; use vortex::stats::{ArrayStatistics, Stat}; use vortex::validity::Validity; use vortex::{Array, ArrayDType, ArrayDef, ArrayTrait, IntoArray, OwnedArray, ToStatic}; use vortex_dtype::PType::U8; use vortex_dtype::{match_each_integer_ptype, NativePType, PType}; use vortex_error::{vortex_bail, vortex_err, VortexResult}; +use vortex_scalar::{ListScalarVec, Scalar}; use crate::{match_integers_by_width, BitPackedArray, BitPackedEncoding}; diff --git a/vortex-fastlanes/src/bitpacking/compute/mod.rs b/vortex-fastlanes/src/bitpacking/compute/mod.rs index 407d58bf02..1806b55611 100644 --- a/vortex-fastlanes/src/bitpacking/compute/mod.rs +++ b/vortex-fastlanes/src/bitpacking/compute/mod.rs @@ -9,10 +9,10 @@ use vortex::compute::scalar_at::{scalar_at, ScalarAtFn}; use vortex::compute::slice::{slice, SliceFn}; use vortex::compute::take::{take, TakeFn}; use vortex::compute::ArrayCompute; -use vortex::scalar::Scalar; use vortex::{Array, ArrayDType, ArrayTrait, IntoArray, OwnedArray}; use vortex_dtype::{match_each_integer_ptype, NativePType}; use vortex_error::{vortex_bail, vortex_err, VortexResult}; +use vortex_scalar::Scalar; use crate::bitpacking::compress::unpack_single; use crate::{match_integers_by_width, unpack_single_primitive, BitPackedArray}; diff --git a/vortex-fastlanes/src/for/compress.rs b/vortex-fastlanes/src/for/compress.rs index dfc02cfda9..c54a3375dd 100644 --- a/vortex-fastlanes/src/for/compress.rs +++ b/vortex-fastlanes/src/for/compress.rs @@ -3,11 +3,11 @@ use num_traits::{PrimInt, WrappingAdd, WrappingSub}; use vortex::array::constant::ConstantArray; use vortex::array::primitive::PrimitiveArray; use vortex::compress::{CompressConfig, CompressCtx, EncodingCompression}; -use vortex::scalar::ListScalarVec; use vortex::stats::{ArrayStatistics, Stat}; use vortex::{Array, ArrayDType, ArrayTrait, IntoArray, OwnedArray}; use vortex_dtype::{match_each_integer_ptype, NativePType, PType}; use vortex_error::{vortex_err, VortexResult}; +use vortex_scalar::ListScalarVec; use crate::{FoRArray, FoREncoding}; diff --git a/vortex-fastlanes/src/for/compute.rs b/vortex-fastlanes/src/for/compute.rs index 5025409422..205dc1c1af 100644 --- a/vortex-fastlanes/src/for/compute.rs +++ b/vortex-fastlanes/src/for/compute.rs @@ -2,10 +2,10 @@ use vortex::compute::scalar_at::{scalar_at, ScalarAtFn}; use vortex::compute::slice::{slice, SliceFn}; use vortex::compute::take::{take, TakeFn}; use vortex::compute::ArrayCompute; -use vortex::scalar::{PrimitiveScalar, Scalar}; use vortex::{Array, IntoArray, OwnedArray}; use vortex_dtype::match_each_integer_ptype; use vortex_error::VortexResult; +use vortex_scalar::{PrimitiveScalar, Scalar}; use crate::FoRArray; diff --git a/vortex-ree/Cargo.toml b/vortex-ree/Cargo.toml index 1184fd8327..2e8d54ba06 100644 --- a/vortex-ree/Cargo.toml +++ b/vortex-ree/Cargo.toml @@ -22,6 +22,7 @@ serde = { workspace = true } vortex-array = { path = "../vortex-array" } vortex-error = { path = "../vortex-error" } vortex-dtype = { path = "../vortex-dtype" } +vortex-scalar = { path = "../vortex-scalar" } [lints] workspace = true diff --git a/vortex-ree/src/compute.rs b/vortex-ree/src/compute.rs index 2e701d11f6..4d2739eef2 100644 --- a/vortex-ree/src/compute.rs +++ b/vortex-ree/src/compute.rs @@ -3,10 +3,10 @@ use vortex::compute::scalar_at::{scalar_at, ScalarAtFn}; use vortex::compute::slice::{slice, SliceFn}; use vortex::compute::take::{take, TakeFn}; use vortex::compute::ArrayCompute; -use vortex::scalar::Scalar; use vortex::{Array, IntoArray, OwnedArray}; use vortex_dtype::match_each_integer_ptype; use vortex_error::VortexResult; +use vortex_scalar::Scalar; use crate::REEArray; diff --git a/vortex-roaring/Cargo.toml b/vortex-roaring/Cargo.toml index 7bd42aac7d..5f609b9346 100644 --- a/vortex-roaring/Cargo.toml +++ b/vortex-roaring/Cargo.toml @@ -16,6 +16,7 @@ arrow-buffer = { workspace = true } vortex-array = { path = "../vortex-array" } vortex-error = { path = "../vortex-error" } vortex-dtype = { path = "../vortex-dtype" } +vortex-scalar = { path = "../vortex-scalar" } linkme = { workspace = true } croaring = { workspace = true } num-traits = { workspace = true } diff --git a/vortex-roaring/src/boolean/compute.rs b/vortex-roaring/src/boolean/compute.rs index 272a9aff11..f252438c66 100644 --- a/vortex-roaring/src/boolean/compute.rs +++ b/vortex-roaring/src/boolean/compute.rs @@ -2,9 +2,9 @@ use croaring::Bitmap; use vortex::compute::scalar_at::ScalarAtFn; use vortex::compute::slice::SliceFn; use vortex::compute::ArrayCompute; -use vortex::scalar::Scalar; use vortex::{IntoArray, OwnedArray}; use vortex_error::VortexResult; +use vortex_scalar::Scalar; use crate::RoaringBoolArray; diff --git a/vortex-roaring/src/boolean/mod.rs b/vortex-roaring/src/boolean/mod.rs index 791ba2f1a0..7956173886 100644 --- a/vortex-roaring/src/boolean/mod.rs +++ b/vortex-roaring/src/boolean/mod.rs @@ -115,9 +115,9 @@ impl ArrayFlatten for RoaringBoolArray<'_> { mod test { use vortex::array::bool::BoolArray; use vortex::compute::scalar_at::scalar_at; - use vortex::scalar::Scalar; use vortex::IntoArray; use vortex_error::VortexResult; + use vortex_scalar::Scalar; use crate::RoaringBoolArray; diff --git a/vortex-roaring/src/integer/compute.rs b/vortex-roaring/src/integer/compute.rs index fe3855ae2a..7d2a0a6287 100644 --- a/vortex-roaring/src/integer/compute.rs +++ b/vortex-roaring/src/integer/compute.rs @@ -1,8 +1,8 @@ use vortex::compute::scalar_at::ScalarAtFn; use vortex::compute::ArrayCompute; -use vortex::scalar::Scalar; use vortex_dtype::PType; use vortex_error::VortexResult; +use vortex_scalar::Scalar; use crate::RoaringIntArray; diff --git a/vortex-scalar/Cargo.toml b/vortex-scalar/Cargo.toml new file mode 100644 index 0000000000..4f4b4605d4 --- /dev/null +++ b/vortex-scalar/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "vortex-scalar" +version = { workspace = true } +description = "Vortex Scalars" +homepage = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +license = { workspace = true } +keywords = { workspace = true } +include = { workspace = true } +edition = { workspace = true } +rust-version = { workspace = true } + +[dependencies] +flatbuffers = { workspace = true } +itertools = { workspace = true } +serde = { workspace = true, optional = true } +vortex-dtype = { path = "../vortex-dtype" } +vortex-error = { path = "../vortex-error" } +vortex-flatbuffers = { path = "../vortex-flatbuffers" } + +[build-dependencies] +flatc = { workspace = true } +walkdir = { workspace = true } + +[lints] +workspace = true diff --git a/vortex-scalar/build.rs b/vortex-scalar/build.rs new file mode 120000 index 0000000000..7cb528993c --- /dev/null +++ b/vortex-scalar/build.rs @@ -0,0 +1 @@ +../flatbuffers.build.rs \ No newline at end of file diff --git a/vortex-array/flatbuffers/scalar.fbs b/vortex-scalar/flatbuffers/scalar.fbs similarity index 100% rename from vortex-array/flatbuffers/scalar.fbs rename to vortex-scalar/flatbuffers/scalar.fbs diff --git a/vortex-array/src/scalar/binary.rs b/vortex-scalar/src/binary.rs similarity index 95% rename from vortex-array/src/scalar/binary.rs rename to vortex-scalar/src/binary.rs index 136d73101b..22129782e6 100644 --- a/vortex-array/src/scalar/binary.rs +++ b/vortex-scalar/src/binary.rs @@ -4,8 +4,8 @@ use vortex_dtype::DType; use vortex_dtype::Nullability::{NonNullable, Nullable}; use vortex_error::{vortex_bail, vortex_err, VortexError, VortexResult}; -use crate::scalar::value::ScalarValue; -use crate::scalar::Scalar; +use crate::value::ScalarValue; +use crate::Scalar; pub type BinaryScalar = ScalarValue>; diff --git a/vortex-array/src/scalar/bool.rs b/vortex-scalar/src/bool.rs similarity index 96% rename from vortex-array/src/scalar/bool.rs rename to vortex-scalar/src/bool.rs index 806640357c..c069750487 100644 --- a/vortex-array/src/scalar/bool.rs +++ b/vortex-scalar/src/bool.rs @@ -3,8 +3,8 @@ use std::fmt::{Display, Formatter}; use vortex_dtype::{DType, Nullability}; use vortex_error::{vortex_bail, vortex_err, VortexError, VortexResult}; -use crate::scalar::value::ScalarValue; -use crate::scalar::Scalar; +use crate::value::ScalarValue; +use crate::Scalar; pub type BoolScalar = ScalarValue; diff --git a/vortex-array/src/scalar/composite.rs b/vortex-scalar/src/composite.rs similarity index 96% rename from vortex-array/src/scalar/composite.rs rename to vortex-scalar/src/composite.rs index 4b0cbceceb..a51f353abc 100644 --- a/vortex-array/src/scalar/composite.rs +++ b/vortex-scalar/src/composite.rs @@ -3,7 +3,7 @@ use std::fmt::{Display, Formatter}; use vortex_dtype::DType; use vortex_error::VortexResult; -use crate::scalar::Scalar; +use crate::Scalar; #[derive(Debug, Clone, PartialEq, PartialOrd)] pub struct CompositeScalar { diff --git a/vortex-array/src/scalar/mod.rs b/vortex-scalar/src/lib.rs similarity index 92% rename from vortex-array/src/scalar/mod.rs rename to vortex-scalar/src/lib.rs index b38ea6201c..dd12c39982 100644 --- a/vortex-array/src/scalar/mod.rs +++ b/vortex-scalar/src/lib.rs @@ -24,6 +24,25 @@ mod struct_; mod utf8; mod value; +pub mod flatbuffers { + pub use gen_scalar::vortex::*; + + #[allow(unused_imports)] + #[allow(dead_code)] + #[allow(non_camel_case_types)] + #[allow(clippy::all)] + mod gen_scalar { + include!(concat!(env!("OUT_DIR"), "/flatbuffers/scalar.rs")); + } + + mod deps { + pub mod dtype { + #[allow(unused_imports)] + pub use vortex_dtype::flatbuffers as dtype; + } + } +} + #[derive(Debug, Clone, PartialEq, PartialOrd)] pub enum Scalar { Binary(BinaryScalar), @@ -156,7 +175,7 @@ impl AsBytes for T { mod test { use std::mem; - use crate::scalar::Scalar; + use crate::Scalar; #[test] fn size_of() { diff --git a/vortex-array/src/scalar/list.rs b/vortex-scalar/src/list.rs similarity index 99% rename from vortex-array/src/scalar/list.rs rename to vortex-scalar/src/list.rs index a0f8e47ca8..77f6b69c3a 100644 --- a/vortex-array/src/scalar/list.rs +++ b/vortex-scalar/src/list.rs @@ -4,7 +4,7 @@ use itertools::Itertools; use vortex_dtype::DType; use vortex_error::{vortex_err, VortexError, VortexResult}; -use crate::scalar::Scalar; +use crate::Scalar; #[derive(Debug, Clone, PartialEq, PartialOrd)] pub struct ListScalar { diff --git a/vortex-array/src/scalar/null.rs b/vortex-scalar/src/null.rs similarity index 96% rename from vortex-array/src/scalar/null.rs rename to vortex-scalar/src/null.rs index 2b157f143f..5e8208a9ba 100644 --- a/vortex-array/src/scalar/null.rs +++ b/vortex-scalar/src/null.rs @@ -3,7 +3,7 @@ use std::fmt::{Display, Formatter}; use vortex_dtype::DType; use vortex_error::VortexResult; -use crate::scalar::Scalar; +use crate::Scalar; #[derive(Debug, Clone, PartialEq, PartialOrd)] pub struct NullScalar; diff --git a/vortex-array/src/scalar/primitive.rs b/vortex-scalar/src/primitive.rs similarity index 99% rename from vortex-array/src/scalar/primitive.rs rename to vortex-scalar/src/primitive.rs index 5570c55370..552100a29d 100644 --- a/vortex-array/src/scalar/primitive.rs +++ b/vortex-scalar/src/primitive.rs @@ -8,7 +8,7 @@ use vortex_dtype::{DType, Nullability}; use vortex_dtype::{NativePType, PType}; use vortex_error::{vortex_bail, vortex_err, VortexError, VortexResult}; -use crate::scalar::Scalar; +use crate::Scalar; pub trait PScalarType: NativePType + Into + TryFrom {} impl + TryFrom> PScalarType for T {} @@ -344,7 +344,7 @@ mod test { use vortex_dtype::{DType, IntWidth, Nullability, Signedness}; use vortex_error::VortexError; - use crate::scalar::Scalar; + use crate::Scalar; #[test] fn into_from() { diff --git a/vortex-array/src/scalar/serde.rs b/vortex-scalar/src/serde.rs similarity index 99% rename from vortex-array/src/scalar/serde.rs rename to vortex-scalar/src/serde.rs index 984490562e..8d953cc878 100644 --- a/vortex-array/src/scalar/serde.rs +++ b/vortex-scalar/src/serde.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "serde")] + use flatbuffers::{root, FlatBufferBuilder, WIPOffset}; use serde::de::Visitor; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -8,7 +10,7 @@ use vortex_error::{vortex_bail, VortexError}; use vortex_flatbuffers::{FlatBufferRoot, FlatBufferToBytes, ReadFlatBuffer, WriteFlatBuffer}; use crate::flatbuffers::scalar as fb; -use crate::scalar::{PScalar, PrimitiveScalar, Scalar, Utf8Scalar}; +use crate::{PScalar, PrimitiveScalar, Scalar, Utf8Scalar}; impl FlatBufferRoot for Scalar {} diff --git a/vortex-array/src/scalar/struct_.rs b/vortex-scalar/src/struct_.rs similarity index 98% rename from vortex-array/src/scalar/struct_.rs rename to vortex-scalar/src/struct_.rs index f234d7a14b..50abe21673 100644 --- a/vortex-array/src/scalar/struct_.rs +++ b/vortex-scalar/src/struct_.rs @@ -6,7 +6,7 @@ use itertools::Itertools; use vortex_dtype::DType; use vortex_error::{vortex_bail, vortex_err, VortexResult}; -use crate::scalar::Scalar; +use crate::Scalar; #[derive(Debug, Clone, PartialEq)] pub struct StructScalar { diff --git a/vortex-array/src/scalar/utf8.rs b/vortex-scalar/src/utf8.rs similarity index 96% rename from vortex-array/src/scalar/utf8.rs rename to vortex-scalar/src/utf8.rs index 9d742205e4..323c61838b 100644 --- a/vortex-array/src/scalar/utf8.rs +++ b/vortex-scalar/src/utf8.rs @@ -3,8 +3,8 @@ use std::fmt::{Display, Formatter}; use vortex_dtype::{DType, Nullability::NonNullable, Nullability::Nullable}; use vortex_error::{vortex_bail, vortex_err, VortexError, VortexResult}; -use crate::scalar::value::ScalarValue; -use crate::scalar::Scalar; +use crate::value::ScalarValue; +use crate::Scalar; pub type Utf8Scalar = ScalarValue; diff --git a/vortex-array/src/scalar/value.rs b/vortex-scalar/src/value.rs similarity index 100% rename from vortex-array/src/scalar/value.rs rename to vortex-scalar/src/value.rs diff --git a/vortex-zigzag/Cargo.toml b/vortex-zigzag/Cargo.toml index 804c44b16f..ce25f8b38a 100644 --- a/vortex-zigzag/Cargo.toml +++ b/vortex-zigzag/Cargo.toml @@ -18,6 +18,7 @@ vortex-array = { path = "../vortex-array" } vortex-error = { path = "../vortex-error" } vortex-fastlanes = { path = "../vortex-fastlanes" } vortex-dtype = { path = "../vortex-dtype" } +vortex-scalar = { path = "../vortex-scalar" } zigzag = { workspace = true } serde = { workspace = true, features = ["derive"] } diff --git a/vortex-zigzag/src/compute.rs b/vortex-zigzag/src/compute.rs index 1aca390a55..fc43083924 100644 --- a/vortex-zigzag/src/compute.rs +++ b/vortex-zigzag/src/compute.rs @@ -1,9 +1,9 @@ use vortex::compute::scalar_at::{scalar_at, ScalarAtFn}; use vortex::compute::slice::{slice, SliceFn}; use vortex::compute::ArrayCompute; -use vortex::scalar::{PScalar, Scalar}; use vortex::{ArrayDType, IntoArray, OwnedArray}; use vortex_error::{vortex_err, VortexResult}; +use vortex_scalar::{PScalar, Scalar}; use zigzag::ZigZag as ExternalZigZag; use crate::ZigZagArray;