merge

Cargo.toml

@@ -7,6 +7,7 @@ members = [
     "pyvortex",
     "vortex-alp",
     "vortex-array",
+    "vortex-buffer",
     "vortex-datetime-parts",
     "vortex-dict",
     "vortex-error",
@@ -48,7 +49,7 @@ arrow-ipc = "51.0.0"
 arrow-schema = "51.0.0"
 arrow-select = "51.0.0"
 bindgen = "0.69.4"
-bytes = "1.0.1"
+bytes = "1.6.0"
 bzip2 = "0.4.4"
 criterion = { version = "0.5.1", features = ["html_reports"] }
 croaring = "1.0.1"

vortex-array/Cargo.toml

@@ -33,6 +33,7 @@ num-traits = { workspace = true }
 num_enum = { workspace = true }
 paste = { workspace = true }
 rand = { workspace = true }
+vortex-buffer = { path = "../vortex-buffer" }
 vortex-dtype = { path = "../vortex-dtype", features = ["flatbuffers", "serde"] }
 vortex-error = { path = "../vortex-error", features = ["flexbuffers"] }
 vortex-flatbuffers = { path = "../vortex-flatbuffers" }

vortex-array/src/array/bool/mod.rs

@@ -1,8 +1,8 @@
 use arrow_buffer::BooleanBuffer;
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
+use vortex_buffer::Buffer;
 
-use crate::buffer::Buffer;
 use crate::validity::{ArrayValidity, ValidityMetadata};
 use crate::validity::{LogicalValidity, Validity};
 use crate::visitor::{AcceptArrayVisitor, ArrayVisitor};
@@ -44,7 +44,7 @@ impl BoolArray<'_> {
                     validity: validity.to_metadata(buffer.len())?,
                     length: buffer.len(),
                 },
-                Some(Buffer::Owned(buffer.into_inner())),
+                Some(Buffer::from(buffer.into_inner())),
                 validity.into_array_data().into_iter().collect_vec().into(),
                 StatsSet::new(),
             )?,

vortex-array/src/array/primitive/compute/as_contiguous.rs

@@ -20,7 +20,7 @@ impl AsContiguousFn for PrimitiveArray<'_> {
             arrays.iter().map(|a| a.len()).sum::<usize>() * self.ptype().byte_width(),
         );
         for array in arrays {
-            buffer.extend_from_slice(array.as_primitive().buffer().as_slice())
+            buffer.extend_from_slice(array.as_primitive().buffer())
         }
         match_each_native_ptype!(self.ptype(), |$T| {
             Ok(PrimitiveArray::try_new(ScalarBuffer::<$T>::from(buffer), validity)

vortex-array/src/array/primitive/mod.rs

@@ -2,10 +2,10 @@ use arrow_buffer::{ArrowNativeType, ScalarBuffer};
 use itertools::Itertools;
 use num_traits::AsPrimitive;
 use serde::{Deserialize, Serialize};
+use vortex_buffer::Buffer;
 use vortex_dtype::{match_each_native_ptype, NativePType, PType};
 use vortex_error::vortex_bail;
 
-use crate::buffer::Buffer;
 use crate::validity::{ArrayValidity, LogicalValidity, Validity, ValidityMetadata};
 use crate::visitor::{AcceptArrayVisitor, ArrayVisitor};
 use crate::ArrayFlatten;
@@ -34,7 +34,7 @@ impl PrimitiveArray<'_> {
                 PrimitiveMetadata {
                     validity: validity.to_metadata(buffer.len())?,
                 },
-                Some(Buffer::Owned(buffer.into_inner())),
+                Some(Buffer::from(buffer.into_inner())),
                 validity.into_array_data().into_iter().collect_vec().into(),
                 StatsSet::new(),
             )?,
@@ -134,11 +134,11 @@ impl PrimitiveArray<'_> {
         }
         Ok(Self::from_vec(own_values, validity))
     }
-}
 
-impl<'a> PrimitiveArray<'a> {
-    pub fn into_buffer(self) -> Buffer<'a> {
-        self.into_array().into_buffer().unwrap()
+    pub fn into_buffer(self) -> Buffer {
+        self.into_array()
+            .into_buffer()
+            .expect("PrimitiveArray must have a buffer")
     }
 }
 

vortex-array/src/array/varbin/compute/mod.rs

@@ -136,8 +136,13 @@ impl AsArrowArray for VarBinArray<'_> {
 impl ScalarAtFn for VarBinArray<'_> {
     fn scalar_at(&self, index: usize) -> VortexResult<Scalar> {
         if self.is_valid(index) {
-            self.bytes_at(index)
-                .map(|bytes| varbin_scalar(bytes, self.dtype()))
+            Ok(varbin_scalar(
+                self.bytes_at(index)?
+                    // TODO(ngates): update to use buffer when we refactor scalars.
+                    .into_vec::<u8>()
+                    .unwrap_or_else(|b| b.as_ref().to_vec()),
+                self.dtype(),
+            ))
         } else {
             Ok(Scalar::null(self.dtype()))
         }

vortex-array/src/array/varbin/mod.rs

@@ -19,6 +19,7 @@ mod flatten;
 mod stats;
 
 pub use stats::compute_stats;
+use vortex_buffer::Buffer;
 
 use crate::array::primitive::PrimitiveArray;
 
@@ -147,11 +148,11 @@ impl VarBinArray<'_> {
             })
     }
 
-    pub fn bytes_at(&self, index: usize) -> VortexResult<Vec<u8>> {
+    pub fn bytes_at(&self, index: usize) -> VortexResult<Buffer> {
         let start = self.offset_at(index);
         let end = self.offset_at(index + 1);
         let sliced = slice(&self.bytes(), start, end)?;
-        Ok(sliced.flatten_primitive()?.buffer().as_slice().to_vec())
+        Ok(sliced.flatten_primitive()?.buffer().clone())
     }
 }
 

vortex-array/src/data.rs

@@ -1,10 +1,10 @@
 use std::sync::{Arc, RwLock};
 
+use vortex_buffer::Buffer;
 use vortex_dtype::DType;
 use vortex_error::{vortex_err, VortexResult};
 use vortex_scalar::Scalar;
 
-use crate::buffer::{Buffer, OwnedBuffer};
 use crate::encoding::EncodingRef;
 use crate::stats::{Stat, Statistics, StatsSet};
 use crate::{Array, ArrayMetadata, IntoArray, OwnedArray, ToArray};
@@ -14,7 +14,7 @@ pub struct ArrayData {
     encoding: EncodingRef,
     dtype: DType, // FIXME(ngates): Arc?
     metadata: Arc<dyn ArrayMetadata>,
-    buffer: Option<OwnedBuffer>,
+    buffer: Option<Buffer>,
     children: Arc<[ArrayData]>,
     stats_map: Arc<RwLock<StatsSet>>,
 }
@@ -24,7 +24,7 @@ impl ArrayData {
         encoding: EncodingRef,
         dtype: DType,
         metadata: Arc<dyn ArrayMetadata>,
-        buffer: Option<OwnedBuffer>,
+        buffer: Option<Buffer>,
         children: Arc<[ArrayData]>,
         statistics: StatsSet,
     ) -> VortexResult<Self> {
@@ -61,7 +61,7 @@ impl ArrayData {
         self.buffer.as_ref()
     }
 
-    pub fn into_buffer(self) -> Option<OwnedBuffer> {
+    pub fn into_buffer(self) -> Option<Buffer> {
         self.buffer
     }
 

vortex-array/src/implementation.rs

@@ -1,14 +1,14 @@
+use vortex_buffer::Buffer;
 use vortex_dtype::DType;
 use vortex_error::{vortex_bail, VortexError, VortexResult};
 
-use crate::buffer::{Buffer, OwnedBuffer};
 use crate::encoding::{ArrayEncoding, ArrayEncodingRef, EncodingRef};
 use crate::encoding::{ArrayEncodingExt, EncodingId};
 use crate::stats::{ArrayStatistics, Statistics};
 use crate::visitor::ArrayVisitor;
 use crate::{
     Array, ArrayDType, ArrayData, ArrayMetadata, AsArray, GetArrayMetadata, IntoArray,
-    IntoArrayData, ToArrayData, ToStatic,
+    IntoArrayData, ToArrayData,
 };
 use crate::{ArrayTrait, TryDeserializeArrayMetadata};
 
@@ -221,7 +221,7 @@ impl<'a, T: IntoArray<'a> + ArrayEncodingRef + ArrayStatistics + GetArrayMetadat
             Array::Data(d) => d,
             Array::View(_) => {
                 struct Visitor {
-                    buffer: Option<OwnedBuffer>,
+                    buffer: Option<Buffer>,
                     children: Vec<ArrayData>,
                 }
                 impl ArrayVisitor for Visitor {
@@ -234,7 +234,7 @@ impl<'a, T: IntoArray<'a> + ArrayEncodingRef + ArrayStatistics + GetArrayMetadat
                         if self.buffer.is_some() {
                             vortex_bail!("Multiple buffers found in view")
                         }
-                        self.buffer = Some(buffer.to_static());
+                        self.buffer = Some(buffer.clone());
                         Ok(())
                     }
                 }

vortex-array/src/lib.rs

@@ -1,7 +1,6 @@
 pub mod accessor;
 pub mod array;
 pub mod arrow;
-pub mod buffer;
 pub mod compress;
 pub mod compute;
 mod context;
@@ -29,10 +28,10 @@ pub use implementation::*;
 pub use metadata::*;
 pub use typed::*;
 pub use view::*;
+use vortex_buffer::Buffer;
 use vortex_dtype::DType;
 use vortex_error::VortexResult;
 
-use crate::buffer::Buffer;
 use crate::compute::ArrayCompute;
 use crate::encoding::{ArrayEncodingRef, EncodingRef};
 use crate::stats::{ArrayStatistics, ArrayStatisticsCompute};
@@ -106,10 +105,10 @@ impl Array<'_> {
 }
 
 impl<'a> Array<'a> {
-    pub fn into_buffer(self) -> Option<Buffer<'a>> {
+    pub fn into_buffer(self) -> Option<Buffer> {
         match self {
             Array::Data(d) => d.into_buffer(),
-            Array::View(v) => v.buffer().map(|b| b.to_static()),
+            Array::View(v) => v.buffer().cloned(),
         }
     }
 }

vortex-array/src/tree.rs

@@ -2,10 +2,10 @@ use std::fmt;
 
 use humansize::{format_size, DECIMAL};
 use serde::ser::Error;
+use vortex_buffer::Buffer;
 use vortex_error::{VortexError, VortexResult};
 
 use crate::array::chunked::ChunkedArray;
-use crate::buffer::Buffer;
 use crate::visitor::ArrayVisitor;
 use crate::{Array, ToArrayData};
 

vortex-array/src/typed.rs

@@ -1,9 +1,9 @@
 use std::sync::Arc;
 
+use vortex_buffer::Buffer;
 use vortex_dtype::DType;
 use vortex_error::{vortex_err, VortexError, VortexResult};
 
-use crate::buffer::OwnedBuffer;
 use crate::stats::StatsSet;
 use crate::{Array, ArrayData, ArrayDef, AsArray, IntoArray, ToArray, TryDeserializeArrayMetadata};
 
@@ -17,7 +17,7 @@ impl<D: ArrayDef> TypedArray<'_, D> {
     pub fn try_from_parts(
         dtype: DType,
         metadata: D::Metadata,
-        buffer: Option<OwnedBuffer>,
+        buffer: Option<Buffer>,
         children: Arc<[ArrayData]>,
         stats: StatsSet,
     ) -> VortexResult<Self> {