Skip to content

Commit

Permalink
vortex-buffer
Browse files Browse the repository at this point in the history
  • Loading branch information
a10y committed Aug 8, 2024
1 parent 98561a3 commit f5bbffa
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 1 deletion.
10 changes: 9 additions & 1 deletion vortex-buffer/src/io_buf.rs
Original file line number Diff line number Diff line change
@@ -1,17 +1,24 @@
//! Provides types that can be used by I/O frameworks to work with byte buffer-shaped data.
use crate::Buffer;

#[allow(clippy::missing_safety_doc)]
/// Trait for types that can provide a readonly byte buffer interface to I/O frameworks.
///
/// # Safety
/// The type must support contiguous raw memory access via pointer, such as `Vec` or `[u8]`.
pub unsafe trait IoBuf: Unpin + 'static {
/// Returns a raw pointer to the vector’s buffer.
fn read_ptr(&self) -> *const u8;

/// Number of initialized bytes.
fn bytes_init(&self) -> usize;

/// Access the buffer as a byte slice
fn as_slice(&self) -> &[u8] {
unsafe { std::slice::from_raw_parts(self.read_ptr(), self.bytes_init()) }
}

/// Access the buffer as a byte slice with begin and end indices
#[inline]
fn slice(self, begin: usize, end: usize) -> Slice<Self>
where
Expand All @@ -33,6 +40,7 @@ pub struct Slice<T> {
}

impl<T> Slice<T> {
/// Unwrap the slice into its underlying type.
pub fn into_inner(self) -> T {
self.buf
}
Expand Down
29 changes: 29 additions & 0 deletions vortex-buffer/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,15 @@
#![deny(missing_docs)]

//! A byte buffer implementation for Vortex.
//!
//! Vortex arrays hold data in a set of buffers.
//!
//! # Alignment
//!
//! We do not currently enforce any alignment guarantees on the buffer. This can be problematic
//! so it is recommended to convert all buffers into Arrow before usage.

use std::cmp::Ordering;
use std::ops::{Deref, Range};

Expand All @@ -8,31 +20,40 @@ mod flexbuffers;
pub mod io_buf;
mod string;

/// Buffer is an owned, cheaply cloneable byte arrays.
///
/// Buffers form the building blocks of all in-memory storage in Vortex.
#[derive(Debug, Clone)]
pub enum Buffer {
// TODO(ngates): we could add Aligned(Arc<AVec>) from aligned-vec package
/// A Buffer that wraps an Apache Arrow buffer
Arrow(ArrowBuffer),

/// A Buffer that wraps an owned [`Bytes`].
Bytes(bytes::Bytes),
}

unsafe impl Send for Buffer {}
unsafe impl Sync for Buffer {}

impl Buffer {
/// Length of the buffer in bytes
pub fn len(&self) -> usize {
match self {
Self::Arrow(b) => b.len(),
Self::Bytes(b) => b.len(),
}
}

/// Predicate for empty buffers
pub fn is_empty(&self) -> bool {
match self {
Self::Arrow(b) => b.is_empty(),
Self::Bytes(b) => b.is_empty(),
}
}

/// Return a new view on the buffer, but limited to the given index range.
pub fn slice(&self, range: Range<usize>) -> Self {
match self {
Self::Arrow(b) => {
Expand All @@ -42,13 +63,21 @@ impl Buffer {
}
}

/// Access the buffer as an immutable byte slice.
pub fn as_slice(&self) -> &[u8] {
match self {
Self::Arrow(b) => b.as_ref(),
Self::Bytes(b) => b.as_ref(),
}
}

/// Convert the buffer into a `Vec` of the given native type `T`.
///
/// # Ownership
/// The caller takes ownership of the underlying memory.
///
/// # Errors
/// This method will fail if the underlying buffer is an owned [`bytes::Bytes`].
pub fn into_vec<T: ArrowNativeType>(self) -> Result<Vec<T>, Self> {
match self {
Self::Arrow(buffer) => buffer.into_vec::<T>().map_err(Buffer::Arrow),
Expand Down
1 change: 1 addition & 0 deletions vortex-buffer/src/string.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ impl BufferString {
Self(buffer)
}

/// Return a view of the contents of BufferString as an immutable `&str`.
pub fn as_str(&self) -> &str {
// SAFETY: We have already validated that the buffer is valid UTF-8
unsafe { std::str::from_utf8_unchecked(self.0.as_ref()) }
Expand Down

0 comments on commit f5bbffa

Please sign in to comment.