diff --git a/utils/buffer/src/buffer.rs b/utils/buffer/src/buffer.rs index 3aa09e20bc..0a9a020c1d 100644 --- a/utils/buffer/src/buffer.rs +++ b/utils/buffer/src/buffer.rs @@ -1,16 +1,16 @@ -//! # Buffer -//! -//! Provides memory management for encoding and transmitting message frames between Sv2 roles when -//! buffer pools have been exhausted. -//! -//! [`BufferFromSystemMemory`] serves as a fallback when a [`crate::BufferPool`] is full or unable -//! to allocate memory fast enough. Instead of relying on pre-allocated memory, it dynamically -//! allocates memory using a [`Vec`], ensuring that message frames can still be processed. -//! -//! This fallback mechanism allows the buffer to resize dynamically based on data needs, making it -//! suitable for scenarios where message sizes vary. However, it introduces performance trade-offs -//! such as slower allocation, increased memory fragmentation, and higher system overhead compared -//! to using pre-allocated buffers. +// # Buffer from System Memory +// +// Provides memory management for encoding and transmitting message frames between Sv2 roles when +// buffer pools have been exhausted. +// +// `BufferFromSystemMemory` serves as a fallback when a `BufferPool` is full or unable to allocate +// memory fast enough. Instead of relying on pre-allocated memory, it dynamically allocates memory +// on the heap using a `Vec`, ensuring that message frames can still be processed. +// +// This fallback mechanism allows the buffer to resize dynamically based on data needs, making it +// suitable for scenarios where message sizes vary. However, it introduces performance trade-offs +// such as slower allocation, increased memory fragmentation, and higher system overhead compared +// to using pre-allocated buffers. use crate::Buffer; use aes_gcm::aead::Buffer as AeadBuffer; @@ -46,7 +46,7 @@ impl BufferFromSystemMemory { } impl Default for BufferFromSystemMemory { - /// Creates a new buffer with no initial data. + // Creates a new buffer with no initial data. fn default() -> Self { Self::new(0) } @@ -55,8 +55,8 @@ impl Default for BufferFromSystemMemory { impl Buffer for BufferFromSystemMemory { type Slice = Vec; - // Dynamically allocates or resizes the internal [`Vec`] to ensure there is enough space - // for writing. + // Dynamically allocates or resizes the internal `Vec` to ensure there is enough space for + // writing. #[inline] fn get_writable(&mut self, len: usize) -> &mut [u8] { let cursor = self.cursor; @@ -75,7 +75,7 @@ impl Buffer for BufferFromSystemMemory { &mut self.inner[cursor..len] } - // Splits off the written portion of the buffer, returning it as a new [`Vec`]. Swaps the + // Splits off the written portion of the buffer, returning it as a new `Vec`. Swaps the // internal buffer with a newly allocated empty one, effectively returning ownership of the // written data while resetting the internal buffer for future use. #[inline] @@ -121,14 +121,14 @@ impl Buffer for BufferFromSystemMemory { self.start = index; } - // Indicates that the buffer is always safe to drop, as [`Vec`] manages memory internally. + // Indicates that the buffer is always safe to drop, as `Vec` manages memory internally. #[inline] fn is_droppable(&self) -> bool { true } } -/// Used to test if [`crate::BufferPool`] tries to allocate from system memory. +// Used to test if `BufferPool` tries to allocate from system memory. #[cfg(test)] pub struct TestBufferFromMemory(pub Vec); @@ -147,6 +147,7 @@ impl Buffer for TestBufferFromMemory { fn get_data_by_ref(&mut self, _len: usize) -> &mut [u8] { &mut self.0[0..0] } + fn get_data_by_ref_(&self, _len: usize) -> &[u8] { &self.0[0..0] } @@ -154,9 +155,11 @@ impl Buffer for TestBufferFromMemory { fn len(&self) -> usize { 0 } + fn danger_set_start(&mut self, _index: usize) { todo!() } + fn is_droppable(&self) -> bool { true } diff --git a/utils/buffer/src/buffer_pool/mod.rs b/utils/buffer/src/buffer_pool/mod.rs index 7985a2f6c6..2fac5b792f 100644 --- a/utils/buffer/src/buffer_pool/mod.rs +++ b/utils/buffer/src/buffer_pool/mod.rs @@ -11,7 +11,7 @@ // Supports different allocation modes to optimize memory usage: // // - **Back Mode**: Allocates from the back of the buffer pool (default). -// - **Front Mode**: Allocates from the front when the back is full. +// - **Front Mode**: Allocates from the front when the back is full the front has space. // - **Alloc Mode**: Falls back to heap allocation when the buffer pool cannot fulfill requests // (with reduced performance). // @@ -20,9 +20,10 @@ // When an incoming Sv2 message is received, it needs to be buffered for processing. The pool first // checks the back part (`PoolBack`) to see if there is enough memory available. If the back is // full, the pool attempts to clear used memory chunks. If clearing fails, it switches to the front -// (`PoolFront`) and tries again. If both back and front are full and no memory can be cleared, the -// pool falls back to allocating fresh memory from the heap (`PoolMode::Alloc`). After processing -// the message, the memory can be cleared, and the buffer pool resets, making the memory available +// (`PoolFront`) and tries again, this time allocating memory from the front of the `BufferPool`. +// If both back and front are full and no memory can be cleared, the pool falls back to allocating +// fresh memory from the heap (`PoolMode::Alloc`) at a performance reduction. After processing the +// message, the memory can be cleared, and the buffer pool resets, making the memory available // for future messages. // **Note**: To prevent leaks or deadlocks, ensure that memory slices are properly released after @@ -48,20 +49,19 @@ use aes_gcm::aead::Buffer as AeadBuffer; mod pool_back; pub use pool_back::PoolBack; -/// Maximum number of memory slices the buffer pool can concurrently manage. -/// -/// This value limits the number of slices the [`BufferPool`] can track and manage at once. Once -/// the pool reaches its capacity of `8` slices, it may need to free memory or switch modes (e.g., -/// to system memory). The choice of `8` ensures a balance between performance and memory -/// management. The use of `usize` allows for compatibility with platform-dependent memory -/// operations. +// Maximum number of memory slices the buffer pool can concurrently manage. +// +// This value limits the number of slices the `BufferPool` can track and manage at once. Once the +// pool reaches its capacity of `8` slices, it may need to free memory or switch modes (e.g., to +// system memory). The choice of `8` ensures a balance between performance and memory management. +// The use of `usize` allows for compatibility with platform-dependent memory operations. pub const POOL_CAPACITY: usize = 8; -/// Manages the "front" section of the [`BufferPool`]. -/// -/// Handles the allocation of memory slices at the front of the buffer pool. It tracks the number -/// of slices in use and attempts to free unused slices when necessary to maximize available -/// memory. The front of the buffer pool is used if the back of the buffer pool is filled up. +// Manages the "front" section of the `BufferPool`. +// +// Handles the allocation of memory slices at the front of the buffer pool. It tracks the number of +// slices in use and attempts to free unused slices when necessary to maximize available memory. +// The front of the buffer pool is used if the back of the buffer pool is filled up. #[derive(Debug, Clone)] pub struct PoolFront { // Starting index of the front section of the buffer pool. @@ -70,7 +70,7 @@ pub struct PoolFront { // Maximum number of bytes that can be allocated in the front section of the buffer pool. // // Helps manage how much memory can be used before triggering memory clearing or switching - // [`PoolMode`]. + // `PoolMode`. byte_capacity: usize, // Number of allocated slices in the front section of the buffer pool. @@ -78,7 +78,7 @@ pub struct PoolFront { } impl PoolFront { - /// Initializes a new [`PoolFront`] with the specified byte capacity and back start position. + // Initializes a new `PoolFront` with the specified byte capacity and back start position. #[inline(always)] fn new(byte_capacity: usize, back_start: usize) -> Self { Self { @@ -88,10 +88,10 @@ impl PoolFront { } } - /// Attempts to clear unused memory slices at the tail of the front section. - /// - /// Returns `true` if slices were successfully cleared, otherwise `false` if no slices could be - /// cleared or memory conditions prevent clearing. + // Attempts to clear unused memory slices at the tail of the front section. + // + // Returns `true` if slices were successfully cleared, otherwise `false` if no slices could be + // cleared or memory conditions prevent clearing. #[inline(always)] fn try_clear_tail(&mut self, memory: &mut InnerMemory, mut shared_state: u8) -> bool { #[cfg(feature = "fuzz")] @@ -129,13 +129,12 @@ impl PoolFront { } } - /// Clears the front memory slices if conditions allow and checks if the memory pool has - /// capacity to allocate `len` bytes in the buffer. - /// - /// Returns `Ok` if memory was successfully cleared and there is sufficient capacity, otherwise - /// an `Err(PoolMode::Back)` if the memory cannot be cleared or lacks capacity. This error - /// indicates the [`BufferPool`] should attempt a transition to use the back of the buffer - /// pool. + // Clears the front memory slices if conditions allow and checks if the memory pool has + // capacity to allocate `len` bytes in the buffer. + // + // Returns `Ok` if memory was successfully cleared and there is sufficient capacity, otherwise + // an `Err(PoolMode::Back)` if the memory cannot be cleared or lacks capacity. This error + // indicates the `BufferPool` should attempt a transition to use the back of the buffer pool. #[inline(always)] fn clear( &mut self, @@ -154,13 +153,12 @@ impl PoolFront { } } - /// Attempts to allocate a writable memory region in the front section of the buffer pool, - /// returning a writable slice if successful, or transitioning to a new pool mode if necessary. - /// - /// Returns a pointer to the writable memory (`Ok(*mut u8)`) if successful, otherwise an - /// `Err(PoolMode::Back)` if the memory cannot be cleared or lacks capacity. This error - /// indicates the [`BufferPool`] should attempt a transition to use the back of the buffer - /// pool. + // Attempts to allocate a writable memory region in the front section of the buffer pool, + // returning a writable slice if successful, or transitioning to a new pool mode if necessary. + // + // Returns a pointer to the writable memory (`Ok(*mut u8)`) if successful, otherwise an + // `Err(PoolMode::Back)` if the memory cannot be cleared or lacks capacity. This error + // indicates the `BufferPool` should attempt a transition to use the back of the buffer pool. #[inline(always)] fn get_writable( &mut self, @@ -180,33 +178,32 @@ impl PoolFront { } } -/// Current mode of operation for the [`BufferPool`]. -/// -/// The pool operates in three modes based on memory availability: it first allocates from the back, -/// then from the front if the back is full, and finally from system memory (with reduced -/// performance) if both sections are exhausted. +// Current mode of operation for the `BufferPool`. +// +// The pool operates in three modes based on memory availability: it first allocates from the back, +// then from the front if the back is full, and finally from system memory (with reduced +// performance) if both sections are exhausted. #[derive(Debug, Clone)] pub enum PoolMode { - /// The buffer pool is operating in "back" mode, where memory is allocated from the back of the - /// buffer pool. + // The buffer pool is operating in "back" mode, where memory is allocated from the back of the + // buffer pool. Back, - /// The buffer pool is operating in "front" mode, where memory is allocated from the front of - /// the buffer pool. Used when the back is full. + // The buffer pool is operating in "front" mode, where memory is allocated from the front of + // the buffer pool. Used when the back is full. Front(PoolFront), - /// The pool has exhausted its internal memory, and it is now allocating directly from the - /// system memory (heap). + // The pool has exhausted its internal memory, and it is now allocating directly from the + // system memory (heap). Alloc, } -/// Internal memory management for the [`BufferPool`]. -/// -/// Handles allocating, tracking, and managing memory slices for manipulating memory offsets, -/// copying data, and managing capacity. It uses a contiguous block of memory ([`Vec`]), -/// tracking its usage through offsets (`raw_offset`, `raw_length`), and manages slice allocations -/// through `slots`. Used by [`BufferPool`] to optimize memory reused and minimize heap -/// allocations. +// Internal memory management for the `BufferPool`. +// +// Handles allocating, tracking, and managing memory slices for manipulating memory offsets, +// copying data, and managing capacity. It uses a contiguous block of memory (`Vec`), tracking +// its usage through offsets (`raw_offset`, `raw_length`), and manages slice allocations through +// `slots`. Used by `BufferPool` to optimize memory reused and minimize heap allocations. #[derive(Debug, Clone)] pub struct InnerMemory { // Underlying contiguous block of memory to be managed. @@ -222,7 +219,7 @@ pub struct InnerMemory { // length of each allocated memory slice. slots: [(usize, usize); POOL_CAPACITY], - // Number of active slices in use, represented by how many slots are currently occupied. + // A pointer to the current slot. Represents how many slots are currently occupied. len: usize, } @@ -410,15 +407,17 @@ impl InnerMemory { self.pool[writable_offset..writable_offset + len].as_mut_ptr() } - // Transfers ownership of the raw memory slice that has been written into the buffer, returning - // it as a `Slice`. This is the main mechanism for passing processed data from the buffer to - // the rest of the system. - // - // After this call, the buffer resets internally, allowing it to write new data into a fresh - // portion of memory. This prevents memory duplication and ensures efficient reuse of the - // buffer. This method is used when the data in the buffer is ready for processing, sending, or - // further manipulation. The returned `Slice` now owns the memory and the buffer no longer has - // responsibility for it, allowing the buffer to handle new incoming data. + /// Provides access to the raw memory slice containing the data written into the buffer, + /// returning it as a [`Slice`]. This method is the primary mechanism for making processed data + /// available from the buffer to the rest of the system. + /// + /// After this call, the buffer advances internally to a new slot, allowing new data to be + /// written into an unused portion of memory. This approach avoids memory duplication and + /// ensures efficient reuse of the buffer without transferring ownership of the memory. + /// + /// This method is typically used when the data in the buffer is ready for processing, sending, + /// or further manipulation. The returned `Slice` contains the data, while the buffer itself + /// remains ready to handle new incoming data by pointing to a fresh memory region. #[inline(always)] fn get_data_owned( &mut self, @@ -463,9 +462,9 @@ impl InnerMemory { /// A pool of reusable memory buffers to optimize memory allocation for Sv2 message frames. /// -/// Manages memory slices across three modes ([`PoolMode`]): back, front, and system memory. It +/// Manages memory slices across three pool modes: back (default), front, and system memory. It /// reuses preallocated memory slices, minimizing the need for frequent memory allocation. The pool -/// is thread-safe, using atomic state tracking with [`SharedState`]. +/// is thread-safe, using atomic state tracking. /// /// Type `T` implements the [`Buffer`] trait, which defines how memory is handled in the buffer /// pool. [`BufferFromSystemMemory`] is used as the default system memory allocator. @@ -477,8 +476,8 @@ pub struct BufferPool { /// Tracks the current mode of memory allocation (back, front, or system). pub mode: PoolMode, - /// Tracks the usage state of memory slices using atomic operations, ensuring memory is not - /// prematurely reused and allowing safe concurrent access across threads. + // Tracks the usage state of memory slices using atomic operations, ensuring memory is not + // prematurely reused and allowing safe concurrent access across threads. shared_state: SharedState, // Core memory area from which slices are allocated and reused. Manages the actual memory @@ -499,8 +498,8 @@ impl BufferPool { /// Creates a new [`BufferPool`] with the specified memory capacity, in bytes. /// /// Initializes the buffer pool with pre-allocated memory (`inner_memory`) and sets the pool - /// mode to [`PoolMode::Back`]. The buffer pool uses [`BufferFromSystemMemory`] as a fallback - /// when the buffer sections are full. + /// mode to the back. The buffer pool uses [`BufferFromSystemMemory`] as a fallback when the + /// buffer sections are full. pub fn new(capacity: usize) -> Self { Self { pool_back: PoolBack::new(), @@ -533,8 +532,8 @@ impl BufferPool { /// Checks if the buffer pool is operating in the front mode. /// /// This mode indicates that the back of the buffer pool has been filled and the system is now - /// using the front section for memory allocation. Returns `true` if the pool is in - /// [`PoolMode::Front`], otherwise `false`. + /// using the front section for memory allocation. Returns `true` if the pool is in front mode, + /// otherwise `false`. pub fn is_front_mode(&self) -> bool { match self.mode { PoolMode::Back => false, @@ -546,7 +545,7 @@ impl BufferPool { /// Checks if the buffer pool is operating in the back mode. /// /// The back mode is the default state, where the buffer pool first tries to allocate memory. - /// Returns `true` if the pool is in [`PoolMode::Back`], otherwise `false`. + /// Returns `true` if the pool is in back mode, otherwise `false`. pub fn is_back_mode(&self) -> bool { match self.mode { PoolMode::Back => true, @@ -559,7 +558,7 @@ impl BufferPool { /// /// This mode is used when both the back and front sections of the buffer pool are full, /// leading the system to allocate memory from the heap, which has performance trade-offs. - /// Returns `true` if the pool is in [`PoolMode::Alloc`], otherwise `false`. + /// Returns `true` if the pool is in alloc mode, otherwise `false`. pub fn is_alloc_mode(&self) -> bool { match self.mode { PoolMode::Back => false, @@ -571,9 +570,8 @@ impl BufferPool { // Resets the buffer pool based on its current mode when the shared state indicates all slices // have been dropped, preparing it for reuse. // - // - In `Back` or `Front` mode, the internal memory is moved to the front, and the back is - // reset. - // - In `Alloc` mode, system memory is checked and, if smaller than pool capacity, transferred + // - In back or front mode, the internal memory is moved to the front, and the back is reset. + // - In alloc mode, system memory is checked and, if smaller than pool capacity, transferred // back into the pool, switching the mode to `Back`. #[inline(always)] fn reset(&mut self) { @@ -613,9 +611,9 @@ impl BufferPool { // fulfill the request. // // Determines whether to allocate memory from system memory or try to clear memory from the - // back section of the buffer pool. If clearing is unsuccessful, it may switch to `Alloc` mode - // or remain in `Back` or `Front` pool modes. When `without_check` is `true`, the function - // bypasses memory checks and allocates directly from the heap. + // back section of the buffer pool. If clearing is unsuccessful, it may switch to alloc mode or + // remain in back or front pool modes. When `without_check` is `true`, the function bypasses + // memory checks and allocates directly from the heap. #[inline(never)] fn get_writable_from_system_memory( &mut self, @@ -664,9 +662,9 @@ impl BufferPool { // Switches the buffer pool to a different mode of operation, adjusting based on the required // memory size (`len`). // - // Depending on the current and target modes (`Back`, `Front`, or `Alloc`), this method adjusts - // the internal buffer pool's state and memory to ensure a smooth transition while allocating - // the necessary buffer space (`len`), ensuring no data is lost. + // Depending on the current and target modes (back, front, or alloc), this method adjusts the + // internal buffer pool's state and memory to ensure a smooth transition while allocating the + // necessary buffer space (`len`), ensuring no data is lost. #[inline(always)] fn change_mode(&mut self, mode: PoolMode, len: usize, shared_state: u8) { match (&mut self.mode, &mode) { @@ -750,9 +748,9 @@ impl BufferPool { // Recursively attempts to allocate writable memory of the specified `len`, switching modes if // necessary. // - // First tries to allocate memory from the current mode (`Back`, `Front`, or `Alloc`), - // and if unsuccessful, switches modes and retries, starting with the memory pool before - // resorting to system memory. + // First tries to allocate memory from the current mode (back, front, or alloc), and if + // unsuccessful, switches modes and retries, starting with the memory pool before resorting to + // system memory. #[inline(always)] fn get_writable_(&mut self, len: usize, shared_state: u8, without_check: bool) -> &mut [u8] { let writable = match &mut self.mode { @@ -807,9 +805,9 @@ impl Buffer for BufferPool { // Transfers ownership of the written data as a `Slice`, handling different pool modes. // - // Depending on the current mode (`Back`, `Front`, or `Alloc`), it retrieves the data from the + // Depending on the current mode (back, front, or alloc), it retrieves the data from the // appropriate memory source. In `Back` or `Front` modes, it updates internal state - // accordingly. In `Alloc` mode, it retrieves data from the heap-allocated system memory. + // accordingly. In alloc mode, it retrieves data from the heap-allocated system memory. #[inline(always)] fn get_data_owned(&mut self) -> Self::Slice { let shared_state = &mut self.shared_state; @@ -867,8 +865,8 @@ impl Buffer for BufferPool { } // Retrieves data differently based on the current buffer pool mode: - // - In `Alloc` mode, it delegates to the system memory buffer. - // - In other modes, it returns a mutable slice of the internal memory buffer. + // - In alloc mode, it delegates to the system memory buffer. + // - In back or front modes, it returns a mutable slice of the internal memory buffer. fn get_data_by_ref(&mut self, len: usize) -> &mut [u8] { match self.mode { PoolMode::Alloc => self.system_memory.get_data_by_ref(len), @@ -880,8 +878,8 @@ impl Buffer for BufferPool { } // Retrieves data differently based on the current pool mode: - // - In `Alloc` mode, it delegates to the system memory buffer. - // - In other modes, it returns an immutable slice of the internal memory buffer. + // - In back or front modes, it returns an immutable slice of the internal memory buffer. + // - In alloc mode, it delegates to the system memory buffer. fn get_data_by_ref_(&self, len: usize) -> &[u8] { match self.mode { PoolMode::Alloc => self.system_memory.get_data_by_ref_(len), @@ -894,9 +892,10 @@ impl Buffer for BufferPool { // Returns the length of the written data in the buffer. // - // The implementation checks the current pool mode to determine where to retrieve the length from: - // - In `Back` or `Front` modes, it returns the length from `inner_memory.raw_len`. - // - In `Alloc` mode, it returns the length from the system memory buffer. + // The implementation checks the current pool mode to determine where to retrieve the length + // from: + // - In back or front modes, it returns the length from `inner_memory.raw_len`. + // - In alloc mode, it returns the length from the system memory buffer. fn len(&self) -> usize { match self.mode { PoolMode::Back => self.inner_memory.raw_len, @@ -935,9 +934,8 @@ impl BufferPool { /// Determines if the [`BufferPool`] can be safely dropped. /// /// Returns `true` if all memory slices managed by the buffer pool have been released (i.e., - /// the `shared_state` is zero), indicating that no other threads or components are using the - /// buffer pool's memory. This check helps prevent dropping the buffer pool while it's still in - /// use. + /// the `shared_state` is zero), indicating that all the slices are dropped. This check helps + /// prevent dropping the buffer pool while it's still in use. pub fn droppable(&self) -> bool { self.shared_state.load(Ordering::Relaxed) == 0 } @@ -948,12 +946,14 @@ impl AsRef<[u8]> for BufferPool { &self.get_data_by_ref_(Buffer::len(self))[self.start..] } } + impl AsMut<[u8]> for BufferPool { fn as_mut(&mut self) -> &mut [u8] { let start = self.start; self.get_data_by_ref(Buffer::len(self))[start..].as_mut() } } + impl AeadBuffer for BufferPool { fn extend_from_slice(&mut self, other: &[u8]) -> aes_gcm::aead::Result<()> { self.get_writable(other.len()).copy_from_slice(other); diff --git a/utils/buffer/src/buffer_pool/pool_back.rs b/utils/buffer/src/buffer_pool/pool_back.rs index 6758f8bcd0..86e8f746d5 100644 --- a/utils/buffer/src/buffer_pool/pool_back.rs +++ b/utils/buffer/src/buffer_pool/pool_back.rs @@ -1,31 +1,34 @@ // # Back of Buffer Pool -//! -//! Manages the "back" section of the buffer pool ([`crate::BufferPool`]). -//! -//! The [`PoolBack`] struct is responsible for allocating, clearing, and managing memory slices -//! in the back part of the pool, allowing efficient memory reuse in high-throughput environments. -//! It tracks the number of allocated slices and attempts to free up memory that is no longer in -//! use, preventing unnecessary memory growth. -//! -//! Key functions in this module handle: -//! - Clearing unused slices from the back of the pool to reclaim memory. -//! - Managing slice allocation and ensuring enough capacity for new operations. -//! - Switching between different pool modes, such as front or back, depending on memory state. -//! -//! Memory in the back of the pool buffer is always favored to be used first. If the back of the -//! pool fills up, the buffer pool switches to use the front of the pool instead. The front of the -//! pool will be used until the back of the pool has been cleared, or until both the back and front -//! are filled, in which case the buffer pool switches to allocate system memory at reduced -//! performance. +// +// Manages the "back" section of the buffer pool (`BufferPool`). +// +// The `PoolBack` struct is responsible for allocating, clearing, and managing memory slices from +// the back of the pool. It tracks the number of allocated slices and attempts to free up memory +// that is no longer in use, preventing unnecessary memory growth. +// +// Key functions in this module handle: +// - Clearing unused slices from the back of the pool to reclaim memory. +// - Managing slice allocation and ensuring enough capacity for new operations. +// - Switching between different pool modes, such as front or back, depending on memory state. +// +// By default, memory is always first allocated from the back of the `BufferPool`. If the all of +// the initially allocated buffer memory is completely filled and a new memory request comes in, +// `BufferPool` checks whether any memory has been freed at the back or the front using +// `SharedState`. If, for example, a slice has been freed that corresponds to the head of +// `SharedState`, `BufferPool` will switch to front mode and start allocating incoming memory +// requests from there. However, if the entire `SharedState` is full, it will switch to alloc mode +// and begin allocating system memory to fulfill incoming requests at a performance reduction. For +// subsequent memory requests, it will continue to check whether the prefix or suffix of +// `SharedState` has been freed. If it has, `BufferPool` will switch modes and start consuming +// pre-allocated `BufferPool` memory; if not, it will remain in alloc mode. use crate::buffer_pool::{InnerMemory, PoolFront, PoolMode, POOL_CAPACITY}; -/// Manages the "back" section of the [`crate::BufferPool`]. -/// -/// Handles the allocation of memory slices at the back of the buffer pool. It tracks the number -/// of slices in use and attempts to free unused slices when necessary to maximize available -/// memory. The back of the buffer pool is used first, if it fills up, the front of the buffer pool -/// is used. +// Manages the "back" section of the `BufferPool`. +// +// Handles the allocation of memory slices at the back of the buffer pool. It tracks the number of +// slices in use and attempts to free unused slices when necessary to maximize available memory. +// The back of the buffer pool is used first, if it fills up, the front of the buffer pool is used. #[derive(Debug, Clone)] pub struct PoolBack { // Starting index of the back section of the buffer pool. @@ -36,7 +39,7 @@ pub struct PoolBack { } impl PoolBack { - /// Initializes a new [`PoolBack`] with no allocated slices. + // Initializes a new `PoolBack` with no allocated slices. pub fn new() -> Self { Self { back_start: 0, @@ -44,13 +47,13 @@ impl PoolBack { } } - /// Returns the number of allocated slices in the back section of the buffer pool. + // Returns the number of allocated slices in the back section of the buffer pool. #[inline(always)] pub fn len(&self) -> usize { self.len } - /// Updates the length of the back section based on the state of the inner memory. + // Updates the length of the back section based on the state of the inner memory. #[inline(always)] pub fn set_len_from_inner_memory(&mut self, len: usize) { let len = len - self.back_start; @@ -61,27 +64,27 @@ impl PoolBack { self.len = len; } - /// Returns the starting index of the back section in the buffer pool. + // Returns the starting index of the back section in the buffer pool. #[inline(always)] pub fn back_start(&self) -> usize { self.back_start } - /// Resets the back section of the pool by clearing its start index and length. + // Resets the back section of the pool by clearing its start index and length. #[inline(always)] pub fn reset(&mut self) { self.back_start = 0; self.len = 0; } - /// Attempts to clear unused slices at the tail of the back section without performing bounds - /// or safety checks. - /// - /// Assumes the caller has already ensured the safety of the operation (such as slice bounds - /// and memory validity). It skips internal safety checks, relying on the caller to manage the - /// state, making it faster but potentially unsafe if misused. - /// - /// Returns `true` if the tail was cleared successfully, otherwise `false`. + // From the back section, checks if there are any unset bits and adjusts the length if there + // are. + // + // Assumes the caller has already ensured the safety of the operation (such as slice bounds and + // memory validity). It skips internal safety checks, relying on the caller to manage the + // state, making it faster but potentially unsafe if misused. + // + // Returns `true` if the tail was cleared successfully, otherwise `false`. #[inline(always)] pub fn try_clear_tail_unchecked( &mut self, @@ -133,10 +136,10 @@ impl PoolBack { } } - /// Checks if the tail of the back section can be cleared. - /// - /// Should always be called before attempting to clear the tail. Returns `true` if the tail can - /// be cleared, otherwise `false`. + // Checks if the tail of the back section can be cleared. + // + // Should always be called before attempting to clear the tail. Returns `true` if the tail can + // be cleared, otherwise `false`. #[inline(always)] pub fn tail_is_clearable(&self, shared_state: u8) -> bool { let element_in_back = POOL_CAPACITY - self.back_start; @@ -159,9 +162,8 @@ impl PoolBack { ) -> Result<(), PoolMode> { // 0b00111110 2 leading zeros // - // the first 2 elements have been dropped so back start at 2 and BufferPool can go in Front - // mode - // + // The first 2 elements have been dropped so back start at 2 and `BufferPool` can go in + // front mode self.back_start = shared_state.leading_zeros() as usize; if self.back_start >= 1 && memory.raw_len < memory.slots[self.back_start].0 { @@ -178,8 +180,13 @@ impl PoolBack { } } - /// Clears the tail or head of the back section, transitioning pool modes if necessary. - /// + // Clears the tail or head of the back section, transitioning pool modes if necessary. + // + // Called when the state of the `BufferPool`, along with both `raw_offset` and `raw_len`, + // reaches its maximum limits. It is used to clear any available space remaining in the + // buffer's suffix or prefix. Based on that, the `BufferPool`'s state is updated by changing + // its pool mode. + // // Returns `Ok` if the tail or head was cleared successfully. Otherwise an // `Err(PoolMode::Front)` if the buffer pool should switch to front mode, or an // `Err(PoolMode::Alloc)` if the buffer pool should switch to allocation mode. @@ -199,11 +206,11 @@ impl PoolBack { self.try_clear_head(shared_state, memory) } - /// Returns a writable slice of memory from the back section or transitions to a new pool mode - /// if necessary. - /// - /// Returns `Ok(*mut u8)` if writable memory is available, otherwise an `Err(PoolMode)` if a - /// mode change is required. + // Returns a writable slice of memory from the back section or transitions to a new pool mode + // if necessary. + // + // Returns `Ok(*mut u8)` if writable memory is available, otherwise an `Err(PoolMode)` if a + // mode change is required. #[inline(always)] pub fn get_writable( &mut self, diff --git a/utils/buffer/src/lib.rs b/utils/buffer/src/lib.rs index 5a95cfebd0..725c739068 100644 --- a/utils/buffer/src/lib.rs +++ b/utils/buffer/src/lib.rs @@ -1,3 +1,44 @@ +//! # `buffer_sv2` +//! +//! Handles memory management for Stratum V2 (Sv2) roles. +//! +//! Provides a memory-efficient buffer pool ([`BufferPool`]) that minimizes allocations and +//! deallocations for high-throughput message frame processing in Sv2 roles. [`Slice`] helps +//! minimize memory allocation overhead by reusing large buffers, improving performance and +//! reducing latency. The [`BufferPool`] tracks the usage of memory slices, using atomic operations +//! and shared state tracking to safely manage memory across multiple threads. +//! +//! ## Memory Structure +//! +//! The [`BufferPool`] manages a contiguous block of memory allocated on the heap, divided into +//! fixed-size slots. Memory allocation within this pool operates in three distinct modes: +//! +//! 1. **Back Mode**: By default, memory is allocated sequentially from the back (end) of the buffer +//! pool. This mode continues until the back slots are fully occupied. +//! 2. **Front Mode**: Once the back slots are exhausted, the [`BufferPool`] checks if any slots at +//! the front (beginning) have been freed. If available, it switches to front mode, allocating +//! memory from the front slots. +//! 3. **Alloc Mode**: If both back and front slots are occupied, the [`BufferPool`] enters alloc +//! mode, where it allocates additional memory directly from the system heap. This mode may +//! introduce performance overhead due to dynamic memory allocation. +//! +//! [`BufferPool`] dynamically transitions between these modes based on slot availability, +//! optimizing memory usage and performance. +//! +//! ## Usage +//! +//! When an incoming Sv2 message is received, it is buffered for processing. The [`BufferPool`] +//! attempts to allocate memory from its internal slots, starting in back mode. If the back slots +//! are full, it checks for available front slots to switch to front mode. If no internal slots are +//! free, it resorts to alloc mode, allocating memory from the system heap. +//! +//! For operations requiring dedicated buffers, the [`Slice`] type manages its own memory using +//! [`Vec`]. In high-performance scenarios, [`Slice`] can reference externally managed memory +//! from the [`BufferPool`], reducing dynamic memory allocations and increasing performance. +//! +//! ### Debug Mode +//! Provides additional tracking for debugging memory management issues. + #![cfg_attr(not(feature = "debug"), no_std)] //#![feature(backtrace)] @@ -83,20 +124,22 @@ impl Write for &mut [u8] { /// Interface for working with memory buffers. /// /// An abstraction for buffer management, allowing implementors to handle either owned memory -/// ([`Slice`] with [`Vec`]) or externally managed memory in a buffer pool ([`Slice`] with -/// [`BufferPool`]. Utilities are provided to borrow writable memory, retrieve data from the -/// buffer, and manage memory slices. +/// ([`Slice`] with [`Vec`]). Utilities are provided to borrow writable memory, retrieve data +/// from the buffer, and manage memory slices. +/// +/// This trait is used during the serialization and deserialization +/// of message types in the [`binary_sv2` crate](https://crates.io/crates/binary_sv2). pub trait Buffer { /// The type of slice that the buffer uses. type Slice: AsMut<[u8]> + AsRef<[u8]> + Into; - /// Borrows a mutable slice of the buffer, allowing the caller to write data into it. The caller - /// specifies the length of the data they need to write. + /// Borrows a mutable slice of the buffer, allowing the caller to write data into it. The + /// caller specifies the length of the data they need to write. fn get_writable(&mut self, len: usize) -> &mut [u8]; - /// Provides ownership of the buffer data that has already been written. This transfers the - /// ownership of the buffer’s written data to the caller, and the buffer is considered cleared - /// after this operation. + /// Provides ownership of a slice in the buffer pool to the caller and updates the buffer + /// pool's state by modifying the position in `shared_state` that the slice occupies. The pool + /// now points to the next set of uninitialized space. fn get_data_owned(&mut self) -> Self::Slice; /// Provides a mutable reference to the written portion of the buffer, up to the specified diff --git a/utils/buffer/src/slice.rs b/utils/buffer/src/slice.rs index 4240fa0250..c5da575d79 100644 --- a/utils/buffer/src/slice.rs +++ b/utils/buffer/src/slice.rs @@ -1,92 +1,92 @@ -//! # Slice -//! -//! Provides efficient memory management for the Sv2 protocol by allowing memory reuse, either -//! through owned memory ([`Vec`]) or externally managed memory in a buffer pool -//! ([`crate::BufferPool`]). -//! -//! [`Slice`] helps minimize memory allocation overhead by reusing large buffers, improving -//! performance and reducing latency in high-throughput environments. Tracks the usage of memory -//! slices, ensuring safe reuse across multiple threads via [`SharedState`]. -//! -//! ## Key Features -//! - **Memory Reuse**: Divides large buffers into smaller slices, reducing the need for frequent -//! allocations. -//! - **Shared Access**: Allows safe concurrent access using atomic state tracking -//! ([`Arc`]). -//! - **Flexible Management**: Supports both owned memory and externally managed memory. -//! -//! ## Usage -//! 1. **Owned Memory**: For isolated operations, [`Slice`] manages its own memory ([`Vec`]). -//! 2. **Buffer Pool**: In high-performance systems, [`Slice`] references externally managed memory -//! from a buffer pool ([`crate::BufferPool`]), reducing dynamic memory allocation. -//! -//! ### Debug Mode -//! Provides additional tracking for debugging memory management issues. +// # Slice +// +// Provides efficient memory management for the Sv2 protocol by allowing memory reuse, either +// through owned memory (`Vec`) or externally managed memory in a buffer pool (`BufferPool`). +// +// `Slice` helps minimize memory allocation overhead by reusing large buffers, improving +// performance and reducing latency in high-throughput environments. Tracks the usage of memory +// slices, ensuring safe reuse across multiple threads via `SharedState`. +// +// ## Key Features +// - **Memory Reuse**: Divides large buffers into smaller slices, reducing the need for frequent +// allocations. +// - **Shared Access**: Allows safe concurrent access using atomic state tracking (`Arc`). +// - **Flexible Management**: Supports both owned memory and externally managed memory. +// +// ## Usage +// 1. **Owned Memory**: For isolated operations, `Slice` manages its own memory (`Vec`). +// 2. **Buffer Pool**: In high-performance systems, `Slice` references externally managed memory +// from a buffer pool (`BufferPool`), reducing dynamic memory allocation. +// +// ### Debug Mode +// Provides additional tracking for debugging memory management issues. use alloc::{sync::Arc, vec::Vec}; use core::sync::atomic::{AtomicU8, Ordering}; #[cfg(feature = "debug")] use std::time::SystemTime; -/// A special index value used to mark [`Slice`] as ignored in certain operations, such as memory -/// pool tracking or state management. -/// -/// It can be used as a sentinel value for slices that should not be processed or tracked, helping -/// differentiate valid slices from those that need to be skipped. When a [`Slice`]'s `index` is -/// set to `INGORE_INDEX`, it is flagged to be ignored and by any logic that processes or tracks -/// slice indices. +// A special index value used to mark `Slice` as ignored in certain operations, such as memory pool +// tracking or state management. +// +// It can be used as a sentinel value for slices that should not be processed or tracked, helping +// differentiate valid slices from those that need to be skipped. When a `Slice`'s `index` is set +// to `INGORE_INDEX`, it is flagged to be ignored and by any logic that processes or tracks slice +// indices. pub const INGORE_INDEX: u8 = 59; -// Allows `Slice` to be safely transferred between threads. -// -// `Slice` contains a raw pointer (`*mut u8`), so Rut cannot automatically implement `Send`. The -// `unsafe` block asserts that memory access is thread-safe, relaying on `SharedState` and atomic -// operations to prevent data races. +/// Allows [`Slice`] to be safely transferred between threads. +/// +/// [`Slice`] contains a raw pointer (`*mut u8`), so Rust cannot automatically implement [`Send`]. +/// The `unsafe` block asserts that memory access is thread-safe, relaying on `SharedState` and +/// atomic operations to prevent data races. unsafe impl Send for Slice {} -/// A contiguous block of memory, either preallocated for dynamically allocated. +/// A contiguous block of memory, either preallocated or dynamically allocated. /// /// It serves as a lightweight handle to a memory buffer, allowing for direct manipulation and /// shared access. It can either hold a reference to a preallocated memory block or own a -/// dynamically allocated buffer (via `Vec`). +/// dynamically allocated buffer (via [`Vec`]). #[derive(Debug, Clone)] pub struct Slice { - /// Raw pointer to the start of the memory block. - /// - /// Allows for efficient access to the underlying memory. Care should be taken when working - /// with raw pointers to avoid memory safety issues. The pointer must be valid and must point - /// to a properly allocated and initialized memory region. + // Raw pointer to the start of the memory block. + // + // Allows for efficient access to the underlying memory. Care should be taken when working with + // raw pointers to avoid memory safety issues. The pointer must be valid and must point to a + // properly allocated and initialized memory region. pub(crate) offset: *mut u8, - /// Length of the memory block in bytes. - /// - /// Represents how much memory is being used. This is critical for safe memory access, as it - /// prevents reading or writing outside the bounds of the buffer. + // Length of the memory block in bytes. + // + // Represents how much memory is being used. This is critical for safe memory access, as it + // prevents reading or writing outside the bounds of the buffer. pub(crate) len: usize, /// Unique identifier (index) of the slice in the shared memory pool. /// - /// Tracks the slice within the pool and manages memory reuse. It allows for quick - /// identification of slices when freeing or reassigning memory. + /// When in back or front mode, tracks the slice within the pool and manages memory reuse. It + /// allows for quick identification of slices when freeing or reassigning memory. If in alloc + /// mode, it is set to `IGNORE_INDEX`. pub index: u8, /// Shared state of the memory pool. /// - /// Tracks how many slices are currently in use and ensures proper synchronization of memory - /// access across multiple contexts. + /// When in back or front mode, tracks how many slices are currently in use and ensures proper + /// synchronization of memory access across multiple contexts. pub shared_state: SharedState, /// Optional dynamically allocated buffer. /// /// If present, the slice owns the memory and is responsible for managing its lifecycle. If - /// [`None`], the slice points to memory managed by the memory pool. + /// [`None`], the buffer pool is in back or front mode and the slice points to memory managed + /// by the memory pool. Is `Some(Vec)` when in alloc mode. pub owned: Option>, - /// Mode flag to track the state of the slice during development. - /// - /// Useful for identifying whether the slice is being used correctly in different modes (e.g., - /// whether is is currently being written to or read from). Typically used for logging and - /// debugging. + // Mode flag to track the state of the slice during development. + // + // Useful for identifying whether the slice is being used correctly in different modes (e.g., + // whether is is currently being written to or read from). Typically used for logging and + // debugging. #[cfg(feature = "debug")] pub mode: u8, @@ -269,67 +269,66 @@ impl From> for Slice { } } -/// The shared state of the buffer pool. -/// -/// Encapsulates an atomic 8-bit value ([`AtomicU8`]) to track the shared state of memory slices -/// in a thread-safe manner. It uses atomic operations to ensure that memory tracking can be -/// done concurrently without locks. -/// -/// Each bit in the [`AtomicU8`] represents the state of a memory slot (e.g., whether it is -/// allocated or free) in the buffer pool, allowing the system to manage and synchronize memory -/// usage across multiple slices. -/// -/// [`SharedState`] acts like a reference counter, helping the buffer pool know when a buffer slice -/// is safe to clear. Each time a memory slice is used or released, the corresponding bit in the -/// shared state is toggled. When no slices are in use (all bits are zero), the buffer pool can -/// safely reclaim or reuse the memory. -/// -/// This system ensures that no memory is prematurely cleared while it is still being referenced. -/// The buffer pool checks whether any slice is still in use before clearing, and only when the -/// shared state indicates that all references have been dropped (i.e., no unprocessed messages -/// remain) can the buffer pool safely clear or reuse the memory. +// The shared state of the buffer pool. +// +// Encapsulates an atomic 8-bit value (`AtomicU8`) to track the shared state of memory slices in a +// thread-safe manner. It uses atomic operations to ensure that memory tracking can be done +// concurrently without locks. +// +// Each bit in the `AtomicU8` represents the state of a memory slot (e.g., whether it is allocated +// or free) in the buffer pool, allowing the system to manage and synchronize memory usage across +// multiple slices. +// +// `SharedState` acts like a reference counter, helping the buffer pool know when a buffer slice is +// safe to clear. Each time a memory slice is used or released, the corresponding bit in the shared +// state is toggled. When no slices are in use (all bits are zero), the buffer pool can safely +// reclaim or reuse the memory. +// +// This system ensures that no memory is prematurely cleared while it is still being referenced. +// The buffer pool checks whether any slice is still in use before clearing, and only when the +// shared state indicates that all references have been dropped (i.e., no unprocessed messages +// remain) can the buffer pool safely clear or reuse the memory. #[derive(Clone, Debug)] pub struct SharedState(Arc); impl Default for SharedState { - /// Creates a new [`SharedState`] with an internal [`AtomicU8`] initialized to `0`, indicating - /// no memory slots are in use. + // Creates a new `SharedState` with an internal `AtomicU8` initialized to `0`, indicating no + // memory slots are in use. fn default() -> Self { Self::new() } } impl SharedState { - /// Creates a new [`SharedState`] with an internal [`AtomicU8`] initialized to `0`, indicating - /// no memory slots are in use. + // Creates a new `SharedState` with an internal `AtomicU8` initialized to `0`, indicating no + // memory slots are in use. pub fn new() -> Self { Self(Arc::new(AtomicU8::new(0))) } - /// Atomically loads and returns the current value of the [`SharedState`] using the specified - /// memory ordering. - /// - /// Returns the current state of the memory slots as an 8-bit value. + // Atomically loads and returns the current value of the `SharedState` using the specified + // memory ordering. + // + // Returns the current state of the memory slots as an 8-bit value. #[inline(always)] pub fn load(&self, ordering: Ordering) -> u8 { self.0.load(ordering) } - /// Toggles the bit at the specified `position` in the [`SharedState`], including logs - /// regarding the shared state of the memory after toggling. The `mode` parameter is used to - /// differentiate between different states or operations (e.g., reading or writing) for - /// debugging purposes. - /// - /// After a message held by a buffer slice has been processed, the corresponding bit in the - /// shared state is toggled (flipped). When the shared state for a given region reaches zero - /// (i.e., all bits are cleared), the buffer pool knows it can safely reclaim or reuse that - /// memory slice. - /// - /// Uses atomic bitwise operations to ensure thread-safe toggling without locks. It manipulates - /// the shared state in-place using the [`AtomicU8::fetch_update`] method, which atomically - /// applies a bitwise XOR (`^`) to toggle the bit at the specified `position`. - /// - /// Panics if the `position` is outside the range of 1-8, as this refers to an invalid bit. + // Toggles the bit at the specified `position` in the `SharedState`, including logs regarding + // the shared state of the memory after toggling. The `mode` parameter is used to differentiate + // between different states or operations (e.g., reading or writing) for debugging purposes. + // + // After a message held by a buffer slice has been processed, the corresponding bit in the + // shared state is toggled (flipped). When the shared state for a given region reaches zero + // (i.e., all bits are cleared), the buffer pool knows it can safely reclaim or reuse that + // memory slice. + // + // Uses atomic bitwise operations to ensure thread-safe toggling without locks. It manipulates + // the shared state in-place using the `AtomicU8::fetch_update` method, which atomically + // applies a bitwise XOR (`^`) to toggle the bit at the specified `position`. + // + // Panics if the `position` is outside the range of 1-8, as this refers to an invalid bit. #[cfg(feature = "debug")] pub fn toogle(&self, position: u8, mode: u8) { let mask: u8 = match position { @@ -358,18 +357,18 @@ impl SharedState { .unwrap(); } - /// Toggles the bit at the specified `position` in the [`SharedState`]. - /// - /// After a message held by a buffer slice has been processed, the corresponding bit in the - /// shared state is toggled (flipped). When the shared state for a given region reaches zero - /// (i.e., all bits are cleared), the buffer pool knows it can safely reclaim or reuse that - /// memory slice. - /// - /// Uses atomic bitwise operations to ensure thread-safe toggling without locks. It manipulates - /// the shared state in-place using the [`AtomicU8::fetch_update`] method, which atomically - /// applies a bitwise XOR (`^`) to toggle the bit at the specified `position`. - /// - /// Panics if the `position` is outside the range of 1-8, as this refers to an invalid bit. + // Toggles the bit at the specified `position` in the `SharedState`. + // + // After a message held by a buffer slice has been processed, the corresponding bit in the + // shared state is toggled (flipped). When the shared state for a given region reaches zero + // (i.e., all bits are cleared), the buffer pool knows it can safely reclaim or reuse that + // memory slice. + // + // Uses atomic bitwise operations to ensure thread-safe toggling without locks. It manipulates + // the shared state in-place using the `AtomicU8::fetch_update` method, which atomically + // applies a bitwise XOR (`^`) to toggle the bit at the specified `position`. + // + // Panics if the `position` is outside the range of 1-8, as this refers to an invalid bit. #[cfg(not(feature = "debug"))] pub fn toogle(&self, position: u8) { let mask: u8 = match position {