From b9361978c70f1ec17733280c08f96e04a723d60c Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 22 May 2024 21:07:59 +1000 Subject: [PATCH 01/23] Update README --- README.md | 6 +++--- examples/malloc-override/example.sh | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 examples/malloc-override/example.sh diff --git a/README.md b/README.md index b4d0822..c77765f 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,13 @@ ## Getting Started ```console -$ cargo build --release -$ env LD_PRELOAD=./target/release/libbump.so cargo # or some other command +$ cargo build -p hoard --release --features malloc +$ env LD_PRELOAD=./target/release/libhoard.so cargo --help # or some other command ``` #### Run on macOS ```console -$ env DYLD_INSERT_LIBRARIES=./target/release/libbump.dylib cargo # or some other command +$ env DYLD_INSERT_LIBRARIES=./target/release/libhoard.dylib cargo --help # or some other command ``` *Note: If you'd like to hijack the system apps and libraries as well, disable System Integrity Protection (SIP). Do this at your own risk 😉* diff --git a/examples/malloc-override/example.sh b/examples/malloc-override/example.sh new file mode 100644 index 0000000..7ad01d5 --- /dev/null +++ b/examples/malloc-override/example.sh @@ -0,0 +1,3 @@ +cargo build -p hoard --release --features malloc + +env LD_PRELOAD=./target/release/libhoard.so cargo --help \ No newline at end of file From 0fcef3ea802eacdfd2e11e6e29e999d681e8f967 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 22 May 2024 21:10:24 +1000 Subject: [PATCH 02/23] Remove __GLOBAL_ALLOCATOR --- mallockit/macros/src/lib.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mallockit/macros/src/lib.rs b/mallockit/macros/src/lib.rs index 2ba8114..16b226e 100644 --- a/mallockit/macros/src/lib.rs +++ b/mallockit/macros/src/lib.rs @@ -22,10 +22,6 @@ pub fn plan(_attr: TokenStream, item: TokenStream) -> TokenStream { } } - #[cfg(any(feature = "malloc", feature = "mallockit/malloc"))] - #[global_allocator] - static __GLOBAL_ALLOCATOR: ::mallockit::space::meta::Meta = ::mallockit::space::meta::Meta; - include!(concat!( env!("CARGO_MANIFEST_DIR"), "/../target/generated_tests.rs" From bd0ee8d5be9313dde6a3b56678610016b5cc5812 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 12:39:42 +1000 Subject: [PATCH 03/23] Refactor --- hoard/src/hoard_space.rs | 20 +-- hoard/src/pool.rs | 7 +- hoard/src/super_block.rs | 55 ++++-- mallockit/src/space/freelist_space.rs | 43 ++++- .../page_resource/block_page_resource.rs | 156 ++++++++++++++---- mallockit/src/util/constants.rs | 10 ++ mallockit/src/util/malloc/malloc_api.rs | 18 +- mallockit/src/util/mod.rs | 1 + mallockit/src/util/sys/hooks.rs | 2 +- 9 files changed, 229 insertions(+), 83 deletions(-) create mode 100644 mallockit/src/util/constants.rs diff --git a/hoard/src/hoard_space.rs b/hoard/src/hoard_space.rs index ff65a31..ab99a2e 100644 --- a/hoard/src/hoard_space.rs +++ b/hoard/src/hoard_space.rs @@ -1,25 +1,28 @@ use super::{page_resource::BlockPageResource, Allocator, Space, SpaceId}; use crate::{pool::Pool, super_block::SuperBlock}; use mallockit::{ - space::meta::{Box, Meta}, + space::{ + meta::{Box, Meta}, + page_resource::Block, + }, util::{mem::alloc::discrete_tlab::DiscreteTLAB, *}, }; /// Global heap pub struct HoardSpace { id: SpaceId, - pr: BlockPageResource, + pr: BlockPageResource, pub(crate) pool: Pool, } impl Space for HoardSpace { const MAX_ALLOCATION_SIZE: usize = SuperBlock::BYTES / 4; - type PR = BlockPageResource; + type PR = BlockPageResource; fn new(id: SpaceId) -> Self { Self { id, - pr: BlockPageResource::new(id, SuperBlock::LOG_BYTES), + pr: BlockPageResource::new(id), pool: Pool::new(true), } } @@ -56,14 +59,11 @@ impl HoardSpace { debug_assert!(!block.is_full()); register(block); debug_assert!(block.is_owned_by(local)); + mallockit::println!("Allocating block1: {:?}", block); return Some(block); } // Acquire new memory - let addr = self - .acquire::(1 << (SuperBlock::LOG_BYTES - Size4K::LOG_BYTES))? - .start - .start(); - let block = SuperBlock::new(addr); + let block = self.pr.acquire_block()?; block.init(local.static_ref(), size_class); debug_assert!(!block.is_full()); debug_assert!(block.is_empty()); @@ -78,7 +78,7 @@ impl HoardSpace { } pub fn release_block(&self, block: SuperBlock) { - self.release::(Page::new(block.start())); + self.pr.release_block(block) } } /// Thread-local heap diff --git a/hoard/src/pool.rs b/hoard/src/pool.rs index 7899ea1..b5b7c13 100644 --- a/hoard/src/pool.rs +++ b/hoard/src/pool.rs @@ -1,6 +1,9 @@ use crate::{hoard_space::HoardSpace, super_block::SuperBlock}; use array_const_fn_init::array_const_fn_init; -use mallockit::util::{mem::size_class::SizeClass, Address, Lazy, Local}; +use mallockit::{ + space::page_resource::Block, + util::{mem::size_class::SizeClass, Address, Lazy, Local}, +}; use spin::{relax::Yield, MutexGuard}; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -36,7 +39,7 @@ impl BlockList { fn group(block: SuperBlock, alloc: bool) -> usize { let u = block.used_bytes() + if alloc { block.size_class.bytes() } else { 0 } - + (Address::ZERO + SuperBlock::HEADER_BYTES) + + (Address::ZERO + SuperBlock::META_BYTES) .align_up(block.size_class.bytes()) .as_usize(); (u << 2) >> SuperBlock::LOG_BYTES diff --git a/hoard/src/super_block.rs b/hoard/src/super_block.rs index 83c4d99..32a1c4e 100644 --- a/hoard/src/super_block.rs +++ b/hoard/src/super_block.rs @@ -1,6 +1,9 @@ -use std::num::NonZeroUsize; +use std::{ + num::NonZeroUsize, + ops::{Deref, DerefMut}, +}; -use mallockit::util::mem::{aligned_block::AlignedBlockConfig, size_class::SizeClass}; +use mallockit::{space::page_resource::Block, util::mem::size_class::SizeClass}; use crate::pool::Pool; @@ -16,35 +19,57 @@ pub struct BlockMeta { pub group: u8, head_cell: Address, pub owner: &'static Pool, + pr_next: Option, } -#[mallockit::aligned_block] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct SuperBlock(NonZeroUsize); -impl AlignedBlockConfig for SuperBlock { - type Header = BlockMeta; +impl Block for SuperBlock { + type Meta = BlockMeta; + const LOG_BYTES: usize = 18; - fn from_address(address: Address) -> Self { - debug_assert!(!address.is_zero()); - debug_assert!(Self::is_aligned(address)); - Self(unsafe { NonZeroUsize::new_unchecked(usize::from(address)) }) + fn start(&self) -> Address { + Address::from(self.0.get()) } - fn into_address(self) -> Address { - Address::from(self.0.get()) + fn from_address(addr: Address) -> Self { + debug_assert!(!addr.is_zero()); + debug_assert!(Self::is_aligned(addr)); + Self(unsafe { NonZeroUsize::new_unchecked(usize::from(addr)) }) + } + + fn set_next(&self, next: Option) { + unsafe { self.meta_mut().pr_next = next }; + } + + fn next(&self) -> Option { + self.pr_next + } +} + +impl Deref for SuperBlock { + type Target = BlockMeta; + + fn deref(&self) -> &Self::Target { + self.meta() + } +} + +impl DerefMut for SuperBlock { + fn deref_mut(&mut self) -> &mut Self::Target { + unsafe { self.meta_mut() } } } impl SuperBlock { pub fn init(mut self, _local: &'static Pool, size_class: SizeClass) { - debug_assert_eq!(SuperBlock::HEADER_BYTES, Address::BYTES * 6); + debug_assert_eq!(Self::META_BYTES, Address::BYTES * 8); self.size_class = size_class; let size = size_class.bytes(); self.head_cell = Address::ZERO; - self.bump_cursor = (Address::ZERO + Self::HEADER_BYTES) - .align_up(size) - .as_usize() as u32; + self.bump_cursor = (Address::ZERO + Self::META_BYTES).align_up(size).as_usize() as u32; self.used_bytes = 0; } diff --git a/mallockit/src/space/freelist_space.rs b/mallockit/src/space/freelist_space.rs index adea5ed..35143c8 100644 --- a/mallockit/src/space/freelist_space.rs +++ b/mallockit/src/space/freelist_space.rs @@ -1,4 +1,7 @@ -use super::{page_resource::BlockPageResource, Allocator, Space, SpaceId}; +use super::{ + page_resource::{Block, BlockPageResource}, + Allocator, Space, SpaceId, +}; use crate::util::bits::{BitField, BitFieldSlot}; use crate::util::mem::freelist::intrusive_freelist::AddressSpaceConfig; use crate::util::mem::freelist::intrusive_freelist::IntrusiveFreeList; @@ -10,6 +13,30 @@ use std::{ops::Range, sync::atomic::AtomicUsize}; // type ActivePageSize = Size4K; type ActivePageSize = Size2M; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(C)] +pub struct Chunk(Address); + +impl Block for Chunk { + const LOG_BYTES: usize = ActivePageSize::LOG_BYTES; + + fn start(&self) -> Address { + self.0 + } + + fn from_address(addr: Address) -> Self { + Self(addr) + } + + fn set_next(&self, _next: Option) { + unreachable!() + } + + fn next(&self) -> Option { + unreachable!() + } +} + pub struct AddressSpace; impl AddressSpaceConfig for AddressSpace { @@ -20,18 +47,18 @@ impl AddressSpaceConfig for AddressSpace { pub struct FreeListSpace { id: SpaceId, - pr: BlockPageResource, + pr: BlockPageResource, pages: Mutex>>, } impl Space for FreeListSpace { const MAX_ALLOCATION_SIZE: usize = Size4K::BYTES; - type PR = BlockPageResource; + type PR = BlockPageResource; fn new(id: SpaceId) -> Self { Self { id, - pr: BlockPageResource::new(id, ActivePageSize::LOG_BYTES), + pr: BlockPageResource::new(id), pages: Mutex::new(None), } } @@ -137,11 +164,11 @@ impl FreeListAllocator { #[cold] fn alloc_cell_slow(&mut self, bytes: usize) -> Option> { - let page = match self.space.get_coalesced_page() { - Some(page) => page, - _ => self.space.acquire::(1)?.start, + let range = match self.space.get_coalesced_page() { + Some(page) => page.range(), + _ => self.space.pr.acquire_block()?.data(), }; - self.freelist.add_units(page.start(), ActivePageSize::BYTES); + self.freelist.add_units(range.start, ActivePageSize::BYTES); self.alloc_cell(bytes) } diff --git a/mallockit/src/space/page_resource/block_page_resource.rs b/mallockit/src/space/page_resource/block_page_resource.rs index 031b607..89fd47a 100644 --- a/mallockit/src/space/page_resource/block_page_resource.rs +++ b/mallockit/src/space/page_resource/block_page_resource.rs @@ -10,41 +10,95 @@ use std::{ sync::atomic::{AtomicUsize, Ordering}, }; -pub struct BlockPageResource { +pub trait Block: 'static + Sized + Clone + Copy { + type Meta = (); + + const LOG_BYTES: usize; + const BYTES: usize = 1 << Self::LOG_BYTES; + + const META_BYTES: usize = std::mem::size_of::().next_power_of_two(); + + fn start(&self) -> Address; + fn from_address(addr: Address) -> Self; + fn set_next(&self, next: Option); + fn next(&self) -> Option; + + fn data_start(&self) -> Address { + self.start() + Self::META_BYTES + } + + fn end(&self) -> Address { + self.start() + Self::BYTES + } + + fn data(&self) -> Range
{ + self.data_start()..self.end() + } + + fn range(&self) -> Range
{ + self.start()..self.end() + } + + fn meta(&self) -> &Self::Meta { + unsafe { &*(self.start().as_ptr::()) } + } + + /// # Safety + /// The caller must ensure that the block is within its corresponding space, and the block is properly aligned. + #[allow(clippy::mut_from_ref)] + unsafe fn meta_mut(&self) -> &mut Self::Meta { + &mut *(self.start().as_mut_ptr::()) + } + + fn is_aligned(addr: Address) -> bool { + addr.is_aligned_to(Self::BYTES) + } + + fn align(addr: Address) -> Address { + addr.align_down(Self::BYTES) + } + + fn containing(addr: Address) -> Self { + let start = Self::align(addr); + Self::from_address(start) + } +} + +pub struct BlockPageResource { pub id: SpaceId, - log_bytes: usize, cursor: Atomic
, highwater: Address, - recycled_blocks: SegQueue
, + recycled_blocks_intrusive: Atomic>, + recycled_blocks_non_intrusive: SegQueue
, reserved_bytes: AtomicUsize, } -impl BlockPageResource { - pub fn new(id: SpaceId, log_bytes: usize) -> Self { +impl BlockPageResource { + pub fn new(id: SpaceId) -> Self { debug_assert!(id.0 < 0b0000_1111); - debug_assert!(log_bytes >= Size4K::LOG_BYTES); + debug_assert!(B::LOG_BYTES >= Size4K::LOG_BYTES); let range = HEAP.get_space_range(id); Self { id, - log_bytes, cursor: Atomic::new(range.start), highwater: range.end, - recycled_blocks: SegQueue::new(), + recycled_blocks_intrusive: Atomic::new(None), + recycled_blocks_non_intrusive: SegQueue::new(), reserved_bytes: AtomicUsize::new(0), } } #[cold] fn acquire_block_slow(&self, pages: usize) -> Option>> { - debug_assert!(self.log_bytes >= S::LOG_BYTES); - debug_assert_eq!(pages, 1 << (self.log_bytes - S::LOG_BYTES)); + debug_assert!(B::LOG_BYTES >= S::LOG_BYTES); + debug_assert_eq!(pages, 1 << (B::LOG_BYTES - S::LOG_BYTES)); let block = self .cursor .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |a| { if a >= self.highwater { None } else { - Some(a + (1usize << self.log_bytes)) + Some(a + (1usize << B::LOG_BYTES)) } }); match block { @@ -56,37 +110,69 @@ impl BlockPageResource { Err(_) => None, } } + + pub fn acquire_block(&self) -> Option { + if INTRUSIVE { + loop { + let head = self.recycled_blocks_intrusive.load(Ordering::Relaxed); + if let Some(block) = head { + if self + .recycled_blocks_intrusive + .compare_exchange(head, block.next(), Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + self.reserved_bytes.fetch_add(B::BYTES, Ordering::Relaxed); + return Some(block); + } + } else { + break; + } + } + } else if let Some(addr) = self.recycled_blocks_non_intrusive.pop() { + let block = B::from_address(addr); + self.reserved_bytes.fetch_add(B::BYTES, Ordering::Relaxed); + return Some(block); + } + let range = self.acquire_block_slow::(B::BYTES >> Size4K::LOG_BYTES)?; + let block = B::from_address(range.start.start()); + if INTRUSIVE { + block.set_next(None); + } + self.reserved_bytes.fetch_add(B::BYTES, Ordering::Relaxed); + Some(block) + } + + pub fn release_block(&self, block: B) { + if INTRUSIVE { + loop { + let head = self.recycled_blocks_intrusive.load(Ordering::Relaxed); + block.set_next(head); + if self + .recycled_blocks_intrusive + .compare_exchange(head, Some(block), Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + break; + } + } + } else { + self.recycled_blocks_non_intrusive.push(block.start()); + } + self.reserved_bytes + .fetch_sub(1 << B::LOG_BYTES, Ordering::Relaxed); + } } -impl PageResource for BlockPageResource { +impl PageResource for BlockPageResource { fn reserved_bytes(&self) -> usize { self.reserved_bytes.load(Ordering::Relaxed) } - fn acquire_pages(&self, pages: usize) -> Option>> { - debug_assert!(self.log_bytes >= S::LOG_BYTES); - debug_assert_eq!(pages, 1 << (self.log_bytes - S::LOG_BYTES)); - if let Some(addr) = self.recycled_blocks.pop() { - let start = Page::::new(addr); - let end = Step::forward(start, pages); - self.reserved_bytes - .fetch_add(1 << self.log_bytes, Ordering::Relaxed); - return Some(start..end); - } - if let Some(result) = self.acquire_block_slow(pages) { - self.reserved_bytes - .fetch_add(1 << self.log_bytes, Ordering::Relaxed); - return Some(result); - } - None + fn acquire_pages(&self, _pages: usize) -> Option>> { + unreachable!("Use `alloc_block` instead") } - fn release_pages(&self, start: Page) { - // NOTE: `SegQueue::push` space expansion indirectly calls `malloc`. - // It's safe for now since we don't call `release_pages` in `malloc`. - // Review this later when we start to work on lazy sweeping. - self.recycled_blocks.push(start.start()); - self.reserved_bytes - .fetch_sub(1 << self.log_bytes, Ordering::Relaxed); + fn release_pages(&self, _start: Page) { + unreachable!("Use `release_block` instead") } } diff --git a/mallockit/src/util/constants.rs b/mallockit/src/util/constants.rs new file mode 100644 index 0000000..27d5aac --- /dev/null +++ b/mallockit/src/util/constants.rs @@ -0,0 +1,10 @@ +#[cfg(not(any( + target_os = "macos", + all(target_os = "windows", target_pointer_width = "64") +)))] +pub const MIN_ALIGNMENT: usize = 16; // should be 8? +#[cfg(any( + target_os = "macos", + all(target_os = "windows", target_pointer_width = "64") +))] +pub const MIN_ALIGNMENT: usize = 16; diff --git a/mallockit/src/util/malloc/malloc_api.rs b/mallockit/src/util/malloc/malloc_api.rs index a2a2301..5a02291 100644 --- a/mallockit/src/util/malloc/malloc_api.rs +++ b/mallockit/src/util/malloc/malloc_api.rs @@ -1,3 +1,4 @@ +use crate::util::constants::MIN_ALIGNMENT; use crate::util::mem::heap::HEAP; use crate::util::Address; use crate::util::Lazy; @@ -18,16 +19,7 @@ impl GetMutatorType for MallocAPI

{ #[allow(unused)] impl MallocAPI

{ - #[cfg(not(any( - target_os = "macos", - all(target_os = "windows", target_pointer_width = "64") - )))] - pub const MIN_ALIGNMENT: usize = 16; // should be 8? - #[cfg(any( - target_os = "macos", - all(target_os = "windows", target_pointer_width = "64") - ))] - pub const MIN_ALIGNMENT: usize = 16; + pub const MIN_ALIGNMENT: usize = MIN_ALIGNMENT; pub const PAGE_SIZE: usize = 4096; pub const fn new(plan: &'static Lazy

) -> Self { @@ -396,8 +388,10 @@ macro_rules! export_rust_global_alloc_api { &self, mut layout: Layout, ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { - if layout.align() < 16 { - layout = layout.align_to(16).unwrap(); + if layout.align() < $crate::util::constants::MIN_ALIGNMENT { + layout = layout + .align_to($crate::util::constants::MIN_ALIGNMENT) + .unwrap(); } layout = unsafe { layout.pad_to_align_unchecked() }; let start = <$plan_ty as $crate::Plan>::Mutator::current() diff --git a/mallockit/src/util/mod.rs b/mallockit/src/util/mod.rs index 053b80f..42e8537 100644 --- a/mallockit/src/util/mod.rs +++ b/mallockit/src/util/mod.rs @@ -18,6 +18,7 @@ macro_rules! name_list { } pub mod bits; +pub mod constants; mod lazy; #[macro_use] pub mod malloc; diff --git a/mallockit/src/util/sys/hooks.rs b/mallockit/src/util/sys/hooks.rs index 604c650..87882d4 100644 --- a/mallockit/src/util/sys/hooks.rs +++ b/mallockit/src/util/sys/hooks.rs @@ -3,7 +3,7 @@ use std::panic::PanicInfo; use crate::Plan; fn panic_handler(panic_info: &PanicInfo<'_>) { - println!("{}", panic_info); + crate::println!("{}", panic_info); std::process::abort(); } From d147ce95a6c5cff53fe773f3a5de87103868399e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 12:40:29 +1000 Subject: [PATCH 04/23] Refactor --- hoard/src/hoard_space.rs | 2 +- hoard/src/pool.rs | 2 +- hoard/src/super_block.rs | 4 ++-- mallockit/src/space/freelist_space.rs | 4 ++-- mallockit/src/space/page_resource/block_page_resource.rs | 8 ++++---- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/hoard/src/hoard_space.rs b/hoard/src/hoard_space.rs index ab99a2e..2195451 100644 --- a/hoard/src/hoard_space.rs +++ b/hoard/src/hoard_space.rs @@ -3,7 +3,7 @@ use crate::{pool::Pool, super_block::SuperBlock}; use mallockit::{ space::{ meta::{Box, Meta}, - page_resource::Block, + page_resource::MemRegion, }, util::{mem::alloc::discrete_tlab::DiscreteTLAB, *}, }; diff --git a/hoard/src/pool.rs b/hoard/src/pool.rs index b5b7c13..1cd4aa5 100644 --- a/hoard/src/pool.rs +++ b/hoard/src/pool.rs @@ -1,7 +1,7 @@ use crate::{hoard_space::HoardSpace, super_block::SuperBlock}; use array_const_fn_init::array_const_fn_init; use mallockit::{ - space::page_resource::Block, + space::page_resource::MemRegion, util::{mem::size_class::SizeClass, Address, Lazy, Local}, }; use spin::{relax::Yield, MutexGuard}; diff --git a/hoard/src/super_block.rs b/hoard/src/super_block.rs index 32a1c4e..a8bc2b5 100644 --- a/hoard/src/super_block.rs +++ b/hoard/src/super_block.rs @@ -3,7 +3,7 @@ use std::{ ops::{Deref, DerefMut}, }; -use mallockit::{space::page_resource::Block, util::mem::size_class::SizeClass}; +use mallockit::{space::page_resource::MemRegion, util::mem::size_class::SizeClass}; use crate::pool::Pool; @@ -25,7 +25,7 @@ pub struct BlockMeta { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct SuperBlock(NonZeroUsize); -impl Block for SuperBlock { +impl MemRegion for SuperBlock { type Meta = BlockMeta; const LOG_BYTES: usize = 18; diff --git a/mallockit/src/space/freelist_space.rs b/mallockit/src/space/freelist_space.rs index 35143c8..bd38838 100644 --- a/mallockit/src/space/freelist_space.rs +++ b/mallockit/src/space/freelist_space.rs @@ -1,5 +1,5 @@ use super::{ - page_resource::{Block, BlockPageResource}, + page_resource::{BlockPageResource, MemRegion}, Allocator, Space, SpaceId, }; use crate::util::bits::{BitField, BitFieldSlot}; @@ -17,7 +17,7 @@ type ActivePageSize = Size2M; #[repr(C)] pub struct Chunk(Address); -impl Block for Chunk { +impl MemRegion for Chunk { const LOG_BYTES: usize = ActivePageSize::LOG_BYTES; fn start(&self) -> Address { diff --git a/mallockit/src/space/page_resource/block_page_resource.rs b/mallockit/src/space/page_resource/block_page_resource.rs index 89fd47a..5c4a0e0 100644 --- a/mallockit/src/space/page_resource/block_page_resource.rs +++ b/mallockit/src/space/page_resource/block_page_resource.rs @@ -10,7 +10,7 @@ use std::{ sync::atomic::{AtomicUsize, Ordering}, }; -pub trait Block: 'static + Sized + Clone + Copy { +pub trait MemRegion: 'static + Sized + Clone + Copy { type Meta = (); const LOG_BYTES: usize; @@ -64,7 +64,7 @@ pub trait Block: 'static + Sized + Clone + Copy { } } -pub struct BlockPageResource { +pub struct BlockPageResource { pub id: SpaceId, cursor: Atomic

, highwater: Address, @@ -73,7 +73,7 @@ pub struct BlockPageResource { reserved_bytes: AtomicUsize, } -impl BlockPageResource { +impl BlockPageResource { pub fn new(id: SpaceId) -> Self { debug_assert!(id.0 < 0b0000_1111); debug_assert!(B::LOG_BYTES >= Size4K::LOG_BYTES); @@ -163,7 +163,7 @@ impl BlockPageResource { } } -impl PageResource for BlockPageResource { +impl PageResource for BlockPageResource { fn reserved_bytes(&self) -> usize { self.reserved_bytes.load(Ordering::Relaxed) } From ee2928a01c31b24369e3135dbef5a9002beb289b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 12:44:24 +1000 Subject: [PATCH 05/23] bump rust version --- hoard/src/super_block.rs | 1 + .../src/space/page_resource/freelist_page_resource.rs | 4 +++- mallockit/src/space/page_table.rs | 10 ++++++++-- mallockit/src/util/mem/size_class.rs | 2 +- rust-toolchain | 2 +- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/hoard/src/super_block.rs b/hoard/src/super_block.rs index a8bc2b5..e090193 100644 --- a/hoard/src/super_block.rs +++ b/hoard/src/super_block.rs @@ -44,6 +44,7 @@ impl MemRegion for SuperBlock { unsafe { self.meta_mut().pr_next = next }; } + #[allow(clippy::misnamed_getters)] fn next(&self) -> Option { self.pr_next } diff --git a/mallockit/src/space/page_resource/freelist_page_resource.rs b/mallockit/src/space/page_resource/freelist_page_resource.rs index 7f05854..60f8253 100644 --- a/mallockit/src/space/page_resource/freelist_page_resource.rs +++ b/mallockit/src/space/page_resource/freelist_page_resource.rs @@ -38,7 +38,9 @@ impl FreelistPageResource { id, freelist: Mutex::new(freelist), reserved_bytes: AtomicUsize::new(0), - meta: RwLock::new(unsafe { std::mem::transmute(meta) }), + meta: RwLock::new(unsafe { + std::mem::transmute::, Vec>(meta) + }), base, } } diff --git a/mallockit/src/space/page_table.rs b/mallockit/src/space/page_table.rs index 442bc58..0c28701 100644 --- a/mallockit/src/space/page_table.rs +++ b/mallockit/src/space/page_table.rs @@ -148,7 +148,11 @@ impl PageTable { Some(_) => unreachable!(), _ => { let table = Box::leak(meta_box!(PageTable:: { - table: unsafe { mem::transmute([0usize; 512]) }, + table: unsafe { + mem::transmute::<[usize; 512], [PageTableEntry; 512]>( + [0usize; 512], + ) + }, phantom: PhantomData, })); self.table[index].set_next_page_table(table); @@ -168,7 +172,9 @@ impl PageTable { impl PageTable { pub(crate) const fn new() -> Self { Self { - table: unsafe { mem::transmute([0usize; 512]) }, + table: unsafe { + mem::transmute::<[usize; 512], [PageTableEntry; 512]>([0usize; 512]) + }, phantom: PhantomData, } } diff --git a/mallockit/src/util/mem/size_class.rs b/mallockit/src/util/mem/size_class.rs index 1b5f9f3..bca886f 100644 --- a/mallockit/src/util/mem/size_class.rs +++ b/mallockit/src/util/mem/size_class.rs @@ -1,4 +1,4 @@ -use std::{alloc::Layout, usize}; +use std::alloc::Layout; use super::layout_utils::LayoutUtils; diff --git a/rust-toolchain b/rust-toolchain index b2833d7..f4ea31c 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2024-03-01 \ No newline at end of file +nightly-2024-05-01 \ No newline at end of file From 582927564fc634a7a47f67abc8600f3864621a95 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 13:27:38 +1000 Subject: [PATCH 06/23] Minor --- buddy/Cargo.toml | 2 +- bump/Cargo.toml | 2 +- hoard/Cargo.toml | 2 +- mallockit/Cargo.toml | 2 +- mallockit/macros/Cargo.toml | 2 +- sanity/Cargo.toml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/buddy/Cargo.toml b/buddy/Cargo.toml index ff5a22a..63cd4cb 100644 --- a/buddy/Cargo.toml +++ b/buddy/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "buddy" version = { workspace = true } -authors = ["Wenyu Zhao "] +authors = ["Wenyu Zhao "] edition = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/bump/Cargo.toml b/bump/Cargo.toml index a5d97e3..95543d6 100644 --- a/bump/Cargo.toml +++ b/bump/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "bump" version = { workspace = true } -authors = ["Wenyu Zhao "] +authors = ["Wenyu Zhao "] edition = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/hoard/Cargo.toml b/hoard/Cargo.toml index de02769..7cc74b9 100644 --- a/hoard/Cargo.toml +++ b/hoard/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "hoard" version = { workspace = true } -authors = ["Wenyu Zhao "] +authors = ["Wenyu Zhao "] edition = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/mallockit/Cargo.toml b/mallockit/Cargo.toml index 1333b13..92404d3 100644 --- a/mallockit/Cargo.toml +++ b/mallockit/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "mallockit" version = { workspace = true } -authors = ["Wenyu Zhao "] +authors = ["Wenyu Zhao "] edition = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/mallockit/macros/Cargo.toml b/mallockit/macros/Cargo.toml index 30c3a79..95f959d 100644 --- a/mallockit/macros/Cargo.toml +++ b/mallockit/macros/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "mallockit-macros" version = { workspace = true } -authors = ["Wenyu Zhao "] +authors = ["Wenyu Zhao "] edition = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/sanity/Cargo.toml b/sanity/Cargo.toml index d481025..91a192a 100644 --- a/sanity/Cargo.toml +++ b/sanity/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "sanity" version = { workspace = true } -authors = ["Wenyu Zhao "] +authors = ["Wenyu Zhao "] edition = { workspace = true } # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 7ff89cf13713f1f14e44d81c6b733c0b7f79052f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 13:31:55 +1000 Subject: [PATCH 07/23] Minor --- mallockit/src/util/constants.rs | 6 ++++-- mallockit/src/util/mem/size_class.rs | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/mallockit/src/util/constants.rs b/mallockit/src/util/constants.rs index 27d5aac..1de30e6 100644 --- a/mallockit/src/util/constants.rs +++ b/mallockit/src/util/constants.rs @@ -2,9 +2,11 @@ target_os = "macos", all(target_os = "windows", target_pointer_width = "64") )))] -pub const MIN_ALIGNMENT: usize = 16; // should be 8? +pub const LOG_MIN_ALIGNMENT: usize = 4; // should be 8? #[cfg(any( target_os = "macos", all(target_os = "windows", target_pointer_width = "64") ))] -pub const MIN_ALIGNMENT: usize = 16; +pub const LOG_MIN_ALIGNMENT: usize = 4; + +pub const MIN_ALIGNMENT: usize = 1 << LOG_MIN_ALIGNMENT; diff --git a/mallockit/src/util/mem/size_class.rs b/mallockit/src/util/mem/size_class.rs index bca886f..c21917d 100644 --- a/mallockit/src/util/mem/size_class.rs +++ b/mallockit/src/util/mem/size_class.rs @@ -1,10 +1,12 @@ use std::alloc::Layout; +use crate::util::constants::LOG_MIN_ALIGNMENT; + use super::layout_utils::LayoutUtils; #[repr(transparent)] #[derive(Debug, Clone, Copy)] -pub struct SizeClass(pub u8); +pub struct SizeClass(pub u8); impl SizeClass { pub const fn as_usize(self) -> usize { From fdbff83c4c9cf560ad4fb43a887d5e9ead4304f3 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 14:48:05 +1000 Subject: [PATCH 08/23] Support all rust allocator apis --- mallockit/src/mutator.rs | 16 ++- mallockit/src/util/malloc/malloc_api.rs | 127 ++++++++++++++++++++++-- 2 files changed, 129 insertions(+), 14 deletions(-) diff --git a/mallockit/src/mutator.rs b/mallockit/src/mutator.rs index d897769..b7a91db 100644 --- a/mallockit/src/mutator.rs +++ b/mallockit/src/mutator.rs @@ -33,25 +33,33 @@ pub trait Mutator: Sized + 'static + TLS { fn dealloc(&mut self, ptr: Address); - fn realloc(&mut self, ptr: Address, new_size: usize) -> Option
{ + fn realloc(&mut self, ptr: Address, new_layout: Layout) -> Option
{ let layout = Self::Plan::get_layout(ptr); - if layout.size() >= new_size { + if layout.size() >= new_layout.size() && layout.align() >= new_layout.align() { return Some(ptr); } - let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; let new_ptr = self.alloc(new_layout); if let Some(new_ptr) = new_ptr { unsafe { ptr::copy_nonoverlapping( ptr.as_ptr::(), new_ptr.as_mut_ptr::(), - usize::min(layout.size(), new_size), + usize::min(layout.size(), new_layout.size()), ); } self.dealloc(ptr); } new_ptr } + + fn realloc_zeroed(&mut self, ptr: Address, new_layout: Layout) -> Option
{ + let size = new_layout.size(); + let new_ptr = self.realloc(ptr, new_layout); + if let Some(new_ptr) = new_ptr { + unsafe { ptr::write_bytes(new_ptr.as_mut_ptr::(), 0, size) }; + } + new_ptr + } } pub(crate) struct InternalTLS { diff --git a/mallockit/src/util/malloc/malloc_api.rs b/mallockit/src/util/malloc/malloc_api.rs index 5a02291..fa9c798 100644 --- a/mallockit/src/util/malloc/malloc_api.rs +++ b/mallockit/src/util/malloc/malloc_api.rs @@ -146,7 +146,8 @@ impl MallocAPI

{ return new_ptr.into(); } - match self.mutator().realloc(ptr.into(), new_size) { + let layout = Layout::from_size_align_unchecked(new_size, Self::MIN_ALIGNMENT); + match self.mutator().realloc(ptr.into(), layout) { Some(ptr) => ptr.into(), None => { if free_if_fail { @@ -383,17 +384,24 @@ macro_rules! export_rust_global_alloc_api { ($plan: expr, $plan_ty: ty) => { pub struct Global; - unsafe impl ::std::alloc::Allocator for Global { - fn allocate( - &self, - mut layout: Layout, - ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { + impl Global { + pub fn __fix_layout(mut layout: ::std::alloc::Layout) -> Layout { if layout.align() < $crate::util::constants::MIN_ALIGNMENT { layout = layout .align_to($crate::util::constants::MIN_ALIGNMENT) .unwrap(); } layout = unsafe { layout.pad_to_align_unchecked() }; + layout + } + } + + unsafe impl ::std::alloc::Allocator for Global { + fn allocate( + &self, + mut layout: ::std::alloc::Layout, + ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { + layout = Self::__fix_layout(layout); let start = <$plan_ty as $crate::Plan>::Mutator::current() .alloc(layout) .unwrap_or($crate::util::Address::ZERO); @@ -402,6 +410,21 @@ macro_rules! export_rust_global_alloc_api { }; Ok(::std::ptr::NonNull::from(slice)) } + + fn allocate_zeroed( + &self, + mut layout: ::std::alloc::Layout, + ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { + layout = Self::__fix_layout(layout); + let start = <$plan_ty as $crate::Plan>::Mutator::current() + .alloc_zeroed(layout) + .unwrap_or($crate::util::Address::ZERO); + let slice = unsafe { + ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, layout.size()) + }; + Ok(::std::ptr::NonNull::from(slice)) + } + unsafe fn deallocate( &self, ptr: ::std::ptr::NonNull, @@ -409,22 +432,106 @@ macro_rules! export_rust_global_alloc_api { ) { <$plan_ty as $crate::Plan>::Mutator::current().dealloc(ptr.as_ptr().into()) } + + unsafe fn grow( + &self, + ptr: ::std::ptr::NonNull, + old_layout: ::std::alloc::Layout, + mut new_layout: ::std::alloc::Layout, + ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { + debug_assert!( + new_layout.size() >= old_layout.size(), + "`new_layout.size()` must be greater than or equal to `old_layout.size()`" + ); + + new_layout = Self::__fix_layout(new_layout); + let start = <$plan_ty as $crate::Plan>::Mutator::current() + .realloc(ptr.as_ptr().into(), new_layout) + .unwrap_or($crate::util::Address::ZERO); + let slice = unsafe { + ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, new_layout.size()) + }; + Ok(::std::ptr::NonNull::from(slice)) + } + + unsafe fn grow_zeroed( + &self, + ptr: ::std::ptr::NonNull, + old_layout: ::std::alloc::Layout, + mut new_layout: ::std::alloc::Layout, + ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { + debug_assert!( + new_layout.size() >= old_layout.size(), + "`new_layout.size()` must be greater than or equal to `old_layout.size()`" + ); + + new_layout = Self::__fix_layout(new_layout); + let start = <$plan_ty as $crate::Plan>::Mutator::current() + .realloc_zeroed(ptr.as_ptr().into(), new_layout) + .unwrap_or($crate::util::Address::ZERO); + let slice = unsafe { + ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, new_layout.size()) + }; + Ok(::std::ptr::NonNull::from(slice)) + } + + unsafe fn shrink( + &self, + ptr: ::std::ptr::NonNull, + old_layout: ::std::alloc::Layout, + mut new_layout: ::std::alloc::Layout, + ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { + debug_assert!( + new_layout.size() <= old_layout.size(), + "`new_layout.size()` must be smaller than or equal to `old_layout.size()`" + ); + + new_layout = Self::__fix_layout(new_layout); + let start = <$plan_ty as $crate::Plan>::Mutator::current() + .realloc(ptr.as_ptr().into(), new_layout) + .unwrap_or($crate::util::Address::ZERO); + let slice = unsafe { + ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, new_layout.size()) + }; + Ok(::std::ptr::NonNull::from(slice)) + } } unsafe impl ::std::alloc::GlobalAlloc for Global { unsafe fn alloc(&self, mut layout: ::std::alloc::Layout) -> *mut u8 { - if layout.align() < 16 { - layout = layout.align_to(16).unwrap(); - } - layout = layout.pad_to_align_unchecked(); + layout = Self::__fix_layout(layout); <$plan_ty as $crate::Plan>::Mutator::current() .alloc(layout) .unwrap_or($crate::util::Address::ZERO) .into() } + + unsafe fn alloc_zeroed(&self, mut layout: ::std::alloc::Layout) -> *mut u8 { + layout = Self::__fix_layout(layout); + <$plan_ty as $crate::Plan>::Mutator::current() + .alloc_zeroed(layout) + .unwrap_or($crate::util::Address::ZERO) + .into() + } + unsafe fn dealloc(&self, ptr: *mut u8, _layout: ::std::alloc::Layout) { <$plan_ty as $crate::Plan>::Mutator::current().dealloc(ptr.into()) } + + unsafe fn realloc( + &self, + ptr: *mut u8, + layout: ::std::alloc::Layout, + new_size: usize, + ) -> *mut u8 { + let mut new_layout = + unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + new_layout = Self::__fix_layout(new_layout); + <$plan_ty as $crate::Plan>::Mutator::current() + .realloc(ptr.into(), new_layout) + .unwrap_or($crate::util::Address::ZERO) + .into() + } } }; } From d1799a8750fc1b2b07011b2f44e9d70d377764f0 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 14:49:08 +1000 Subject: [PATCH 09/23] refactor --- mallockit/src/util/malloc/malloc_api.rs | 158 ------------------------ mallockit/src/util/malloc/mod.rs | 2 + mallockit/src/util/malloc/rust_alloc.rs | 157 +++++++++++++++++++++++ 3 files changed, 159 insertions(+), 158 deletions(-) create mode 100644 mallockit/src/util/malloc/rust_alloc.rs diff --git a/mallockit/src/util/malloc/malloc_api.rs b/mallockit/src/util/malloc/malloc_api.rs index fa9c798..688e72a 100644 --- a/mallockit/src/util/malloc/malloc_api.rs +++ b/mallockit/src/util/malloc/malloc_api.rs @@ -377,161 +377,3 @@ macro_rules! export_malloc_api_macos { } }; } - -#[macro_export] -#[doc(hidden)] -macro_rules! export_rust_global_alloc_api { - ($plan: expr, $plan_ty: ty) => { - pub struct Global; - - impl Global { - pub fn __fix_layout(mut layout: ::std::alloc::Layout) -> Layout { - if layout.align() < $crate::util::constants::MIN_ALIGNMENT { - layout = layout - .align_to($crate::util::constants::MIN_ALIGNMENT) - .unwrap(); - } - layout = unsafe { layout.pad_to_align_unchecked() }; - layout - } - } - - unsafe impl ::std::alloc::Allocator for Global { - fn allocate( - &self, - mut layout: ::std::alloc::Layout, - ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { - layout = Self::__fix_layout(layout); - let start = <$plan_ty as $crate::Plan>::Mutator::current() - .alloc(layout) - .unwrap_or($crate::util::Address::ZERO); - let slice = unsafe { - ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, layout.size()) - }; - Ok(::std::ptr::NonNull::from(slice)) - } - - fn allocate_zeroed( - &self, - mut layout: ::std::alloc::Layout, - ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { - layout = Self::__fix_layout(layout); - let start = <$plan_ty as $crate::Plan>::Mutator::current() - .alloc_zeroed(layout) - .unwrap_or($crate::util::Address::ZERO); - let slice = unsafe { - ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, layout.size()) - }; - Ok(::std::ptr::NonNull::from(slice)) - } - - unsafe fn deallocate( - &self, - ptr: ::std::ptr::NonNull, - layout: ::std::alloc::Layout, - ) { - <$plan_ty as $crate::Plan>::Mutator::current().dealloc(ptr.as_ptr().into()) - } - - unsafe fn grow( - &self, - ptr: ::std::ptr::NonNull, - old_layout: ::std::alloc::Layout, - mut new_layout: ::std::alloc::Layout, - ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { - debug_assert!( - new_layout.size() >= old_layout.size(), - "`new_layout.size()` must be greater than or equal to `old_layout.size()`" - ); - - new_layout = Self::__fix_layout(new_layout); - let start = <$plan_ty as $crate::Plan>::Mutator::current() - .realloc(ptr.as_ptr().into(), new_layout) - .unwrap_or($crate::util::Address::ZERO); - let slice = unsafe { - ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, new_layout.size()) - }; - Ok(::std::ptr::NonNull::from(slice)) - } - - unsafe fn grow_zeroed( - &self, - ptr: ::std::ptr::NonNull, - old_layout: ::std::alloc::Layout, - mut new_layout: ::std::alloc::Layout, - ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { - debug_assert!( - new_layout.size() >= old_layout.size(), - "`new_layout.size()` must be greater than or equal to `old_layout.size()`" - ); - - new_layout = Self::__fix_layout(new_layout); - let start = <$plan_ty as $crate::Plan>::Mutator::current() - .realloc_zeroed(ptr.as_ptr().into(), new_layout) - .unwrap_or($crate::util::Address::ZERO); - let slice = unsafe { - ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, new_layout.size()) - }; - Ok(::std::ptr::NonNull::from(slice)) - } - - unsafe fn shrink( - &self, - ptr: ::std::ptr::NonNull, - old_layout: ::std::alloc::Layout, - mut new_layout: ::std::alloc::Layout, - ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { - debug_assert!( - new_layout.size() <= old_layout.size(), - "`new_layout.size()` must be smaller than or equal to `old_layout.size()`" - ); - - new_layout = Self::__fix_layout(new_layout); - let start = <$plan_ty as $crate::Plan>::Mutator::current() - .realloc(ptr.as_ptr().into(), new_layout) - .unwrap_or($crate::util::Address::ZERO); - let slice = unsafe { - ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, new_layout.size()) - }; - Ok(::std::ptr::NonNull::from(slice)) - } - } - - unsafe impl ::std::alloc::GlobalAlloc for Global { - unsafe fn alloc(&self, mut layout: ::std::alloc::Layout) -> *mut u8 { - layout = Self::__fix_layout(layout); - <$plan_ty as $crate::Plan>::Mutator::current() - .alloc(layout) - .unwrap_or($crate::util::Address::ZERO) - .into() - } - - unsafe fn alloc_zeroed(&self, mut layout: ::std::alloc::Layout) -> *mut u8 { - layout = Self::__fix_layout(layout); - <$plan_ty as $crate::Plan>::Mutator::current() - .alloc_zeroed(layout) - .unwrap_or($crate::util::Address::ZERO) - .into() - } - - unsafe fn dealloc(&self, ptr: *mut u8, _layout: ::std::alloc::Layout) { - <$plan_ty as $crate::Plan>::Mutator::current().dealloc(ptr.into()) - } - - unsafe fn realloc( - &self, - ptr: *mut u8, - layout: ::std::alloc::Layout, - new_size: usize, - ) -> *mut u8 { - let mut new_layout = - unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; - new_layout = Self::__fix_layout(new_layout); - <$plan_ty as $crate::Plan>::Mutator::current() - .realloc(ptr.into(), new_layout) - .unwrap_or($crate::util::Address::ZERO) - .into() - } - } - }; -} diff --git a/mallockit/src/util/malloc/mod.rs b/mallockit/src/util/malloc/mod.rs index 8648133..5cef82e 100644 --- a/mallockit/src/util/malloc/mod.rs +++ b/mallockit/src/util/malloc/mod.rs @@ -2,5 +2,7 @@ pub mod macos_malloc_zone; #[macro_use] mod malloc_api; +#[macro_use] +mod rust_alloc; pub use malloc_api::*; diff --git a/mallockit/src/util/malloc/rust_alloc.rs b/mallockit/src/util/malloc/rust_alloc.rs new file mode 100644 index 0000000..eab4c18 --- /dev/null +++ b/mallockit/src/util/malloc/rust_alloc.rs @@ -0,0 +1,157 @@ +#[macro_export] +#[doc(hidden)] +macro_rules! export_rust_global_alloc_api { + ($plan: expr, $plan_ty: ty) => { + pub struct Global; + + impl Global { + pub fn __fix_layout(mut layout: ::std::alloc::Layout) -> Layout { + if layout.align() < $crate::util::constants::MIN_ALIGNMENT { + layout = layout + .align_to($crate::util::constants::MIN_ALIGNMENT) + .unwrap(); + } + layout = unsafe { layout.pad_to_align_unchecked() }; + layout + } + } + + unsafe impl ::std::alloc::Allocator for Global { + fn allocate( + &self, + mut layout: ::std::alloc::Layout, + ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { + layout = Self::__fix_layout(layout); + let start = <$plan_ty as $crate::Plan>::Mutator::current() + .alloc(layout) + .unwrap_or($crate::util::Address::ZERO); + let slice = unsafe { + ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, layout.size()) + }; + Ok(::std::ptr::NonNull::from(slice)) + } + + fn allocate_zeroed( + &self, + mut layout: ::std::alloc::Layout, + ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { + layout = Self::__fix_layout(layout); + let start = <$plan_ty as $crate::Plan>::Mutator::current() + .alloc_zeroed(layout) + .unwrap_or($crate::util::Address::ZERO); + let slice = unsafe { + ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, layout.size()) + }; + Ok(::std::ptr::NonNull::from(slice)) + } + + unsafe fn deallocate( + &self, + ptr: ::std::ptr::NonNull, + layout: ::std::alloc::Layout, + ) { + <$plan_ty as $crate::Plan>::Mutator::current().dealloc(ptr.as_ptr().into()) + } + + unsafe fn grow( + &self, + ptr: ::std::ptr::NonNull, + old_layout: ::std::alloc::Layout, + mut new_layout: ::std::alloc::Layout, + ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { + debug_assert!( + new_layout.size() >= old_layout.size(), + "`new_layout.size()` must be greater than or equal to `old_layout.size()`" + ); + + new_layout = Self::__fix_layout(new_layout); + let start = <$plan_ty as $crate::Plan>::Mutator::current() + .realloc(ptr.as_ptr().into(), new_layout) + .unwrap_or($crate::util::Address::ZERO); + let slice = unsafe { + ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, new_layout.size()) + }; + Ok(::std::ptr::NonNull::from(slice)) + } + + unsafe fn grow_zeroed( + &self, + ptr: ::std::ptr::NonNull, + old_layout: ::std::alloc::Layout, + mut new_layout: ::std::alloc::Layout, + ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { + debug_assert!( + new_layout.size() >= old_layout.size(), + "`new_layout.size()` must be greater than or equal to `old_layout.size()`" + ); + + new_layout = Self::__fix_layout(new_layout); + let start = <$plan_ty as $crate::Plan>::Mutator::current() + .realloc_zeroed(ptr.as_ptr().into(), new_layout) + .unwrap_or($crate::util::Address::ZERO); + let slice = unsafe { + ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, new_layout.size()) + }; + Ok(::std::ptr::NonNull::from(slice)) + } + + unsafe fn shrink( + &self, + ptr: ::std::ptr::NonNull, + old_layout: ::std::alloc::Layout, + mut new_layout: ::std::alloc::Layout, + ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { + debug_assert!( + new_layout.size() <= old_layout.size(), + "`new_layout.size()` must be smaller than or equal to `old_layout.size()`" + ); + + new_layout = Self::__fix_layout(new_layout); + let start = <$plan_ty as $crate::Plan>::Mutator::current() + .realloc(ptr.as_ptr().into(), new_layout) + .unwrap_or($crate::util::Address::ZERO); + let slice = unsafe { + ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, new_layout.size()) + }; + Ok(::std::ptr::NonNull::from(slice)) + } + } + + unsafe impl ::std::alloc::GlobalAlloc for Global { + unsafe fn alloc(&self, mut layout: ::std::alloc::Layout) -> *mut u8 { + layout = Self::__fix_layout(layout); + <$plan_ty as $crate::Plan>::Mutator::current() + .alloc(layout) + .unwrap_or($crate::util::Address::ZERO) + .into() + } + + unsafe fn alloc_zeroed(&self, mut layout: ::std::alloc::Layout) -> *mut u8 { + layout = Self::__fix_layout(layout); + <$plan_ty as $crate::Plan>::Mutator::current() + .alloc_zeroed(layout) + .unwrap_or($crate::util::Address::ZERO) + .into() + } + + unsafe fn dealloc(&self, ptr: *mut u8, _layout: ::std::alloc::Layout) { + <$plan_ty as $crate::Plan>::Mutator::current().dealloc(ptr.into()) + } + + unsafe fn realloc( + &self, + ptr: *mut u8, + layout: ::std::alloc::Layout, + new_size: usize, + ) -> *mut u8 { + let mut new_layout = + unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + new_layout = Self::__fix_layout(new_layout); + <$plan_ty as $crate::Plan>::Mutator::current() + .realloc(ptr.into(), new_layout) + .unwrap_or($crate::util::Address::ZERO) + .into() + } + } + }; +} From 7c4db9f0b15df6ede5ba141a1a445f07664c58e9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 14:51:25 +1000 Subject: [PATCH 10/23] Minor --- mallockit/src/util/malloc/rust_alloc.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/mallockit/src/util/malloc/rust_alloc.rs b/mallockit/src/util/malloc/rust_alloc.rs index eab4c18..a5141da 100644 --- a/mallockit/src/util/malloc/rust_alloc.rs +++ b/mallockit/src/util/malloc/rust_alloc.rs @@ -20,7 +20,7 @@ macro_rules! export_rust_global_alloc_api { fn allocate( &self, mut layout: ::std::alloc::Layout, - ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { + ) -> ::std::result::Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { layout = Self::__fix_layout(layout); let start = <$plan_ty as $crate::Plan>::Mutator::current() .alloc(layout) @@ -28,13 +28,13 @@ macro_rules! export_rust_global_alloc_api { let slice = unsafe { ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, layout.size()) }; - Ok(::std::ptr::NonNull::from(slice)) + ::std::result::Result::Ok(::std::ptr::NonNull::from(slice)) } fn allocate_zeroed( &self, mut layout: ::std::alloc::Layout, - ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { + ) -> ::std::result::Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { layout = Self::__fix_layout(layout); let start = <$plan_ty as $crate::Plan>::Mutator::current() .alloc_zeroed(layout) @@ -42,7 +42,7 @@ macro_rules! export_rust_global_alloc_api { let slice = unsafe { ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, layout.size()) }; - Ok(::std::ptr::NonNull::from(slice)) + ::std::result::Result::Ok(::std::ptr::NonNull::from(slice)) } unsafe fn deallocate( @@ -58,7 +58,7 @@ macro_rules! export_rust_global_alloc_api { ptr: ::std::ptr::NonNull, old_layout: ::std::alloc::Layout, mut new_layout: ::std::alloc::Layout, - ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { + ) -> ::std::result::Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { debug_assert!( new_layout.size() >= old_layout.size(), "`new_layout.size()` must be greater than or equal to `old_layout.size()`" @@ -71,7 +71,7 @@ macro_rules! export_rust_global_alloc_api { let slice = unsafe { ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, new_layout.size()) }; - Ok(::std::ptr::NonNull::from(slice)) + ::std::result::Result::Ok(::std::ptr::NonNull::from(slice)) } unsafe fn grow_zeroed( @@ -79,7 +79,7 @@ macro_rules! export_rust_global_alloc_api { ptr: ::std::ptr::NonNull, old_layout: ::std::alloc::Layout, mut new_layout: ::std::alloc::Layout, - ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { + ) -> ::std::result::Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { debug_assert!( new_layout.size() >= old_layout.size(), "`new_layout.size()` must be greater than or equal to `old_layout.size()`" @@ -92,7 +92,7 @@ macro_rules! export_rust_global_alloc_api { let slice = unsafe { ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, new_layout.size()) }; - Ok(::std::ptr::NonNull::from(slice)) + ::std::result::Result::Ok(::std::ptr::NonNull::from(slice)) } unsafe fn shrink( @@ -100,7 +100,7 @@ macro_rules! export_rust_global_alloc_api { ptr: ::std::ptr::NonNull, old_layout: ::std::alloc::Layout, mut new_layout: ::std::alloc::Layout, - ) -> Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { + ) -> ::std::result::Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { debug_assert!( new_layout.size() <= old_layout.size(), "`new_layout.size()` must be smaller than or equal to `old_layout.size()`" @@ -113,7 +113,7 @@ macro_rules! export_rust_global_alloc_api { let slice = unsafe { ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, new_layout.size()) }; - Ok(::std::ptr::NonNull::from(slice)) + ::std::result::Result::Ok(::std::ptr::NonNull::from(slice)) } } From be7dc544a203d703ef656c4580e1e26a609a7109 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 14:52:10 +1000 Subject: [PATCH 11/23] Fix --- mallockit/src/util/mem/size_class.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mallockit/src/util/mem/size_class.rs b/mallockit/src/util/mem/size_class.rs index c21917d..1862ddb 100644 --- a/mallockit/src/util/mem/size_class.rs +++ b/mallockit/src/util/mem/size_class.rs @@ -4,9 +4,11 @@ use crate::util::constants::LOG_MIN_ALIGNMENT; use super::layout_utils::LayoutUtils; +const LOG_MIN_ALIGNMENT_U8: u8 = LOG_MIN_ALIGNMENT as u8; + #[repr(transparent)] #[derive(Debug, Clone, Copy)] -pub struct SizeClass(pub u8); +pub struct SizeClass(pub u8); impl SizeClass { pub const fn as_usize(self) -> usize { From ad9897b876600c45c2e1f029c230a2a495d00a28 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 15:40:20 +1000 Subject: [PATCH 12/23] Refactor --- mallockit/macros/src/lib.rs | 48 ++-- mallockit/src/util/malloc/malloc_api.rs | 187 +++++++--------- mallockit/src/util/malloc/rust_alloc.rs | 285 +++++++++++++----------- mallockit/src/util/sys/hooks.rs | 5 +- 4 files changed, 273 insertions(+), 252 deletions(-) diff --git a/mallockit/macros/src/lib.rs b/mallockit/macros/src/lib.rs index 16b226e..9c932a3 100644 --- a/mallockit/macros/src/lib.rs +++ b/mallockit/macros/src/lib.rs @@ -9,24 +9,42 @@ pub fn plan(_attr: TokenStream, item: TokenStream) -> TokenStream { #input mod __mallockit_plan { - pub(super) static PLAN: ::mallockit::util::Lazy = ::mallockit::util::Lazy::new(|| ::new()); + type Plan = super::#name; - ::mallockit::export_malloc_api!(PLAN, super::super::#name); - } + pub(super) static PLAN: ::mallockit::util::Lazy = ::mallockit::util::Lazy::new(|| ::new()); - ::mallockit::export_rust_global_alloc_api!(PLAN, #name); + #[cfg(any(feature = "malloc", feature = "mallockit/malloc"))] + #[::mallockit::ctor] + unsafe fn ctor() { + ::mallockit::util::sys::hooks::process_start(&*PLAN); + } + + #[cfg(target_os = "macos")] + #[no_mangle] + pub extern "C" fn mallockit_initialize_macos_tls() -> *mut u8 { + ::Mutator::current() as *mut ::mallockit::Mutator as _ + } - impl ::mallockit::plan::Singleton for #name { - fn singleton() -> &'static Self { - unsafe { &__mallockit_plan::PLAN } + impl ::mallockit::plan::Singleton for super::#name { + fn singleton() -> &'static Self { + unsafe { &PLAN } + } } + + ::mallockit::export_malloc_api!(PLAN, super::super::#name); + ::mallockit::export_rust_global_alloc_api!(super::super::#name); } - include!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../target/generated_tests.rs" - )); - ::mallockit::rust_allocator_tests!(Global); + pub use __mallockit_plan::__mallockit_rust_api::Global; + + #[cfg(test)] + mod tests { + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../target/generated_tests.rs" + )); + ::mallockit::rust_allocator_tests!(crate::Global); + } }; result.into() } @@ -42,12 +60,12 @@ pub fn mutator(_attr: TokenStream, item: TokenStream) -> TokenStream { #[cfg(not(target_os = "macos"))] mod __mallockit_mutator { #[thread_local] - pub(super) static mut MUTATOR: mallockit::util::Lazy = mallockit::util::Lazy::new(|| ::new()); + pub(super) static mut MUTATOR: ::mallockit::util::Lazy = ::mallockit::util::Lazy::new(|| ::new()); } - impl mallockit::mutator::TLS for #name { + impl ::mallockit::mutator::TLS for #name { fn new() -> Self { - ::new() + ::new() } #[cfg(not(target_os = "macos"))] diff --git a/mallockit/src/util/malloc/malloc_api.rs b/mallockit/src/util/malloc/malloc_api.rs index 688e72a..054936e 100644 --- a/mallockit/src/util/malloc/malloc_api.rs +++ b/mallockit/src/util/malloc/malloc_api.rs @@ -224,128 +224,109 @@ impl MallocAPI

{ #[doc(hidden)] macro_rules! export_malloc_api { ($plan: expr, $plan_ty: ty) => { - pub mod __mallockit { + #[cfg(any(feature = "malloc", feature = "mallockit/malloc"))] + pub mod __mallockit_malloc_api { use super::*; use $crate::Plan; - type ConcretePlan = $plan_ty; - type Malloc = $crate::util::malloc::MallocAPI; - static MALLOC_IMPL: Malloc = - $crate::util::malloc::MallocAPI::::new(&$plan); - - #[cfg(any(feature = "malloc", feature = "mallockit/malloc"))] - #[$crate::ctor] - unsafe fn ctor() { - $crate::util::sys::hooks::process_start(&*$plan); - $crate::libc::atexit($crate::util::sys::hooks::process_exit); + type Malloc = $crate::util::malloc::MallocAPI<$plan_ty>; + static MALLOC_IMPL: Malloc = $crate::util::malloc::MallocAPI::<$plan_ty>::new(&$plan); + + #[$crate::interpose] + pub unsafe extern "C" fn malloc(size: usize) -> *mut u8 { + MALLOC_IMPL.alloc_or_enomem(size, Malloc::MIN_ALIGNMENT) } #[cfg(target_os = "macos")] - #[no_mangle] - pub extern "C" fn mallockit_initialize_macos_tls() -> *mut u8 { - MALLOC_IMPL.mutator() as *mut _ as _ + #[$crate::interpose] + pub unsafe extern "C" fn malloc_size(ptr: *mut u8) -> usize { + MALLOC_IMPL.malloc_size(ptr.into()) } - #[cfg(any(feature = "malloc", feature = "mallockit/malloc"))] - pub mod __malloc_api { - use super::{Malloc, MALLOC_IMPL}; - - #[$crate::interpose] - pub unsafe extern "C" fn malloc(size: usize) -> *mut u8 { - MALLOC_IMPL.alloc_or_enomem(size, Malloc::MIN_ALIGNMENT) - } - - #[cfg(target_os = "macos")] - #[$crate::interpose] - pub unsafe extern "C" fn malloc_size(ptr: *mut u8) -> usize { - MALLOC_IMPL.malloc_size(ptr.into()) - } - - // #[cfg(target_os = "macos")] - // #[$crate::interpose] - // pub unsafe fn malloc_good_size(ptr: *mut u8) -> usize { - // MALLOC_IMPL.malloc_size(ptr.into()) - // } + // #[cfg(target_os = "macos")] + // #[$crate::interpose] + // pub unsafe fn malloc_good_size(ptr: *mut u8) -> usize { + // MALLOC_IMPL.malloc_size(ptr.into()) + // } - #[cfg(target_os = "linux")] - #[$crate::interpose] - pub unsafe extern "C" fn malloc_usable_size(ptr: *mut u8) -> usize { - MALLOC_IMPL.malloc_size(ptr.into()) - } + #[cfg(target_os = "linux")] + #[$crate::interpose] + pub unsafe extern "C" fn malloc_usable_size(ptr: *mut u8) -> usize { + MALLOC_IMPL.malloc_size(ptr.into()) + } - #[$crate::interpose] - pub unsafe extern "C" fn free(ptr: *mut u8) { - MALLOC_IMPL.free(ptr) - } + #[$crate::interpose] + pub unsafe extern "C" fn free(ptr: *mut u8) { + MALLOC_IMPL.free(ptr) + } - #[cfg(target_os = "linux")] - #[$crate::interpose] - pub unsafe extern "C" fn cfree(ptr: *mut u8) { - MALLOC_IMPL.free(ptr) - } + #[cfg(target_os = "linux")] + #[$crate::interpose] + pub unsafe extern "C" fn cfree(ptr: *mut u8) { + MALLOC_IMPL.free(ptr) + } - #[$crate::interpose] - pub unsafe extern "C" fn calloc(count: usize, size: usize) -> *mut u8 { - let size = count * size; - let ptr = MALLOC_IMPL.alloc_or_enomem(size, Malloc::MIN_ALIGNMENT); - std::ptr::write_bytes(ptr, 0, size); - ptr - } + #[$crate::interpose] + pub unsafe extern "C" fn calloc(count: usize, size: usize) -> *mut u8 { + let size = count * size; + let ptr = MALLOC_IMPL.alloc_or_enomem(size, Malloc::MIN_ALIGNMENT); + std::ptr::write_bytes(ptr, 0, size); + ptr + } - #[cfg(any(target_os = "linux", target_os = "macos"))] - #[$crate::interpose] - pub unsafe extern "C" fn valloc(size: usize) -> *mut u8 { - MALLOC_IMPL.alloc_or_enomem(size, Malloc::PAGE_SIZE) - } + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[$crate::interpose] + pub unsafe extern "C" fn valloc(size: usize) -> *mut u8 { + MALLOC_IMPL.alloc_or_enomem(size, Malloc::PAGE_SIZE) + } - #[cfg(target_os = "linux")] - #[$crate::interpose] - pub unsafe extern "C" fn pvalloc(size: usize) -> *mut u8 { - MALLOC_IMPL.alloc_or_enomem(size, Malloc::PAGE_SIZE) - } + #[cfg(target_os = "linux")] + #[$crate::interpose] + pub unsafe extern "C" fn pvalloc(size: usize) -> *mut u8 { + MALLOC_IMPL.alloc_or_enomem(size, Malloc::PAGE_SIZE) + } - #[$crate::interpose] - pub unsafe extern "C" fn realloc(ptr: *mut u8, size: usize) -> *mut u8 { - MALLOC_IMPL.reallocate_or_enomem( - ptr, - size, - cfg!(any(target_os = "linux", target_os = "windows")), - false, - ) - } + #[$crate::interpose] + pub unsafe extern "C" fn realloc(ptr: *mut u8, size: usize) -> *mut u8 { + MALLOC_IMPL.reallocate_or_enomem( + ptr, + size, + cfg!(any(target_os = "linux", target_os = "windows")), + false, + ) + } - #[cfg(target_os = "macos")] - #[$crate::interpose] - pub unsafe extern "C" fn reallocf(ptr: *mut u8, size: usize) -> *mut u8 { - MALLOC_IMPL.reallocate_or_enomem(ptr, size, false, true) - } + #[cfg(target_os = "macos")] + #[$crate::interpose] + pub unsafe extern "C" fn reallocf(ptr: *mut u8, size: usize) -> *mut u8 { + MALLOC_IMPL.reallocate_or_enomem(ptr, size, false, true) + } - #[cfg(any(target_os = "linux", target_os = "macos"))] - #[$crate::interpose] - pub unsafe extern "C" fn posix_memalign( - ptr: *mut *mut u8, - alignment: usize, - size: usize, - ) -> i32 { - MALLOC_IMPL.posix_memalign(ptr, alignment, size) - } + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[$crate::interpose] + pub unsafe extern "C" fn posix_memalign( + ptr: *mut *mut u8, + alignment: usize, + size: usize, + ) -> i32 { + MALLOC_IMPL.posix_memalign(ptr, alignment, size) + } - #[cfg(target_os = "linux")] - #[$crate::interpose] - pub unsafe extern "C" fn memalign(alignment: usize, size: usize) -> *mut u8 { - MALLOC_IMPL.memalign(alignment, size) - } + #[cfg(target_os = "linux")] + #[$crate::interpose] + pub unsafe extern "C" fn memalign(alignment: usize, size: usize) -> *mut u8 { + MALLOC_IMPL.memalign(alignment, size) + } - #[cfg(target_os = "linux")] - #[$crate::interpose] - pub unsafe extern "C" fn aligned_alloc(alignment: usize, size: usize) -> *mut u8 { - MALLOC_IMPL.aligned_alloc(size, alignment, true, false) - } + #[cfg(target_os = "linux")] + #[$crate::interpose] + pub unsafe extern "C" fn aligned_alloc(alignment: usize, size: usize) -> *mut u8 { + MALLOC_IMPL.aligned_alloc(size, alignment, true, false) + } - #[cfg(target_os = "windows")] - #[$crate::interpose] - pub unsafe extern "C" fn _aligned_malloc(size: usize, alignment: usize) -> *mut u8 { - MALLOC_IMPL.aligned_alloc(size, alignment, false, true) - } + #[cfg(target_os = "windows")] + #[$crate::interpose] + pub unsafe extern "C" fn _aligned_malloc(size: usize, alignment: usize) -> *mut u8 { + MALLOC_IMPL.aligned_alloc(size, alignment, false, true) } } }; diff --git a/mallockit/src/util/malloc/rust_alloc.rs b/mallockit/src/util/malloc/rust_alloc.rs index a5141da..994e206 100644 --- a/mallockit/src/util/malloc/rust_alloc.rs +++ b/mallockit/src/util/malloc/rust_alloc.rs @@ -1,156 +1,175 @@ #[macro_export] #[doc(hidden)] macro_rules! export_rust_global_alloc_api { - ($plan: expr, $plan_ty: ty) => { - pub struct Global; + ($plan_ty: ty) => { + pub mod __mallockit_rust_api { + use $crate::util::{Layout, LayoutUtils}; + use $crate::{Mutator, Plan}; - impl Global { - pub fn __fix_layout(mut layout: ::std::alloc::Layout) -> Layout { - if layout.align() < $crate::util::constants::MIN_ALIGNMENT { - layout = layout - .align_to($crate::util::constants::MIN_ALIGNMENT) - .unwrap(); + pub struct Global; + + impl Global { + fn __fix_layout(mut layout: ::std::alloc::Layout) -> Layout { + if layout.align() < $crate::util::constants::MIN_ALIGNMENT { + layout = layout + .align_to($crate::util::constants::MIN_ALIGNMENT) + .unwrap(); + } + layout = unsafe { layout.pad_to_align_unchecked() }; + layout } - layout = unsafe { layout.pad_to_align_unchecked() }; - layout } - } - unsafe impl ::std::alloc::Allocator for Global { - fn allocate( - &self, - mut layout: ::std::alloc::Layout, - ) -> ::std::result::Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { - layout = Self::__fix_layout(layout); - let start = <$plan_ty as $crate::Plan>::Mutator::current() - .alloc(layout) - .unwrap_or($crate::util::Address::ZERO); - let slice = unsafe { - ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, layout.size()) - }; - ::std::result::Result::Ok(::std::ptr::NonNull::from(slice)) - } + unsafe impl ::std::alloc::Allocator for Global { + fn allocate( + &self, + mut layout: ::std::alloc::Layout, + ) -> ::std::result::Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> + { + layout = Self::__fix_layout(layout); + let start = <$plan_ty as $crate::Plan>::Mutator::current() + .alloc(layout) + .unwrap_or($crate::util::Address::ZERO); + let slice = unsafe { + ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, layout.size()) + }; + ::std::result::Result::Ok(::std::ptr::NonNull::from(slice)) + } - fn allocate_zeroed( - &self, - mut layout: ::std::alloc::Layout, - ) -> ::std::result::Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { - layout = Self::__fix_layout(layout); - let start = <$plan_ty as $crate::Plan>::Mutator::current() - .alloc_zeroed(layout) - .unwrap_or($crate::util::Address::ZERO); - let slice = unsafe { - ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, layout.size()) - }; - ::std::result::Result::Ok(::std::ptr::NonNull::from(slice)) - } + fn allocate_zeroed( + &self, + mut layout: ::std::alloc::Layout, + ) -> ::std::result::Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> + { + layout = Self::__fix_layout(layout); + let start = <$plan_ty as $crate::Plan>::Mutator::current() + .alloc_zeroed(layout) + .unwrap_or($crate::util::Address::ZERO); + let slice = unsafe { + ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, layout.size()) + }; + ::std::result::Result::Ok(::std::ptr::NonNull::from(slice)) + } - unsafe fn deallocate( - &self, - ptr: ::std::ptr::NonNull, - layout: ::std::alloc::Layout, - ) { - <$plan_ty as $crate::Plan>::Mutator::current().dealloc(ptr.as_ptr().into()) - } + unsafe fn deallocate( + &self, + ptr: ::std::ptr::NonNull, + layout: ::std::alloc::Layout, + ) { + <$plan_ty as $crate::Plan>::Mutator::current().dealloc(ptr.as_ptr().into()) + } - unsafe fn grow( - &self, - ptr: ::std::ptr::NonNull, - old_layout: ::std::alloc::Layout, - mut new_layout: ::std::alloc::Layout, - ) -> ::std::result::Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { - debug_assert!( - new_layout.size() >= old_layout.size(), - "`new_layout.size()` must be greater than or equal to `old_layout.size()`" - ); + unsafe fn grow( + &self, + ptr: ::std::ptr::NonNull, + old_layout: ::std::alloc::Layout, + mut new_layout: ::std::alloc::Layout, + ) -> ::std::result::Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> + { + debug_assert!( + new_layout.size() >= old_layout.size(), + "`new_layout.size()` must be greater than or equal to `old_layout.size()`" + ); - new_layout = Self::__fix_layout(new_layout); - let start = <$plan_ty as $crate::Plan>::Mutator::current() - .realloc(ptr.as_ptr().into(), new_layout) - .unwrap_or($crate::util::Address::ZERO); - let slice = unsafe { - ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, new_layout.size()) - }; - ::std::result::Result::Ok(::std::ptr::NonNull::from(slice)) - } + new_layout = Self::__fix_layout(new_layout); + let start = <$plan_ty as $crate::Plan>::Mutator::current() + .realloc(ptr.as_ptr().into(), new_layout) + .unwrap_or($crate::util::Address::ZERO); + let slice = unsafe { + ::std::slice::from_raw_parts_mut( + start.as_mut() as *mut u8, + new_layout.size(), + ) + }; + ::std::result::Result::Ok(::std::ptr::NonNull::from(slice)) + } - unsafe fn grow_zeroed( - &self, - ptr: ::std::ptr::NonNull, - old_layout: ::std::alloc::Layout, - mut new_layout: ::std::alloc::Layout, - ) -> ::std::result::Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { - debug_assert!( - new_layout.size() >= old_layout.size(), - "`new_layout.size()` must be greater than or equal to `old_layout.size()`" - ); + unsafe fn grow_zeroed( + &self, + ptr: ::std::ptr::NonNull, + old_layout: ::std::alloc::Layout, + mut new_layout: ::std::alloc::Layout, + ) -> ::std::result::Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> + { + debug_assert!( + new_layout.size() >= old_layout.size(), + "`new_layout.size()` must be greater than or equal to `old_layout.size()`" + ); - new_layout = Self::__fix_layout(new_layout); - let start = <$plan_ty as $crate::Plan>::Mutator::current() - .realloc_zeroed(ptr.as_ptr().into(), new_layout) - .unwrap_or($crate::util::Address::ZERO); - let slice = unsafe { - ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, new_layout.size()) - }; - ::std::result::Result::Ok(::std::ptr::NonNull::from(slice)) - } + new_layout = Self::__fix_layout(new_layout); + let start = <$plan_ty as $crate::Plan>::Mutator::current() + .realloc_zeroed(ptr.as_ptr().into(), new_layout) + .unwrap_or($crate::util::Address::ZERO); + let slice = unsafe { + ::std::slice::from_raw_parts_mut( + start.as_mut() as *mut u8, + new_layout.size(), + ) + }; + ::std::result::Result::Ok(::std::ptr::NonNull::from(slice)) + } - unsafe fn shrink( - &self, - ptr: ::std::ptr::NonNull, - old_layout: ::std::alloc::Layout, - mut new_layout: ::std::alloc::Layout, - ) -> ::std::result::Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> { - debug_assert!( - new_layout.size() <= old_layout.size(), - "`new_layout.size()` must be smaller than or equal to `old_layout.size()`" - ); + unsafe fn shrink( + &self, + ptr: ::std::ptr::NonNull, + old_layout: ::std::alloc::Layout, + mut new_layout: ::std::alloc::Layout, + ) -> ::std::result::Result<::std::ptr::NonNull<[u8]>, ::std::alloc::AllocError> + { + debug_assert!( + new_layout.size() <= old_layout.size(), + "`new_layout.size()` must be smaller than or equal to `old_layout.size()`" + ); - new_layout = Self::__fix_layout(new_layout); - let start = <$plan_ty as $crate::Plan>::Mutator::current() - .realloc(ptr.as_ptr().into(), new_layout) - .unwrap_or($crate::util::Address::ZERO); - let slice = unsafe { - ::std::slice::from_raw_parts_mut(start.as_mut() as *mut u8, new_layout.size()) - }; - ::std::result::Result::Ok(::std::ptr::NonNull::from(slice)) + new_layout = Self::__fix_layout(new_layout); + let start = <$plan_ty as $crate::Plan>::Mutator::current() + .realloc(ptr.as_ptr().into(), new_layout) + .unwrap_or($crate::util::Address::ZERO); + let slice = unsafe { + ::std::slice::from_raw_parts_mut( + start.as_mut() as *mut u8, + new_layout.size(), + ) + }; + ::std::result::Result::Ok(::std::ptr::NonNull::from(slice)) + } } - } - unsafe impl ::std::alloc::GlobalAlloc for Global { - unsafe fn alloc(&self, mut layout: ::std::alloc::Layout) -> *mut u8 { - layout = Self::__fix_layout(layout); - <$plan_ty as $crate::Plan>::Mutator::current() - .alloc(layout) - .unwrap_or($crate::util::Address::ZERO) - .into() - } + unsafe impl ::std::alloc::GlobalAlloc for Global { + unsafe fn alloc(&self, mut layout: ::std::alloc::Layout) -> *mut u8 { + layout = Self::__fix_layout(layout); + <$plan_ty as $crate::Plan>::Mutator::current() + .alloc(layout) + .unwrap_or($crate::util::Address::ZERO) + .into() + } - unsafe fn alloc_zeroed(&self, mut layout: ::std::alloc::Layout) -> *mut u8 { - layout = Self::__fix_layout(layout); - <$plan_ty as $crate::Plan>::Mutator::current() - .alloc_zeroed(layout) - .unwrap_or($crate::util::Address::ZERO) - .into() - } + unsafe fn alloc_zeroed(&self, mut layout: ::std::alloc::Layout) -> *mut u8 { + layout = Self::__fix_layout(layout); + <$plan_ty as $crate::Plan>::Mutator::current() + .alloc_zeroed(layout) + .unwrap_or($crate::util::Address::ZERO) + .into() + } - unsafe fn dealloc(&self, ptr: *mut u8, _layout: ::std::alloc::Layout) { - <$plan_ty as $crate::Plan>::Mutator::current().dealloc(ptr.into()) - } + unsafe fn dealloc(&self, ptr: *mut u8, _layout: ::std::alloc::Layout) { + <$plan_ty as $crate::Plan>::Mutator::current().dealloc(ptr.into()) + } - unsafe fn realloc( - &self, - ptr: *mut u8, - layout: ::std::alloc::Layout, - new_size: usize, - ) -> *mut u8 { - let mut new_layout = - unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; - new_layout = Self::__fix_layout(new_layout); - <$plan_ty as $crate::Plan>::Mutator::current() - .realloc(ptr.into(), new_layout) - .unwrap_or($crate::util::Address::ZERO) - .into() + unsafe fn realloc( + &self, + ptr: *mut u8, + layout: ::std::alloc::Layout, + new_size: usize, + ) -> *mut u8 { + let mut new_layout = + unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + new_layout = Self::__fix_layout(new_layout); + <$plan_ty as $crate::Plan>::Mutator::current() + .realloc(ptr.into(), new_layout) + .unwrap_or($crate::util::Address::ZERO) + .into() + } } } }; diff --git a/mallockit/src/util/sys/hooks.rs b/mallockit/src/util/sys/hooks.rs index 87882d4..30323c7 100644 --- a/mallockit/src/util/sys/hooks.rs +++ b/mallockit/src/util/sys/hooks.rs @@ -13,11 +13,14 @@ pub fn set_panic_handler() { pub extern "C" fn process_start(plan: &'static impl Plan) { set_panic_handler(); + unsafe { + libc::atexit(process_exit); + } #[cfg(target_os = "macos")] crate::util::malloc::macos_malloc_zone::init(); plan.init(); } -pub extern "C" fn process_exit() { +extern "C" fn process_exit() { crate::stat::report(); } From 429d239a0f7f9be81fb1c3122aa732c4018a764c Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 15:46:01 +1000 Subject: [PATCH 13/23] refactor --- mallockit/macros/src/lib.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/mallockit/macros/src/lib.rs b/mallockit/macros/src/lib.rs index 9c932a3..1a26df8 100644 --- a/mallockit/macros/src/lib.rs +++ b/mallockit/macros/src/lib.rs @@ -11,7 +11,7 @@ pub fn plan(_attr: TokenStream, item: TokenStream) -> TokenStream { mod __mallockit_plan { type Plan = super::#name; - pub(super) static PLAN: ::mallockit::util::Lazy = ::mallockit::util::Lazy::new(|| ::new()); + static PLAN: ::mallockit::util::Lazy = ::mallockit::util::Lazy::new(|| ::new()); #[cfg(any(feature = "malloc", feature = "mallockit/malloc"))] #[::mallockit::ctor] @@ -21,7 +21,7 @@ pub fn plan(_attr: TokenStream, item: TokenStream) -> TokenStream { #[cfg(target_os = "macos")] #[no_mangle] - pub extern "C" fn mallockit_initialize_macos_tls() -> *mut u8 { + extern "C" fn mallockit_initialize_macos_tls() -> *mut u8 { ::Mutator::current() as *mut ::mallockit::Mutator as _ } @@ -60,17 +60,17 @@ pub fn mutator(_attr: TokenStream, item: TokenStream) -> TokenStream { #[cfg(not(target_os = "macos"))] mod __mallockit_mutator { #[thread_local] - pub(super) static mut MUTATOR: ::mallockit::util::Lazy = ::mallockit::util::Lazy::new(|| ::new()); - } + static mut MUTATOR: ::mallockit::util::Lazy = ::mallockit::util::Lazy::new(|| ::new()); - impl ::mallockit::mutator::TLS for #name { - fn new() -> Self { - ::new() - } + impl ::mallockit::mutator::TLS for super::#name { + fn new() -> Self { + ::new() + } - #[cfg(not(target_os = "macos"))] - fn current() -> &'static mut Self { - unsafe { &mut *__mallockit_mutator::MUTATOR } + #[cfg(not(target_os = "macos"))] + fn current() -> &'static mut Self { + unsafe { &mut *MUTATOR } + } } } }; From cd53af6415904737c70cf4321ee592b4569c0d8f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 15:54:50 +1000 Subject: [PATCH 14/23] Fix stat --- mallockit/src/space/meta/meta_allocator.rs | 1 + mallockit/src/stat.rs | 27 ++++++++++++---------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/mallockit/src/space/meta/meta_allocator.rs b/mallockit/src/space/meta/meta_allocator.rs index a39bd06..6e38e0b 100644 --- a/mallockit/src/space/meta/meta_allocator.rs +++ b/mallockit/src/space/meta/meta_allocator.rs @@ -131,6 +131,7 @@ impl MetaLocal { } } +#[derive(Clone, Copy, Debug, Default)] pub struct Meta; unsafe impl Allocator for Meta { diff --git a/mallockit/src/stat.rs b/mallockit/src/stat.rs index 483f7f0..2c69e8e 100644 --- a/mallockit/src/stat.rs +++ b/mallockit/src/stat.rs @@ -6,26 +6,29 @@ use std::{ }, }; -use crossbeam::queue::SegQueue; +use spin::Mutex; -use crate::util::Lazy; +use crate::{ + space::meta::{Meta, Vec}, + util::Lazy, +}; -static COUNTERS: SegQueue> = SegQueue::new(); +static COUNTERS: Mutex>> = Mutex::new(Vec::new_in(Meta)); -pub type CounterRef = Lazy>; +pub type CounterRef = Lazy>; -pub const fn define_counter() -> Lazy> { +pub const fn define_counter() -> Lazy> { Lazy::new(|| { - let c = Arc::new(Counter::new(NAME)); - COUNTERS.push(c.clone()); + let c = Arc::new_in(Counter::new(NAME), Meta); + COUNTERS.lock().push(c.clone()); c }) } -static TOTAL_ALLOCATIONS: Lazy> = define_counter::<"total-allocations">(); -static LARGE_ALLOCATIONS: Lazy> = define_counter::<"large-allocations">(); -static TOTAL_DEALLOCATIONS: Lazy> = define_counter::<"total-deallocations">(); -static LARGE_DEALLOCATIONS: Lazy> = define_counter::<"large-deallocations">(); +static TOTAL_ALLOCATIONS: Lazy> = define_counter::<"total-allocations">(); +static LARGE_ALLOCATIONS: Lazy> = define_counter::<"large-allocations">(); +static TOTAL_DEALLOCATIONS: Lazy> = define_counter::<"total-deallocations">(); +static LARGE_DEALLOCATIONS: Lazy> = define_counter::<"large-deallocations">(); static ALIGNMENTS: [Counter; 11] = [ Counter::new(""), // 1 @@ -138,7 +141,7 @@ pub(crate) fn report() { } println!(" - others = {}", OTHER_SIZE.get()); println!(""); - while let Some(c) = COUNTERS.pop() { + while let Some(c) = COUNTERS.lock().pop() { println!("{}: {}", c.0, c.get()); } } From f521857989820a9e1585a1e07543cd85fb076a7d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 15:56:41 +1000 Subject: [PATCH 15/23] Remove crossbeam --- Cargo.lock | 57 ------------------- Cargo.toml | 1 - mallockit/Cargo.toml | 1 - .../page_resource/block_page_resource.rs | 13 +++-- 4 files changed, 8 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1b923ee..9cc09dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -117,62 +117,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "crossbeam" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" -dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-epoch", - "crossbeam-queue", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-queue" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" - [[package]] name = "ctor" version = "0.1.26" @@ -275,7 +219,6 @@ version = "0.1.0" dependencies = [ "atomic", "cargo_metadata", - "crossbeam", "ctor", "errno", "libc", diff --git a/Cargo.toml b/Cargo.toml index 71fa5b7..16eaf83 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,7 +29,6 @@ errno = "0.2.8" spin = { version = "0.9.3", features = ["std"] } ctor = "0.1" num_cpus = "1.13.1" -crossbeam = "0.8.1" atomic = "0.5.1" quote = "1.0.20" syn = "1.0.98" diff --git a/mallockit/Cargo.toml b/mallockit/Cargo.toml index 92404d3..cfbf3a9 100644 --- a/mallockit/Cargo.toml +++ b/mallockit/Cargo.toml @@ -12,7 +12,6 @@ errno = { workspace = true } spin = { workspace = true } ctor = { workspace = true } num_cpus = { workspace = true } -crossbeam = { workspace = true } atomic = { workspace = true } shell-words = { workspace = true } mallockit-macros = { path = "./macros" } diff --git a/mallockit/src/space/page_resource/block_page_resource.rs b/mallockit/src/space/page_resource/block_page_resource.rs index 5c4a0e0..86c9882 100644 --- a/mallockit/src/space/page_resource/block_page_resource.rs +++ b/mallockit/src/space/page_resource/block_page_resource.rs @@ -1,9 +1,10 @@ use super::super::SpaceId; use super::PageResource; +use crate::space::meta::{Meta, Vec}; use crate::util::mem::heap::HEAP; use crate::util::*; use atomic::Atomic; -use crossbeam::queue::SegQueue; +use spin::Mutex; use std::iter::Step; use std::{ ops::Range, @@ -69,7 +70,7 @@ pub struct BlockPageResource { cursor: Atomic

, highwater: Address, recycled_blocks_intrusive: Atomic>, - recycled_blocks_non_intrusive: SegQueue
, + recycled_blocks_non_intrusive: Mutex>, reserved_bytes: AtomicUsize, } @@ -83,7 +84,7 @@ impl BlockPageResource { cursor: Atomic::new(range.start), highwater: range.end, recycled_blocks_intrusive: Atomic::new(None), - recycled_blocks_non_intrusive: SegQueue::new(), + recycled_blocks_non_intrusive: Mutex::new(Vec::new_in(Meta)), reserved_bytes: AtomicUsize::new(0), } } @@ -128,7 +129,7 @@ impl BlockPageResource { break; } } - } else if let Some(addr) = self.recycled_blocks_non_intrusive.pop() { + } else if let Some(addr) = self.recycled_blocks_non_intrusive.lock().pop() { let block = B::from_address(addr); self.reserved_bytes.fetch_add(B::BYTES, Ordering::Relaxed); return Some(block); @@ -156,7 +157,9 @@ impl BlockPageResource { } } } else { - self.recycled_blocks_non_intrusive.push(block.start()); + self.recycled_blocks_non_intrusive + .lock() + .push(block.start()); } self.reserved_bytes .fetch_sub(1 << B::LOG_BYTES, Ordering::Relaxed); From bf83e8ee90b357663f3a7d9356d495f55eae5c02 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 16:02:07 +1000 Subject: [PATCH 16/23] cleanup --- hoard/src/hoard_space.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/hoard/src/hoard_space.rs b/hoard/src/hoard_space.rs index 2195451..68f722b 100644 --- a/hoard/src/hoard_space.rs +++ b/hoard/src/hoard_space.rs @@ -59,7 +59,6 @@ impl HoardSpace { debug_assert!(!block.is_full()); register(block); debug_assert!(block.is_owned_by(local)); - mallockit::println!("Allocating block1: {:?}", block); return Some(block); } // Acquire new memory From 6b13da1497af722432a251095a0ebac3bfa17ca5 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 16:05:50 +1000 Subject: [PATCH 17/23] minor --- mallockit/src/stat.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/mallockit/src/stat.rs b/mallockit/src/stat.rs index 2c69e8e..72b60cf 100644 --- a/mallockit/src/stat.rs +++ b/mallockit/src/stat.rs @@ -129,19 +129,19 @@ pub(crate) fn report() {} #[cfg(feature = "stat")] pub(crate) fn report() { - println!("alignment:"); + eprintln!("alignment:"); for i in 0..ALIGNMENTS.len() { - println!(" - {} = {}", i, ALIGNMENTS[i].get()); + eprintln!(" - {} = {}", i, ALIGNMENTS[i].get()); } - println!(" - others = {}", OTHER_ALIGNMENT.get()); - println!(""); - println!("size:"); + eprintln!(" - others = {}", OTHER_ALIGNMENT.get()); + eprintln!(""); + eprintln!("size:"); for i in 0..SIZES.len() { - println!(" - {} = {}", i, SIZES[i].get()); + eprintln!(" - {} = {}", i, SIZES[i].get()); } - println!(" - others = {}", OTHER_SIZE.get()); - println!(""); + eprintln!(" - others = {}", OTHER_SIZE.get()); + eprintln!(""); while let Some(c) = COUNTERS.lock().pop() { - println!("{}: {}", c.0, c.get()); + eprintln!("{}: {}", c.0, c.get()); } } From ddaaa84b1384584d4a0ae7816aba7fb03ef8bea2 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 17:25:27 +1000 Subject: [PATCH 18/23] Simple dev tools --- .cargo/config.toml | 6 +- Cargo.lock | 200 ++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + mallockit/dev/Cargo.toml | 11 +++ mallockit/dev/src/main.rs | 89 +++++++++++++++++ 5 files changed, 306 insertions(+), 1 deletion(-) create mode 100644 mallockit/dev/Cargo.toml create mode 100644 mallockit/dev/src/main.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index 7a41013..a520d27 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,6 @@ [build] -rustflags = ["-Z", "tls_model=initial-exec"] \ No newline at end of file +rustflags = ["-Z", "tls_model=initial-exec"] + +[alias] +# Helper command to run a program with a malloc implementation +x = "run -p mallockit-dev-tools --quiet --" diff --git a/Cargo.lock b/Cargo.lock index 9cc09dc..cfe5d16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,55 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "anstream" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" + +[[package]] +name = "anstyle-parse" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +dependencies = [ + "anstyle", + "windows-sys", +] + [[package]] name = "anyhow" version = "1.0.86" @@ -117,6 +166,52 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clap" +version = "4.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "clap_lex" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" + +[[package]] +name = "colorchoice" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" + [[package]] name = "ctor" version = "0.1.26" @@ -166,6 +261,12 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.3.9" @@ -191,6 +292,12 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" + [[package]] name = "itoa" version = "1.0.10" @@ -228,6 +335,14 @@ dependencies = [ "spin", ] +[[package]] +name = "mallockit-dev-tools" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", +] + [[package]] name = "mallockit-macros" version = "0.1.0" @@ -386,6 +501,12 @@ dependencies = [ "lock_api", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "syn" version = "1.0.109" @@ -468,6 +589,12 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "winapi" version = "0.3.9" @@ -490,6 +617,79 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + [[package]] name = "winnow" version = "0.6.8" diff --git a/Cargo.toml b/Cargo.toml index 16eaf83..8696c09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ resolver = "2" members = [ "mallockit", "mallockit/macros", + "mallockit/dev", "bump", "buddy", "hoard", diff --git a/mallockit/dev/Cargo.toml b/mallockit/dev/Cargo.toml new file mode 100644 index 0000000..3f67845 --- /dev/null +++ b/mallockit/dev/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "mallockit-dev-tools" +version = "0.1.0" +edition = "2021" +publish = false + + + +[dependencies] +anyhow = { version = "1.0.86", features = ["backtrace"] } +clap = { version = "4.5.4", features = ["derive"] } diff --git a/mallockit/dev/src/main.rs b/mallockit/dev/src/main.rs new file mode 100644 index 0000000..17bdea1 --- /dev/null +++ b/mallockit/dev/src/main.rs @@ -0,0 +1,89 @@ +use std::process::Command; + +use clap::Parser; + +/// Run a program with a custom allocator +#[derive(Parser, Debug)] +struct Options { + /// Malloc implementation to use + malloc: String, + + /// Space or comma separated list of features to activate + #[arg(short = 'F', long)] + features: Vec, + + /// Activate all available features + #[arg(long, default_value_t = false)] + all_features: bool, + + /// Do not activate the `default` feature + #[arg(long, default_value_t = false)] + no_default_features: bool, + + /// Build artifacts in release mode, with optimizations + #[arg(short, long, default_value_t = false)] + release: bool, + + /// Build artifacts with the specified profile + #[arg(short, long)] + profile: Option, + + /// The program to run, with arguments + #[arg(last = true, allow_hyphen_values = true)] + command: Vec, +} + +fn build_crate(opts: &Options) -> anyhow::Result<()> { + let mut cmd = Command::new("cargo"); + let mut cmd = cmd.arg("build").args(["-p", &opts.malloc]); + if opts.features.len() > 0 { + cmd = cmd.arg("--features").arg(opts.features.join(",")); + } + if opts.all_features { + cmd = cmd.arg("--all-features"); + } + if opts.no_default_features { + cmd = cmd.arg("--no-default-features"); + } + if opts.release { + cmd = cmd.arg("--release"); + } + if let Some(profile) = &opts.profile { + cmd = cmd.arg("--profile").arg(profile); + } + let status = cmd.status()?; + if !status.success() { + anyhow::bail!("cargo run failed: {}", status); + } + Ok(()) +} + +fn main() -> anyhow::Result<()> { + let options = Options::parse(); + // 1. Build the crate + build_crate(&options)?; + // 2. Run the program + let args = options.command.clone(); + let mut cmd = Command::new(&args[0]); + cmd.args(&args[1..]); + let profile = if options.release { "release" } else { "debug" }; + let ext = if cfg!(target_os = "macos") { + "dylib" + } else { + "so" + }; + let ld_preload_env = if cfg!(target_os = "macos") { + "DYLD_INSERT_LIBRARIES" + } else { + "LD_PRELOAD" + }; + cmd.env( + ld_preload_env, + format!("target/{}/lib{}.{}", profile, options.malloc, ext), + ); + let status = cmd.status()?; + if !status.success() { + anyhow::bail!("program failed: {}", status); + } + Ok(()) +} From bd37b26b084b1754acb51c1c8a5523c4b6f83431 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 19:05:06 +1000 Subject: [PATCH 19/23] Update tools --- mallockit/dev/src/main.rs | 74 +++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 23 deletions(-) diff --git a/mallockit/dev/src/main.rs b/mallockit/dev/src/main.rs index 17bdea1..b0e58b6 100644 --- a/mallockit/dev/src/main.rs +++ b/mallockit/dev/src/main.rs @@ -28,6 +28,14 @@ struct Options { #[arg(short, long)] profile: Option, + /// Run the program under GDB + #[arg(long, default_value_t = false)] + gdb: bool, + + /// Run the program under LLDB + #[arg(long, default_value_t = false)] + lldb: bool, + /// The program to run, with arguments #[arg(last = true, allow_hyphen_values = true)] command: Vec, @@ -35,55 +43,75 @@ struct Options { fn build_crate(opts: &Options) -> anyhow::Result<()> { let mut cmd = Command::new("cargo"); - let mut cmd = cmd.arg("build").args(["-p", &opts.malloc]); - if opts.features.len() > 0 { - cmd = cmd.arg("--features").arg(opts.features.join(",")); + cmd.arg("build").args(["-p", &opts.malloc]); + let mut features = opts.features.clone(); + if !features.contains(&"malloc".to_owned()) && !opts.all_features { + features.push("malloc".to_string()); + } + if features.len() > 0 { + cmd.arg("--features").arg(features.join(",")); } if opts.all_features { - cmd = cmd.arg("--all-features"); + cmd.arg("--all-features"); } if opts.no_default_features { - cmd = cmd.arg("--no-default-features"); + cmd.arg("--no-default-features"); } if opts.release { - cmd = cmd.arg("--release"); + cmd.arg("--release"); } if let Some(profile) = &opts.profile { - cmd = cmd.arg("--profile").arg(profile); + cmd.arg("--profile").arg(profile); } let status = cmd.status()?; if !status.success() { - anyhow::bail!("cargo run failed: {}", status); + std::process::exit(status.code().unwrap_or(1)); } Ok(()) } +const ENV: &str = if cfg!(target_os = "macos") { + "DYLD_INSERT_LIBRARIES" +} else { + "LD_PRELOAD" +}; + +const EXT: &str = if cfg!(target_os = "macos") { + "dylib" +} else { + "so" +}; + fn main() -> anyhow::Result<()> { let options = Options::parse(); // 1. Build the crate build_crate(&options)?; // 2. Run the program let args = options.command.clone(); - let mut cmd = Command::new(&args[0]); - cmd.args(&args[1..]); let profile = if options.release { "release" } else { "debug" }; - let ext = if cfg!(target_os = "macos") { - "dylib" - } else { - "so" - }; - let ld_preload_env = if cfg!(target_os = "macos") { - "DYLD_INSERT_LIBRARIES" + let dylib = format!("target/{}/lib{}.{}", profile, options.malloc, EXT); + let mut cmd = if !options.gdb && !options.lldb { + let mut cmd = Command::new(&args[0]); + cmd.args(&args[1..]); + cmd.env(ENV, dylib); + println!("{:?}", cmd); + cmd + } else if options.gdb { + let mut cmd = Command::new("rust-gdb"); + cmd.args(&["-ex", &format!("set environment {ENV}={dylib}")]); + cmd.arg("--args").args(&args); + cmd } else { - "LD_PRELOAD" + let mut cmd = Command::new("rust-lldb"); + cmd.args(&["-o", &format!("env {ENV}={dylib}")]); + cmd.arg("--source-quietly"); + cmd.arg("--").args(&args); + cmd }; - cmd.env( - ld_preload_env, - format!("target/{}/lib{}.{}", profile, options.malloc, ext), - ); + cmd.env("RUST_BACKTRACE", "1"); let status = cmd.status()?; if !status.success() { - anyhow::bail!("program failed: {}", status); + std::process::exit(status.code().unwrap_or(1)); } Ok(()) } From f869b9e8bbbdcef67509fb6d5bf540eabbb3a7ae Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 19:05:44 +1000 Subject: [PATCH 20/23] cleanup --- mallockit/dev/src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/mallockit/dev/src/main.rs b/mallockit/dev/src/main.rs index b0e58b6..25ab05a 100644 --- a/mallockit/dev/src/main.rs +++ b/mallockit/dev/src/main.rs @@ -94,7 +94,6 @@ fn main() -> anyhow::Result<()> { let mut cmd = Command::new(&args[0]); cmd.args(&args[1..]); cmd.env(ENV, dylib); - println!("{:?}", cmd); cmd } else if options.gdb { let mut cmd = Command::new("rust-gdb"); From 9efa3c9e443b30cd4b4e55fa03afeca40ba194bc Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 19:16:04 +1000 Subject: [PATCH 21/23] Fix macos error --- mallockit/macros/src/lib.rs | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/mallockit/macros/src/lib.rs b/mallockit/macros/src/lib.rs index 1a26df8..38f9bfb 100644 --- a/mallockit/macros/src/lib.rs +++ b/mallockit/macros/src/lib.rs @@ -22,7 +22,8 @@ pub fn plan(_attr: TokenStream, item: TokenStream) -> TokenStream { #[cfg(target_os = "macos")] #[no_mangle] extern "C" fn mallockit_initialize_macos_tls() -> *mut u8 { - ::Mutator::current() as *mut ::mallockit::Mutator as _ + use ::mallockit::mutator::TLS; + ::Mutator::current() as *mut ::Mutator as _ } impl ::mallockit::plan::Singleton for super::#name { @@ -57,22 +58,19 @@ pub fn mutator(_attr: TokenStream, item: TokenStream) -> TokenStream { #[repr(align(256))] #input - #[cfg(not(target_os = "macos"))] - mod __mallockit_mutator { - #[thread_local] - static mut MUTATOR: ::mallockit::util::Lazy = ::mallockit::util::Lazy::new(|| ::new()); - - impl ::mallockit::mutator::TLS for super::#name { - fn new() -> Self { - ::new() - } + impl ::mallockit::mutator::TLS for #name { + fn new() -> Self { + ::new() + } - #[cfg(not(target_os = "macos"))] - fn current() -> &'static mut Self { - unsafe { &mut *MUTATOR } - } + #[cfg(not(target_os = "macos"))] + fn current() -> &'static mut Self { + #[thread_local] + static mut MUTATOR: ::mallockit::util::Lazy<#name, ::mallockit::util::Local> = ::mallockit::util::Lazy::new(|| <#name as ::mallockit::Mutator>::new()); + unsafe { &mut * MUTATOR } } } + }; result.into() } From c96feae96d26147b3f6dcd8ef160ab6cbeac58bd Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 19:16:57 +1000 Subject: [PATCH 22/23] Fix clippy --- mallockit/dev/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mallockit/dev/src/main.rs b/mallockit/dev/src/main.rs index 25ab05a..283369d 100644 --- a/mallockit/dev/src/main.rs +++ b/mallockit/dev/src/main.rs @@ -48,7 +48,7 @@ fn build_crate(opts: &Options) -> anyhow::Result<()> { if !features.contains(&"malloc".to_owned()) && !opts.all_features { features.push("malloc".to_string()); } - if features.len() > 0 { + if !features.is_empty() { cmd.arg("--features").arg(features.join(",")); } if opts.all_features { @@ -97,12 +97,12 @@ fn main() -> anyhow::Result<()> { cmd } else if options.gdb { let mut cmd = Command::new("rust-gdb"); - cmd.args(&["-ex", &format!("set environment {ENV}={dylib}")]); + cmd.args(["-ex", &format!("set environment {ENV}={dylib}")]); cmd.arg("--args").args(&args); cmd } else { let mut cmd = Command::new("rust-lldb"); - cmd.args(&["-o", &format!("env {ENV}={dylib}")]); + cmd.args(["-o", &format!("env {ENV}={dylib}")]); cmd.arg("--source-quietly"); cmd.arg("--").args(&args); cmd From 576f804d2c77b31557e39a6ace19768b09b36c28 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 May 2024 19:58:55 +1000 Subject: [PATCH 23/23] cleanup --- buddy/src/lib.rs | 6 ++---- bump/src/lib.rs | 2 +- hoard/src/hoard_space.rs | 12 ++++++------ hoard/src/lib.rs | 4 ++-- hoard/src/pool.rs | 12 ++++++------ mallockit/src/space/freelist_space.rs | 10 ++++------ mallockit/src/space/immortal_space.rs | 4 ++-- mallockit/src/space/large_object_space.rs | 8 ++++---- sanity/src/lib.rs | 2 +- 9 files changed, 28 insertions(+), 32 deletions(-) diff --git a/buddy/src/lib.rs b/buddy/src/lib.rs index 841ed4a..95d25f3 100644 --- a/buddy/src/lib.rs +++ b/buddy/src/lib.rs @@ -49,10 +49,8 @@ impl Mutator for BuddyMutator { fn new() -> Self { Self { - freelist: FreeListAllocator::new::(Lazy::new(|| { - &Self::plan().freelist_space - })), - los: LargeObjectAllocator::new(Lazy::new(|| &Self::plan().large_object_space)), + freelist: FreeListAllocator::new::(&Self::plan().freelist_space), + los: LargeObjectAllocator::new(&Self::plan().large_object_space), } } diff --git a/bump/src/lib.rs b/bump/src/lib.rs index 22fb816..573d985 100644 --- a/bump/src/lib.rs +++ b/bump/src/lib.rs @@ -41,7 +41,7 @@ impl Mutator for BumpMutator { fn new() -> Self { Self { - bump: BumpAllocator::new(Lazy::new(|| &Self::plan().immortal)), + bump: BumpAllocator::new(&Self::plan().immortal), } } diff --git a/hoard/src/hoard_space.rs b/hoard/src/hoard_space.rs index 68f722b..df68ff5 100644 --- a/hoard/src/hoard_space.rs +++ b/hoard/src/hoard_space.rs @@ -82,20 +82,20 @@ impl HoardSpace { } /// Thread-local heap pub struct HoardAllocator { - space: Lazy<&'static HoardSpace, Local>, + space: &'static HoardSpace, tlab: DiscreteTLAB<{ SizeClass::<4>::from_bytes(Self::LARGEST_SMALL_OBJECT).as_usize() + 1 }>, - local: Lazy, Local>, + local: Box, } impl HoardAllocator { const LOCAL_HEAP_THRESHOLD: usize = 16 * 1024 * 1024; const LARGEST_SMALL_OBJECT: usize = 1024; - pub const fn new(space: Lazy<&'static HoardSpace, Local>, _space_id: SpaceId) -> Self { + pub fn new(space: &'static HoardSpace, _space_id: SpaceId) -> Self { Self { space, tlab: DiscreteTLAB::new(), - local: Lazy::new(|| Box::new_in(Pool::new(false), Meta)), + local: Box::new_in(Pool::new(false), Meta), } } } @@ -108,7 +108,7 @@ impl Allocator for HoardAllocator { return Some(cell); } } - self.local.alloc_cell(size_class, &self.space) + self.local.alloc_cell(size_class, self.space) } fn dealloc(&mut self, cell: Address) { @@ -119,7 +119,7 @@ impl Allocator for HoardAllocator { { self.tlab.push(block.size_class, cell); } else { - self.local.free_cell(cell, &self.space); + self.local.free_cell(cell, self.space); } } } diff --git a/hoard/src/lib.rs b/hoard/src/lib.rs index 81fce97..f367c38 100644 --- a/hoard/src/lib.rs +++ b/hoard/src/lib.rs @@ -55,8 +55,8 @@ impl Mutator for HoardMutator { fn new() -> Self { Self { - hoard: HoardAllocator::new(Lazy::new(|| &Self::plan().hoard_space), HOARD_SPACE), - los: LargeObjectAllocator::new(Lazy::new(|| &Self::plan().large_object_space)), + hoard: HoardAllocator::new(&Self::plan().hoard_space, HOARD_SPACE), + los: LargeObjectAllocator::new(&Self::plan().large_object_space), } } diff --git a/hoard/src/pool.rs b/hoard/src/pool.rs index 1cd4aa5..c34981f 100644 --- a/hoard/src/pool.rs +++ b/hoard/src/pool.rs @@ -2,7 +2,7 @@ use crate::{hoard_space::HoardSpace, super_block::SuperBlock}; use array_const_fn_init::array_const_fn_init; use mallockit::{ space::page_resource::MemRegion, - util::{mem::size_class::SizeClass, Address, Lazy, Local}, + util::{mem::size_class::SizeClass, Address}, }; use spin::{relax::Yield, MutexGuard}; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -255,7 +255,7 @@ impl Pool { &self, size_class: SizeClass, blocks: &mut BlockList, - space: &Lazy<&'static HoardSpace, Local>, + space: &'static HoardSpace, ) -> SuperBlock { // Get a block from global pool let block = space @@ -276,7 +276,7 @@ impl Pool { pub fn alloc_cell( &mut self, size_class: SizeClass, - space: &Lazy<&'static HoardSpace, Local>, + space: &'static HoardSpace, ) -> Option
{ debug_assert!(!self.global); let mut blocks = unsafe { self.blocks.get_unchecked(size_class.as_usize()).lock() }; @@ -292,7 +292,7 @@ impl Pool { } #[cold] - pub fn free_cell(&self, cell: Address, space: &Lazy<&'static HoardSpace, Local>) { + pub fn free_cell(&self, cell: Address, space: &'static HoardSpace) { let block = SuperBlock::containing(cell); let mut owner = block.owner; let mut blocks = owner.lock_blocks(block.size_class); @@ -308,7 +308,7 @@ impl Pool { fn free_cell_slow_impl( &self, cell: Address, - space: &Lazy<&'static HoardSpace, Local>, + space: &'static HoardSpace, blocks: &mut BlockList, ) { let block = SuperBlock::containing(cell); @@ -331,7 +331,7 @@ impl Pool { fn flush_block_slow( &self, size_class: SizeClass, - space: &Lazy<&'static HoardSpace, Local>, + space: &'static HoardSpace, blocks: &mut BlockList, ) { // Transit a mostly-empty block to the global pool diff --git a/mallockit/src/space/freelist_space.rs b/mallockit/src/space/freelist_space.rs index bd38838..c3f09ca 100644 --- a/mallockit/src/space/freelist_space.rs +++ b/mallockit/src/space/freelist_space.rs @@ -148,17 +148,15 @@ impl Cell { } pub struct FreeListAllocator { - space: Lazy<&'static FreeListSpace, Local>, - freelist: Lazy, Local>, + space: &'static FreeListSpace, + freelist: IntrusiveFreeList, } impl FreeListAllocator { - pub const fn new(space: Lazy<&'static FreeListSpace, Local>) -> Self { + pub fn new(space: &'static FreeListSpace) -> Self { Self { space, - freelist: Lazy::new(|| { - IntrusiveFreeList::new(false, HEAP.get_space_range(SPACE_ID).start) - }), + freelist: IntrusiveFreeList::new(false, HEAP.get_space_range(SPACE_ID).start), } } diff --git a/mallockit/src/space/immortal_space.rs b/mallockit/src/space/immortal_space.rs index 6e0ec57..f1cbf63 100644 --- a/mallockit/src/space/immortal_space.rs +++ b/mallockit/src/space/immortal_space.rs @@ -30,13 +30,13 @@ impl Space for ImmortalSpace { } pub struct BumpAllocator { - space: Lazy<&'static ImmortalSpace, Local>, + space: &'static ImmortalSpace, allocation_area: AllocationArea, retry: bool, } impl BumpAllocator { - pub const fn new(space: Lazy<&'static ImmortalSpace, Local>) -> Self { + pub const fn new(space: &'static ImmortalSpace) -> Self { Self { space, allocation_area: AllocationArea::EMPTY, diff --git a/mallockit/src/space/large_object_space.rs b/mallockit/src/space/large_object_space.rs index a6c983c..d205ba9 100644 --- a/mallockit/src/space/large_object_space.rs +++ b/mallockit/src/space/large_object_space.rs @@ -4,7 +4,7 @@ use super::{ page_resource::{FreelistPageResource, PageResource}, Allocator, Space, SpaceId, }; -use crate::util::{Address, Lazy, Local, Page, PageSize, Size4K}; +use crate::util::{Address, Page, PageSize, Size4K}; pub struct LargeObjectSpace { id: SpaceId, @@ -62,7 +62,7 @@ pub struct LargeObjectAllocator< > where [(); bins::(MAX_CACHEABLE_SIZE)]: Sized, { - space: Lazy<&'static LargeObjectSpace, Local>, + space: &'static LargeObjectSpace, bins: [Address; bins::(MAX_CACHEABLE_SIZE)], max_live: usize, live: usize, @@ -77,7 +77,7 @@ where { const CACHE_ENABLED: bool = bins::(MAX_CACHEABLE_SIZE) > 0; - pub const fn new(los: Lazy<&'static LargeObjectSpace, Local>) -> Self { + pub fn new(los: &'static LargeObjectSpace) -> Self { Self { space: los, bins: [Address::ZERO; bins::(MAX_CACHEABLE_SIZE)], @@ -89,7 +89,7 @@ where } fn space(&self) -> &'static LargeObjectSpace { - *self.space + self.space } fn alloc_slow(&mut self, layout: Layout) -> Option
{ diff --git a/sanity/src/lib.rs b/sanity/src/lib.rs index ae08beb..26a542b 100644 --- a/sanity/src/lib.rs +++ b/sanity/src/lib.rs @@ -41,7 +41,7 @@ impl Mutator for SanityMutator { fn new() -> Self { Self { - los: LargeObjectAllocator::new(Lazy::new(|| &Self::plan().large_object_space)), + los: LargeObjectAllocator::new(&Self::plan().large_object_space), } }