NFC: sizeclass: differentiate minimum step size and minimum allocation sizes

CMakeLists.txt

@@ -62,6 +62,9 @@ if (SNMALLOC_SANITIZER)
  message(STATUS "Using sanitizer=${SNMALLOC_SANITIZER}")
 endif()
 
+set(SNMALLOC_MIN_ALLOC_SIZE "" CACHE STRING "Minimum allocation bytes (power of 2)")
+set(SNMALLOC_MIN_ALLOC_STEP_SIZE "" CACHE STRING "Minimum allocation step (power of 2)")
+
 if(MSVC AND SNMALLOC_STATIC_LIBRARY AND (SNMALLOC_STATIC_LIBRARY_PREFIX STREQUAL ""))
  message(FATAL_ERROR "Empty static library prefix not supported on MSVC")
 endif()
@@ -226,6 +229,11 @@ endif()
 function(add_as_define FLAG)
  target_compile_definitions(snmalloc INTERFACE $<$<BOOL:${${FLAG}}>:${FLAG}>)
 endfunction()
+function(add_as_define_value KEY)
+ if (NOT ${${KEY}} STREQUAL "")
+ target_compile_definitions(snmalloc INTERFACE ${KEY}=${${KEY}})
+ endif ()
+endfunction()
 
 add_as_define(SNMALLOC_QEMU_WORKAROUND)
 add_as_define(SNMALLOC_TRACING)
@@ -238,6 +246,8 @@ endif()
 if (SNMALLOC_NO_REALLOCARR)
  add_as_define(SNMALLOC_NO_REALLOCARR)
 endif()
+add_as_define_value(SNMALLOC_MIN_ALLOC_SIZE)
+add_as_define_value(SNMALLOC_MIN_ALLOC_STEP_SIZE)
 
 target_compile_definitions(snmalloc INTERFACE $<$<BOOL:CONST_QUALIFIED_MALLOC_USABLE_SIZE>:MALLOC_USABLE_SIZE_QUALIFIER=const>)
 

src/snmalloc/ds/allocconfig.h

@@ -20,10 +20,31 @@ namespace snmalloc
  // Used to isolate values on cache lines to prevent false sharing.
  static constexpr size_t CACHELINE_SIZE = 64;
 
- // Minimum allocation size is space for two pointers.
- static_assert(bits::next_pow2_const(sizeof(void*)) == sizeof(void*));
- static constexpr size_t MIN_ALLOC_SIZE = 2 * sizeof(void*);
- static constexpr size_t MIN_ALLOC_BITS = bits::ctz_const(MIN_ALLOC_SIZE);
+ /// The "machine epsilon" for the small sizeclass machinery.
+ static constexpr size_t MIN_ALLOC_STEP_SIZE =
+#if defined(SNMALLOC_MIN_ALLOC_STEP_SIZE)
+ SNMALLOC_MIN_ALLOC_STEP_SIZE;
+#else
+ 2 * sizeof(void*);
+#endif
+
+ /// Derived from MIN_ALLOC_STEP_SIZE
+ static constexpr size_t MIN_ALLOC_STEP_BITS =
+ bits::ctz_const(MIN_ALLOC_STEP_SIZE);
+ static_assert(bits::is_pow2(MIN_ALLOC_STEP_SIZE));
+
+ /**
+ * Minimum allocation size is space for two pointers. If the small sizeclass
+ * machinery permits smaller values (that is, if MIN_ALLOC_STEP_SIZE is
+ * smaller than MIN_ALLOC_SIZE), which may be useful if MIN_ALLOC_SIZE must
+ * be large or not a power of two, those smaller size classes will be unused.
+ */
+ static constexpr size_t MIN_ALLOC_SIZE =
+#if defined(SNMALLOC_MIN_ALLOC_SIZE)
+ SNMALLOC_MIN_ALLOC_SIZE;
+#else
+ 2 * sizeof(void*);
+#endif
 
  // Minimum slab size.
 #if defined(SNMALLOC_QEMU_WORKAROUND) && defined(SNMALLOC_VA_BITS_64)
@@ -78,11 +99,18 @@ namespace snmalloc
  static constexpr size_t REMOTE_MASK = REMOTE_SLOTS - 1;
 
  static_assert(
- INTERMEDIATE_BITS < MIN_ALLOC_BITS,
+ INTERMEDIATE_BITS < MIN_ALLOC_STEP_BITS,
  "INTERMEDIATE_BITS must be less than MIN_ALLOC_BITS");
  static_assert(
  MIN_ALLOC_SIZE >= (sizeof(void*) * 2),
  "MIN_ALLOC_SIZE must be sufficient for two pointers");
+ static_assert(
+ 1 << (INTERMEDIATE_BITS + MIN_ALLOC_STEP_BITS) >=
+ bits::next_pow2_const(MIN_ALLOC_SIZE),
+ "Entire sizeclass exponent is below MIN_ALLOC_SIZE; adjust STEP_SIZE");
+ static_assert(
+ MIN_ALLOC_SIZE >= MIN_ALLOC_STEP_SIZE,
+ "Minimum alloc sizes below minimum step size; raise MIN_ALLOC_SIZE");
 
  // Return remote small allocs when the local cache reaches this size.
  static constexpr int64_t REMOTE_CACHE =

src/snmalloc/ds_core/bits.h

@@ -322,22 +322,6 @@ namespace snmalloc
  *
  * Does not work for value=0.
  ***********************************************/
- template<size_t MANTISSA_BITS, size_t LOW_BITS = 0>
- static size_t to_exp_mant(size_t value)
- {
- constexpr size_t LEADING_BIT = one_at_bit(MANTISSA_BITS + LOW_BITS) >> 1;
- constexpr size_t MANTISSA_MASK = one_at_bit(MANTISSA_BITS) - 1;
-
- value = value - 1;
-
- size_t e =
- bits::BITS - MANTISSA_BITS - LOW_BITS - clz(value | LEADING_BIT);
- size_t b = (e == 0) ? 0 : 1;
- size_t m = (value >> (LOW_BITS + e - b)) & MANTISSA_MASK;
-
- return (e << MANTISSA_BITS) + m;
- }
-
  template<size_t MANTISSA_BITS, size_t LOW_BITS = 0>
  constexpr size_t to_exp_mant_const(size_t value)
  {

src/snmalloc/mem/corealloc.h

@@ -597,23 +597,6 @@ namespace snmalloc
  init_message_queue();
  message_queue().invariant();
  }
-
- if constexpr (DEBUG)
- {
- for (smallsizeclass_t i = 0; i < NUM_SMALL_SIZECLASSES; i++)
- {
- size_t size = sizeclass_to_size(i);
- smallsizeclass_t sc1 = size_to_sizeclass(size);
- smallsizeclass_t sc2 = size_to_sizeclass_const(size);
- size_t size1 = sizeclass_to_size(sc1);
- size_t size2 = sizeclass_to_size(sc2);
-
- SNMALLOC_CHECK(sc1 == i);
- SNMALLOC_CHECK(sc1 == sc2);
- SNMALLOC_CHECK(size1 == size);
- SNMALLOC_CHECK(size1 == size2);
- }
- }
  }
 
  public:

src/snmalloc/mem/sizeclasstable.h

@@ -24,7 +24,7 @@ namespace snmalloc
  // For example, 24 byte allocations can be
  // problematic for some data due to alignment issues.
  auto sc = static_cast<smallsizeclass_t>(
- bits::to_exp_mant_const<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(size));
+ bits::to_exp_mant_const<INTERMEDIATE_BITS, MIN_ALLOC_STEP_BITS>(size));
 
  SNMALLOC_ASSERT(sc == static_cast<uint8_t>(sc));
 
@@ -214,7 +214,8 @@ namespace snmalloc
  auto& meta = fast_small(sizeclass);
 
  size_t rsize =
- bits::from_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(sizeclass);
+ bits::from_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_STEP_BITS>(
+ sizeclass);
  meta.size = rsize;
  size_t slab_bits = bits::max(
  bits::next_pow2_bits_const(MIN_OBJECT_COUNT * rsize), MIN_CHUNK_BITS);
@@ -405,7 +406,7 @@ namespace snmalloc
  {
  // We subtract and shift to reduce the size of the table, i.e. we don't have
  // to store a value for every size.
- return (s - 1) >> MIN_ALLOC_BITS;
+ return (s - 1) >> MIN_ALLOC_STEP_BITS;
  }
 
  constexpr size_t sizeclass_lookup_size =
@@ -421,13 +422,29 @@ namespace snmalloc
 
  constexpr SizeClassLookup()
  {
+ constexpr sizeclass_compress_t minimum_class =
+ static_cast<sizeclass_compress_t>(
+ size_to_sizeclass_const(MIN_ALLOC_SIZE));
+
+ /* Some unused sizeclasses is OK, but keep it within reason! */
+ static_assert(minimum_class < sizeclass_lookup_size);
+
  size_t curr = 1;
- for (sizeclass_compress_t sizeclass = 0;
- sizeclass < NUM_SMALL_SIZECLASSES;
- sizeclass++)
+
+ sizeclass_compress_t sizeclass = 0;
+ for (; sizeclass < minimum_class; sizeclass++)
+ {
+ for (; curr <= sizeclass_metadata.fast_small(sizeclass).size;
+ curr += MIN_ALLOC_STEP_SIZE)
+ {
+ table[sizeclass_lookup_index(curr)] = minimum_class;
+ }
+ }
+
+ for (; sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++)
  {
  for (; curr <= sizeclass_metadata.fast_small(sizeclass).size;
- curr += 1 << MIN_ALLOC_BITS)
+ curr += MIN_ALLOC_STEP_SIZE)
  {
  auto i = sizeclass_lookup_index(curr);
  if (i == sizeclass_lookup_size)

src/test/func/memcpy/func-memcpy.cc

@@ -57,6 +57,9 @@ extern "C" void abort()
  {
  longjmp(jmp, 1);
  }
+# if __has_builtin(__builtin_trap)
+ __builtin_trap();
+# endif
  exit(-1);
 }
 
@@ -152,7 +155,11 @@ int main()
  // Some sizes to check for out-of-bounds access. As we are only able to
  // catch overflows past the end of the sizeclass-padded allocation, make
  // sure we don't try to test on smaller allocations.
- std::initializer_list<size_t> sizes = {MIN_ALLOC_SIZE, 1024, 2 * 1024 * 1024};
+
+ static constexpr size_t min_class_size =
+ sizeclass_to_size(size_to_sizeclass(MIN_ALLOC_SIZE));
+
+ std::initializer_list<size_t> sizes = {min_class_size, 1024, 2 * 1024 * 1024};
  static_assert(
  MIN_ALLOC_SIZE < 1024,
  "Can't detect overflow except at sizeclass boundaries");

src/test/func/memory/memory.cc

@@ -237,7 +237,9 @@ void test_external_pointer()
  // Malloc does not have an external pointer querying mechanism.
  auto& alloc = ThreadAlloc::get();
 
- for (uint8_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++)
+ for (snmalloc::smallsizeclass_t sc = size_to_sizeclass(MIN_ALLOC_SIZE);
+ sc < NUM_SMALL_SIZECLASSES;
+ sc++)
  {
  size_t size = sizeclass_to_size(sc);
  void* p1 = alloc.alloc(size);
@@ -470,7 +472,9 @@ void test_static_sized_allocs()
 void test_remaining_bytes()
 {
  auto& alloc = ThreadAlloc::get();
- for (size_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++)
+ for (snmalloc::smallsizeclass_t sc = size_to_sizeclass(MIN_ALLOC_SIZE);
+ sc < NUM_SMALL_SIZECLASSES;
+ sc++)
  {
  auto size = sizeclass_to_size(sc);
  char* p = (char*)alloc.alloc(size);

src/test/func/sizeclass/sizeclass.cc

@@ -8,6 +8,9 @@ snmalloc::smallsizeclass_t size_to_sizeclass(size_t size)
  return snmalloc::size_to_sizeclass(size);
 }
 
+static constexpr snmalloc::smallsizeclass_t minimum_sizeclass =
+ snmalloc::size_to_sizeclass_const(snmalloc::MIN_ALLOC_SIZE);
+
 void test_align_size()
 {
  bool failed = false;
@@ -72,6 +75,10 @@ int main(int, char**)
  bool failed = false;
  size_t size_low = 0;
 
+ std::cout << "Configured with minimum allocation size "
+ << snmalloc::MIN_ALLOC_SIZE << " and step size "
+ << snmalloc::MIN_ALLOC_STEP_SIZE << std::endl;
+
  std::cout << "0 has sizeclass: " << (size_t)snmalloc::size_to_sizeclass(0)
  << std::endl;
 
@@ -86,12 +93,14 @@ int main(int, char**)
  slab_size != snmalloc::sizeclass_to_slab_size(sz))
  {
  slab_size = snmalloc::sizeclass_to_slab_size(sz);
- std::cout << std::endl;
+ std::cout << std::endl << "slab size: " << slab_size << std::endl;
  }
 
  size_t size = snmalloc::sizeclass_to_size(sz);
  std::cout << (size_t)sz << " |-> "
- << "[" << size_low + 1 << ", " << size << "]" << std::endl;
+ << "[" << size_low + 1 << ", " << size << "]"
+ << (sz == minimum_sizeclass ? " is minimum class" : "")
+ << std::endl;
 
  if (size < size_low)
  {
@@ -102,7 +111,30 @@ int main(int, char**)
 
  for (size_t i = size_low + 1; i <= size; i++)
  {
- if (size_to_sizeclass(i) != sz)
+ /* All sizes should, via bit-math, come back to their class value */
+ if (snmalloc::size_to_sizeclass_const(i) != sz)
+ {
+ std::cout << "Size " << i << " has _const sizeclass "
+ << (size_t)snmalloc::size_to_sizeclass_const(i)
+ << " but expected sizeclass " << (size_t)sz << std::endl;
+ failed = true;
+ }
+
+ if (size < snmalloc::MIN_ALLOC_SIZE)
+ {
+ /*
+ * It is expected that these sizes have the "wrong" class from tabular
+ * lookup: they will have been clipped up to the minimum class.
+ */
+ if (size_to_sizeclass(i) != minimum_sizeclass)
+ {
+ std::cout << "Size " << i << " below minimum size; sizeclass "
+ << (size_t)size_to_sizeclass(i) << " not expected minimum "
+ << (size_t)minimum_sizeclass << std::endl;
+ failed = true;
+ }
+ }
+ else if (size_to_sizeclass(i) != sz)
  {
  std::cout << "Size " << i << " has sizeclass "
  << (size_t)size_to_sizeclass(i) << " but expected sizeclass "