Switch libc's malloc with mimalloc

CMakeLists.txt

@@ -35,6 +35,7 @@ if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64")
 endif()
 
 include("${CMAKE_SOURCE_DIR}/cmake/sabre_add_component.cmake")
+include("${CMAKE_SOURCE_DIR}/cmake/mimalloc.cmake")
 
 # Sources
 add_subdirectory("arch/${CMAKE_SYSTEM_PROCESSOR}" arch)

cmake/mimalloc.cmake

@@ -0,0 +1,28 @@
+include(ExternalProject)
+
+set(MIMALLOC_INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/mimalloc/install)
+
+ExternalProject_Add(
+  mimalloc
+  PREFIX mimalloc
+  GIT_REPOSITORY https://github.com/microsoft/mimalloc.git
+  GIT_TAG 81bd1b70b086d872bda515f99cfcf0d78f8ac86c # dev branch v1.6.3 + #240
+  INSTALL_DIR ${MIMALLOC_INSTALL_DIR}
+  # We need to patch mimalloc's default mmap ranges as they interfere with
+  # sanitizers
+  # https://github.com/llvm/llvm-project/blob/master/compiler-rt/lib/tsan/rtl/tsan_platform.h#L67-L72
+  PATCH_COMMAND
+    patch -p1 <
+    ${CMAKE_CURRENT_SOURCE_DIR}/mimalloc-sanitizer-compatibility.patch
+  CMAKE_ARGS
+    "-DCMAKE_INSTALL_PREFIX=${MIMALLOC_INSTALL_DIR};-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE};-DMI_LOCAL_DYNAMIC_TLS=ON;"
+)
+
+set(IS_DEBUG "")
+string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC)
+if(CMAKE_BUILD_TYPE_LC STREQUAL "debug")
+  set(IS_DEBUG "-debug")
+endif()
+
+set(MIMALLOC_STATIC_LIB
+    ${MIMALLOC_INSTALL_DIR}/lib/mimalloc-1.6/libmimalloc${IS_DEBUG}.a pthread)

cmake/sabre_add_component.cmake

@@ -15,13 +15,15 @@ function(sabre_add_executable exe_name)
   else()
     target_link_options(${exe_name} PRIVATE ${SABRE_EXE_LINK_FLAGS})
   endif()
-  target_link_libraries(${exe_name} loader)
+  add_dependencies(${exe_name} mimalloc)
+  target_link_libraries(${exe_name} loader ${MIMALLOC_STATIC_LIB})
 endfunction()
 
 function(sabre_add_plugin plugin_name)
   add_library(${plugin_name} MODULE ${ARGN})
   target_include_directories(${plugin_name} PUBLIC ${SABRE_PLUGIN_INCLUDE_DIRS})
   target_compile_definitions(${plugin_name} PUBLIC ${SABRE_PLUGIN_C_DEFINES})
   target_compile_options(${plugin_name} PUBLIC ${SABRE_PLUGIN_C_FLAGS})
-  target_link_libraries(${plugin_name} realsyscall)
+  add_dependencies(${plugin_name} mimalloc)
+  target_link_libraries(${plugin_name} realsyscall ${MIMALLOC_STATIC_LIB})
 endfunction()

loader/loader.c

@@ -154,15 +154,12 @@ void load(int argc, char *argv[], void **new_entry, void **new_stack_top)
 	exit(-1);
   }
 
-  // There 2 mallocs in memory because SaBRe loads 2 libcs (one for SaBRe) and
-  // one for the client (which is intercepted). These two mallocs will overlap
-  // their arenas as malloc uses brk(NULL) to initialize its arena, and this
-  // brk(NULL) will always return the same pointer. To avoid this we force
-  // SaBRe's malloc to completely skip the arena initialization and keep objects
-  // into separate mmap() pages. This of course comes with a small performance
-  // decrease, and the potential to OOM if we allocate too many items.
-  int ret = mallopt(M_MMAP_THRESHOLD, 0);
-  assert(ret == 1);
+  // SaBRe and its plugins statically link with mimalloc now, and stay separate
+  // from client's libc malloc. Before, there were 2 libc mallocs (one for SaBRe
+  // + plugin) and one for the client (which is intercepted). These two mallocs
+  // had overlapping arenas as malloc uses brk(NULL) to initialize its arena,
+  // and this brk(NULL) will always return the same pointer. Now, mimalloc is
+  // using mmap and this is no longer a problem.
 
   // Setup our custom TLS
   register_first_tid();

mimalloc-sanitizer-compatibility.patch

@@ -0,0 +1,14 @@
+diff --git a/src/os.c b/src/os.c
+index f33cfbc..536269e 100644
+--- a/src/os.c
++++ b/src/os.c
+@@ -405,7 +405,8 @@ static void* mi_unix_mmap(void* addr, size_t size, size_t try_alignment, int pro
+
+ // On 64-bit systems, we can do efficient aligned allocation by using
+ // the 4TiB to 30TiB area to allocate them.
+-#if (MI_INTPTR_SIZE >= 8) && (defined(_WIN32) || (defined(MI_OS_USE_MMAP) && !defined(MAP_ALIGNED)))
++// #if (MI_INTPTR_SIZE >= 8) && (defined(_WIN32) || (defined(MI_OS_USE_MMAP) && !defined(MAP_ALIGNED)))
++#if 0
+ static volatile mi_decl_cache_align _Atomic(uintptr_t) aligned_base;
+
+ // Return a 4MiB aligned address that is probably available