Implement mimalloc (#230)

This PR switch xalloc to use mimalloc internally instead of using it via
LD_PRELOAD

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "3rdparty/nodejs-http-parser"]
 	path = 3rdparty/nodejs-http-parser
 	url = https://github.com/nodejs/http-parser
+[submodule "3rdparty/mimalloc"]
+	path = 3rdparty/mimalloc
+	url = https://github.com/microsoft/mimalloc.git

3rdparty/CMakeLists.txt

@@ -9,6 +9,7 @@ include("libnuma")
 include("libcurl")
 
 # Dependencies built as part of the build process
+include("mimalloc")
 include("liburing")
 include("libcyaml")
 include("sentry")

3rdparty/mimalloc.cmake

@@ -0,0 +1,21 @@
+set(MI_BUILD_OBJECT FALSE CACHE BOOL "mimalloc option overridden" FORCE)
+set(MI_BUILD_SHARED FALSE CACHE BOOL "mimalloc option overridden" FORCE)
+set(MI_BUILD_TESTS FALSE CACHE BOOL "mimalloc option overridden" FORCE)
+set(MI_OVERRIDE FALSE CACHE BOOL "mimalloc option overridden" FORCE)
+
+if (CMAKE_BUILD_TYPE MATCHES Debug)
+    set(MI_SECURE TRUE CACHE BOOL "mimalloc option overridden" FORCE)
+    set(MI_DEBUG_FULL TRUE CACHE BOOL "mimalloc option overridden" FORCE)
+    set(MI_PADDING TRUE CACHE BOOL "mimalloc option overridden" FORCE)
+else()
+    set(MI_SECURE FALSE CACHE BOOL "mimalloc option overridden" FORCE)
+    set(MI_DEBUG_FULL FALSE CACHE BOOL "mimalloc option overridden" FORCE)
+    set(MI_PADDING FALSE CACHE BOOL "mimalloc option overridden" FORCE)
+    set(MI_PADDING FALSE CACHE BOOL "mimalloc option overridden" FORCE)
+endif()
+
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/mimalloc/ EXCLUDE_FROM_ALL)
+
+
+list(APPEND DEPS_LIST_LIBRARIES "mimalloc-static")
+list(APPEND DEPS_LIST_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/mimalloc")

src/config.c

@@ -76,7 +76,7 @@ void config_internal_cyaml_log(
 
     // If any, clone the message to skip them
     if (fmt_has_newline) {
-        fmt_to_print = malloc(fmt_len + 1);
+        fmt_to_print = xalloc_alloc(fmt_len + 1);
         strncpy(fmt_to_print, fmt, fmt_len);
         fmt_to_print[fmt_len] = 0;
     }

src/data_structures/hashtable/mcmp/hashtable_data.c

@@ -113,7 +113,7 @@ void hashtable_mcmp_data_keys_free(
             continue;
         }
 
-        ffma_mem_free(key_value->external_key.data);
+        xalloc_free(key_value->external_key.data);
     }
 }
 

src/data_structures/hashtable/mcmp/hashtable_op_delete.c

@@ -6,7 +6,6 @@
  * of the BSD license.  See the LICENSE file for details.
  **/
 
-#include <stdlib.h>
 #include <stdint.h>
 #include <stdbool.h>
 #include <stdatomic.h>
@@ -16,13 +15,11 @@
 #include "misc.h"
 #include "exttypes.h"
 #include "memory_fences.h"
+#include "xalloc.h"
 #include "log/log.h"
 #include "spinlock.h"
 #include "transaction.h"
 #include "transaction_spinlock.h"
-#include "data_structures/double_linked_list/double_linked_list.h"
-#include "data_structures/queue_mpmc/queue_mpmc.h"
-#include "memory_allocator/ffma.h"
 
 #include "hashtable.h"
 #include "hashtable_support_index.h"
@@ -121,7 +118,7 @@ bool hashtable_mcmp_op_delete(
 #if HASHTABLE_FLAG_ALLOW_KEY_INLINE == 1
             if (!HASHTABLE_KEY_VALUE_HAS_FLAG(key_value_flags, HASHTABLE_KEY_VALUE_FLAG_KEY_INLINE)) {
 #endif
-                ffma_mem_free(key_value->external_key.data);
+                xalloc_free(key_value->external_key.data);
                 key_value->external_key.data = NULL;
                 key_value->external_key.size = 0;
 #if HASHTABLE_FLAG_ALLOW_KEY_INLINE == 1

src/data_structures/hashtable/mcmp/hashtable_op_get_key.c

@@ -16,6 +16,7 @@
 
 #include "misc.h"
 #include "exttypes.h"
+#include "xalloc.h"
 #include "log/log.h"
 #include "memory_fences.h"
 #include "spinlock.h"
@@ -65,7 +66,7 @@ bool hashtable_mcmp_op_get_key(
     assert(source_key != NULL);
     assert(source_key_size > 0);
 
-    *key = ffma_mem_alloc(source_key_size);
+    *key = xalloc_alloc(source_key_size);
     memcpy(*key, source_key, source_key_size);
     *key_size = source_key_size;
 
@@ -99,7 +100,7 @@ bool hashtable_mcmp_op_get_key(
 #endif
 
     if (unlikely(key_deleted_or_different)) {
-        ffma_mem_free(key);
+        xalloc_free(key);
         *key = NULL;
         *key_size = 0;
     }

src/data_structures/hashtable/mcmp/hashtable_op_rmw.c

@@ -6,7 +6,6 @@
  * of the BSD license.  See the LICENSE file for details.
  **/
 
-#include <stdlib.h>
 #include <stdint.h>
 #include <stdbool.h>
 #include <stdatomic.h>
@@ -16,14 +15,11 @@
 
 #include "misc.h"
 #include "memory_fences.h"
-#include "exttypes.h"
+#include "xalloc.h"
 #include "spinlock.h"
 #include "transaction.h"
 #include "transaction_spinlock.h"
 #include "log/log.h"
-#include "data_structures/double_linked_list/double_linked_list.h"
-#include "data_structures/queue_mpmc/queue_mpmc.h"
-#include "memory_allocator/ffma.h"
 
 #include "hashtable.h"
 #include "hashtable_op_rmw.h"
@@ -134,7 +130,8 @@ void hashtable_mcmp_op_rmw_commit_update(
 
     // Validate if the passed key can be freed because unused or because inlined
     if (!rmw_status->created_new || key_inlined) {
-        ffma_mem_free(rmw_status->key);
+        xalloc_free(rmw_status->key);
+
     }
 
     // Decrement the size counter if deleted is true
@@ -159,7 +156,7 @@ void hashtable_mcmp_op_rmw_commit_delete(
 #if HASHTABLE_FLAG_ALLOW_KEY_INLINE == 1
         if (!HASHTABLE_KEY_VALUE_HAS_FLAG(key_value_flags, HASHTABLE_KEY_VALUE_FLAG_KEY_INLINE)) {
 #endif
-        ffma_mem_free(rmw_status->key_value->external_key.data);
+        xalloc_free(rmw_status->key_value->external_key.data);
         rmw_status->key_value->external_key.data = NULL;
         rmw_status->key_value->external_key.size = 0;
 #if HASHTABLE_FLAG_ALLOW_KEY_INLINE == 1

src/data_structures/hashtable/mcmp/hashtable_op_set.c

@@ -6,24 +6,19 @@
  * of the BSD license.  See the LICENSE file for details.
  **/
 
-#include <stdlib.h>
 #include <stdint.h>
 #include <stdbool.h>
 #include <stdatomic.h>
 #include <string.h>
 #include <assert.h>
 #include <numa.h>
 
-#include "misc.h"
 #include "memory_fences.h"
-#include "exttypes.h"
+#include "xalloc.h"
 #include "spinlock.h"
 #include "transaction.h"
 #include "transaction_spinlock.h"
 #include "log/log.h"
-#include "data_structures/double_linked_list/double_linked_list.h"
-#include "data_structures/queue_mpmc/queue_mpmc.h"
-#include "memory_allocator/ffma.h"
 
 #include "hashtable.h"
 #include "hashtable_op_set.h"
@@ -140,7 +135,7 @@ bool hashtable_mcmp_op_set(
 
     // Validate if the passed key can be freed because unused or because inlined
     if (!created_new || key_inlined) {
-        ffma_mem_free(key);
+        xalloc_free(key);
     }
 
     // Increment the size counter

src/data_structures/small_circular_queue/small_circular_queue.c

@@ -107,4 +107,4 @@ void *small_circular_queue_dequeue(
     scq->count--;
 
     return value;
-}
+}

src/log/log.c

@@ -18,6 +18,7 @@
 
 #include "misc.h"
 #include "xalloc.h"
+#include "mimalloc.h"
 
 #include "log.h"
 #include "log/sink/log_sink.h"

src/module/redis/command/module_redis_command_randomkey.c

@@ -15,6 +15,7 @@
 
 #include "misc.h"
 #include "exttypes.h"
+#include "xalloc.h"
 #include "clock.h"
 #include "spinlock.h"
 #include "transaction.h"
@@ -48,7 +49,7 @@ MODULE_REDIS_COMMAND_FUNCPTR_COMMAND_END(randomkey) {
 
     if (likely(key)) {
         return_res = module_redis_connection_send_blob_string(connection_context, key, key_size);
-        ffma_mem_free(key);
+        xalloc_free(key);
     } else {
         return_res = module_redis_connection_send_string_null(connection_context);
     }

src/module/redis/module_redis_command.c

@@ -20,6 +20,7 @@
 
 #include "misc.h"
 #include "exttypes.h"
+#include "xalloc.h"
 #include "log/log.h"
 #include "spinlock.h"
 #include "transaction.h"
@@ -608,7 +609,7 @@ bool module_redis_command_process_argument_full(
                        guessed_argument->name);
             }
 
-            string_value = ffma_mem_alloc(chunk_length);
+            string_value = xalloc_alloc(chunk_length);
 
             if (!string_value) {
                 LOG_E(TAG, "Failed to allocate memory for the incoming data");

src/program.c

@@ -20,6 +20,7 @@
 #include <sys/resource.h>
 #include <string.h>
 #include <stdlib.h>
+#include <mimalloc.h>
 
 #include "misc.h"
 #include "pow2.h"
@@ -433,7 +434,7 @@ bool program_use_huge_pages(
 
     // use_huge_pages is optional
     if (program_context->config->use_huge_pages == NULL) {
-        use_huge_pages = true;
+        use_huge_pages = false;
     } else {
         use_huge_pages = *program_context->config->use_huge_pages;
     }
@@ -449,10 +450,11 @@ bool program_use_huge_pages(
 
     if (use_huge_pages) {
         hugepage_cache_init();
+        ffma_enable(use_huge_pages);
+    } else {
+        ffma_enable(false);
     }
 
-    ffma_enable(use_huge_pages);
-
     program_context->use_huge_pages = use_huge_pages;
 
     return use_huge_pages;

src/storage/db/storage_db.c

@@ -1379,7 +1379,7 @@ bool storage_db_op_flush_sync(
             // The bucket might have been deleted in the meantime so get_key has to return true
             if (hashtable_mcmp_op_get_key(db->hashtable, bucket_index, &key, &key_size)) {
                 storage_db_op_delete(db, key, key_size);
-                ffma_mem_free(key);
+                xalloc_free(key);
             }
         }
     }
@@ -1445,7 +1445,7 @@ storage_db_key_and_key_length_t *storage_db_op_get_keys(
 
         if (likely(pattern_length > 0)) {
             if (!utils_string_glob_match(key, key_size, pattern, pattern_length)) {
-                ffma_mem_free(key);
+                xalloc_free(key);
                 continue;
             }
         }
@@ -1471,7 +1471,7 @@ void storage_db_free_key_and_key_length_list(
         storage_db_key_and_key_length_t *keys,
         uint64_t keys_count) {
     for(uint64_t index = 0; index < keys_count; index++) {
-        ffma_mem_free(keys[index].key);
+        xalloc_free(keys[index].key);
     }
     xalloc_free(keys);
 }

src/xalloc.c

@@ -21,11 +21,10 @@
 #include "misc.h"
 #include "log/log.h"
 #include "fatal.h"
+#include "mimalloc.h"
 
 #include "xalloc.h"
 
-// TODO: should use libnuma set_mempolicy the be able to use mmap and allocate interleaved memory across numa nodes
-
 #define MAP_HUGE_2MB    (21 << MAP_HUGE_SHIFT)   /* 2 MB hugepages */
 
 #define TAG "xalloc"
@@ -34,7 +33,7 @@ void* xalloc_alloc(
         size_t size) {
     void* memptr;
 
-    memptr = malloc(size);
+    memptr = mi_malloc(size);
 
     if (memptr == NULL) {
         FATAL(TAG, "Unable to allocate the requested memory %lu", size);
@@ -46,7 +45,7 @@ void* xalloc_alloc(
 void* xalloc_realloc(
         void* memptr,
         size_t size) {
-    memptr = realloc(memptr, size);
+    memptr = mi_realloc(memptr, size);
 
     if (memptr == NULL) {
         FATAL(TAG, "Unable to allocate the requested memory %lu to resize the pointer 0x%p", size, memptr);
@@ -59,7 +58,12 @@ void* xalloc_alloc_zero(
         size_t size) {
     void* memptr;
 
-    memptr = xalloc_alloc(size);
+    memptr = mi_zalloc(size);
+
+    if (memptr == NULL) {
+        FATAL(TAG, "Unable to allocate the requested memory %lu", size);
+    }
+
     if (memset(memptr, 0, size) != memptr) {
         FATAL(TAG, "Unable to zero the requested memory %lu", size);
     }
@@ -74,7 +78,7 @@ void* xalloc_alloc_aligned(
     bool failed = false;
 
 #if defined(__linux__)
-    memptr = aligned_alloc(alignment, size);
+    memptr = mi_malloc_aligned(size, alignment);
 
     if (memptr == NULL) {
         failed = true;
@@ -95,7 +99,12 @@ void* xalloc_alloc_aligned_zero(
         size_t size) {
     void* memptr;
 
-    memptr = xalloc_alloc_aligned(alignment, size);
+    memptr = mi_zalloc_aligned(size, alignment);
+
+    if (memptr == NULL) {
+        FATAL(TAG, "Unable to allocate the requested memory %lu", size);
+    }
+
     if (memset(memptr, 0, size) != memptr) {
         FATAL(TAG, "Unable to zero the requested memory %lu", size);
     }
@@ -105,7 +114,7 @@ void* xalloc_alloc_aligned_zero(
 
 void xalloc_free(
         void *memptr) {
-    free(memptr);
+    mi_free(memptr);
 }
 
 size_t xalloc_get_page_size() {