From fb6ad369a8215e4f60732483493adb5196e156d1 Mon Sep 17 00:00:00 2001 From: SchrodingerZhu Date: Sun, 17 May 2020 01:30:48 +0800 Subject: [PATCH 1/6] add snmalloc and mimalloc --- .gitmodules | 9 +++++++++ deps/Makefile | 18 ++++++++++++++++++ deps/mimalloc | 1 + deps/snmalloc | 1 + src/Makefile | 30 ++++++++++++++++++++++++++++++ src/zmalloc.c | 6 ++++++ src/zmalloc.h | 18 +++++++++++++++++- 7 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 .gitmodules create mode 160000 deps/mimalloc create mode 160000 deps/snmalloc diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..ef82b5771 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,9 @@ +[submodule "deps/snmalloc"] + path = deps/snmalloc + url = https://github.com/microsoft/snmalloc +[submodule "deps/mimalloc"] + path = deps/mimalloc + url = https://github.com/microsoft/mimalloc +[submodule "temp-snmalloc"] + path = deps/snmalloc + url = https://github.com/schrodingerzhu/snmalloc diff --git a/deps/Makefile b/deps/Makefile index 700867f3b..f3b14a610 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -85,3 +85,21 @@ jemalloc: .make-prerequisites cd jemalloc && $(MAKE) CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" lib/libjemalloc.a .PHONY: jemalloc + +mimalloc: .make-prerequisites + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) + cd mimalloc && cmake . -DCMAKE_BUILD_TYPE=Release && cmake --build . --target mimalloc-static + +.PHONY: mimalloc + +snmalloc: .make-prerequisites + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) + cd snmalloc && cmake . -DCMAKE_BUILD_TYPE=Release -DSNMALLOC_STATIC_LIBRARY=On && cmake --build . --target snmallocshim-static + +.PHONY: snmalloc + +snmalloc-1mib: .make-prerequisites + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) + cd snmalloc && cmake . -DCMAKE_BUILD_TYPE=Release -DSNMALLOC_STATIC_LIBRARY=On && cmake --build . --target snmallocshim-1mib-static + +.PHONY: snmalloc-1mib \ No newline at end of file diff --git a/deps/mimalloc b/deps/mimalloc new file mode 160000 index 000000000..e6c7b778f --- /dev/null +++ b/deps/mimalloc @@ -0,0 +1 @@ +Subproject commit e6c7b778fb53663985e0a36016a3a087af71d783 diff --git a/deps/snmalloc b/deps/snmalloc new file mode 160000 index 000000000..e7b7e8a73 --- /dev/null +++ b/deps/snmalloc @@ -0,0 +1 @@ +Subproject commit e7b7e8a739e87e6ffc59bba54ff698115279640f diff --git a/src/Makefile b/src/Makefile index b8c05c32b..6d12a8cd8 100644 --- a/src/Makefile +++ b/src/Makefile @@ -54,6 +54,18 @@ endif endif # Backwards compatibility for selecting an allocator +ifeq ($(USE_MIMALLOC),yes) + MALLOC=mimalloc +endif + +ifeq ($(USE_SNMALLOC),yes) + MALLOC=snmalloc +endif + +ifeq ($(USE_SNMALLOC_1MIB),yes) + MALLOC=snmalloc-1mib +endif + ifeq ($(USE_TCMALLOC),yes) MALLOC=tcmalloc endif @@ -175,6 +187,24 @@ ifeq ($(BUILD_WITH_SYSTEMD),yes) FINAL_CFLAGS+= -DHAVE_LIBSYSTEMD endif +ifeq ($(MALLOC),snmalloc) + DEPENDENCY_TARGETS+= snmalloc + FINAL_CFLAGS+= -mcx16 -DUSE_SNMALLOC + FINAL_LIBS := ../deps/snmalloc/libsnmallocshim-static.a -lstdc++ -latomic $(FINAL_LIBS) +endif + +ifeq ($(MALLOC),snmalloc-1mib) + DEPENDENCY_TARGETS+= snmalloc-1mib + FINAL_CFLAGS+= -mcx16 -DUSE_SNMALLOC + FINAL_LIBS := ../deps/snmalloc/libsnmallocshim-1mib-static.a -lstdc++ -latomic $(FINAL_LIBS) +endif + +ifeq ($(MALLOC),mimalloc) + DEPENDENCY_TARGETS+= mimalloc + FINAL_CFLAGS+= -DUSE_MIMALLOC -I../deps/mimalloc/include + FINAL_LIBS := ../deps/mimalloc/libmimalloc.a $(FINAL_LIBS) +endif + ifeq ($(MALLOC),tcmalloc) FINAL_CFLAGS+= -DUSE_TCMALLOC FINAL_LIBS+= -ltcmalloc diff --git a/src/zmalloc.c b/src/zmalloc.c index 639a5fe2b..e81e5cfba 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -57,6 +57,7 @@ void zlibc_free(void *ptr) { #endif /* Explicitly override malloc/free etc when using tcmalloc. */ +/* Not overriding the malloc here if using snmalloc */ #if defined(USE_TCMALLOC) #define malloc(size) tc_malloc(size) #define calloc(count,size) tc_calloc(count,size) @@ -69,6 +70,11 @@ void zlibc_free(void *ptr) { #define free(ptr) je_free(ptr) #define mallocx(size,flags) je_mallocx(size,flags) #define dallocx(ptr,flags) je_dallocx(ptr,flags) +#elif defined(USE_MIMALLOC) +#define malloc(size) mi_malloc(size) +#define calloc(count, size) mi_calloc(count, size) +#define realloc(ptr, size) mi_realloc(ptr, size) +#define free(ptr) mi_free(ptr) #endif #define update_zmalloc_stat_alloc(__n) do { \ diff --git a/src/zmalloc.h b/src/zmalloc.h index b136a910d..36f94f5cd 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -35,7 +35,23 @@ #define __xstr(s) __str(s) #define __str(s) #s -#if defined(USE_TCMALLOC) +#if defined(USE_MIMALLOC) +#include +#define ZMALLOC_LIB ("mimalloc-" __xstr(MI_MALLOC_VERSION)) +#define HAVE_MALLOC_SIZE 1 +#define zmalloc_size(p) mi_malloc_usable_size(p) + +#elif defined(USE_SNMALLOC) +#define ZMALLOC_LIB ("snmalloc-git") // TODO: FIXME +#define HAVE_MALLOC_SIZE 1 +#define zmalloc_size(p) malloc_usable_size(p) +extern size_t malloc_usable_size(void *); +extern void * malloc(size_t); +extern void free(void *); +extern void * realloc(void *, size_t); +extern void * calloc(size_t, size_t); + +#elif defined(USE_TCMALLOC) #define ZMALLOC_LIB ("tcmalloc-" __xstr(TC_VERSION_MAJOR) "." __xstr(TC_VERSION_MINOR)) #include #if (TC_VERSION_MAJOR == 1 && TC_VERSION_MINOR >= 6) || (TC_VERSION_MAJOR > 1) From aa933f3a19f32616b0367868b06973ba97fafd90 Mon Sep 17 00:00:00 2001 From: SchrodingerZhu Date: Sun, 17 May 2020 16:31:18 +0800 Subject: [PATCH 2/6] use shell to determine the version --- src/Makefile | 9 ++++++--- src/zmalloc.h | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Makefile b/src/Makefile index 6d12a8cd8..97d957583 100644 --- a/src/Makefile +++ b/src/Makefile @@ -188,20 +188,23 @@ ifeq ($(BUILD_WITH_SYSTEMD),yes) endif ifeq ($(MALLOC),snmalloc) + __version := $(shell sh -c 'git submodule status | grep "snmalloc" | grep -oP "(?<=\()(.+)(?=\))"') DEPENDENCY_TARGETS+= snmalloc - FINAL_CFLAGS+= -mcx16 -DUSE_SNMALLOC + FINAL_CFLAGS+= -mcx16 -DUSE_SNMALLOC -DSNMALLOC_VERSION=\"$(__version)\" FINAL_LIBS := ../deps/snmalloc/libsnmallocshim-static.a -lstdc++ -latomic $(FINAL_LIBS) endif ifeq ($(MALLOC),snmalloc-1mib) + __version := $(shell sh -c 'git submodule status | grep "snmalloc" | grep -oP "(?<=\()(.+)(?=\))"') DEPENDENCY_TARGETS+= snmalloc-1mib - FINAL_CFLAGS+= -mcx16 -DUSE_SNMALLOC + FINAL_CFLAGS+= -mcx16 -DUSE_SNMALLOC -DSNMALLOC_VERSION='"$(__version) (1mib)"' FINAL_LIBS := ../deps/snmalloc/libsnmallocshim-1mib-static.a -lstdc++ -latomic $(FINAL_LIBS) endif ifeq ($(MALLOC),mimalloc) + __version := $(shell sh -c 'git submodule status | grep "mimalloc" | grep -oP "(?<=\()(.+)(?=\))"') DEPENDENCY_TARGETS+= mimalloc - FINAL_CFLAGS+= -DUSE_MIMALLOC -I../deps/mimalloc/include + FINAL_CFLAGS+= -DUSE_MIMALLOC -I../deps/mimalloc/include -DMIMALLOC_VERSION=\"$(__version)\" FINAL_LIBS := ../deps/mimalloc/libmimalloc.a $(FINAL_LIBS) endif diff --git a/src/zmalloc.h b/src/zmalloc.h index 36f94f5cd..6dd6180a0 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -37,12 +37,12 @@ #if defined(USE_MIMALLOC) #include -#define ZMALLOC_LIB ("mimalloc-" __xstr(MI_MALLOC_VERSION)) +#define ZMALLOC_LIB ("mimalloc-" MIMALLOC_VERSION) #define HAVE_MALLOC_SIZE 1 #define zmalloc_size(p) mi_malloc_usable_size(p) #elif defined(USE_SNMALLOC) -#define ZMALLOC_LIB ("snmalloc-git") // TODO: FIXME +#define ZMALLOC_LIB ("snmalloc-" SNMALLOC_VERSION) #define HAVE_MALLOC_SIZE 1 #define zmalloc_size(p) malloc_usable_size(p) extern size_t malloc_usable_size(void *); From 9cb712c4b683158321e53f99a1e3aab8068cdc49 Mon Sep 17 00:00:00 2001 From: SchrodingerZhu Date: Sun, 17 May 2020 16:39:28 +0800 Subject: [PATCH 3/6] remove unused cflag --- src/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index 97d957583..bc8728615 100644 --- a/src/Makefile +++ b/src/Makefile @@ -190,14 +190,14 @@ endif ifeq ($(MALLOC),snmalloc) __version := $(shell sh -c 'git submodule status | grep "snmalloc" | grep -oP "(?<=\()(.+)(?=\))"') DEPENDENCY_TARGETS+= snmalloc - FINAL_CFLAGS+= -mcx16 -DUSE_SNMALLOC -DSNMALLOC_VERSION=\"$(__version)\" + FINAL_CFLAGS+= -DUSE_SNMALLOC -DSNMALLOC_VERSION=\"$(__version)\" FINAL_LIBS := ../deps/snmalloc/libsnmallocshim-static.a -lstdc++ -latomic $(FINAL_LIBS) endif ifeq ($(MALLOC),snmalloc-1mib) __version := $(shell sh -c 'git submodule status | grep "snmalloc" | grep -oP "(?<=\()(.+)(?=\))"') DEPENDENCY_TARGETS+= snmalloc-1mib - FINAL_CFLAGS+= -mcx16 -DUSE_SNMALLOC -DSNMALLOC_VERSION='"$(__version) (1mib)"' + FINAL_CFLAGS+= -DUSE_SNMALLOC -DSNMALLOC_VERSION='"$(__version) (1mib)"' FINAL_LIBS := ../deps/snmalloc/libsnmallocshim-1mib-static.a -lstdc++ -latomic $(FINAL_LIBS) endif From 10074797aa86bac9741493004cefeb24af342027 Mon Sep 17 00:00:00 2001 From: SchrodingerZhu Date: Sun, 17 May 2020 19:35:18 +0800 Subject: [PATCH 4/6] fix extra override --- deps/Makefile | 6 +++--- src/zmalloc.c | 5 +++++ src/zmalloc.h | 12 ++++++------ 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index f3b14a610..a8aeb92d2 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -88,18 +88,18 @@ jemalloc: .make-prerequisites mimalloc: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) - cd mimalloc && cmake . -DCMAKE_BUILD_TYPE=Release && cmake --build . --target mimalloc-static + cd mimalloc && cmake . -DCMAKE_BUILD_TYPE=Release -DMI_OVERRIDE=OFF && cmake --build . --target mimalloc-static .PHONY: mimalloc snmalloc: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) - cd snmalloc && cmake . -DCMAKE_BUILD_TYPE=Release -DSNMALLOC_STATIC_LIBRARY=On && cmake --build . --target snmallocshim-static + cd snmalloc && cmake -E env CXXFLAGS='-D"SNMALLOC_NAME_MANGLE(x)=sn_##x"' cmake . -DCMAKE_BUILD_TYPE=Release -DSNMALLOC_STATIC_LIBRARY=On && cmake --build . --target snmallocshim-static .PHONY: snmalloc snmalloc-1mib: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) - cd snmalloc && cmake . -DCMAKE_BUILD_TYPE=Release -DSNMALLOC_STATIC_LIBRARY=On && cmake --build . --target snmallocshim-1mib-static + cd snmalloc && cmake -E env CXXFLAGS='-D"SNMALLOC_NAME_MANGLE(x)=sn_##x"' cmake . -DCMAKE_BUILD_TYPE=Release -DSNMALLOC_STATIC_LIBRARY=On && cmake --build . --target snmallocshim-1mib-static .PHONY: snmalloc-1mib \ No newline at end of file diff --git a/src/zmalloc.c b/src/zmalloc.c index e81e5cfba..9e0eba7bd 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -75,6 +75,11 @@ void zlibc_free(void *ptr) { #define calloc(count, size) mi_calloc(count, size) #define realloc(ptr, size) mi_realloc(ptr, size) #define free(ptr) mi_free(ptr) +#elif defined(USE_SNMALLOC) +#define malloc(size) sn_malloc(size) +#define calloc(count, size) sn_calloc(count, size) +#define realloc(ptr, size) sn_realloc(ptr, size) +#define free(ptr) sn_free(ptr) #endif #define update_zmalloc_stat_alloc(__n) do { \ diff --git a/src/zmalloc.h b/src/zmalloc.h index 6dd6180a0..253d09f5d 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -44,12 +44,12 @@ #elif defined(USE_SNMALLOC) #define ZMALLOC_LIB ("snmalloc-" SNMALLOC_VERSION) #define HAVE_MALLOC_SIZE 1 -#define zmalloc_size(p) malloc_usable_size(p) -extern size_t malloc_usable_size(void *); -extern void * malloc(size_t); -extern void free(void *); -extern void * realloc(void *, size_t); -extern void * calloc(size_t, size_t); +#define zmalloc_size(p) sn_malloc_usable_size(p) +extern size_t sn_malloc_usable_size(void *); +extern void * sn_malloc(size_t); +extern void sn_free(void *); +extern void * sn_realloc(void *, size_t); +extern void * sn_calloc(size_t, size_t); #elif defined(USE_TCMALLOC) #define ZMALLOC_LIB ("tcmalloc-" __xstr(TC_VERSION_MAJOR) "." __xstr(TC_VERSION_MINOR)) From 96f15a6aa1aabd3906ffa2f62fce2d61aa1e7750 Mon Sep 17 00:00:00 2001 From: SchrodingerZhu Date: Wed, 20 May 2020 08:29:56 +0800 Subject: [PATCH 5/6] remove temp repo --- .gitmodules | 7 ++----- deps/Makefile | 4 ++-- deps/snmalloc | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.gitmodules b/.gitmodules index ef82b5771..353c3a724 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,6 @@ -[submodule "deps/snmalloc"] - path = deps/snmalloc - url = https://github.com/microsoft/snmalloc [submodule "deps/mimalloc"] path = deps/mimalloc url = https://github.com/microsoft/mimalloc -[submodule "temp-snmalloc"] +[submodule "snmalloc"] path = deps/snmalloc - url = https://github.com/schrodingerzhu/snmalloc + url = https://github.com/microsoft/snmalloc diff --git a/deps/Makefile b/deps/Makefile index a8aeb92d2..764e3433b 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -94,12 +94,12 @@ mimalloc: .make-prerequisites snmalloc: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) - cd snmalloc && cmake -E env CXXFLAGS='-D"SNMALLOC_NAME_MANGLE(x)=sn_##x"' cmake . -DCMAKE_BUILD_TYPE=Release -DSNMALLOC_STATIC_LIBRARY=On && cmake --build . --target snmallocshim-static + cd snmalloc && cmake . && cmake --build . --target snmallocshim-static .PHONY: snmalloc snmalloc-1mib: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) - cd snmalloc && cmake -E env CXXFLAGS='-D"SNMALLOC_NAME_MANGLE(x)=sn_##x"' cmake . -DCMAKE_BUILD_TYPE=Release -DSNMALLOC_STATIC_LIBRARY=On && cmake --build . --target snmallocshim-1mib-static + cd snmalloc && cmake . && cmake --build . --target snmallocshim-1mib-static .PHONY: snmalloc-1mib \ No newline at end of file diff --git a/deps/snmalloc b/deps/snmalloc index e7b7e8a73..958de73f5 160000 --- a/deps/snmalloc +++ b/deps/snmalloc @@ -1 +1 @@ -Subproject commit e7b7e8a739e87e6ffc59bba54ff698115279640f +Subproject commit 958de73f5bfb5ab28b798031627af5400e048edc From 08471d5a98f6bc87c5149f7e21f5a31cafccc9e2 Mon Sep 17 00:00:00 2001 From: SchrodingerZhu Date: Mon, 8 Jun 2020 17:05:51 +0800 Subject: [PATCH 6/6] update deps --- deps/mimalloc | 2 +- deps/snmalloc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deps/mimalloc b/deps/mimalloc index e6c7b778f..a09a64e29 160000 --- a/deps/mimalloc +++ b/deps/mimalloc @@ -1 +1 @@ -Subproject commit e6c7b778fb53663985e0a36016a3a087af71d783 +Subproject commit a09a64e29b5535f6f4ea70fc58ac41c91c73a440 diff --git a/deps/snmalloc b/deps/snmalloc index 958de73f5..61afa7789 160000 --- a/deps/snmalloc +++ b/deps/snmalloc @@ -1 +1 @@ -Subproject commit 958de73f5bfb5ab28b798031627af5400e048edc +Subproject commit 61afa77898d233586acf8567fd2dcc24caa611ab