malloc hook for jemalloc and c memory info dump

Makefile

@@ -1,32 +1,55 @@
 include platform.mk
 
-LUA_STATICLIB := 3rd/lua/liblua.a
-LUA_LIB ?= $(LUA_STATICLIB)
-LUA_INC ?= 3rd/lua
 LUA_CLIB_PATH ?= luaclib
 CSERVICE_PATH ?= cservice
+
 SKYNET_BUILD_PATH ?= .
 
 CFLAGS = -g -O2 -Wall -I$(LUA_INC) $(MYCFLAGS)
 
+# lua
+
+LUA_STATICLIB := 3rd/lua/liblua.a
+LUA_LIB ?= $(LUA_STATICLIB)
+LUA_INC ?= 3rd/lua
+
 $(LUA_STATICLIB) :
 	cd 3rd/lua && $(MAKE) CC=$(CC) $(PLAT)
 
+# jemalloc 
+
+JEMALLOC_STATICLIB := 3rd/jemalloc/lib/libjemalloc_pic.a
+JEMALLOC_INC := 3rd/jemalloc/include/jemalloc
+NOBUILDIN_MALLOC := -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free -fno-builtin-memalign
+
+all : jemalloc
+.PHONY : jemalloc
+
+$(JEMALLOC_STATICLIB) : 3rd/jemalloc/Makefile
+	cd 3rd/jemalloc && $(MAKE) CC=$(CC) 
+
+3rd/jemalloc/Makefile :
+	cd 3rd/jemalloc && ./autogen.sh --with-jemalloc-prefix=je_ --disable-valgrind
+
+jemalloc : $(JEMALLOC_STATICLIB)
+
+# skynet
+
 CSERVICE = snlua logger gate master harbor
-LUA_CLIB = skynet socketdriver int64 bson mongo md5 netpack cjson clientsocket
+LUA_CLIB = skynet socketdriver int64 bson mongo md5 netpack cjson clientsocket memory
 
 SKYNET_SRC = skynet_main.c skynet_handle.c skynet_module.c skynet_mq.c \
   skynet_server.c skynet_start.c skynet_timer.c skynet_error.c \
-  skynet_harbor.c skynet_env.c \
-  skynet_monitor.c skynet_socket.c socket_server.c
+  skynet_harbor.c skynet_env.c skynet_monitor.c skynet_socket.c socket_server.c \
+  malloc_hook.c
 
 all : \
   $(SKYNET_BUILD_PATH)/skynet \
   $(foreach v, $(CSERVICE), $(CSERVICE_PATH)/$(v).so) \
   $(foreach v, $(LUA_CLIB), $(LUA_CLIB_PATH)/$(v).so) 
 
-$(SKYNET_BUILD_PATH)/skynet : $(foreach v, $(SKYNET_SRC), skynet-src/$(v)) $(LUA_LIB)
-	$(CC) $(CFLAGS) -o $@ $^ -Iskynet-src $(LDFLAGS) $(EXPORT) $(LIBS)
+$(SKYNET_BUILD_PATH)/skynet : $(foreach v, $(SKYNET_SRC), skynet-src/$(v)) $(LUA_LIB) $(JEMALLOC_STATICLIB)
+	$(CC) $(CFLAGS) $(NOBUILDIN_MALLOC) -o $@ $^ -Iskynet-src -I$(JEMALLOC_INC) $(LDFLAGS) $(EXPORT) $(LIBS)
 
 $(LUA_CLIB_PATH) :
 	mkdir $(LUA_CLIB_PATH)
@@ -68,9 +91,14 @@ $(LUA_CLIB_PATH)/cjson.so : | $(LUA_CLIB_PATH)
 $(LUA_CLIB_PATH)/clientsocket.so : lualib-src/lua-clientsocket.c | $(LUA_CLIB_PATH)
 	$(CC) $(CFLAGS) $(SHARED) $^ -o $@ -lpthread
 
+$(LUA_CLIB_PATH)/memory.so : lualib-src/lua-memory.c | $(LUA_CLIB_PATH)
+	$(CC) $(CFLAGS) $(SHARED) -Iskynet-src $^ -o $@ 
+
 clean :
 	rm -f $(SKYNET_BUILD_PATH)/skynet $(CSERVICE_PATH)/*.so $(LUA_CLIB_PATH)/*.so
 
 cleanall: clean
 	cd 3rd/lua-cjson && $(MAKE) clean
+	cd 3rd/jemalloc && $(MAKE) clean
 	rm -f $(LUA_STATICLIB)
+

README.md

@@ -1,5 +1,7 @@
 ## Build
 
+Install autoconf first for jemalloc
+
 ```
 make 'PLATFORM'  # PLATFORM can be linux, macosx, freebsd now
 ```

lualib-src/lua-memory.c

@@ -0,0 +1,51 @@
+#include <lua.h>
+#include <lauxlib.h>
+
+#include "malloc_hook.h"
+
+static int
+ltotal(lua_State *L) {
+	size_t t = malloc_used_memory();
+	lua_pushunsigned(L, t);
+
+	return 1;
+}
+
+static int
+lblock(lua_State *L) {
+	size_t t = malloc_memory_block();
+	lua_pushunsigned(L, t);
+
+	return 1;
+}
+
+static int
+ldumpinfo(lua_State *L) {
+	memory_info_dump();
+
+	return 0;
+}
+
+static int
+ldump(lua_State *L) {
+	dump_c_mem();
+
+	return 0;
+}
+
+int
+luaopen_memory(lua_State *L) {
+	luaL_checkversion(L);
+
+	luaL_Reg l[] = {
+		{ "total", ltotal },
+		{ "block", lblock },
+		{ "dumpinfo", ldumpinfo },
+		{ "dump", ldump },
+		{ NULL, NULL },
+	};
+
+	luaL_newlib(L,l);
+
+	return 1;
+}

service/cmemory.lua

@@ -0,0 +1,10 @@
+local skynet = require "skynet"
+local memory = require "memory"
+
+memory.dumpinfo()
+memory.dump()
+
+print("Total memory:", memory.total())
+print("Total block:", memory.block())
+
+skynet.exit()

skynet-src/malloc_hook.c

@@ -0,0 +1,196 @@
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+//#include <malloc.h>
+
+#include "malloc_hook.h"
+
+#include "jemalloc.h"
+
+#include "skynet.h"
+
+static size_t _used_memory = 0;
+static size_t _memory_block = 0;
+typedef struct _mem_data {
+    uint32_t handle;
+    size_t   allocated;
+} mem_data;
+
+#define SLOT_SIZE 0xffff
+#define PREFIX_SIZE sizeof(uint32_t)
+
+static mem_data mem_stats[SLOT_SIZE];
+
+static void _init() {
+    memset(mem_stats, 0, sizeof(mem_stats));
+}
+
+static size_t*
+get_allocated_field(handle) {
+    int h = (handle & SLOT_SIZE) - 1;
+    mem_data *data = &mem_stats[h];
+    if(data->handle == 0 || data->allocated == 0) {
+        __sync_synchronize();
+        if(!__sync_bool_compare_and_swap(&data->handle, 0, handle)) {
+            return 0;
+        }
+    }
+    if(data->handle != handle) {
+        return 0;
+    }
+    return &data->allocated;
+}
+
+inline static void 
+update_xmalloc_stat_alloc(uint32_t handle, size_t __n) {
+    __sync_add_and_fetch(&_used_memory, __n);
+    __sync_add_and_fetch(&_memory_block, 1); 
+    size_t* allocated = get_allocated_field(handle);
+    if(allocated) {
+        __sync_add_and_fetch(allocated, __n);
+    }
+}
+
+inline static void
+update_xmalloc_stat_free(uint32_t handle, size_t __n) {
+    __sync_sub_and_fetch(&_used_memory, __n);
+    __sync_sub_and_fetch(&_memory_block, 1);
+    size_t* allocated = get_allocated_field(handle);
+    if(allocated) {
+        __sync_sub_and_fetch(allocated, __n);
+    }
+}
+
+inline static void*
+fill_prefix(char* ptr) {
+    uint32_t handle = skynet_current_handle();
+    size_t size = je_malloc_usable_size(ptr);
+    uint32_t *p = (uint32_t *)(ptr + size - sizeof(uint32_t));
+    *p = handle;
+
+	update_xmalloc_stat_alloc(handle, size);
+    return ptr;
+}
+
+inline static void*
+clean_prefix(char* ptr) {
+    uint32_t* rawptr = (uint32_t*)ptr - 1;
+    size_t size = je_malloc_usable_size(rawptr);
+    uint32_t *p = (uint32_t *)(ptr + size - sizeof(uint32_t));
+    uint32_t handle = *p;
+    update_xmalloc_stat_free(handle, size);
+    return ptr;
+}
+
+static void malloc_oom(size_t size) {
+    fprintf(stderr, "xmalloc: Out of memory trying to allocate %zu bytes\n",
+        size);
+    fflush(stderr);
+    abort();
+}
+
+// hook : malloc, realloc, memalign, free, calloc
+
+void *
+malloc(size_t size) {
+    void* ptr = je_malloc(size + PREFIX_SIZE);
+    if(!ptr) malloc_oom(size);
+    return fill_prefix(ptr);
+}
+
+void *
+realloc(void *ptr, size_t size) {
+    if (ptr == NULL) return malloc(size);
+
+    void* rawptr = clean_prefix(ptr);
+    void *newptr = je_realloc(rawptr, size+PREFIX_SIZE);
+    if(!newptr) malloc_oom(size);
+    return fill_prefix(newptr);
+}
+
+void *
+memalign(size_t alignment, size_t size) {
+    void *ptr = je_memalign(alignment, size+PREFIX_SIZE);
+    if(!ptr) malloc_oom(size);
+    return fill_prefix(ptr);
+}
+
+void
+free(void *ptr) {
+    if (ptr == NULL) return;
+    void* rawptr = clean_prefix(ptr);
+    je_free(rawptr);
+}
+
+void *
+calloc(size_t nmemb,size_t size) {
+    void* ptr = je_calloc(nmemb + ((PREFIX_SIZE+size-1)/size), size );
+    if(!ptr) malloc_oom(size);
+    return fill_prefix(ptr);
+}
+
+size_t
+malloc_used_memory(void) {
+    return _used_memory;
+}
+
+size_t
+malloc_memory_block(void) {
+    return _memory_block;
+}
+
+void memory_info_dump(void) {
+    je_malloc_stats_print(0,0,0);
+}
+
+size_t mallctl_int64(const char* name, size_t* newval) {
+    size_t v = 0;
+    size_t len = sizeof(v);
+    if(newval) {
+        je_mallctl(name, &v, &len, newval, sizeof(size_t));
+    } else {
+        je_mallctl(name, &v, &len, NULL, 0);
+    }
+    // printf("name: %s, value: %zd\n", name, v);
+    return v;
+}
+
+int mallctl_opt(const char* name, int* newval) {
+    int    v   = 0;
+    size_t len = sizeof(v);
+    if(newval) {
+        int ret = je_mallctl(name, &v, &len, newval, sizeof(int));
+        if(ret == 0) {
+            printf("set new value(%d) for (%s) succeed\n", *newval, name);
+        } else {
+            printf("set new value(%d) for (%s) failed: error -> %d\n", *newval, name, ret);
+        }
+    } else {
+        je_mallctl(name, &v, &len, NULL, 0);
+    }
+
+    return v;
+}
+
+void
+dump_c_mem() {
+    int i;
+    size_t total = 0;
+    printf("dump all service mem:\n");
+    for(i=0; i<SLOT_SIZE; i++) {
+        mem_data* data = &mem_stats[i];
+        if(data->handle != 0 && data->allocated != 0) {
+            total += data->allocated;
+            printf("0x%x -> %zdkb\n", data->handle, data->allocated >> 10);
+        }
+    }
+    printf("+total: %zdkb\n",total >> 10);
+}
+
+/*
+ * init 
+ */
+void __attribute__ ((constructor)) malloc_hook_init(void){
+    _init();
+}
+

skynet-src/malloc_hook.h

@@ -0,0 +1,13 @@
+#ifndef __MALLOC_HOOK_H
+#define __MALLOC_HOOK_H
+#include <stdlib.h>
+
+extern size_t malloc_used_memory(void);
+extern size_t malloc_memory_block(void);
+extern void   memory_info_dump(void);
+extern size_t mallctl_int64(const char* name, size_t* newval);
+extern int    mallctl_opt(const char* name, int* newval);
+extern void   dump_c_mem(void);
+
+#endif /* __MALLOC_HOOK_H */
+

skynet-src/skynet.h

@@ -34,4 +34,6 @@ int skynet_isremote(struct skynet_context *, uint32_t handle, int * harbor);
 typedef int (*skynet_cb)(struct skynet_context * context, void *ud, int type, int session, uint32_t source , const void * msg, size_t sz);
 void skynet_callback(struct skynet_context * context, void *ud, skynet_cb cb);
 
+uint32_t skynet_current_handle(void);
+
 #endif

skynet-src/skynet_server.c

@@ -52,6 +52,7 @@ struct skynet_node {
 };
 
 static struct skynet_node G_NODE = { 0,0 };
+static __thread uint32_t handle_tls = 0xffffffff;
 
 int 
 skynet_context_total() {
@@ -68,6 +69,11 @@ _context_dec() {
 	__sync_fetch_and_sub(&G_NODE.total,1);
 }
 
+uint32_t 
+skynet_current_handle(void) {
+	return handle_tls;
+}
+
 static void
 _id_to_hex(char * str, uint32_t id) {
 	int i;
@@ -192,11 +198,13 @@ static void
 _dispatch_message(struct skynet_context *ctx, struct skynet_message *msg) {
 	assert(ctx->init);
 	CHECKCALLING_BEGIN(ctx)
+	handle_tls = ctx->handle;
 	int type = msg->sz >> HANDLE_REMOTE_SHIFT;
 	size_t sz = msg->sz & HANDLE_MASK;
 	if (!ctx->cb(ctx, ctx->cb_ud, type, msg->session, msg->source, msg->data, sz)) {
 		free(msg->data);
 	}
+	handle_tls = 0xffffffff;
 	CHECKCALLING_END(ctx)
 }