From ac1a0a4c9923583b24c792d4beb2774b5e62d4b6 Mon Sep 17 00:00:00 2001 From: Cloud Wu Date: Fri, 18 Apr 2014 17:38:18 +0800 Subject: [PATCH] malloc hook for jemalloc and c memory info dump --- Makefile | 44 +++++++-- README.md | 2 + lualib-src/lua-memory.c | 51 ++++++++++ service/cmemory.lua | 10 ++ skynet-src/malloc_hook.c | 196 +++++++++++++++++++++++++++++++++++++ skynet-src/malloc_hook.h | 13 +++ skynet-src/skynet.h | 2 + skynet-src/skynet_server.c | 8 ++ 8 files changed, 318 insertions(+), 8 deletions(-) create mode 100644 lualib-src/lua-memory.c create mode 100644 service/cmemory.lua create mode 100644 skynet-src/malloc_hook.c create mode 100644 skynet-src/malloc_hook.h diff --git a/Makefile b/Makefile index 90d9a8cf..61eb01fb 100644 --- a/Makefile +++ b/Makefile @@ -1,32 +1,55 @@ include platform.mk -LUA_STATICLIB := 3rd/lua/liblua.a -LUA_LIB ?= $(LUA_STATICLIB) -LUA_INC ?= 3rd/lua LUA_CLIB_PATH ?= luaclib CSERVICE_PATH ?= cservice + SKYNET_BUILD_PATH ?= . CFLAGS = -g -O2 -Wall -I$(LUA_INC) $(MYCFLAGS) +# lua + +LUA_STATICLIB := 3rd/lua/liblua.a +LUA_LIB ?= $(LUA_STATICLIB) +LUA_INC ?= 3rd/lua + $(LUA_STATICLIB) : cd 3rd/lua && $(MAKE) CC=$(CC) $(PLAT) +# jemalloc + +JEMALLOC_STATICLIB := 3rd/jemalloc/lib/libjemalloc_pic.a +JEMALLOC_INC := 3rd/jemalloc/include/jemalloc +NOBUILDIN_MALLOC := -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free -fno-builtin-memalign + +all : jemalloc +.PHONY : jemalloc + +$(JEMALLOC_STATICLIB) : 3rd/jemalloc/Makefile + cd 3rd/jemalloc && $(MAKE) CC=$(CC) + +3rd/jemalloc/Makefile : + cd 3rd/jemalloc && ./autogen.sh --with-jemalloc-prefix=je_ --disable-valgrind + +jemalloc : $(JEMALLOC_STATICLIB) + +# skynet + CSERVICE = snlua logger gate master harbor -LUA_CLIB = skynet socketdriver int64 bson mongo md5 netpack cjson clientsocket +LUA_CLIB = skynet socketdriver int64 bson mongo md5 netpack cjson clientsocket memory SKYNET_SRC = skynet_main.c skynet_handle.c skynet_module.c skynet_mq.c \ skynet_server.c skynet_start.c skynet_timer.c skynet_error.c \ - skynet_harbor.c skynet_env.c \ - skynet_monitor.c skynet_socket.c socket_server.c + skynet_harbor.c skynet_env.c skynet_monitor.c skynet_socket.c socket_server.c \ + malloc_hook.c all : \ $(SKYNET_BUILD_PATH)/skynet \ $(foreach v, $(CSERVICE), $(CSERVICE_PATH)/$(v).so) \ $(foreach v, $(LUA_CLIB), $(LUA_CLIB_PATH)/$(v).so) -$(SKYNET_BUILD_PATH)/skynet : $(foreach v, $(SKYNET_SRC), skynet-src/$(v)) $(LUA_LIB) - $(CC) $(CFLAGS) -o $@ $^ -Iskynet-src $(LDFLAGS) $(EXPORT) $(LIBS) +$(SKYNET_BUILD_PATH)/skynet : $(foreach v, $(SKYNET_SRC), skynet-src/$(v)) $(LUA_LIB) $(JEMALLOC_STATICLIB) + $(CC) $(CFLAGS) $(NOBUILDIN_MALLOC) -o $@ $^ -Iskynet-src -I$(JEMALLOC_INC) $(LDFLAGS) $(EXPORT) $(LIBS) $(LUA_CLIB_PATH) : mkdir $(LUA_CLIB_PATH) @@ -68,9 +91,14 @@ $(LUA_CLIB_PATH)/cjson.so : | $(LUA_CLIB_PATH) $(LUA_CLIB_PATH)/clientsocket.so : lualib-src/lua-clientsocket.c | $(LUA_CLIB_PATH) $(CC) $(CFLAGS) $(SHARED) $^ -o $@ -lpthread +$(LUA_CLIB_PATH)/memory.so : lualib-src/lua-memory.c | $(LUA_CLIB_PATH) + $(CC) $(CFLAGS) $(SHARED) -Iskynet-src $^ -o $@ + clean : rm -f $(SKYNET_BUILD_PATH)/skynet $(CSERVICE_PATH)/*.so $(LUA_CLIB_PATH)/*.so cleanall: clean cd 3rd/lua-cjson && $(MAKE) clean + cd 3rd/jemalloc && $(MAKE) clean rm -f $(LUA_STATICLIB) + diff --git a/README.md b/README.md index 991666dd..d05b9330 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ ## Build +Install autoconf first for jemalloc + ``` make 'PLATFORM' # PLATFORM can be linux, macosx, freebsd now ``` diff --git a/lualib-src/lua-memory.c b/lualib-src/lua-memory.c new file mode 100644 index 00000000..fa7c026c --- /dev/null +++ b/lualib-src/lua-memory.c @@ -0,0 +1,51 @@ +#include +#include + +#include "malloc_hook.h" + +static int +ltotal(lua_State *L) { + size_t t = malloc_used_memory(); + lua_pushunsigned(L, t); + + return 1; +} + +static int +lblock(lua_State *L) { + size_t t = malloc_memory_block(); + lua_pushunsigned(L, t); + + return 1; +} + +static int +ldumpinfo(lua_State *L) { + memory_info_dump(); + + return 0; +} + +static int +ldump(lua_State *L) { + dump_c_mem(); + + return 0; +} + +int +luaopen_memory(lua_State *L) { + luaL_checkversion(L); + + luaL_Reg l[] = { + { "total", ltotal }, + { "block", lblock }, + { "dumpinfo", ldumpinfo }, + { "dump", ldump }, + { NULL, NULL }, + }; + + luaL_newlib(L,l); + + return 1; +} diff --git a/service/cmemory.lua b/service/cmemory.lua new file mode 100644 index 00000000..e2c1b4b1 --- /dev/null +++ b/service/cmemory.lua @@ -0,0 +1,10 @@ +local skynet = require "skynet" +local memory = require "memory" + +memory.dumpinfo() +memory.dump() + +print("Total memory:", memory.total()) +print("Total block:", memory.block()) + +skynet.exit() diff --git a/skynet-src/malloc_hook.c b/skynet-src/malloc_hook.c new file mode 100644 index 00000000..5ace81d5 --- /dev/null +++ b/skynet-src/malloc_hook.c @@ -0,0 +1,196 @@ +#include +#include +#include +//#include + +#include "malloc_hook.h" + +#include "jemalloc.h" + +#include "skynet.h" + +static size_t _used_memory = 0; +static size_t _memory_block = 0; +typedef struct _mem_data { + uint32_t handle; + size_t allocated; +} mem_data; + +#define SLOT_SIZE 0xffff +#define PREFIX_SIZE sizeof(uint32_t) + +static mem_data mem_stats[SLOT_SIZE]; + +static void _init() { + memset(mem_stats, 0, sizeof(mem_stats)); +} + +static size_t* +get_allocated_field(handle) { + int h = (handle & SLOT_SIZE) - 1; + mem_data *data = &mem_stats[h]; + if(data->handle == 0 || data->allocated == 0) { + __sync_synchronize(); + if(!__sync_bool_compare_and_swap(&data->handle, 0, handle)) { + return 0; + } + } + if(data->handle != handle) { + return 0; + } + return &data->allocated; +} + +inline static void +update_xmalloc_stat_alloc(uint32_t handle, size_t __n) { + __sync_add_and_fetch(&_used_memory, __n); + __sync_add_and_fetch(&_memory_block, 1); + size_t* allocated = get_allocated_field(handle); + if(allocated) { + __sync_add_and_fetch(allocated, __n); + } +} + +inline static void +update_xmalloc_stat_free(uint32_t handle, size_t __n) { + __sync_sub_and_fetch(&_used_memory, __n); + __sync_sub_and_fetch(&_memory_block, 1); + size_t* allocated = get_allocated_field(handle); + if(allocated) { + __sync_sub_and_fetch(allocated, __n); + } +} + +inline static void* +fill_prefix(char* ptr) { + uint32_t handle = skynet_current_handle(); + size_t size = je_malloc_usable_size(ptr); + uint32_t *p = (uint32_t *)(ptr + size - sizeof(uint32_t)); + *p = handle; + + update_xmalloc_stat_alloc(handle, size); + return ptr; +} + +inline static void* +clean_prefix(char* ptr) { + uint32_t* rawptr = (uint32_t*)ptr - 1; + size_t size = je_malloc_usable_size(rawptr); + uint32_t *p = (uint32_t *)(ptr + size - sizeof(uint32_t)); + uint32_t handle = *p; + update_xmalloc_stat_free(handle, size); + return ptr; +} + +static void malloc_oom(size_t size) { + fprintf(stderr, "xmalloc: Out of memory trying to allocate %zu bytes\n", + size); + fflush(stderr); + abort(); +} + +// hook : malloc, realloc, memalign, free, calloc + +void * +malloc(size_t size) { + void* ptr = je_malloc(size + PREFIX_SIZE); + if(!ptr) malloc_oom(size); + return fill_prefix(ptr); +} + +void * +realloc(void *ptr, size_t size) { + if (ptr == NULL) return malloc(size); + + void* rawptr = clean_prefix(ptr); + void *newptr = je_realloc(rawptr, size+PREFIX_SIZE); + if(!newptr) malloc_oom(size); + return fill_prefix(newptr); +} + +void * +memalign(size_t alignment, size_t size) { + void *ptr = je_memalign(alignment, size+PREFIX_SIZE); + if(!ptr) malloc_oom(size); + return fill_prefix(ptr); +} + +void +free(void *ptr) { + if (ptr == NULL) return; + void* rawptr = clean_prefix(ptr); + je_free(rawptr); +} + +void * +calloc(size_t nmemb,size_t size) { + void* ptr = je_calloc(nmemb + ((PREFIX_SIZE+size-1)/size), size ); + if(!ptr) malloc_oom(size); + return fill_prefix(ptr); +} + +size_t +malloc_used_memory(void) { + return _used_memory; +} + +size_t +malloc_memory_block(void) { + return _memory_block; +} + +void memory_info_dump(void) { + je_malloc_stats_print(0,0,0); +} + +size_t mallctl_int64(const char* name, size_t* newval) { + size_t v = 0; + size_t len = sizeof(v); + if(newval) { + je_mallctl(name, &v, &len, newval, sizeof(size_t)); + } else { + je_mallctl(name, &v, &len, NULL, 0); + } + // printf("name: %s, value: %zd\n", name, v); + return v; +} + +int mallctl_opt(const char* name, int* newval) { + int v = 0; + size_t len = sizeof(v); + if(newval) { + int ret = je_mallctl(name, &v, &len, newval, sizeof(int)); + if(ret == 0) { + printf("set new value(%d) for (%s) succeed\n", *newval, name); + } else { + printf("set new value(%d) for (%s) failed: error -> %d\n", *newval, name, ret); + } + } else { + je_mallctl(name, &v, &len, NULL, 0); + } + + return v; +} + +void +dump_c_mem() { + int i; + size_t total = 0; + printf("dump all service mem:\n"); + for(i=0; ihandle != 0 && data->allocated != 0) { + total += data->allocated; + printf("0x%x -> %zdkb\n", data->handle, data->allocated >> 10); + } + } + printf("+total: %zdkb\n",total >> 10); +} + +/* + * init + */ +void __attribute__ ((constructor)) malloc_hook_init(void){ + _init(); +} + diff --git a/skynet-src/malloc_hook.h b/skynet-src/malloc_hook.h new file mode 100644 index 00000000..c26701a1 --- /dev/null +++ b/skynet-src/malloc_hook.h @@ -0,0 +1,13 @@ +#ifndef __MALLOC_HOOK_H +#define __MALLOC_HOOK_H +#include + +extern size_t malloc_used_memory(void); +extern size_t malloc_memory_block(void); +extern void memory_info_dump(void); +extern size_t mallctl_int64(const char* name, size_t* newval); +extern int mallctl_opt(const char* name, int* newval); +extern void dump_c_mem(void); + +#endif /* __MALLOC_HOOK_H */ + diff --git a/skynet-src/skynet.h b/skynet-src/skynet.h index 636c0b87..cab05e62 100644 --- a/skynet-src/skynet.h +++ b/skynet-src/skynet.h @@ -34,4 +34,6 @@ int skynet_isremote(struct skynet_context *, uint32_t handle, int * harbor); typedef int (*skynet_cb)(struct skynet_context * context, void *ud, int type, int session, uint32_t source , const void * msg, size_t sz); void skynet_callback(struct skynet_context * context, void *ud, skynet_cb cb); +uint32_t skynet_current_handle(void); + #endif diff --git a/skynet-src/skynet_server.c b/skynet-src/skynet_server.c index 1b9fa159..9ca53d0f 100644 --- a/skynet-src/skynet_server.c +++ b/skynet-src/skynet_server.c @@ -52,6 +52,7 @@ struct skynet_node { }; static struct skynet_node G_NODE = { 0,0 }; +static __thread uint32_t handle_tls = 0xffffffff; int skynet_context_total() { @@ -68,6 +69,11 @@ _context_dec() { __sync_fetch_and_sub(&G_NODE.total,1); } +uint32_t +skynet_current_handle(void) { + return handle_tls; +} + static void _id_to_hex(char * str, uint32_t id) { int i; @@ -192,11 +198,13 @@ static void _dispatch_message(struct skynet_context *ctx, struct skynet_message *msg) { assert(ctx->init); CHECKCALLING_BEGIN(ctx) + handle_tls = ctx->handle; int type = msg->sz >> HANDLE_REMOTE_SHIFT; size_t sz = msg->sz & HANDLE_MASK; if (!ctx->cb(ctx, ctx->cb_ud, type, msg->session, msg->source, msg->data, sz)) { free(msg->data); } + handle_tls = 0xffffffff; CHECKCALLING_END(ctx) }