mem: 优化 C 内存统计, 缓解多线程竞争导致的 skynet_malloc 性能下降 (#2083)

* mem: 优化 C 内存统计, 缓解多线程竞争导致的 skynet_malloc 性能下降

* rm unused SPMC_STEP_LG macro

---------

Co-authored-by: efve.zff <efve.zff@alibaba-inc.com>

Makefile

@@ -29,13 +29,13 @@ JEMALLOC_STATICLIB := 3rd/jemalloc/lib/libjemalloc_pic.a
 JEMALLOC_INC := 3rd/jemalloc/include/jemalloc
 
 all : jemalloc
-	
+
 .PHONY : jemalloc update3rd
 
 MALLOC_STATICLIB := $(JEMALLOC_STATICLIB)
 
 $(JEMALLOC_STATICLIB) : 3rd/jemalloc/Makefile
-	cd 3rd/jemalloc && $(MAKE) CC=$(CC) 
+	cd 3rd/jemalloc && $(MAKE) CC=$(CC)
 
 3rd/jemalloc/autogen.sh :
 	git submodule update --init
@@ -74,12 +74,12 @@ LUA_CLIB_SKYNET = \
 SKYNET_SRC = skynet_main.c skynet_handle.c skynet_module.c skynet_mq.c \
   skynet_server.c skynet_start.c skynet_timer.c skynet_error.c \
   skynet_harbor.c skynet_env.c skynet_monitor.c skynet_socket.c socket_server.c \
-  malloc_hook.c skynet_daemon.c skynet_log.c
+  mem_info.c malloc_hook.c skynet_daemon.c skynet_log.c
 
 all : \
   $(SKYNET_BUILD_PATH)/skynet \
   $(foreach v, $(CSERVICE), $(CSERVICE_PATH)/$(v).so) \
-  $(foreach v, $(LUA_CLIB), $(LUA_CLIB_PATH)/$(v).so) 
+  $(foreach v, $(LUA_CLIB), $(LUA_CLIB_PATH)/$(v).so)
 
 $(SKYNET_BUILD_PATH)/skynet : $(foreach v, $(SKYNET_SRC), skynet-src/$(v)) $(LUA_LIB) $(MALLOC_STATICLIB)
 	$(CC) $(CFLAGS) -o $@ $^ -Iskynet-src -I$(JEMALLOC_INC) $(LDFLAGS) $(EXPORT) $(SKYNET_LIBS) $(SKYNET_DEFINES)
@@ -104,19 +104,19 @@ $(LUA_CLIB_PATH)/bson.so : lualib-src/lua-bson.c | $(LUA_CLIB_PATH)
 	$(CC) $(CFLAGS) $(SHARED) -Iskynet-src $^ -o $@
 
 $(LUA_CLIB_PATH)/md5.so : 3rd/lua-md5/md5.c 3rd/lua-md5/md5lib.c 3rd/lua-md5/compat-5.2.c | $(LUA_CLIB_PATH)
-	$(CC) $(CFLAGS) $(SHARED) -I3rd/lua-md5 $^ -o $@ 
+	$(CC) $(CFLAGS) $(SHARED) -I3rd/lua-md5 $^ -o $@
 
 $(LUA_CLIB_PATH)/client.so : lualib-src/lua-clientsocket.c lualib-src/lua-crypt.c lualib-src/lsha1.c | $(LUA_CLIB_PATH)
 	$(CC) $(CFLAGS) $(SHARED) $^ -o $@ -lpthread
 
 $(LUA_CLIB_PATH)/sproto.so : lualib-src/sproto/sproto.c lualib-src/sproto/lsproto.c | $(LUA_CLIB_PATH)
-	$(CC) $(CFLAGS) $(SHARED) -Ilualib-src/sproto $^ -o $@ 
+	$(CC) $(CFLAGS) $(SHARED) -Ilualib-src/sproto $^ -o $@
 
 $(LUA_CLIB_PATH)/ltls.so : lualib-src/ltls.c | $(LUA_CLIB_PATH)
 	$(CC) $(CFLAGS) $(SHARED) -Iskynet-src -L$(TLS_LIB) -I$(TLS_INC) $^ -o $@ -lssl
 
 $(LUA_CLIB_PATH)/lpeg.so : 3rd/lpeg/lpcap.c 3rd/lpeg/lpcode.c 3rd/lpeg/lpprint.c 3rd/lpeg/lptree.c 3rd/lpeg/lpvm.c 3rd/lpeg/lpcset.c | $(LUA_CLIB_PATH)
-	$(CC) $(CFLAGS) $(SHARED) -I3rd/lpeg $^ -o $@ 
+	$(CC) $(CFLAGS) $(SHARED) -I3rd/lpeg $^ -o $@
 
 clean :
 	rm -f $(SKYNET_BUILD_PATH)/skynet $(CSERVICE_PATH)/*.so $(LUA_CLIB_PATH)/*.so && \
@@ -128,4 +128,3 @@ ifneq (,$(wildcard 3rd/jemalloc/Makefile))
 endif
 	cd 3rd/lua && $(MAKE) clean
 	rm -f $(LUA_STATICLIB)
-

skynet-src/malloc_hook.c

@@ -1,27 +1,27 @@
-#include <stdio.h>
 #include <string.h>
 #include <assert.h>
 #include <stdlib.h>
-#include <lua.h>
 #include <stdio.h>
 
-#include "malloc_hook.h"
+#include <lauxlib.h>
+
 #include "skynet.h"
 #include "atomic.h"
 
+#include "malloc_hook.h"
+
 // turn on MEMORY_CHECK can do more memory check, such as double free
 // #define MEMORY_CHECK
 
 #define MEMORY_ALLOCTAG 0x20140605
 #define MEMORY_FREETAG 0x0badf00d
 
-static ATOM_SIZET _used_memory = 0;
-static ATOM_SIZET _memory_block = 0;
-
 struct mem_data {
-	ATOM_ULONG handle;
+    alignas(CACHE_LINE_SIZE)
-	ATOM_SIZET allocated;
+	ATOM_ULONG     handle;
+    AtomicMemInfo  info;
 };
+_Static_assert(sizeof(struct mem_data) % CACHE_LINE_SIZE == 0, "mem_data must be cache-line aligned");
 
 struct mem_cookie {
 	size_t size;
@@ -36,7 +36,14 @@ struct mem_cookie {
 #define PREFIX_SIZE sizeof(struct mem_cookie)
 
 static struct mem_data mem_stats[SLOT_SIZE];
+_Static_assert(alignof(mem_stats) % CACHE_LINE_SIZE == 0, "mem_stats must be cache-line aligned");
 
+static struct mem_data *
+get_mem_stat(uint32_t handle) {
+	int h = (int)(handle & (SLOT_SIZE - 1));
+	struct mem_data *data = &mem_stats[h];
+	return data;
+}
 
 #ifndef NOUSE_JEMALLOC
 
@@ -46,45 +53,19 @@ static struct mem_data mem_stats[SLOT_SIZE];
 #define raw_realloc je_realloc
 #define raw_free je_free
 
-static ATOM_SIZET *
-get_allocated_field(uint32_t handle) {
-	int h = (int)(handle & (SLOT_SIZE - 1));
-	struct mem_data *data = &mem_stats[h];
-	uint32_t old_handle = data->handle;
-	ssize_t old_alloc = (ssize_t)data->allocated;
-	if(old_handle == 0 || old_alloc <= 0) {
-		// data->allocated may less than zero, because it may not count at start.
-		if(!ATOM_CAS_ULONG(&data->handle, old_handle, handle)) {
-			return 0;
-		}
-		if (old_alloc < 0) {
-			ATOM_CAS_SIZET(&data->allocated, (size_t)old_alloc, 0);
-		}
-	}
-	if(data->handle != handle) {
-		return 0;
-	}
-	return &data->allocated;
-}
-
 inline static void
 update_xmalloc_stat_alloc(uint32_t handle, size_t __n) {
-	ATOM_FADD(&_used_memory, __n);
+	struct mem_data *data = get_mem_stat(handle);
-	ATOM_FINC(&_memory_block);
+    // 当两个不同的 handle 被哈希到同一个槽位时, 新的服务会覆盖旧服务的数据
-	ATOM_SIZET * allocated = get_allocated_field(handle);
+    // 这种情况在实际运行中非常罕见, 因为同时存在的服务数量很难超过 65536
-	if(allocated) {
+    ATOM_STORE(&data->handle, handle);
-		ATOM_FADD(allocated, __n);
+	atomic_meminfo_alloc(&data->info, __n);
-	}
 }
 
 inline static void
 update_xmalloc_stat_free(uint32_t handle, size_t __n) {
-	ATOM_FSUB(&_used_memory, __n);
+	struct mem_data *data = get_mem_stat(handle);
-	ATOM_FDEC(&_memory_block);
+	atomic_meminfo_free(&data->info, __n);
-	ATOM_SIZET * allocated = get_allocated_field(handle);
-	if(allocated) {
-		ATOM_FSUB(allocated, __n);
-	}
 }
 
 inline static void*
@@ -92,6 +73,7 @@ fill_prefix(char* ptr, size_t sz, uint32_t cookie_size) {
 	uint32_t handle = skynet_current_handle();
 	struct mem_cookie *p = (struct mem_cookie *)ptr;
 	char * ret = ptr + cookie_size;
+	sz += cookie_size;
 	p->size = sz;
 	p->handle = handle;
 #ifdef MEMORY_CHECK
@@ -300,27 +282,55 @@ mallctl_cmd(const char* name) {
 
 size_t
 malloc_used_memory(void) {
-	return ATOM_LOAD(&_used_memory);
+	MemInfo total = {};
+	for(int i = 0; i < SLOT_SIZE; i++) {
+		struct mem_data* data = &mem_stats[i];
+		const uint32_t handle = ATOM_LOAD(&data->handle);
+		if (handle != 0) {
+			atomic_meminfo_merge(&total, &data->info);
+		}
+	}
+	return total.alloc - total.free;
 }
 
 size_t
 malloc_memory_block(void) {
-	return ATOM_LOAD(&_memory_block);
+	MemInfo total = {};
+	for(int i = 0; i < SLOT_SIZE; i++) {
+		struct mem_data* data = &mem_stats[i];
+		const uint32_t handle = ATOM_LOAD(&data->handle);
+		if (handle != 0) {
+			atomic_meminfo_merge(&total, &data->info);
+		}
+	}
+	return total.alloc_count - total.free_count;
 }
 
 void
 dump_c_mem() {
-	int i;
-	size_t total = 0;
 	skynet_error(NULL, "dump all service mem:");
-	for(i=0; i<SLOT_SIZE; i++) {
+	MemInfo total = {};
+	for(int i = 0; i < SLOT_SIZE; i++) {
 		struct mem_data* data = &mem_stats[i];
-		if(data->handle != 0 && data->allocated != 0) {
+		const uint32_t handle = ATOM_LOAD(&data->handle);
-			total += data->allocated;
+		if (handle != 0) {
-			skynet_error(NULL, ":%08x -> %zdkb %db", data->handle, data->allocated >> 10, (int)(data->allocated % 1024));
+			MemInfo info = {};
+			atomic_meminfo_merge(&info, &data->info);
+			meminfo_merge(&total, &info);
+			const size_t using = info.alloc - info.free;
+			skynet_error(NULL, ":%08x -> %zukb %zub", handle, using >> 10, using);
 		}
 	}
-	skynet_error(NULL, "+total: %zdkb",total >> 10);
+	const size_t using = total.alloc - total.free;
+	skynet_error(NULL, "+total: %zukb", using >> 10);
+}
+
+char *
+skynet_strdup(const char *str) {
+	size_t sz = strlen(str);
+	char * ret = skynet_malloc(sz+1);
+	memcpy(ret, str, sz+1);
+	return ret;
 }
 
 void *
@@ -339,9 +349,12 @@ dump_mem_lua(lua_State *L) {
 	lua_newtable(L);
 	for(i=0; i<SLOT_SIZE; i++) {
 		struct mem_data* data = &mem_stats[i];
-		if(data->handle != 0 && data->allocated != 0) {
+		const uint32_t handle = ATOM_LOAD(&data->handle);
-			lua_pushinteger(L, data->allocated);
+		if (handle != 0) {
-			lua_rawseti(L, -2, (lua_Integer)data->handle);
+			MemInfo info = {};
+			atomic_meminfo_merge(&info, &data->info);
+			lua_pushinteger(L, info.alloc - info.free);
+			lua_rawseti(L, -2, handle);
 		}
 	}
 	return 1;
@@ -350,14 +363,13 @@ dump_mem_lua(lua_State *L) {
 size_t
 malloc_current_memory(void) {
 	uint32_t handle = skynet_current_handle();
-	int i;
+	struct mem_data *data = get_mem_stat(handle);
-	for(i=0; i<SLOT_SIZE; i++) {
+	if (ATOM_LOAD(&data->handle) != handle) {
-		struct mem_data* data = &mem_stats[i];
+		return 0;
-		if(data->handle == (uint32_t)handle && data->allocated != 0) {
-			return (size_t) data->allocated;
-		}
 	}
-	return 0;
+	MemInfo info = {};
+	atomic_meminfo_merge(&info, &data->info);
+	return info.alloc - info.free;
 }
 
 void

skynet-src/malloc_hook.h

@@ -5,6 +5,8 @@
 #include <stdbool.h>
 #include <lua.h>
 
+#include "mem_info.h"
+
 extern size_t malloc_used_memory(void);
 extern size_t malloc_memory_block(void);
 extern void   memory_info_dump(const char *opts);
@@ -17,4 +19,3 @@ extern int    dump_mem_lua(lua_State *L);
 extern size_t malloc_current_memory(void);
 
 #endif /* SKYNET_MALLOC_HOOK_H */
-

skynet-src/mem_info.c

@@ -0,0 +1,59 @@
+#include <string.h>
+
+#include "mem_info.h"
+
+void
+meminfo_init(MemInfo *info) {
+    memset(info, 0, sizeof(*info));
+}
+
+void
+atomic_meminfo_init(AtomicMemInfo *info) {
+    ATOM_INIT(&info->alloc, 0);
+    ATOM_INIT(&info->alloc_count, 0);
+    ATOM_INIT(&info->free, 0);
+    ATOM_INIT(&info->free_count, 0);
+}
+
+void
+meminfo_alloc(MemInfo *info, size_t size) {
+    info->alloc += size;
+    ++info->alloc_count;
+}
+
+void
+atomic_meminfo_alloc(AtomicMemInfo *info, size_t size) {
+    ATOM_FADD(&info->alloc, size);
+    ATOM_FADD(&info->alloc_count, 1);
+}
+
+void
+meminfo_free(MemInfo *info, size_t size) {
+    info->free += size;
+    ++info->free_count;
+}
+
+void
+atomic_meminfo_free(AtomicMemInfo *info, size_t size) {
+    ATOM_FADD(&info->free, size);
+    ATOM_FADD(&info->free_count, 1);
+}
+
+void
+meminfo_merge(MemInfo *dest, const MemInfo *src) {
+    dest->alloc += src->alloc;
+    dest->alloc_count += src->alloc_count;
+    dest->free += src->free;
+    dest->free_count += src->free_count;
+}
+
+void
+atomic_meminfo_merge(MemInfo *dest, const AtomicMemInfo *src) {
+    MemInfo info;
+    // 先加载 free 后加载 alloc 避免大小错乱
+    info.free_count = ATOM_LOAD(&src->free_count);
+    info.free = ATOM_LOAD(&src->free);
+    info.alloc_count = ATOM_LOAD(&src->alloc_count);
+    info.alloc = ATOM_LOAD(&src->alloc);
+    meminfo_merge(dest, &info);
+}

skynet-src/mem_info.h

@@ -0,0 +1,42 @@
+#ifndef _MEM_INFO_H_
+#define _MEM_INFO_H_
+
+#include <stdalign.h>
+#include <stddef.h>
+
+#include "atomic.h"
+
+#define CACHE_LINE_SIZE 64
+
+typedef struct {
+    size_t alloc;
+    size_t alloc_count;
+    size_t free;
+    size_t free_count;
+} MemInfo;
+
+typedef struct {
+    // alloc 与 free 可能不在一个线程
+    // 为了避免竞争，这里也拆成两个 CacheLine 大小
+    alignas(CACHE_LINE_SIZE)
+    ATOM_SIZET alloc;
+    ATOM_SIZET alloc_count;
+
+    alignas(CACHE_LINE_SIZE)
+    ATOM_SIZET free;
+    ATOM_SIZET free_count;
+} AtomicMemInfo;
+
+void meminfo_init(MemInfo *info);
+void atomic_meminfo_init(AtomicMemInfo *info);
+
+void meminfo_alloc(MemInfo *info, size_t size);
+void atomic_meminfo_alloc(AtomicMemInfo *info, size_t size);
+
+void meminfo_free(MemInfo *info, size_t size);
+void atomic_meminfo_free(AtomicMemInfo *info, size_t size);
+
+void meminfo_merge(MemInfo *dest, const MemInfo *src);
+void atomic_meminfo_merge(MemInfo *dest, const AtomicMemInfo *src);
+
+#endif // _MEM_INFO_H_