From c0862d8445bf6e20f104b0e171e8e03ca98c58e5 Mon Sep 17 00:00:00 2001 From: Cloud Wu Date: Fri, 21 Aug 2015 11:41:01 +0800 Subject: [PATCH] add global share string table --- 3rd/lua/Makefile | 2 +- 3rd/lua/lapi.c | 4 +- 3rd/lua/lgc.c | 3 +- 3rd/lua/lstring.c | 224 +++++++++++++++++++++++++++++++++++++- 3rd/lua/lstring.h | 5 + 3rd/lua/luaconf.h | 1 + lualib-src/lua-memory.c | 10 ++ service/bootstrap.lua | 4 + service/debug_console.lua | 19 ++++ skynet-src/luashrtbl.h | 17 +++ skynet-src/skynet_main.c | 4 + 11 files changed, 287 insertions(+), 6 deletions(-) create mode 100644 skynet-src/luashrtbl.h diff --git a/3rd/lua/Makefile b/3rd/lua/Makefile index 6c680538..ddf7b349 100644 --- a/3rd/lua/Makefile +++ b/3rd/lua/Makefile @@ -19,7 +19,7 @@ SYSCFLAGS= SYSLDFLAGS= SYSLIBS= -MYCFLAGS= +MYCFLAGS=-I../../skynet-src MYLDFLAGS= MYLIBS= MYOBJS= diff --git a/3rd/lua/lapi.c b/3rd/lua/lapi.c index 953a304b..3f47f7ee 100644 --- a/3rd/lua/lapi.c +++ b/3rd/lua/lapi.c @@ -1000,7 +1000,7 @@ static Proto * cloneproto (lua_State *L, const Proto *src) { const TValue *s=&src->k[i]; TValue *o=&f->k[i]; if (ttisstring(s)) { - TString * str = luaS_newlstr(L,svalue(s),vslen(s)); + TString * str = luaS_clonestring(L,tsvalue(s)); setsvalue2n(L,o,str); } else { setobj(L,o,s); @@ -1288,7 +1288,7 @@ static UpVal **getupvalref (lua_State *L, int fidx, int n, LClosure **pf) { StkId fi = index2addr(L, fidx); api_check(L, ttisLclosure(fi), "Lua function expected"); f = clLvalue(fi); - api_check(L, (1 <= n && n <= f->p->sizeupvalues), "invalid upvalue index"); + api_check(L, (1 <= n && n <= f->p->sp->sizeupvalues), "invalid upvalue index"); if (pf) *pf = f; return &f->upvals[n - 1]; /* get its upvalue pointer */ } diff --git a/3rd/lua/lgc.c b/3rd/lua/lgc.c index e73c58bf..a1ce0245 100644 --- a/3rd/lua/lgc.c +++ b/3rd/lua/lgc.c @@ -188,7 +188,8 @@ void luaC_upvalbarrier_ (lua_State *L, UpVal *uv) { void luaC_fix (lua_State *L, GCObject *o) { global_State *g = G(L); - lua_assert(g->allgc == o); /* object must be 1st in 'allgc' list! */ + if (g->allgc != o) + return; /* if object is not 1st in 'allgc' list, it is in global short string table */ white2gray(o); /* they will be gray forever */ g->allgc = o->next; /* remove object from 'allgc' list */ o->next = g->fixedgc; /* link it to 'fixedgc' list */ diff --git a/3rd/lua/lstring.c b/3rd/lua/lstring.c index 5e0e3c40..dd36eb87 100644 --- a/3rd/lua/lstring.c +++ b/3rd/lua/lstring.c @@ -148,10 +148,9 @@ void luaS_remove (lua_State *L, TString *ts) { /* ** checks whether short string exists and reuses it or creates a new one */ -static TString *internshrstr (lua_State *L, const char *str, size_t l) { +static TString *queryshrstr (lua_State *L, const char *str, size_t l, unsigned int h) { TString *ts; global_State *g = G(L); - unsigned int h = luaS_hash(str, l, g->seed); TString **list = &g->strt.hash[lmod(h, g->strt.size)]; for (ts = *list; ts != NULL; ts = ts->u.hnext) { if (l == ts->shrlen && @@ -162,6 +161,13 @@ static TString *internshrstr (lua_State *L, const char *str, size_t l) { return ts; } } + return NULL; +} + +static TString *addshrstr (lua_State *L, const char *str, size_t l, unsigned int h) { + TString *ts; + global_State *g = G(L); + TString **list = &g->strt.hash[lmod(h, g->strt.size)]; if (g->strt.nuse >= g->strt.size && g->strt.size <= MAX_INT/2) { luaS_resize(L, g->strt.size * 2); list = &g->strt.hash[lmod(h, g->strt.size)]; /* recompute with new size */ @@ -174,6 +180,7 @@ static TString *internshrstr (lua_State *L, const char *str, size_t l) { return ts; } +static TString *internshrstr (lua_State *L, const char *str, size_t l); /* ** new string (with explicit length) @@ -224,3 +231,216 @@ Udata *luaS_newudata (lua_State *L, size_t s) { return u; } +/* + * global shared table + */ + +#include "rwlock.h" +#include "atomic.h" +#include + +#define SHRSTR_SLOT 0x10000 +#define HASH_NODE(h) ((h) % SHRSTR_SLOT) + +struct shrmap_slot { + struct rwlock lock; + TString *str; +}; + +struct shrmap { + struct shrmap_slot h[SHRSTR_SLOT]; + int n; +}; + +static struct shrmap *SSM = NULL; + +LUA_API void +luaS_initshr() { + struct shrmap * s = malloc(sizeof(*s)); + memset(s, 0, sizeof(*s)); + int i; + for (i=0;ih[i].lock); + } + SSM = s; +} + +LUA_API void +luaS_exitshr() { + int i; + for (i=0;ih[i].str; + while (str) { + TString * next = str->u.hnext; + free(str); + str = next; + } + } + free(SSM); +} + +static TString * +query_string(unsigned int h, const char *str, lu_byte l) { + struct shrmap_slot *s = &SSM->h[HASH_NODE(h)]; + rwlock_rlock(&s->lock); + TString *ts = s->str; + while (ts) { + if (ts->hash == h && + ts->shrlen == l && + memcmp(str, ts+1, l) == 0) { + break; + } + ts = ts->u.hnext; + } + rwlock_runlock(&s->lock); + return ts; +} + +static TString * +query_ptr(TString *t) { + unsigned int h = t->hash; + struct shrmap_slot *s = &SSM->h[HASH_NODE(h)]; + rwlock_rlock(&s->lock); + TString *ts = s->str; + while (ts) { + if (ts == t) + break; + ts = ts->u.hnext; + } + rwlock_runlock(&s->lock); + return ts; +} + +static TString * +new_string(unsigned int h, const char *str, lu_byte l) { + size_t sz = sizelstring(l); + TString *ts = malloc(sz); + memset(ts, 0, sz); + ts->tt = LUA_TSHRSTR; + ts->hash = h; + ts->shrlen = l; + memcpy(ts+1, str, l); + return ts; +} + +static TString * +add_string(unsigned int h, const char *str, lu_byte l) { + TString * tmp = new_string(h, str, l); + struct shrmap_slot *s = &SSM->h[HASH_NODE(h)]; + rwlock_wlock(&s->lock); + TString *ts = s->str; + while (ts) { + if (ts->hash == h && + ts->shrlen == l && + memcmp(str, ts+1, l) == 0) { + break; + } + ts = ts->u.hnext; + } + if (ts == NULL) { + ts = tmp; + ts->u.hnext = s->str; + s->str = ts; + tmp = NULL; + } + rwlock_wunlock(&s->lock); + if (tmp) { + // string is create by other thread, so free tmp + free(tmp); + } + return ts; +} + +static TString * +internshrstr (lua_State *L, const char *str, size_t l) { + TString *ts; + global_State *g = G(L); + unsigned int h = luaS_hash(str, l, g->seed); + unsigned int h0; + // lookup global state of this L first + ts = queryshrstr (L, str, l, h); + if (ts) + return ts; + // lookup SSM again + h0 = luaS_hash(str, l, 0); + ts = query_string(h0, str, l); + if (ts) + return ts; + // If SSM->n greate than 0, add it to SSM + if (SSM->n > 0) { + ATOM_DEC(&SSM->n); + return add_string(h0, str, l); + } + // Else add it to global state (local) + return addshrstr (L, str, l, h); +} + +LUA_API void +luaS_expandshr(int n) { + ATOM_ADD(&SSM->n, n); +} + +LUAI_FUNC TString * +luaS_clonestring(lua_State *L, TString *ts) { + unsigned int h; + int l; + const char * str = getaddrstr(ts); + global_State *g = G(L); + TString *result; + if (ts->tt == LUA_TLNGSTR) + return luaS_newlstr(L, str, ts->u.lnglen); + // look up global state of this L first + l = ts->shrlen; + h = luaS_hash(str, l, g->seed); + result = queryshrstr (L, str, l, h); + if (result) + return result; + // look up SSM by ptr + result = query_ptr(ts); + if (result) + return result; + // ts is not in SSM, so recalc hash, and add it to SSM + h = luaS_hash(str, l, 0); + return add_string(h, str, l); +} + +struct slotinfo { + int len; + int size; +}; + +static void +getslot(struct shrmap_slot *s, struct slotinfo *info) { + memset(info, 0, sizeof(*info)); + rwlock_rlock(&s->lock); + TString *ts = s->str; + while (ts) { + ++info->len; + info->size += ts->shrlen; + ts = ts->u.hnext; + } + rwlock_runlock(&s->lock); +} + +LUA_API int +luaS_shrinfo(lua_State *L) { + struct slotinfo total; + struct slotinfo tmp; + memset(&total, 0, sizeof(total)); + int i; + int len = 0; + for (i=0;ih[i]; + getslot(s, &tmp); + len += tmp.len; + if (tmp.len > total.len) { + total.len = tmp.len; + } + total.size += tmp.size; + } + lua_pushinteger(L, len); + lua_pushinteger(L, total.size); + lua_pushinteger(L, total.len); + lua_pushinteger(L, SSM->n); + return 4; +} diff --git a/3rd/lua/lstring.h b/3rd/lua/lstring.h index e746f5fc..b3cfa1b7 100644 --- a/3rd/lua/lstring.h +++ b/3rd/lua/lstring.h @@ -43,5 +43,10 @@ LUAI_FUNC Udata *luaS_newudata (lua_State *L, size_t s); LUAI_FUNC TString *luaS_newlstr (lua_State *L, const char *str, size_t l); LUAI_FUNC TString *luaS_new (lua_State *L, const char *str); +LUA_API void luaS_initshr(); +LUA_API void luaS_exitshr(); +LUA_API void luaS_expandshr(int n); +LUAI_FUNC TString *luaS_clonestring(lua_State *L, TString *); +LUA_API int luaS_shrinfo(lua_State *L); #endif diff --git a/3rd/lua/luaconf.h b/3rd/lua/luaconf.h index 7cfa4faf..5e88dcb0 100644 --- a/3rd/lua/luaconf.h +++ b/3rd/lua/luaconf.h @@ -11,6 +11,7 @@ #include #include +#define LUA_USE_APICHECK /* ** =================================================================== diff --git a/lualib-src/lua-memory.c b/lualib-src/lua-memory.c index 67a3acf6..a3afecfc 100644 --- a/lualib-src/lua-memory.c +++ b/lualib-src/lua-memory.c @@ -2,6 +2,7 @@ #include #include "malloc_hook.h" +#include "luashrtbl.h" static int ltotal(lua_State *L) { @@ -33,6 +34,13 @@ ldump(lua_State *L) { return 0; } +static int +lexpandshrtbl(lua_State *L) { + int n = luaL_checkinteger(L, 1); + luaS_expandshr(n); + return 0; +} + int luaopen_memory(lua_State *L) { luaL_checkversion(L); @@ -43,6 +51,8 @@ luaopen_memory(lua_State *L) { { "dumpinfo", ldumpinfo }, { "dump", ldump }, { "info", dump_mem_lua }, + { "ssinfo", luaS_shrinfo }, + { "ssexpand", lexpandshrtbl }, { NULL, NULL }, }; diff --git a/service/bootstrap.lua b/service/bootstrap.lua index 269654c3..db8d8581 100644 --- a/service/bootstrap.lua +++ b/service/bootstrap.lua @@ -1,8 +1,12 @@ local skynet = require "skynet" local harbor = require "skynet.harbor" require "skynet.manager" -- import skynet.launch, ... +local memory = require "memory" skynet.start(function() + local sharestring = tonumber(skynet.getenv "sharestring") + memory.ssexpand(sharestring or 4096) + local standalone = skynet.getenv "standalone" local launcher = assert(skynet.launch("snlua","launcher")) diff --git a/service/debug_console.lua b/service/debug_console.lua index 97b35806..37b20c30 100644 --- a/service/debug_console.lua +++ b/service/debug_console.lua @@ -3,6 +3,7 @@ local codecache = require "skynet.codecache" local core = require "skynet.core" local socket = require "socket" local snax = require "snax" +local memory = require "memory" local port = tonumber(...) local COMMAND = {} @@ -130,6 +131,8 @@ function COMMAND.help() log = "launch a new lua service with log", debug = "debug address : debug a lua service", signal = "signal address sig", + cmem = "Show C memory info", + shrtbl = "Show shared short string table info", } end @@ -258,3 +261,19 @@ function COMMAND.signal(address, sig) core.command("SIGNAL", address) end end + +function COMMAND.cmem() + local info = memory.info() + local tmp = {} + for k,v in pairs(info) do + tmp[skynet.address(k)] = v + end + return tmp +end + +function COMMAND.shrtbl() + local n, total, longest, space = memory.ssinfo() + return { n = n, total = total, longest = longest, space = space } +end + + diff --git a/skynet-src/luashrtbl.h b/skynet-src/luashrtbl.h new file mode 100644 index 00000000..6f566038 --- /dev/null +++ b/skynet-src/luashrtbl.h @@ -0,0 +1,17 @@ +#ifndef LUA_SHORT_STRING_TABLE_H +#define LUA_SHORT_STRING_TABLE_H + +#ifndef DISABLE_SHORT_STRING + +#include "lstring.h" + +#else + +static inline int luaS_shrinfo(lua_State *L) { return 0; } +static inline void luaS_initshr() {} +static inline void luaS_exitshr() {} +static inline void luaS_expandshr(int n); + +#endif + +#endif diff --git a/skynet-src/skynet_main.c b/skynet-src/skynet_main.c index 59e344f8..eb6f4429 100644 --- a/skynet-src/skynet_main.c +++ b/skynet-src/skynet_main.c @@ -3,6 +3,7 @@ #include "skynet_imp.h" #include "skynet_env.h" #include "skynet_server.h" +#include "luashrtbl.h" #include #include @@ -105,6 +106,8 @@ main(int argc, char *argv[]) { "usage: skynet configfilename\n"); return 1; } + + luaS_initshr(); skynet_globalinit(); skynet_env_init(); @@ -139,6 +142,7 @@ main(int argc, char *argv[]) { skynet_start(&config); skynet_globalexit(); + luaS_exitshr(); return 0; }