From 76b166f04af1b228d0a2a0c217f124fd0727b865 Mon Sep 17 00:00:00 2001 From: Cloud Wu Date: Wed, 17 Apr 2019 21:30:47 +0800 Subject: [PATCH] Add new sharetable --- 3rd/lua/lgc.c | 2 +- 3rd/lua/lstring.c | 2 +- 3rd/lua/lvm.c | 2 + 3rd/lua/lvm.h | 2 +- Makefile | 1 + lualib-src/lua-sharetable.c | 203 +++++++++++++++++++++++++++++++++++ lualib/skynet/sharetable.lua | 173 +++++++++++++++++++++++++++++ test/testsharetable.lua | 10 ++ 8 files changed, 392 insertions(+), 3 deletions(-) create mode 100644 lualib-src/lua-sharetable.c create mode 100644 lualib/skynet/sharetable.lua create mode 100644 test/testsharetable.lua diff --git a/3rd/lua/lgc.c b/3rd/lua/lgc.c index e56f2922..92d38f54 100644 --- a/3rd/lua/lgc.c +++ b/3rd/lua/lgc.c @@ -1018,6 +1018,7 @@ static l_mem atomic (lua_State *L) { clearvalues(g, g->allweak, origall); luaS_clearcache(g); g->currentwhite = cast_byte(otherwhite(g)); /* flip current white */ + luaS_collect(g, 0); /* send short strings set to gc thread */ work += g->GCmemtrav; /* complete counting */ return work; /* estimate of memory marked by 'atomic' */ } @@ -1084,7 +1085,6 @@ static lu_mem singlestep (lua_State *L) { return (n * GCFINALIZECOST); } else { /* emergency mode or no more finalizers */ - luaS_collect(g, 0); /* send short strings set to gc thread */ g->gcstate = GCSpause; /* finish collection */ return 0; } diff --git a/3rd/lua/lstring.c b/3rd/lua/lstring.c index 5368bde0..c9f30fde 100644 --- a/3rd/lua/lstring.c +++ b/3rd/lua/lstring.c @@ -509,7 +509,7 @@ exist(struct ssm_ref *r, TString *s) { TString *t = r->array[mid]; if (t == s) return 1; - if (s > t) + if (memcmp(&s,&t,sizeof(TString *)) > 0) begin = mid + 1; else end = mid - 1; diff --git a/3rd/lua/lvm.c b/3rd/lua/lvm.c index a2888848..334cc9d8 100644 --- a/3rd/lua/lvm.c +++ b/3rd/lua/lvm.c @@ -207,6 +207,8 @@ void luaV_finishset (lua_State *L, const TValue *t, TValue *key, const TValue *tm; /* '__newindex' metamethod */ if (slot != NULL) { /* is 't' a table? */ Table *h = hvalue(t); /* save 't' table */ + if (isshared(h)) + luaG_typeerror(L, t, "change"); lua_assert(ttisnil(slot)); /* old value must be nil */ tm = fasttm(L, h->metatable, TM_NEWINDEX); /* get metamethod */ if (tm == NULL) { /* no metamethod? */ diff --git a/3rd/lua/lvm.h b/3rd/lua/lvm.h index a8f954f0..94b2a62d 100644 --- a/3rd/lua/lvm.h +++ b/3rd/lua/lvm.h @@ -81,7 +81,7 @@ (!ttistable(t) \ ? (slot = NULL, 0) \ : (slot = f(hvalue(t), k), \ - ttisnil(slot) ? 0 \ + (ttisnil(slot) || isshared(hvalue(t))) ? 0 \ : (luaC_barrierback(L, hvalue(t), v), \ setobj2t(L, cast(TValue *,slot), v), \ 1))) diff --git a/Makefile b/Makefile index f2b7df0d..10076bab 100644 --- a/Makefile +++ b/Makefile @@ -70,6 +70,7 @@ LUA_CLIB_SKYNET = \ lua-debugchannel.c \ lua-datasheet.c \ lua-ssm.c \ + lua-sharetable.c \ \ SKYNET_SRC = skynet_main.c skynet_handle.c skynet_module.c skynet_mq.c \ diff --git a/lualib-src/lua-sharetable.c b/lualib-src/lua-sharetable.c new file mode 100644 index 00000000..720274ae --- /dev/null +++ b/lualib-src/lua-sharetable.c @@ -0,0 +1,203 @@ +#define LUA_LIB + +#include +#include +#include + +#include "lstring.h" +#include "lobject.h" +#include "ltable.h" +#include "lstate.h" +#include "lapi.h" +#include "lgc.h" + +#ifdef ENABLE_SHORT_STRING_TABLE + +static void +mark_shared(lua_State *L) { + if (lua_type(L, -1) != LUA_TTABLE) { + luaL_error(L, "Not a table, it's a %s.", lua_typename(L, lua_type(L, -1))); + } + Table * t = (Table *)lua_topointer(L, -1); + if (isshared(t)) + return; + makeshared(t); + luaL_checkstack(L, 4, NULL); + if (lua_getmetatable(L, -1)) { + luaL_error(L, "Can't share metatable"); + } + lua_pushnil(L); + while (lua_next(L, -2) != 0) { + int i; + for (i=0;i<2;i++) { + int idx = -i-1; + int t = lua_type(L, idx); + switch (t) { + case LUA_TTABLE: + mark_shared(L); + break; + case LUA_TNUMBER: + case LUA_TBOOLEAN: + case LUA_TLIGHTUSERDATA: + break; + case LUA_TFUNCTION: + if (!lua_iscfunction(L, idx) || lua_getupvalue(L, idx, 1) != NULL) + luaL_error(L, "Invalid function"); + break; + case LUA_TSTRING: { + const char *str = lua_tostring(L, idx); + TString *ts = (TString *)(str - sizeof(UTString)); + makeshared(ts); + break; + } + default: + luaL_error(L, "Invalid type [%s]", lua_typename(L, t)); + break; + } + } + lua_pop(L, 1); + } +} + +static int +make_matrix(lua_State *L) { + // turn off gc , because marking shared will prevent gc mark. + lua_gc(L, LUA_GCSTOP, 0); + mark_shared(L); + Table * t = (Table *)lua_topointer(L, -1); + lua_pushlightuserdata(L, t); + return 1; +} + +static int +clone_table(lua_State *L) { + Table * t = (Table *)lua_touserdata(L, 1); + if (!isshared(t)) + return luaL_error(L, "Not a shared table"); + + sethvalue(L, L->top, t); + api_incr_top(L); + + return 1; +} + +struct state_ud { + lua_State *L; +}; + +static int +close_state(lua_State *L) { + struct state_ud *ud = (struct state_ud *)luaL_checkudata(L, 1, "BOXMATRIXSTATE"); + if (ud->L) { + lua_close(ud->L); + ud->L = NULL; + } + return 0; +} + +static int +get_matrix(lua_State *L) { + struct state_ud *ud = (struct state_ud *)luaL_checkudata(L, 1, "BOXMATRIXSTATE"); + if (ud->L) { + const void * v = lua_topointer(ud->L, 1); + lua_pushlightuserdata(L, (void *)v); + return 1; + } + return 0; +} + +static int +get_size(lua_State *L) { + struct state_ud *ud = (struct state_ud *)luaL_checkudata(L, 1, "BOXMATRIXSTATE"); + if (ud->L) { + lua_Integer sz = lua_gc(ud->L, LUA_GCCOUNT, 0); + sz *= 1024; + sz += lua_gc(ud->L, LUA_GCCOUNTB, 0); + lua_pushinteger(L, sz); + } else { + lua_pushinteger(L, 0); + } + return 1; +} + + +static int +box_state(lua_State *L, lua_State *mL) { + struct state_ud *ud = (struct state_ud *)lua_newuserdata(L, sizeof(*ud)); + ud->L = mL; + if (luaL_newmetatable(L, "BOXMATRIXSTATE")) { + lua_pushvalue(L, -1); + lua_setfield(L, -2, "__index"); + lua_pushcfunction(L, close_state); + lua_setfield(L, -2, "close"); + lua_pushcfunction(L, get_matrix); + lua_setfield(L, -2, "getptr"); + lua_pushcfunction(L, get_size); + lua_setfield(L, -2, "size"); + } + lua_setmetatable(L, -2); + + return 1; +} + +static int +load_matrixfile(lua_State *L) { + luaL_openlibs(L); + const char * source = (const char *)lua_touserdata(L, 1); + if (source[0] == '@') { + if (luaL_loadfilex_(L, source+1, NULL) || lua_pcall(L, 0, LUA_MULTRET, 0)) { + lua_error(L); + } + } else { + if (luaL_dostring(L, source) != LUA_OK) { + lua_error(L); + } + } + if (lua_gettop(L) == 0) { + luaL_error(L, "No table returns"); + } + lua_gc(L, LUA_GCCOLLECT, 0); + lua_pushcfunction(L, make_matrix); + lua_insert(L, -2); + lua_call(L, 1, 1); + return 1; +} + +static int +matrix_from_file(lua_State *L) { + lua_State *mL = luaL_newstate(); + if (mL == NULL) { + return luaL_error(L, "luaL_newstate failed"); + } + const char * source = luaL_checkstring(L, 1); + lua_pushcfunction(mL, load_matrixfile); + lua_pushlightuserdata(mL, (void *)source); + int ok = lua_pcall(mL, 1, 1, 0); + if (ok != LUA_OK) { + lua_pushstring(L, lua_tostring(mL, -1)); + lua_close(mL); + lua_error(L); + } + return box_state(L, mL); +} + +LUAMOD_API int +luaopen_skynet_sharetable_core(lua_State *L) { + luaL_checkversion(L); + luaL_Reg l[] = { + { "clone", clone_table }, + { "matrix", matrix_from_file }, + { NULL, NULL }, + }; + luaL_newlib(L, l); + return 1; +} + +#else + +LUAMOD_API int +luaopen_skynet_sharetable_core(lua_State *L) { + return luaL_error(L, "No share string table support"); +} + +#endif \ No newline at end of file diff --git a/lualib/skynet/sharetable.lua b/lualib/skynet/sharetable.lua new file mode 100644 index 00000000..58aa56e5 --- /dev/null +++ b/lualib/skynet/sharetable.lua @@ -0,0 +1,173 @@ +local skynet = require "skynet" +local service = require "skynet.service" +local core = require "skynet.sharetable.core" + +local function sharetable_service() + local skynet = require "skynet" + local core = require "skynet.sharetable.core" + + local matrix = {} -- all the matrix + local files = {} -- filename : matrix + local clients = {} + + local sharetable = {} + + local function close_matrix(m) + if m == nil then + return + end + local ptr = m:getptr() + local ref = matrix[ptr] + if ref == nil or ref.count == 0 then + matrix[ptr] = nil + m:close() + end + end + + function sharetable.load(source, filename, datasource) + close_matrix(files[filename]) + if datasource == nil then + skynet.error("Load file : " .. filename) + datasource = "@" .. filename + else + skynet.error("Load chunk with name : " .. filename) + end + + local m = core.matrix(datasource) + files[filename] = m + skynet.ret() + end + + local function query_file(source, filename) + local m = files[filename] + local ptr = m:getptr() + local ref = matrix[ptr] + if ref == nil then + ref = { + filename = filename, + count = 0, + matrix = m, + refs = {}, + } + matrix[ptr] = ref + end + if ref.refs[source] == nil then + ref.refs[source] = true + local list = clients[source] + if not list then + clients[source] = { ptr } + else + table.insert(list, ptr) + end + ref.count = ref.count + 1 + end + return ptr + end + + function sharetable.query(source, filename) + local m = files[filename] + if m == nil then + skynet.ret() + end + local ptr = query_file(source, filename) + skynet.ret(skynet.pack(ptr)) + end + + function sharetable.close(source) + local list = clients[source] + if list then + for _, ptr in ipairs(list) do + local ref = matrix[ptr] + if ref and ref.refs[source] then + ref.refs[source] = nil + ref.count = ref.count - 1 + if ref.count == 0 then + if files[ref.filename] ~= ref.matrix then + -- It's a history version + skynet.error(string.format("Delete a version (%s) of %s", ptr, ref.filename)) + ref.matrix:close() + matrix[ptr] = nil + end + end + end + end + clients[source] = nil + end + -- no return + end + + skynet.dispatch("lua", function(_,source,cmd,...) + sharetable[cmd](source,...) + end) + + skynet.info_func(function() + local info = {} + + for filename, m in pairs(files) do + info[filename] = { + current = m:getptr(), + size = m:size(), + } + end + + local function address(refs) + local keys = {} + for addr in pairs(refs) do + table.insert(keys, skynet.address(addr)) + end + table.sort(keys) + return table.concat(keys, ",") + end + + for ptr, copy in pairs(matrix) do + local v = info[copy.filename] + local h = v.history + if h == nil then + h = {} + v.history = h + end + table.insert(h, string.format("%s [%d]: (%s)", copy.matrix:getptr(), copy.matrix:size(), address(copy.refs))) + end + for _, v in pairs(info) do + if v.history then + v.history = table.concat(v.history, "\n\t") + end + end + + return info + end) + +end + +local function load_service(t, key) + if key == "address" then + t.address = service.new("sharetable", sharetable_service) + return t.address + else + return nil + end +end + +local function report_close(t) + local addr = rawget(t, "address") + if addr then + skynet.send(addr, "lua", "close") + end +end + +local sharetable = setmetatable ( {} , { + __index = load_service, + __gc = report_close, +}) + +function sharetable.load(filename, source) + skynet.call(sharetable.address, "lua", "load", filename, source) +end + +function sharetable.query(filename) + local newptr = skynet.call(sharetable.address, "lua", "query", filename) + return core.clone(newptr) +end + +return sharetable + diff --git a/test/testsharetable.lua b/test/testsharetable.lua new file mode 100644 index 00000000..61f0ee37 --- /dev/null +++ b/test/testsharetable.lua @@ -0,0 +1,10 @@ +local skynet = require "skynet" +local sharetable = require "skynet.sharetable" + +skynet.start(function() + sharetable.load("test", "return { x=1,y={ 'hello world' },['hello world'] = true }") + local t = sharetable.query("test") + for k,v in pairs(t) do + print(k,v) + end +end)