1.0.0 rc5

auto determine primary host

3rd/lua/lparser.c

@@ -325,6 +325,8 @@ static void adjust_assign (LexState *ls, int nvars, int nexps, expdesc *e) {
       luaK_nil(fs, reg, extra);
     }
   }
+  if (nexps > nvars)
+    ls->fs->freereg -= nexps - nvars;  /* remove extra values */
 }
 
 
@@ -1163,11 +1165,8 @@ static void assignment (LexState *ls, struct LHS_assign *lh, int nvars) {
     int nexps;
     checknext(ls, '=');
     nexps = explist(ls, &e);
-    if (nexps != nvars) {
+    if (nexps != nvars)
       adjust_assign(ls, nvars, nexps, &e);
-      if (nexps > nvars)
-        ls->fs->freereg -= nexps - nvars;  /* remove extra values */
-    }
     else {
       luaK_setoneret(ls->fs, &e);  /* close last expression */
       luaK_storevar(ls->fs, &lh->v, &e);

HISTORY.md

@@ -1,3 +1,11 @@
+v1.0.0-rc5 (2016-7-4)
+-----------
+* MongoDB : Support auth_scram_sha1
+* MongoDB : Auto determine primary host
+* Bugfix : memory leak in multicast
+* Bugfix : Lua 5.3.3
+* Bson : support meta array
+
 v1.0.0-rc4 (2016-6-13)
 -----------
 * Update lua to 5.3.3

examples/config

@@ -9,7 +9,9 @@ start = "main"	-- main script
 bootstrap = "snlua bootstrap"	-- The service for bootstrap
 standalone = "0.0.0.0:2013"
 luaservice = root.."service/?.lua;"..root.."test/?.lua;"..root.."examples/?.lua"
-lualoader = "lualib/loader.lua"
+lualoader = root .. "lualib/loader.lua"
+lua_path = root.."lualib/?.lua;"..root.."lualib/?/init.lua"
+lua_cpath = root .. "luaclib/?.so"
 -- preload = "./examples/preload.lua"	-- run preload.lua before every lua service run
 snax = root.."examples/?.lua;"..root.."test/?.lua"
 -- snax_interface_g = "snax_g"

examples/main.lua

@@ -6,7 +6,9 @@ local max_client = 64
 skynet.start(function()
 	skynet.error("Server start")
 	skynet.uniqueservice("protoloader")
-	local console = skynet.newservice("console")
+	if not skynet.getenv "daemon" then
+		local console = skynet.newservice("console")
+	end
 	skynet.newservice("debug_console",8000)
 	skynet.newservice("simpledb")
 	local watchdog = skynet.newservice("watchdog")

lualib-src/lua-bson.c

@@ -238,8 +238,6 @@ write_double(struct bson *b, lua_Number d) {
 	}
 }
 
-static void pack_dict(lua_State *L, struct bson *b, bool array, int depth);
-
 static inline void
 append_key(struct bson *bs, int type, const char *key, size_t sz) {
 	write_byte(bs, type);
@@ -269,25 +267,7 @@ append_number(struct bson *bs, lua_State *L, const char *key, size_t sz) {
 	}
 }
 
-static void
-append_table(struct bson *bs, lua_State *L, const char *key, size_t sz, int depth) {
-	size_t len = lua_rawlen(L, -1);
-	bool isarray = false;
-	if (len > 0) {
-		lua_pushinteger(L, len);
-		if (lua_next(L,-2) == 0) {
-			isarray = true;
-		} else {
-			lua_pop(L,2);
-		}
-	}
-	if (isarray) {
-		append_key(bs, BSON_ARRAY, key, sz);
-	} else {
-		append_key(bs, BSON_DOCUMENT, key, sz);
-	}
-	pack_dict(L, bs, isarray, depth);
-}
+static void append_table(struct bson *bs, lua_State *L, const char *key, size_t sz, int depth);
 
 static void
 write_binary(struct bson *b, const void * buffer, size_t sz) {
@@ -409,65 +389,59 @@ bson_numstr( char *str, unsigned int i ) {
 }
 
 static void
-pack_dict_data(lua_State *L, struct bson *b, bool isarray, int depth, int kt) {
-	char numberkey[32];
-	const char * key = NULL;
-	size_t sz;
-	if (isarray) {
-		if (kt != LUA_TNUMBER) {
-			luaL_error(L, "Invalid array key type : %s", lua_typename(L, kt));
-			return;
-		}
-		sz = bson_numstr(numberkey, (unsigned int)lua_tointeger(L,-2)-1);
-		key = numberkey;
-
-		append_one(b, L, key, sz, depth);
-		lua_pop(L,1);
-	} else {
-		switch(kt) {
-		case LUA_TNUMBER:
-			// copy key, don't change key type
-			lua_pushvalue(L,-2);
-			lua_insert(L,-2);
-			key = lua_tolstring(L,-2,&sz);
-			append_one(b, L, key, sz, depth);
-			lua_pop(L,2);
-			break;
-		case LUA_TSTRING:
-			key = lua_tolstring(L,-2,&sz);
-			append_one(b, L, key, sz, depth);
-			lua_pop(L,1);
-			break;
-		default:
-			luaL_error(L, "Invalid key type : %s", lua_typename(L, kt));
-			return;
-		}
-	}
-}
-
-static void
-pack_simple_dict(lua_State *L, struct bson *b, bool isarray, int depth) {
-	if (depth > MAX_DEPTH) {
-		luaL_error(L, "Too depth while encoding bson");
-	}
-	luaL_checkstack(L, 16, NULL);	// reserve enough stack space to pack table
+pack_array(lua_State *L, struct bson *b, int depth, size_t len) {
 	int length = reserve_length(b);
-	lua_pushnil(L);
-	while(lua_next(L,-2) != 0) {
-		int kt = lua_type(L, -2);
-		pack_dict_data(L, b, isarray, depth, kt);
+	size_t i;
+	for (i=1;i<=len;i++) {
+		char numberkey[32];
+		size_t sz = bson_numstr(numberkey, i - 1);
+		const char * key = numberkey;
+		lua_geti(L, -1, i);
+		append_one(b, L, key, sz, depth);
+		lua_pop(L, 1);
 	}
 	write_byte(b,0);
 	write_length(b, b->size - length, length);
 }
 
+static void
+pack_dict_data(lua_State *L, struct bson *b, int depth, int kt) {
+	const char * key = NULL;
+	size_t sz;
+	switch(kt) {
+	case LUA_TNUMBER:
+		// copy key, don't change key type
+		lua_pushvalue(L,-2);
+		lua_insert(L,-2);
+		key = lua_tolstring(L,-2,&sz);
+		append_one(b, L, key, sz, depth);
+		lua_pop(L,2);
+		break;
+	case LUA_TSTRING:
+		key = lua_tolstring(L,-2,&sz);
+		append_one(b, L, key, sz, depth);
+		lua_pop(L,1);
+		break;
+	default:
+		luaL_error(L, "Invalid key type : %s", lua_typename(L, kt));
+		return;
+	}
+}
 
 static void
-pack_meta_dict(lua_State *L, struct bson *b, bool isarray, int depth) {
-	if (depth > MAX_DEPTH) {
-		luaL_error(L, "Too depth while encoding bson");
+pack_simple_dict(lua_State *L, struct bson *b, int depth) {
+	int length = reserve_length(b);
+	lua_pushnil(L);
+	while(lua_next(L,-2) != 0) {
+		int kt = lua_type(L, -2);
+		pack_dict_data(L, b, depth, kt);
 	}
-	luaL_checkstack(L, 16, NULL);	// reserve enough stack space to pack table
+	write_byte(b,0);
+	write_length(b, b->size - length, length);
+}
+
+static void
+pack_meta_dict(lua_State *L, struct bson *b, int depth) {
 	int length = reserve_length(b);
 
 	lua_pushvalue(L, -2); // push meta_obj
@@ -483,18 +457,51 @@ pack_meta_dict(lua_State *L, struct bson *b, bool isarray, int depth) {
 			lua_pop(L, 4); // pop all k, v, next_func, obj
 			break;
 		}
-		pack_dict_data(L, b, isarray, depth, kt);
+		pack_dict_data(L, b, depth, kt);
 	}
 	write_byte(b,0);
 	write_length(b, b->size - length, length);
 }
 
+static bool
+is_rawarray(lua_State *L) {
+	size_t len = lua_rawlen(L, -1);
+	if (len > 0) {
+		lua_pushinteger(L, len);
+		if (lua_next(L,-2) == 0) {
+			return true;
+		} else {
+			lua_pop(L,2);
+		}
+	}
+	return false;
+}
+
 static void
-pack_dict(lua_State *L, struct bson *b, bool isarray, int depth) {
-	if (luaL_getmetafield(L, -1, "__pairs") != LUA_TNIL) {
-		pack_meta_dict(L, b, isarray, depth);
+append_table(struct bson *bs, lua_State *L, const char *key, size_t sz, int depth) {
+	if (depth > MAX_DEPTH) {
+		luaL_error(L, "Too depth while encoding bson");
+	}
+	luaL_checkstack(L, 16, NULL);	// reserve enough stack space to pack table
+	if (luaL_getmetafield(L, -1, "__len") != LUA_TNIL) {
+		lua_pushvalue(L, -2);
+		lua_call(L, 1, 1);
+		if (!lua_isinteger(L, -1)) {
+			luaL_error(L, "__len should return integer");
+		}
+		size_t len = lua_tointeger(L, -1);
+		lua_pop(L, 1);
+		append_key(bs, BSON_ARRAY, key, sz);
+		pack_array(L, bs, depth, len);
+	} else if (luaL_getmetafield(L, -1, "__pairs") != LUA_TNIL) {
+		append_key(bs, BSON_DOCUMENT, key, sz);
+		pack_meta_dict(L, bs, depth);
+	} else if (is_rawarray(L)) {
+		append_key(bs, BSON_ARRAY, key, sz);
+		pack_array(L, bs, depth, lua_rawlen(L, -1));
 	} else {
-		pack_simple_dict(L, b, isarray, depth);
+		append_key(bs, BSON_DOCUMENT, key, sz);
+		pack_simple_dict(L, bs, depth);
 	}
 }
 
@@ -895,7 +902,11 @@ lencode(lua_State *L) {
 	bson_create(&b);
 	lua_settop(L,1);
 	luaL_checktype(L, 1, LUA_TTABLE);
-	pack_dict(L, &b, false, 0);
+	if (luaL_getmetafield(L, -1, "__pairs") != LUA_TNIL) {
+		pack_meta_dict(L, &b, 0);
+	} else {
+		pack_simple_dict(L, &b, 0);
+	}
 	void * ud = lua_newuserdata(L, b.size);
 	memcpy(ud, b.ptr, b.size);
 	bson_destroy(&b);

lualib-src/lua-crypt.c

@@ -751,7 +751,7 @@ ldhexchange(lua_State *L) {
 	if (x64 == 0)
 		return luaL_error(L, "Can't be 0");
 
-	uint64_t r = powmodp(5,	x64);
+	uint64_t r = powmodp(G,	x64);
 	push64(L, r);
 	return 1;
 }
@@ -878,6 +878,25 @@ lb64decode(lua_State *L) {
 	return 1;
 }
 
+static int
+lxor_str(lua_State *L) {
+	size_t len1,len2;
+	const char *s1 = luaL_checklstring(L,1,&len1);
+	const char *s2 = luaL_checklstring(L,2,&len2);
+	if (len2 == 0) {
+		return luaL_error(L, "Can't xor empty string");
+	}
+	luaL_Buffer b;
+	char * buffer = luaL_buffinitsize(L, &b, len1);
+	int i;
+	for (i=0;i<len1;i++) {
+		buffer[i] = s1[i] ^ s2[i % len2];
+	}
+	luaL_addsize(&b, len1);
+	luaL_pushresult(&b);
+	return 1;
+}
+
 // defined in lsha1.c
 int lsha1(lua_State *L);
 int lhmac_sha1(lua_State *L);
@@ -906,6 +925,7 @@ luaopen_crypt(lua_State *L) {
 		{ "sha1", lsha1 },
 		{ "hmac_sha1", lhmac_sha1 },
 		{ "hmac_hash", lhmac_hash },
+		{ "xor_str", lxor_str },
 		{ NULL, NULL },
 	};
 	luaL_newlib(L,l);

lualib-src/lua-multicast.c

@@ -60,18 +60,6 @@ mc_packremote(lua_State *L) {
 	return pack(L, msg, size);
 }
 
-static int
-mc_packstring(lua_State *L) {
-	size_t size;
-	const char * msg = luaL_checklstring(L, 1, &size);
-	if (size != (uint32_t)size) {
-		return luaL_error(L, "string is too long");
-	}
-	void * data = skynet_malloc(size);
-	memcpy(data, msg, size);
-	return pack(L, data, size);
-}
-
 /*
 	lightuserdata struct mc_package **
 	integer size (must be sizeof(struct mc_package *)
@@ -82,7 +70,7 @@ static int
 mc_unpacklocal(lua_State *L) {
 	struct mc_package ** pack = lua_touserdata(L,1);
 	int sz = luaL_checkinteger(L,2);
-	if (sz != sizeof(*pack)) {
+	if (sz != sizeof(pack)) {
 		return luaL_error(L, "Invalid multicast package size %d", sz);
 	}
 	lua_pushlightuserdata(L, *pack);
@@ -108,6 +96,8 @@ mc_bindrefer(lua_State *L) {
 
 	lua_pushlightuserdata(L, *pack);
 
+	skynet_free(pack);
+
 	return 1;
 }
 
@@ -161,7 +151,6 @@ luaopen_multicast_core(lua_State *L) {
 		{ "bind", mc_bindrefer },
 		{ "close", mc_closelocal },
 		{ "remote", mc_remote },
-		{ "packstring", mc_packstring },
 		{ "packremote", mc_packremote },
 		{ "nextid", mc_nextid },
 		{ NULL, NULL },

lualib/mongo.lua

@@ -4,6 +4,7 @@ local socketchannel	= require "socketchannel"
 local skynet = require "skynet"
 local driver = require "mongo.driver"
 local md5 =	require	"md5"
+local crypt = require "crypt"
 local rawget = rawget
 local assert = assert
 
@@ -83,10 +84,15 @@ end
 local function mongo_auth(mongoc)
 	local user = rawget(mongoc,	"username")
 	local pass = rawget(mongoc,	"password")
+	local authmod = rawget(mongoc, "authmod") or "scram_sha1"
+	authmod = "auth_" ..  authmod
 
 	return function()
 		if user	~= nil and pass	~= nil then
-			assert(mongoc:auth(user, pass))
+			-- autmod can be "mongodb_cr" or "scram_sha1"
+			local auth_func = mongoc[authmod]
+			assert(auth_func , "Invalid authmod")
+			assert(auth_func(mongoc,user, pass))
 		end
 		local rs_data =	mongoc:runCommand("ismaster")
 		if rs_data.ok == 1 then
@@ -99,12 +105,38 @@ local function mongo_auth(mongoc)
 				mongoc.__sock:changebackup(backup)
 			end
 			if rs_data.ismaster	then
+				if rawget(mongoc, "__pickserver") then
+					rawset(mongoc, "__pickserver", nil)
+				end
 				return
 			else
-				local host,	port = __parse_addr(rs_data.primary)
-				mongoc.host	= host
-				mongoc.port	= port
-				mongoc.__sock:changehost(host, port)
+				if rs_data.primary then
+					local host,	port = __parse_addr(rs_data.primary)
+					mongoc.host	= host
+					mongoc.port	= port
+					mongoc.__sock:changehost(host, port)
+				else
+					skynet.error("WARNING: NO PRIMARY RETURN " .. rs_data.me)
+					-- determine the primary db using hosts
+					local pickserver = {}
+					if rawget(mongoc, "__pickserver") == nil then
+						for _, v in ipairs(rs_data.hosts) do
+							if v ~= rs_data.me then
+								table.insert(pickserver, v)
+							end
+							rawset(mongoc, "__pickserver", pickserver)
+						end
+					end
+					if #mongoc.__pickserver <= 0 then
+						error("CAN NOT DETERMINE THE PRIMARY DB")
+					end
+					skynet.error("INFO: TRY TO CONNECT " .. mongoc.__pickserver[1])
+					local host, port = __parse_addr(mongoc.__pickserver[1])
+					table.remove(mongoc.__pickserver, 1)
+					mongoc.host	= host
+					mongoc.port	= port
+					mongoc.__sock:changehost(host, port)
+				end
 			end
 		end
 	end
@@ -122,6 +154,7 @@ function mongo.client( conf	)
 		port = first.port or 27017,
 		username = first.username,
 		password = first.password,
+		authmod = first.authmod,
 	}
 
 	obj.__id = 0
@@ -172,7 +205,7 @@ function mongo_client:runCommand(...)
 	return self.admin:runCommand(...)
 end
 
-function mongo_client:auth(user,password)
+function mongo_client:auth_mongodb_cr(user,password)
 	local password = md5.sumhexa(string.format("%s:mongo:%s",user,password))
 	local result= self:runCommand "getnonce"
 	if result.ok ~=1 then
@@ -184,6 +217,83 @@ function mongo_client:auth(user,password)
 	return result.ok ==	1
 end
 
+local function salt_password(password, salt, iter)
+	salt = salt .. "\0\0\0\1"
+	local output = crypt.hmac_sha1(password, salt)
+	local inter = output
+	for i=2,iter do
+		inter = crypt.hmac_sha1(password, inter)
+		output = crypt.xor_str(output, inter)
+	end
+	return output
+end
+
+function mongo_client:auth_scram_sha1(username,password)
+	local user = string.gsub(string.gsub(username, '=', '=3D'), ',' , '=2C')
+	local nonce = crypt.base64encode(crypt.randomkey())
+	local first_bare = "n="  .. user .. ",r="  .. nonce
+	local sasl_start_payload = crypt.base64encode("n,," .. first_bare)
+	local r
+
+	r = self:runCommand("saslStart",1,"autoAuthorize",1,"mechanism","SCRAM-SHA-1","payload",sasl_start_payload)
+	if r.ok ~= 1 then
+		return false
+	end
+
+	local conversationId = r['conversationId']
+	local server_first = r['payload']
+	local parsed_s = crypt.base64decode(server_first)
+	local parsed_t = {}
+	for k, v in string.gmatch(parsed_s, "(%w+)=([^,]*)") do
+		parsed_t[k] = v
+	end
+	local iterations = tonumber(parsed_t['i'])
+	local salt = parsed_t['s']
+	local rnonce = parsed_t['r']
+
+	if not string.sub(rnonce, 1, 12) == nonce then
+		skynet.error("Server returned an invalid nonce.")
+		return false
+	end
+	local without_proof = "c=biws,r=" .. rnonce
+	local pbkdf2_key = md5.sumhexa(string.format("%s:mongo:%s",username,password))
+	local salted_pass = salt_password(pbkdf2_key, crypt.base64decode(salt), iterations)
+	local client_key = crypt.hmac_sha1(salted_pass, "Client Key")
+	local stored_key = crypt.sha1(client_key)
+	local auth_msg = first_bare .. ',' .. parsed_s .. ',' .. without_proof
+	local client_sig = crypt.hmac_sha1(stored_key, auth_msg)
+	local client_key_xor_sig = crypt.xor_str(client_key, client_sig)
+	local client_proof = "p=" .. crypt.base64encode(client_key_xor_sig)
+	local client_final = crypt.base64encode(without_proof .. ',' .. client_proof)
+	local server_key = crypt.hmac_sha1(salted_pass, "Server Key")
+	local server_sig = crypt.base64encode(crypt.hmac_sha1(server_key, auth_msg))
+
+	r = self:runCommand("saslContinue",1,"conversationId",conversationId,"payload",client_final)
+	if r.ok ~= 1 then
+		return false
+	end
+	parsed_s = crypt.base64decode(r['payload'])
+	parsed_t = {}
+	for k, v in string.gmatch(parsed_s, "(%w+)=([^,]*)") do
+		parsed_t[k] = v
+	end
+	if parsed_t['v'] ~= server_sig then
+		skynet.error("Server returned an invalid signature.")
+		return false
+	end
+	if not r.done then
+		r = self:runCommand("saslContinue",1,"conversationId",conversationId,"payload","")
+		if r.ok ~= 1 then
+			return false
+		end
+		if not r.done then
+			skynet.error("SASL conversation failed to complete.")
+			return false
+		end
+	end
+	return true
+end
+
 function mongo_client:logout()
 	local result = self:runCommand "logout"
 	return result.ok ==	1

service/multicastd.lua

@@ -78,13 +78,13 @@ local function publish(c , source, pack, size)
 
 	local group = channel[c]
 	if group == nil or next(group) == nil then
-		-- dead channel, delete the pack. mc.bind returns the pointer in pack
+		-- dead channel, delete the pack. mc.bind returns the pointer in pack and free the pack (struct mc_package **)
 		local pack = mc.bind(pack, 1)
 		mc.close(pack)
 		return
 	end
-	mc.bind(pack, channel_n[c])
-	local msg = skynet.tostring(pack, size)
+	local msg = skynet.tostring(pack, size)	-- copy (pack,size) to a string
+	mc.bind(pack, channel_n[c])	-- mc.bind will free the pack(struct mc_package **)
 	for k in pairs(group) do
 		-- the msg is a pointer to the real message, publish pointer in local is ok.
 		skynet.redirect(k, source, "multicast", c , msg)

test/testbson.lua

@@ -0,0 +1,95 @@
+local bson = require "bson"
+
+sub = bson.encode_order( "hello", 1, "world", 2 )
+
+local function tbl_next(...)
+	print("--- next.a", ...)
+	local k, v = next(...)
+	print("--- next.b", k, v)
+	return k, v
+end
+
+local function tbl_pairs(obj)
+	return tbl_next, obj.__data, nil
+end
+
+local obj_a = {
+	__data = {
+		[1] = 2,
+		[3] = 4,
+		[5] = 6,
+	}
+}
+
+setmetatable(
+	obj_a,
+	{
+		__index = obj_a.__data,
+		__pairs = tbl_pairs,
+	}
+)
+
+local obj_b = {
+	__data = {
+		[7] = 8,
+		[9] = 10,
+		[11] = obj_a,
+	}
+}
+
+setmetatable(
+	obj_b,
+	{
+		__index = obj_b.__data,
+		__pairs = tbl_pairs,
+	}
+)
+
+local metaarray = setmetatable({ n = 5 }, {
+	__len = function(self) return self.n end,
+	__index = function(self, idx) return tostring(idx) end,
+})
+
+b = bson.encode {
+	a = 1,
+	b = true,
+	c = bson.null,
+	d = { 1,2,3,4 },
+	e = bson.binary "hello",
+	f = bson.regex ("*","i"),
+	g = bson.regex "hello",
+	h = bson.date (os.time()),
+	i = bson.timestamp(os.time()),
+	j = bson.objectid(),
+	k = { a = false, b = true },
+	l = {},
+	m = bson.minkey,
+	n = bson.maxkey,
+	o = sub,
+	p = 2^32-1,
+	q = obj_b,
+	r = metaarray,
+}
+
+print "\n[before replace]"
+t = b:decode()
+
+for k, v in pairs(t) do
+	print(k,type(v))
+end
+
+for k,v in ipairs(t.r) do
+	print(k,v)
+end
+
+b:makeindex()
+b.a = 2
+b.b = false
+b.h = bson.date(os.time())
+b.i = bson.timestamp(os.time())
+b.j = bson.objectid()
+
+print "\n[after replace]"
+t = b:decode()
+
+print("o.hello", bson.type(t.o.hello))