release 1.0.0 beta

repeated “struct table" of line 19

3rd/lpeg/HISTORY

@@ -1,4 +1,10 @@
-HISTORY for LPeg 0.12
+HISTORY for LPeg 1.0
+
+* Changes from version 0.12 to 1.0
+  ---------------------------------
+  + group "names" can be any Lua value
+  + some bugs fixed
+  + other small improvements
 
 * Changes from version 0.11 to 0.12
   ---------------------------------

3rd/lpeg/lpcap.c

@@ -1,5 +1,5 @@
 /*
-** $Id: lpcap.c,v 1.5 2014/12/12 16:58:47 roberto Exp $
+** $Id: lpcap.c,v 1.6 2015/06/15 16:09:57 roberto Exp $
 ** Copyright 2007, Lua.org & PUC-Rio  (see 'lpeg.html' for license)
 */
 
@@ -126,7 +126,7 @@ static Capture *findback (CapState *cs, Capture *cap) {
       continue; /* opening an enclosing capture: skip and get previous */
     if (captype(cap) == Cgroup) {
       getfromktable(cs, cap->idx);  /* get group name */
-      if (lua_equal(L, -2, -1)) {  /* right group? */
+      if (lp_equal(L, -2, -1)) {  /* right group? */
         lua_pop(L, 2);  /* remove reference name and group name */
         return cap;
       }

3rd/lpeg/lpcap.h

@@ -1,5 +1,5 @@
 /*
-** $Id: lpcap.h,v 1.1 2013/03/21 20:25:12 roberto Exp $
+** $Id: lpcap.h,v 1.2 2015/02/27 17:13:17 roberto Exp $
 */
 
 #if !defined(lpcap_h)
@@ -18,7 +18,7 @@ typedef enum CapKind {
 
 typedef struct Capture {
   const char *s;  /* subject position */
-  short idx;  /* extra info about capture (group name, arg index, etc.) */
+  unsigned short idx;  /* extra info (group name, arg index, etc.) */
   byte kind;  /* kind of capture */
   byte siz;  /* size of full capture + 1 (0 = not a full capture) */
 } Capture;

3rd/lpeg/lpcode.c

@@ -1,5 +1,5 @@
 /*
-** $Id: lpcode.c,v 1.21 2014/12/12 17:01:29 roberto Exp $
+** $Id: lpcode.c,v 1.23 2015/06/12 18:36:47 roberto Exp $
 ** Copyright 2007, Lua.org & PUC-Rio  (see 'lpeg.html' for license)
 */
 
@@ -431,11 +431,11 @@ typedef struct CompileState {
 
 
 /*
-** code generation is recursive; 'opt' indicates that the code is
-** being generated under a 'IChoice' operator jumping to its end
-** (that is, the match is "optional").
-** 'tt' points to a previous test protecting this code. 'fl' is
-** the follow set of the pattern.
+** code generation is recursive; 'opt' indicates that the code is being
+** generated as the last thing inside an optional pattern (so, if that
+** code is optional too, it can reuse the 'IChoice' already in place for
+** the outer pattern). 'tt' points to a previous test protecting this
+** code (or NOINST). 'fl' is the follow set of the pattern.
 */
 static void codegen (CompileState *compst, TTree *tree, int opt, int tt,
                      const Charset *fl);
@@ -638,13 +638,13 @@ static void codebehind (CompileState *compst, TTree *tree) {
 
 /*
 ** Choice; optimizations:
-** - when p1 is headfail
-** - when first(p1) and first(p2) are disjoint; than
+** - when p1 is headfail or
+** when first(p1) and first(p2) are disjoint, than
 ** a character not in first(p1) cannot go to p1, and a character
 ** in first(p1) cannot go to p2 (at it is not in first(p2)).
 ** (The optimization is not valid if p1 accepts the empty string,
 ** as then there is no character at all...)
-** - when p2 is empty and opt is true; a IPartialCommit can resuse
+** - when p2 is empty and opt is true; a IPartialCommit can reuse
 ** the Choice already active in the stack.
 */
 static void codechoice (CompileState *compst, TTree *p1, TTree *p2, int opt,
@@ -671,7 +671,7 @@ static void codechoice (CompileState *compst, TTree *p1, TTree *p2, int opt,
   }
   else {
     /* <p1 / p2> == 
-        test(fail(p1)) -> L1; choice L1; <p1>; commit L2; L1: <p2>; L2: */
+        test(first(p1)) -> L1; choice L1; <p1>; commit L2; L1: <p2>; L2: */
     int pcommit;
     int test = codetestset(compst, &cs1, e1);
     int pchoice = addoffsetinst(compst, IChoice);
@@ -759,7 +759,7 @@ static void coderep (CompileState *compst, TTree *tree, int opt,
       /* L1: test (fail(p1)) -> L2; <p>; jmp L1; L2: */
       int jmp;
       int test = codetestset(compst, &st, 0);
-      codegen(compst, tree, opt, test, fullset);
+      codegen(compst, tree, 0, test, fullset);
       jmp = addoffsetinst(compst, IJmp);
       jumptohere(compst, test);
       jumptothere(compst, jmp, test);

3rd/lpeg/lpcode.h

@@ -1,5 +1,5 @@
 /*
-** $Id: lpcode.h,v 1.6 2013/11/28 14:56:02 roberto Exp $
+** $Id: lpcode.h,v 1.7 2015/06/12 18:24:45 roberto Exp $
 */
 
 #if !defined(lpcode_h)
@@ -24,7 +24,15 @@ int sizei (const Instruction *i);
 #define PEnullable      0
 #define PEnofail        1
 
+/*
+** nofail(t) implies that 't' cannot fail with any input
+*/
 #define nofail(t)	checkaux(t, PEnofail)
+
+/*
+** (not nullable(t)) implies 't' cannot match without consuming
+** something
+*/
 #define nullable(t)	checkaux(t, PEnullable)
 
 #define fixedlen(t)     fixedlenx(t, 0, 0)

3rd/lpeg/lpeg.html

@@ -10,7 +10,7 @@
 </head>
 <body>
 
-<!-- $Id: lpeg.html,v 1.72 2014/12/12 17:11:35 roberto Exp $ -->
+<!-- $Id: lpeg.html,v 1.75 2015/09/28 17:17:41 roberto Exp $ -->
 
 <div id="container">
 	
@@ -22,7 +22,7 @@
   </div>
   <div id="product_name"><big><strong>LPeg</strong></big></div>
   <div id="product_description">
-     Parsing Expression Grammars For Lua, version 0.12
+     Parsing Expression Grammars For Lua, version 1.0
   </div>
 </div> <!-- id="product" -->
 
@@ -195,13 +195,16 @@ Returns a string with the running version of LPeg.
 
 <h3><a name="f-setstack"></a><code>lpeg.setmaxstack (max)</code></h3>
 <p>
-Sets the maximum size for the backtrack stack used by LPeg to
+Sets a limit for the size of the backtrack stack used by LPeg to
 track calls and choices.
+(The default limit is 400.)
 Most well-written patterns need little backtrack levels and
-therefore you seldom need to change this maximum;
-but a few useful patterns may need more space.
-Before changing this maximum you should try to rewrite your
+therefore you seldom need to change this limit;
+before changing it you should try to rewrite your
 pattern to avoid the need for extra space.
+Nevertheless, a few useful patterns may overflow.
+Also, with recursive grammars,
+subjects with deep recursion may also need larger limits.
 </p>
 
 
@@ -682,7 +685,8 @@ argument given in the call to <code>lpeg.match</code>.
 Creates a <em>back capture</em>.
 This pattern matches the empty string and
 produces the values produced by the <em>most recent</em>
-<a href="#cap-g">group capture</a> named <code>name</code>.
+<a href="#cap-g">group capture</a> named <code>name</code>
+(where <code>name</code> can be any Lua value).
 </p>
 
 <p>
@@ -762,7 +766,8 @@ Creates a <em>group capture</em>.
 It groups all values returned by <code>patt</code>
 into a single capture.
 The group may be anonymous (if no name is given)
-or named with the given name.
+or named with the given name
+(which can be any non-nil Lua value).
 </p>
 
 <p>
@@ -1375,13 +1380,13 @@ and the new term for each repetition.
 <h2><a name="download"></a>Download</h2>
 
 <p>LPeg 
-<a href="http://www.inf.puc-rio.br/~roberto/lpeg/lpeg-0.12.1.tar.gz">source code</a>.</p>
+<a href="http://www.inf.puc-rio.br/~roberto/lpeg/lpeg-1.0.0.tar.gz">source code</a>.</p>
 
 
 <h2><a name="license">License</a></h2>
 
 <p>
-Copyright &copy; 2014 Lua.org, PUC-Rio.
+Copyright &copy; 2007-2015 Lua.org, PUC-Rio.
 </p>
 <p>
 Permission is hereby granted, free of charge,
@@ -1419,7 +1424,7 @@ THE SOFTWARE.
 
 <div id="about">
 <p><small>
-$Id: lpeg.html,v 1.72 2014/12/12 17:11:35 roberto Exp $
+$Id: lpeg.html,v 1.75 2015/09/28 17:17:41 roberto Exp $
 </small></p>
 </div> <!-- id="about" -->
 

3rd/lpeg/lpprint.c

@@ -1,5 +1,5 @@
 /*
-** $Id: lpprint.c,v 1.7 2013/04/12 16:29:49 roberto Exp $
+** $Id: lpprint.c,v 1.9 2015/06/15 16:09:57 roberto Exp $
 ** Copyright 2007, Lua.org & PUC-Rio  (see 'lpeg.html' for license)
 */
 
@@ -52,7 +52,7 @@ static void printjmp (const Instruction *op, const Instruction *p) {
 }
 
 
-static void printinst (const Instruction *op, const Instruction *p) {
+void printinst (const Instruction *op, const Instruction *p) {
   const char *const names[] = {
     "any", "char", "set",
     "testany", "testchar", "testset",
@@ -221,10 +221,10 @@ void printtree (TTree *tree, int ident) {
 
 void printktable (lua_State *L, int idx) {
   int n, i;
-  lua_getfenv(L, idx);
+  lua_getuservalue(L, idx);
   if (lua_isnil(L, -1))  /* no ktable? */
     return;
-  n = lua_objlen(L, -1);
+  n = lua_rawlen(L, -1);
   printf("[");
   for (i = 1; i <= n; i++) {
     printf("%d = ", i);

3rd/lpeg/lpprint.h

@@ -1,5 +1,5 @@
 /*
-** $Id: lpprint.h,v 1.1 2013/03/21 20:25:12 roberto Exp $
+** $Id: lpprint.h,v 1.2 2015/06/12 18:18:08 roberto Exp $
 */
 
 
@@ -18,6 +18,7 @@ void printtree (TTree *tree, int ident);
 void printktable (lua_State *L, int idx);
 void printcharset (const byte *st);
 void printcaplist (Capture *cap, Capture *limit);
+void printinst (const Instruction *op, const Instruction *p);
 
 #else
 

3rd/lpeg/lptree.c

@@ -1,5 +1,5 @@
 /*
-** $Id: lptree.c,v 1.13 2014/12/12 16:59:10 roberto Exp $
+** $Id: lptree.c,v 1.21 2015/09/28 17:01:25 roberto Exp $
 ** Copyright 2013, Lua.org & PUC-Rio  (see 'lpeg.html' for license)
 */
 
@@ -126,6 +126,189 @@ static void finalfix (lua_State *L, int postable, TTree *g, TTree *t) {
 }
 
 
+
+/*
+** {===================================================================
+** KTable manipulation
+**
+** - The ktable of a pattern 'p' can be shared by other patterns that
+** contain 'p' and no other constants. Because of this sharing, we
+** should not add elements to a 'ktable' unless it was freshly created
+** for the new pattern.
+**
+** - The maximum index in a ktable is USHRT_MAX, because trees and
+** patterns use unsigned shorts to store those indices.
+** ====================================================================
+*/
+
+/*
+** Create a new 'ktable' to the pattern at the top of the stack.
+*/
+static void newktable (lua_State *L, int n) {
+  lua_createtable(L, n, 0);  /* create a fresh table */
+  lua_setuservalue(L, -2);  /* set it as 'ktable' for pattern */
+}
+
+
+/*
+** Add element 'idx' to 'ktable' of pattern at the top of the stack;
+** Return index of new element.
+** If new element is nil, does not add it to table (as it would be
+** useless) and returns 0, as ktable[0] is always nil.
+*/
+static int addtoktable (lua_State *L, int idx) {
+  if (lua_isnil(L, idx))  /* nil value? */
+    return 0;
+  else {
+    int n;
+    lua_getuservalue(L, -1);  /* get ktable from pattern */
+    n = lua_rawlen(L, -1);
+    if (n >= USHRT_MAX)
+      luaL_error(L, "too many Lua values in pattern");
+    lua_pushvalue(L, idx);  /* element to be added */
+    lua_rawseti(L, -2, ++n);
+    lua_pop(L, 1);  /* remove 'ktable' */
+    return n;
+  }
+}
+
+
+/*
+** Return the number of elements in the ktable at 'idx'.
+** In Lua 5.2/5.3, default "environment" for patterns is nil, not
+** a table. Treat it as an empty table. In Lua 5.1, assumes that
+** the environment has no numeric indices (len == 0)
+*/
+static int ktablelen (lua_State *L, int idx) {
+  if (!lua_istable(L, idx)) return 0;
+  else return lua_rawlen(L, idx);
+}
+
+
+/*
+** Concatentate the contents of table 'idx1' into table 'idx2'.
+** (Assume that both indices are negative.)
+** Return the original length of table 'idx2' (or 0, if no
+** element was added, as there is no need to correct any index).
+*/
+static int concattable (lua_State *L, int idx1, int idx2) {
+  int i;
+  int n1 = ktablelen(L, idx1);
+  int n2 = ktablelen(L, idx2);
+  if (n1 + n2 > USHRT_MAX)
+    luaL_error(L, "too many Lua values in pattern");
+  if (n1 == 0) return 0;  /* nothing to correct */
+  for (i = 1; i <= n1; i++) {
+    lua_rawgeti(L, idx1, i);
+    lua_rawseti(L, idx2 - 1, n2 + i);  /* correct 'idx2' */
+  }
+  return n2;
+}
+
+
+/*
+** When joining 'ktables', constants from one of the subpatterns must
+** be renumbered; 'correctkeys' corrects their indices (adding 'n'
+** to each of them)
+*/
+static void correctkeys (TTree *tree, int n) {
+  if (n == 0) return;  /* no correction? */
+ tailcall:
+  switch (tree->tag) {
+    case TOpenCall: case TCall: case TRunTime: case TRule: {
+      if (tree->key > 0)
+        tree->key += n;
+      break;
+    }
+    case TCapture: {
+      if (tree->key > 0 && tree->cap != Carg && tree->cap != Cnum)
+        tree->key += n;
+      break;
+    }
+    default: break;
+  }
+  switch (numsiblings[tree->tag]) {
+    case 1:  /* correctkeys(sib1(tree), n); */
+      tree = sib1(tree); goto tailcall;
+    case 2:
+      correctkeys(sib1(tree), n);
+      tree = sib2(tree); goto tailcall;  /* correctkeys(sib2(tree), n); */
+    default: assert(numsiblings[tree->tag] == 0); break;
+  }
+}
+
+
+/*
+** Join the ktables from p1 and p2 the ktable for the new pattern at the
+** top of the stack, reusing them when possible.
+*/
+static void joinktables (lua_State *L, int p1, TTree *t2, int p2) {
+  int n1, n2;
+  lua_getuservalue(L, p1);  /* get ktables */
+  lua_getuservalue(L, p2);
+  n1 = ktablelen(L, -2);
+  n2 = ktablelen(L, -1);
+  if (n1 == 0 && n2 == 0)  /* are both tables empty? */
+    lua_pop(L, 2);  /* nothing to be done; pop tables */
+  else if (n2 == 0 || lp_equal(L, -2, -1)) {  /* 2nd table empty or equal? */
+    lua_pop(L, 1);  /* pop 2nd table */
+    lua_setuservalue(L, -2);  /* set 1st ktable into new pattern */
+  }
+  else if (n1 == 0) {  /* first table is empty? */
+    lua_setuservalue(L, -3);  /* set 2nd table into new pattern */
+    lua_pop(L, 1);  /* pop 1st table */
+  }
+  else {
+    lua_createtable(L, n1 + n2, 0);  /* create ktable for new pattern */
+    /* stack: new p; ktable p1; ktable p2; new ktable */
+    concattable(L, -3, -1);  /* from p1 into new ktable */
+    concattable(L, -2, -1);  /* from p2 into new ktable */
+    lua_setuservalue(L, -4);  /* new ktable becomes 'p' environment */
+    lua_pop(L, 2);  /* pop other ktables */
+    correctkeys(t2, n1);  /* correction for indices from p2 */
+  }
+}
+
+
+/*
+** copy 'ktable' of element 'idx' to new tree (on top of stack)
+*/
+static void copyktable (lua_State *L, int idx) {
+  lua_getuservalue(L, idx);
+  lua_setuservalue(L, -2);
+}
+
+
+/*
+** merge 'ktable' from 'stree' at stack index 'idx' into 'ktable'
+** from tree at the top of the stack, and correct corresponding
+** tree.
+*/
+static void mergektable (lua_State *L, int idx, TTree *stree) {
+  int n;
+  lua_getuservalue(L, -1);  /* get ktables */
+  lua_getuservalue(L, idx);
+  n = concattable(L, -1, -2);
+  lua_pop(L, 2);  /* remove both ktables */
+  correctkeys(stree, n);
+}
+
+
+/*
+** Create a new 'ktable' to the pattern at the top of the stack, adding
+** all elements from pattern 'p' (if not 0) plus element 'idx' to it.
+** Return index of new element.
+*/
+static int addtonewktable (lua_State *L, int p, int idx) {
+  newktable(L, 1);
+  if (p)
+    mergektable(L, p, NULL);
+  return addtoktable(L, idx);
+}
+
+/* }====================================================== */
+
+
 /*
 ** {======================================================
 ** Tree generation
@@ -155,7 +338,7 @@ static Pattern *getpattern (lua_State *L, int idx) {
 
 
 static int getsize (lua_State *L, int idx) {
-  return (lua_objlen(L, idx) - sizeof(Pattern)) / sizeof(TTree) + 1;
+  return (lua_rawlen(L, idx) - sizeof(Pattern)) / sizeof(TTree) + 1;
 }
 
 
@@ -168,12 +351,16 @@ static TTree *gettree (lua_State *L, int idx, int *len) {
 
 
 /*
-** create a pattern
+** create a pattern. Set its uservalue (the 'ktable') equal to its
+** metatable. (It could be any empty sequence; the metatable is at
+** hand here, so we use it.)
 */
 static TTree *newtree (lua_State *L, int len) {
   size_t size = (len - 1) * sizeof(TTree) + sizeof(Pattern);
   Pattern *p = (Pattern *)lua_newuserdata(L, size);
   luaL_getmetatable(L, PATTERN_T);
+  lua_pushvalue(L, -1);
+  lua_setuservalue(L, -3);
   lua_setmetatable(L, -2);
   p->code = NULL;  p->codesize = 0;
   return p->tree;
@@ -206,35 +393,6 @@ static TTree *seqaux (TTree *tree, TTree *sib, int sibsize) {
 }
 
 
-/*
-** Add element 'idx' to 'ktable' of pattern at the top of the stack;
-** create new 'ktable' if necessary. Return index of new element.
-** If new element is nil, does not add it to table (as it would be
-** useless) and returns 0, as ktable[0] is always nil.
-*/
-static int addtoktable (lua_State *L, int idx) {
-  if (idx == 0)  /* no actual value to insert? */
-    return 0;
-  else {
-    int n;
-    lua_getfenv(L, -1);  /* get ktable from pattern */
-    n = lua_objlen(L, -1);
-    if (n == 0) {  /* is it empty/non-existent? */
-      lua_pop(L, 1);  /* remove it */
-      lua_createtable(L, 1, 0);  /* create a fresh table */
-      lua_pushvalue(L, -1);  /* make a copy */
-      lua_setfenv(L, -3);  /* set it as 'ktable' for pattern */
-    }
-    if (!lua_isnil(L, idx)) {  /* non-nil value? */
-      lua_pushvalue(L, idx);  /* element to be added */
-      lua_rawseti(L, -2, ++n);
-    }
-    lua_pop(L, 1);  /* remove 'ktable' */
-    return n;
-  }
-}
-
-
 /*
 ** Build a sequence of 'n' nodes, each with tag 'tag' and 'u.n' got
 ** from the array 's' (or 0 if array is NULL). (TSeq is binary, so it
@@ -310,7 +468,7 @@ static TTree *getpatt (lua_State *L, int idx, int *len) {
     case LUA_TFUNCTION: {
       tree = newtree(L, 2);
       tree->tag = TRunTime;
-      tree->key = addtoktable(L, idx);
+      tree->key = addtonewktable(L, 0, idx);
       sib1(tree)->tag = TTrue;
       break;
     }
@@ -325,123 +483,6 @@ static TTree *getpatt (lua_State *L, int idx, int *len) {
 }
 
 
-/*
-** Return the number of elements in the ktable of pattern at 'idx'.
-** In Lua 5.2, default "environment" for patterns is nil, not
-** a table. Treat it as an empty table. In Lua 5.1, assumes that
-** the environment has no numeric indices (len == 0)
-*/
-static int ktablelen (lua_State *L, int idx) {
-  if (!lua_istable(L, idx)) return 0;
-  else return lua_objlen(L, idx);
-}
-
-
-/*
-** Concatentate the contents of table 'idx1' into table 'idx2'.
-** (Assume that both indices are negative.)
-** Return the original length of table 'idx2'
-*/
-static int concattable (lua_State *L, int idx1, int idx2) {
-  int i;
-  int n1 = ktablelen(L, idx1);
-  int n2 = ktablelen(L, idx2);
-  if (n1 == 0) return 0;  /* nothing to correct */
-  for (i = 1; i <= n1; i++) {
-    lua_rawgeti(L, idx1, i);
-    lua_rawseti(L, idx2 - 1, n2 + i);  /* correct 'idx2' */
-  }
-  return n2;
-}
-
-
-/*
-** Make a merge of ktables from p1 and p2 the ktable for the new
-** pattern at the top of the stack.
-*/
-static int joinktables (lua_State *L, int p1, int p2) {
-  int n1, n2;
-  lua_getfenv(L, p1);  /* get ktables */
-  lua_getfenv(L, p2);
-  n1 = ktablelen(L, -2);
-  n2 = ktablelen(L, -1);
-  if (n1 == 0 && n2 == 0) {  /* are both tables empty? */
-    lua_pop(L, 2);  /* nothing to be done; pop tables */
-    return 0;  /* nothing to correct */
-  }
-  if (n2 == 0 || lua_equal(L, -2, -1)) {  /* second table is empty or equal? */
-    lua_pop(L, 1);  /* pop 2nd table */
-    lua_setfenv(L, -2);  /* set 1st ktable into new pattern */
-    return 0;  /* nothing to correct */
-  }
-  if (n1 == 0) {  /* first table is empty? */
-    lua_setfenv(L, -3);  /* set 2nd table into new pattern */
-    lua_pop(L, 1);  /* pop 1st table */
-    return 0;  /* nothing to correct */
-  }
-  else {
-    lua_createtable(L, n1 + n2, 0);  /* create ktable for new pattern */
-    /* stack: new p; ktable p1; ktable p2; new ktable */
-    concattable(L, -3, -1);  /* from p1 into new ktable */
-    concattable(L, -2, -1);  /* from p2 into new ktable */
-    lua_setfenv(L, -4);  /* new ktable becomes p env */
-    lua_pop(L, 2);  /* pop other ktables */
-    return n1;  /* correction for indices from p2 */
-  }
-}
-
-
-static void correctkeys (TTree *tree, int n) {
-  if (n == 0) return;  /* no correction? */
- tailcall:
-  switch (tree->tag) {
-    case TOpenCall: case TCall: case TRunTime: case TRule: {
-      if (tree->key > 0)
-        tree->key += n;
-      break;
-    }
-    case TCapture: {
-      if (tree->key > 0 && tree->cap != Carg && tree->cap != Cnum)
-        tree->key += n;
-      break;
-    }
-    default: break;
-  }
-  switch (numsiblings[tree->tag]) {
-    case 1:  /* correctkeys(sib1(tree), n); */
-      tree = sib1(tree); goto tailcall;
-    case 2:
-      correctkeys(sib1(tree), n);
-      tree = sib2(tree); goto tailcall;  /* correctkeys(sib2(tree), n); */
-    default: assert(numsiblings[tree->tag] == 0); break;
-  }
-}
-
-
-/*
-** copy 'ktable' of element 'idx' to new tree (on top of stack)
-*/
-static void copyktable (lua_State *L, int idx) {
-  lua_getfenv(L, idx);
-  lua_setfenv(L, -2);
-}
-
-
-/*
-** merge 'ktable' from rule at stack index 'idx' into 'ktable'
-** from tree at the top of the stack, and correct corresponding
-** tree.
-*/
-static void mergektable (lua_State *L, int idx, TTree *rule) {
-  int n;
-  lua_getfenv(L, -1);  /* get ktables */
-  lua_getfenv(L, idx);
-  n = concattable(L, -1, -2);
-  lua_pop(L, 2);  /* remove both ktables */
-  correctkeys(rule, n);
-}
-
-
 /*
 ** create a new tree, whith a new root and one sibling.
 ** Sibling must be on the Lua stack, at index 1.
@@ -470,7 +511,7 @@ static TTree *newroot2sib (lua_State *L, int tag) {
   tree->u.ps =  1 + s1;
   memcpy(sib1(tree), tree1, s1 * sizeof(TTree));
   memcpy(sib2(tree), tree2, s2 * sizeof(TTree));
-  correctkeys(sib2(tree), joinktables(L, 1, 2));
+  joinktables(L, 1, sib2(tree), 2);
   return tree;
 }
 
@@ -599,7 +640,7 @@ static int lp_sub (lua_State *L) {
     sib1(tree)->tag = TNot;  /* ...not... */
     memcpy(sib1(sib1(tree)), t2, s2 * sizeof(TTree));  /* ...t2 */
     memcpy(sib2(tree), t1, s1 * sizeof(TTree));  /* ... and t1 */
-    correctkeys(sib1(tree), joinktables(L, 1, 2));
+    joinktables(L, 1, sib1(tree), 2);
   }
   return 1;
 }
@@ -640,7 +681,7 @@ static int lp_behind (lua_State *L) {
   TTree *tree;
   TTree *tree1 = getpatt(L, 1, NULL);
   int n = fixedlen(tree1);
-  luaL_argcheck(L, n > 0, 1, "pattern may not have fixed length");
+  luaL_argcheck(L, n >= 0, 1, "pattern may not have fixed length");
   luaL_argcheck(L, !hascaptures(tree1), 1, "pattern have captures");
   luaL_argcheck(L, n <= MAXBEHIND, 1, "pattern too long to look behind");
   tree = newroot1sib(L, TBehind);
@@ -655,7 +696,7 @@ static int lp_behind (lua_State *L) {
 static int lp_V (lua_State *L) {
   TTree *tree = newleaf(L, TOpenCall);
   luaL_argcheck(L, !lua_isnoneornil(L, 1), 1, "non-nil value expected");
-  tree->key = addtoktable(L, 1);
+  tree->key = addtonewktable(L, 0, 1);
   return 1;
 }
 
@@ -668,7 +709,7 @@ static int lp_V (lua_State *L) {
 static int capture_aux (lua_State *L, int cap, int labelidx) {
   TTree *tree = newroot1sib(L, TCapture);
   tree->cap = cap;
-  tree->key = addtoktable(L, labelidx);
+  tree->key = (labelidx == 0) ? 0 : addtonewktable(L, 1, labelidx);
   return 1;
 }
 
@@ -676,10 +717,9 @@ static int capture_aux (lua_State *L, int cap, int labelidx) {
 /*
 ** Fill a tree with an empty capture, using an empty (TTrue) sibling.
 */
-static TTree *auxemptycap (lua_State *L, TTree *tree, int cap, int idx) {
+static TTree *auxemptycap (TTree *tree, int cap) {
   tree->tag = TCapture;
   tree->cap = cap;
-  tree->key = addtoktable(L, idx);
   sib1(tree)->tag = TTrue;
   return tree;
 }
@@ -688,8 +728,18 @@ static TTree *auxemptycap (lua_State *L, TTree *tree, int cap, int idx) {
 /*
 ** Create a tree for an empty capture
 */
-static TTree *newemptycap (lua_State *L, int cap, int idx) {
-  return auxemptycap(L, newtree(L, 2), cap, idx);
+static TTree *newemptycap (lua_State *L, int cap) {
+  return auxemptycap(newtree(L, 2), cap);
+}
+
+
+/*
+** Create a tree for an empty capture with an associated Lua value
+*/
+static TTree *newemptycapkey (lua_State *L, int cap, int idx) {
+  TTree *tree = auxemptycap(newtree(L, 2), cap);
+  tree->key = addtonewktable(L, 0, idx);
+  return tree;
 }
 
 
@@ -728,10 +778,8 @@ static int lp_tablecapture (lua_State *L) {
 static int lp_groupcapture (lua_State *L) {
   if (lua_isnoneornil(L, 2))
     return capture_aux(L, Cgroup, 0);
-  else {
-    luaL_checkstring(L, 2);
+  else
     return capture_aux(L, Cgroup, 2);
-  }
 }
 
 
@@ -747,14 +795,14 @@ static int lp_simplecapture (lua_State *L) {
 
 
 static int lp_poscapture (lua_State *L) {
-  newemptycap(L, Cposition, 0);
+  newemptycap(L, Cposition);
   return 1;
 }
 
 
 static int lp_argcapture (lua_State *L) {
   int n = (int)luaL_checkinteger(L, 1);
-  TTree *tree = newemptycap(L, Carg, 0);
+  TTree *tree = newemptycap(L, Carg);
   tree->key = n;
   luaL_argcheck(L, 0 < n && n <= SHRT_MAX, 1, "invalid argument index");
   return 1;
@@ -762,8 +810,8 @@ static int lp_argcapture (lua_State *L) {
 
 
 static int lp_backref (lua_State *L) {
-  luaL_checkstring(L, 1);
-  newemptycap(L, Cbackref, 1);
+  luaL_checkany(L, 1);
+  newemptycapkey(L, Cbackref, 1);
   return 1;
 }
 
@@ -777,9 +825,10 @@ static int lp_constcapture (lua_State *L) {
   if (n == 0)  /* no values? */
     newleaf(L, TTrue);  /* no capture */
   else if (n == 1)
-    newemptycap(L, Cconst, 1);  /* single constant capture */
+    newemptycapkey(L, Cconst, 1);  /* single constant capture */
   else {  /* create a group capture with all values */
     TTree *tree = newtree(L, 1 + 3 * (n - 1) + 2);
+    newktable(L, n);  /* create a 'ktable' for new tree */
     tree->tag = TCapture;
     tree->cap = Cgroup;
     tree->key = 0;
@@ -787,10 +836,12 @@ static int lp_constcapture (lua_State *L) {
     for (i = 1; i <= n - 1; i++) {
       tree->tag = TSeq;
       tree->u.ps = 3;  /* skip TCapture and its sibling */
-      auxemptycap(L, sib1(tree), Cconst, i);
+      auxemptycap(sib1(tree), Cconst);
+      sib1(tree)->key = addtoktable(L, i);
       tree = sib2(tree);
     }
-    auxemptycap(L, tree, Cconst, i);
+    auxemptycap(tree, Cconst);
+    tree->key = addtoktable(L, i);
   }
   return 1;
 }
@@ -800,7 +851,7 @@ static int lp_matchtime (lua_State *L) {
   TTree *tree;
   luaL_checktype(L, 2, LUA_TFUNCTION);
   tree = newroot1sib(L, TRunTime);
-  tree->key = addtoktable(L, 2);
+  tree->key = addtonewktable(L, 1, 2);
   return 1;
 }
 
@@ -857,7 +908,7 @@ static int collectrules (lua_State *L, int arg, int *totalsize) {
   lua_pushnil(L);  /* prepare to traverse grammar table */
   while (lua_next(L, arg) != 0) {
     if (lua_tonumber(L, -2) == 1 ||
-        lua_equal(L, -2, postab + 1)) {  /* initial rule? */
+        lp_equal(L, -2, postab + 1)) {  /* initial rule? */
       lua_pop(L, 1);  /* remove value (keep key for lua_next) */
       continue;
     }
@@ -934,36 +985,40 @@ static int verifyerror (lua_State *L, int *passed, int npassed) {
 
 /*
 ** Check whether a rule can be left recursive; raise an error in that
-** case; otherwise return 1 iff pattern is nullable. Assume ktable at
-** the top of the stack.
+** case; otherwise return 1 iff pattern is nullable.
+** The return value is used to check sequences, where the second pattern
+** is only relevant if the first is nullable.
+** Parameter 'nb' works as an accumulator, to allow tail calls in
+** choices. ('nb' true makes function returns true.)
+** Assume ktable at the top of the stack.
 */
 static int verifyrule (lua_State *L, TTree *tree, int *passed, int npassed,
-                       int nullable) {
+                       int nb) {
  tailcall:
   switch (tree->tag) {
     case TChar: case TSet: case TAny:
     case TFalse:
-      return nullable;  /* cannot pass from here */
+      return nb;  /* cannot pass from here */
     case TTrue:
     case TBehind:  /* look-behind cannot have calls */
       return 1;
     case TNot: case TAnd: case TRep:
       /* return verifyrule(L, sib1(tree), passed, npassed, 1); */
-      tree = sib1(tree); nullable = 1; goto tailcall;
+      tree = sib1(tree); nb = 1; goto tailcall;
     case TCapture: case TRunTime:
-      /* return verifyrule(L, sib1(tree), passed, npassed); */
+      /* return verifyrule(L, sib1(tree), passed, npassed, nb); */
       tree = sib1(tree); goto tailcall;
     case TCall:
-      /* return verifyrule(L, sib2(tree), passed, npassed); */
+      /* return verifyrule(L, sib2(tree), passed, npassed, nb); */
       tree = sib2(tree); goto tailcall;
-    case TSeq:  /* only check 2nd child if first is nullable */
+    case TSeq:  /* only check 2nd child if first is nb */
       if (!verifyrule(L, sib1(tree), passed, npassed, 0))
-        return nullable;
-      /* else return verifyrule(L, sib2(tree), passed, npassed); */
+        return nb;
+      /* else return verifyrule(L, sib2(tree), passed, npassed, nb); */
       tree = sib2(tree); goto tailcall;
     case TChoice:  /* must check both children */
-      nullable = verifyrule(L, sib1(tree), passed, npassed, nullable);
-      /* return verifyrule(L, sib2(tree), passed, npassed, nullable); */
+      nb = verifyrule(L, sib1(tree), passed, npassed, nb);
+      /* return verifyrule(L, sib2(tree), passed, npassed, nb); */
       tree = sib2(tree); goto tailcall;
     case TRule:
       if (npassed >= MAXRULES)
@@ -1006,7 +1061,7 @@ static void verifygrammar (lua_State *L, TTree *grammar) {
 */
 static void initialrulename (lua_State *L, TTree *grammar, int frule) {
   if (sib1(grammar)->key == 0) {  /* initial rule is not referenced? */
-    int n = lua_objlen(L, -1) + 1;  /* index for name */
+    int n = lua_rawlen(L, -1) + 1;  /* index for name */
     lua_pushvalue(L, frule);  /* rule's name */
     lua_rawseti(L, -2, n);  /* ktable was on the top of the stack */
     sib1(grammar)->key = n;
@@ -1022,9 +1077,9 @@ static TTree *newgrammar (lua_State *L, int arg) {
   luaL_argcheck(L, n <= MAXRULES, arg, "grammar has too many rules");
   g->tag = TGrammar;  g->u.n = n;
   lua_newtable(L);  /* create 'ktable' */
-  lua_setfenv(L, -2);
+  lua_setuservalue(L, -2);
   buildgrammar(L, g, frule, n);
-  lua_getfenv(L, -1);  /* get 'ktable' for new tree */
+  lua_getuservalue(L, -1);  /* get 'ktable' for new tree */
   finalfix(L, frule - 1, g, sib1(g));
   initialrulename(L, g, frule);
   verifygrammar(L, g);
@@ -1038,7 +1093,7 @@ static TTree *newgrammar (lua_State *L, int arg) {
 
 
 static Instruction *prepcompile (lua_State *L, Pattern *p, int idx) {
-  lua_getfenv(L, idx);  /* push 'ktable' (may be used by 'finalfix') */
+  lua_getuservalue(L, idx);  /* push 'ktable' (may be used by 'finalfix') */
   finalfix(L, 0, NULL, p->tree);
   lua_pop(L, 1);  /* remove 'ktable' */
   return compile(L, p);
@@ -1049,7 +1104,7 @@ static int lp_printtree (lua_State *L) {
   TTree *tree = getpatt(L, 1, NULL);
   int c = lua_toboolean(L, 2);
   if (c) {
-    lua_getfenv(L, 1);  /* push 'ktable' (may be used by 'finalfix') */
+    lua_getuservalue(L, 1);  /* push 'ktable' (may be used by 'finalfix') */
     finalfix(L, 0, NULL, tree);
     lua_pop(L, 1);  /* remove 'ktable' */
   }
@@ -1102,7 +1157,7 @@ static int lp_match (lua_State *L) {
   int ptop = lua_gettop(L);
   lua_pushnil(L);  /* initialize subscache */
   lua_pushlightuserdata(L, capture);  /* initialize caplistidx */
-  lua_getfenv(L, 1);  /* initialize penvidx */
+  lua_getuservalue(L, 1);  /* initialize penvidx */
   r = match(L, s, s + i, s + l, code, capture, ptop);
   if (r == NULL) {
     lua_pushnil(L);
@@ -1119,8 +1174,12 @@ static int lp_match (lua_State *L) {
 ** =======================================================
 */
 
+/* maximum limit for stack size */
+#define MAXLIM		(INT_MAX / 100)
+
 static int lp_setmax (lua_State *L) {
-  luaL_optinteger(L, 1, -1);
+  lua_Integer lim = luaL_checkinteger(L, 1);
+  luaL_argcheck(L, 0 < lim && lim <= MAXLIM, 1, "out of range");
   lua_settop(L, 1);
   lua_setfield(L, LUA_REGISTRYINDEX, MAXSTACKIDX);
   return 0;
@@ -1144,8 +1203,7 @@ static int lp_type (lua_State *L) {
 
 int lp_gc (lua_State *L) {
   Pattern *p = getpattern(L, 1);
-  if (p->codesize > 0)
-    realloccode(L, p, 0);
+  realloccode(L, p, 0);  /* delete code block */
   return 0;
 }
 
@@ -1228,8 +1286,8 @@ int luaopen_lpeg (lua_State *L) {
   luaL_newmetatable(L, PATTERN_T);
   lua_pushnumber(L, MAXBACK);  /* initialize maximum backtracking */
   lua_setfield(L, LUA_REGISTRYINDEX, MAXSTACKIDX);
-  luaL_register(L, NULL, metareg);
-  luaL_register(L, "lpeg", pattreg);
+  luaL_setfuncs(L, metareg, 0);
+  luaL_newlib(L, pattreg);
   lua_pushvalue(L, -1);
   lua_setfield(L, -3, "__index");
   return 1;

3rd/lpeg/lptypes.h

@@ -1,7 +1,7 @@
 /*
-** $Id: lptypes.h,v 1.10 2014/12/12 17:11:35 roberto Exp $
+** $Id: lptypes.h,v 1.14 2015/09/28 17:17:41 roberto Exp $
 ** LPeg - PEG pattern matching for Lua
-** Copyright 2007-2014, Lua.org & PUC-Rio  (see 'lpeg.html' for license)
+** Copyright 2007-2015, Lua.org & PUC-Rio  (see 'lpeg.html' for license)
 ** written by Roberto Ierusalimschy
 */
 
@@ -19,7 +19,7 @@
 #include "lua.h"
 
 
-#define VERSION         "0.12.1"
+#define VERSION         "1.0.0"
 
 
 #define PATTERN_T	"lpeg-pattern"
@@ -27,31 +27,31 @@
 
 
 /*
-** compatibility with Lua 5.2
+** compatibility with Lua 5.1
 */
-#if (LUA_VERSION_NUM >= 502)
+#if (LUA_VERSION_NUM == 501)
 
-#undef lua_equal
-#define lua_equal(L,idx1,idx2)  lua_compare(L,(idx1),(idx2),LUA_OPEQ)
+#define lp_equal	lua_equal
 
-#undef lua_getfenv
-#define lua_getfenv	lua_getuservalue
-#undef lua_setfenv
-#define lua_setfenv	lua_setuservalue
+#define lua_getuservalue	lua_getfenv
+#define lua_setuservalue	lua_setfenv
 
-#undef lua_objlen
-#define lua_objlen	lua_rawlen
+#define lua_rawlen		lua_objlen
 
-#undef luaL_register
-#define luaL_register(L,n,f) \
-	{ if ((n) == NULL) luaL_setfuncs(L,f,0); else luaL_newlib(L,f); }
+#define luaL_setfuncs(L,f,n)	luaL_register(L,NULL,f)
+#define luaL_newlib(L,f)	luaL_register(L,"lpeg",f)
 
 #endif
 
 
+#if !defined(lp_equal)
+#define lp_equal(L,idx1,idx2)  lua_compare(L,(idx1),(idx2),LUA_OPEQ)
+#endif
+
+
 /* default maximum size for call/backtrack stack */
 #if !defined(MAXBACK)
-#define MAXBACK         100
+#define MAXBACK         400
 #endif
 
 

3rd/lpeg/lpvm.c

@@ -1,5 +1,5 @@
 /*
-** $Id: lpvm.c,v 1.5 2013/04/12 16:29:49 roberto Exp $
+** $Id: lpvm.c,v 1.6 2015/09/28 17:01:25 roberto Exp $
 ** Copyright 2007, Lua.org & PUC-Rio  (see 'lpeg.html' for license)
 */
 
@@ -18,7 +18,7 @@
 
 /* initial size for call/backtrack stack */
 #if !defined(INITBACK)
-#define INITBACK	100
+#define INITBACK	MAXBACK
 #endif
 
 
@@ -70,7 +70,7 @@ static Stack *doublestack (lua_State *L, Stack **stacklimit, int ptop) {
   max = lua_tointeger(L, -1);  /* maximum allowed size */
   lua_pop(L, 1);
   if (n >= max)  /* already at maximum size? */
-    luaL_error(L, "too many pending calls/choices");
+    luaL_error(L, "backtrack stack overflow (current limit is %d)", max);
   newn = 2 * n;  /* new size */
   if (newn > max) newn = max;
   newstack = (Stack *)lua_newuserdata(L, newn * sizeof(Stack));

3rd/lpeg/re.html

@@ -10,7 +10,7 @@
 </head>
 <body>
 
-<!-- $Id: re.html,v 1.21 2013/03/28 20:43:30 roberto Exp $ -->
+<!-- $Id: re.html,v 1.23 2015/09/28 17:17:41 roberto Exp $ -->
 
 <div id="container">
 	
@@ -296,7 +296,7 @@ it would be useful if each table had
 a <code>tag</code> field telling what non terminal
 that table represents.
 We can add such a tag using
-<a href="lpeg.html/#cap-g">named group captures</a>:
+<a href="lpeg.html#cap-g">named group captures</a>:
 </p>
 <pre class="example">
 x = re.compile[[
@@ -450,7 +450,7 @@ print(re.match(p, p))   -- a self description must match itself
 <h2><a name="license">License</a></h2>
 
 <p>
-Copyright &copy; 2008-2010 Lua.org, PUC-Rio.
+Copyright &copy; 2008-2015 Lua.org, PUC-Rio.
 </p>
 <p>
 Permission is hereby granted, free of charge,
@@ -488,7 +488,7 @@ THE SOFTWARE.
 
 <div id="about">
 <p><small>
-$Id: re.html,v 1.21 2013/03/28 20:43:30 roberto Exp $
+$Id: re.html,v 1.23 2015/09/28 17:17:41 roberto Exp $
 </small></p>
 </div> <!-- id="about" -->
 

3rd/lpeg/test.lua

@@ -1,6 +1,6 @@
-#!/usr/bin/env lua5.1
+#!/usr/bin/env lua
 
--- $Id: test.lua,v 1.105 2014/12/12 17:00:39 roberto Exp $
+-- $Id: test.lua,v 1.109 2015/09/28 17:01:25 roberto Exp $
 
 -- require"strict"    -- just to be pedantic
 
@@ -16,9 +16,6 @@ local unpack = rawget(table, "unpack") or unpack
 local loadstring = rawget(_G, "loadstring") or load
 
 
--- most tests here do not need much stack space
-m.setmaxstack(5)
-
 local any = m.P(1)
 local space = m.S" \t\n"^0
 
@@ -291,6 +288,13 @@ assert(m.match(m.P"ab"^-1 - "c", "abcd") == 3)
 
 p = ('Aa' * ('Bb' * ('Cc' * m.P'Dd'^0)^0)^0)^-1
 assert(p:match("AaBbCcDdBbCcDdDdDdBb") == 21)
+
+
+-- bug in 0.12.2
+-- p = { ('ab' ('c' 'ef'?)*)? }
+p = m.C(('ab' * ('c' * m.P'ef'^-1)^0)^-1)
+s = "abcefccefc"
+assert(s == p:match(s))
  
 
 pi = "3.14159 26535 89793 23846 26433 83279 50288 41971 69399 37510"
@@ -352,6 +356,11 @@ checkeq(t, {hi = 10, ho = 20})
 t = p:match'abc'
 checkeq(t, {hi = 10, ho = 20, 'a', 'b', 'c'})
 
+-- non-string group names
+p = m.Ct(m.Cg(1, print) * m.Cg(1, 23.5) * m.Cg(1, io))
+t = p:match('abcdefghij')
+assert(t[print] == 'a' and t[23.5] == 'b' and t[io] == 'c')
+
 
 -- test for error messages
 local function checkerr (msg, f, ...)
@@ -388,6 +397,25 @@ p = m.P { (m.P {m.P'abc'} + 'ayz') * m.V'y'; y = m.P'x' }
 assert(p:match('abcx') == 5 and p:match('ayzx') == 5 and not p:match'abc')
 
 
+do
+  -- large dynamic Cc
+  local lim = 2^16 - 1
+  local c = 0
+  local function seq (n) 
+    if n == 1 then c = c + 1; return m.Cc(c)
+    else
+      local m = math.floor(n / 2)
+      return seq(m) * seq(n - m)
+    end
+  end
+  p = m.Ct(seq(lim))
+  t = p:match('')
+  assert(t[lim] == lim)
+  checkerr("too many", function () p = p / print end)
+  checkerr("too many", seq, lim + 1)
+end
+
+
 -- tests for non-pattern as arguments to pattern functions
 
 p = { ('a' * m.V(1))^-1 } * m.P'b' * { 'a' * m.V(2); m.V(1)^-1 }
@@ -575,9 +603,9 @@ assert(not p:match(string.rep("011", 10001)))
 -- this grammar does need backtracking info.
 local lim = 10000
 p = m.P{ '0' * m.V(1) + '0' }
-checkerr("too many pending", m.match, p, string.rep("0", lim))
+checkerr("stack overflow", m.match, p, string.rep("0", lim))
 m.setmaxstack(2*lim)
-checkerr("too many pending", m.match, p, string.rep("0", lim))
+checkerr("stack overflow", m.match, p, string.rep("0", lim))
 m.setmaxstack(2*lim + 4)
 assert(m.match(p, string.rep("0", lim)) == lim + 1)
 
@@ -586,7 +614,7 @@ p = m.P{ ('a' * m.V(1))^0 * 'b' + 'c' }
 m.setmaxstack(200)
 assert(p:match(string.rep('a', 180) .. 'c' .. string.rep('b', 180)) == 362)
 
-m.setmaxstack(5)   -- restore original limit
+m.setmaxstack(100)   -- restore low limit
 
 -- tests for optional start position
 assert(m.match("a", "abc", 1))
@@ -718,6 +746,10 @@ t = {m.match(m.Cc(nil,nil,4) * m.Cc(nil,3) * m.Cc(nil, nil) / g / g, "")}
 t1 = {1,1,nil,nil,4,nil,3,nil,nil}
 for i=1,10 do assert(t[i] == t1[i]) end
 
+-- bug in 0.12.2: ktable with only nil could be eliminated when joining
+-- with a pattern without ktable
+assert((m.P"aaa" * m.Cc(nil)):match"aaa" == nil)
+
 t = {m.match((m.C(1) / function (x) return x, x.."x" end)^0, "abc")}
 checkeq(t, {"a", "ax", "b", "bx", "c", "cx"})
 
@@ -925,6 +957,13 @@ p = m.Cg(m.C(1) * m.C(1), "k") * m.Ct(m.Cb("k"))
 t = p:match("ab")
 checkeq(t, {"a", "b"})
 
+p = m.P(true)
+for i = 1, 10 do p = p * m.Cg(1, i) end
+for i = 1, 10 do
+  local p = p * m.Cb(i)
+  assert(p:match('abcdefghij') == string.sub('abcdefghij', i, i))
+end
+
 
 t = {}
 function foo (p) t[#t + 1] = p; return p .. "x" end

3rd/lua/Makefile

@@ -19,7 +19,7 @@ SYSCFLAGS=
 SYSLDFLAGS=
 SYSLIBS=
 
-MYCFLAGS=
+MYCFLAGS=-I../../skynet-src
 MYLDFLAGS=
 MYLIBS=
 MYOBJS=

3rd/lua/README

@@ -1,5 +1,6 @@
 This is a modify version of lua 5.3.1 (http://www.lua.org/ftp/lua-5.3.1.tar.gz) .
 
-For detail : http://lua-users.org/lists/lua-l/2014-03/msg00489.html
-
-
+For detail , 
+  Shared Proto : http://lua-users.org/lists/lua-l/2014-03/msg00489.html
+  Shared short string table : http://blog.codingnow.com/2015/08/lua_vm_share_string.html
+  

3rd/lua/lapi.c

@@ -1000,7 +1000,7 @@ static Proto * cloneproto (lua_State *L, const Proto *src) {
     const TValue *s=&src->k[i];
     TValue *o=&f->k[i];
     if (ttisstring(s)) {
-      TString * str = luaS_newlstr(L,svalue(s),vslen(s));
+      TString * str = luaS_clonestring(L,tsvalue(s));
       setsvalue2n(L,o,str);
     } else {
       setobj(L,o,s);
@@ -1288,7 +1288,7 @@ static UpVal **getupvalref (lua_State *L, int fidx, int n, LClosure **pf) {
   StkId fi = index2addr(L, fidx);
   api_check(L, ttisLclosure(fi), "Lua function expected");
   f = clLvalue(fi);
-  api_check(L, (1 <= n && n <= f->p->sizeupvalues), "invalid upvalue index");
+  api_check(L, (1 <= n && n <= f->p->sp->sizeupvalues), "invalid upvalue index");
   if (pf) *pf = f;
   return &f->upvals[n - 1];  /* get its upvalue pointer */
 }

3rd/lua/lauxlib.c

@@ -972,29 +972,28 @@ LUALIB_API void luaL_checkversion_ (lua_State *L, lua_Number ver, size_t sz) {
 
 // use clonefunction
 
-#define LOCK(q) while (__sync_lock_test_and_set(&(q)->lock,1)) {}
-#define UNLOCK(q) __sync_lock_release(&(q)->lock);
+#include "spinlock.h"
 
 struct codecache {
-	int lock;
+	struct spinlock lock;
 	lua_State *L;
 };
 
-static struct codecache CC = { 0 , NULL };
+static struct codecache CC;
 
 static void
 clearcache() {
 	if (CC.L == NULL)
 		return;
-	LOCK(&CC)
+	SPIN_LOCK(&CC)
 		lua_close(CC.L);
 		CC.L = luaL_newstate();
-	UNLOCK(&CC)
+	SPIN_UNLOCK(&CC)
 }
 
 static void
 init() {
-	CC.lock = 0;
+	SPIN_INIT(&CC);
 	CC.L = luaL_newstate();
 }
 
@@ -1002,13 +1001,13 @@ static const void *
 load(const char *key) {
   if (CC.L == NULL)
     return NULL;
-  LOCK(&CC)
+  SPIN_LOCK(&CC)
     lua_State *L = CC.L;
     lua_pushstring(L, key);
     lua_rawget(L, LUA_REGISTRYINDEX);
     const void * result = lua_touserdata(L, -1);
     lua_pop(L, 1);
-  UNLOCK(&CC)
+  SPIN_UNLOCK(&CC)
 
   return result;
 }
@@ -1018,7 +1017,7 @@ save(const char *key, const void * proto) {
   lua_State *L;
   const void * result = NULL;
 
-  LOCK(&CC)
+  SPIN_LOCK(&CC)
     if (CC.L == NULL) {
       init();
       L = CC.L;
@@ -1036,17 +1035,66 @@ save(const char *key, const void * proto) {
         lua_pop(L,2);
       }
     }
-  UNLOCK(&CC)
+  SPIN_UNLOCK(&CC)
   return result;
 }
 
+#define CACHE_OFF 0
+#define CACHE_EXIST 1
+#define CACHE_ON 2
+
+static int cache_key = 0;
+
+static int cache_level(lua_State *L) {
+	int t = lua_rawgetp(L, LUA_REGISTRYINDEX, &cache_key);
+	int r = lua_tointeger(L, -1);
+	lua_pop(L,1);
+	if (t == LUA_TNUMBER) {
+		return r;
+	}
+	return CACHE_ON;
+}
+
+static int cache_mode(lua_State *L) {
+	static const char * lst[] = {
+		"OFF",
+		"EXIST",
+		"ON",
+		NULL,
+	};
+	if (lua_isnoneornil(L,1)) {
+		int t = lua_rawgetp(L, LUA_REGISTRYINDEX, &cache_key);
+		int r = lua_tointeger(L, -1);
+		if (t == LUA_TNUMBER) {
+			if (r < 0  || r >= CACHE_ON) {
+				r = CACHE_ON;
+			}
+		} else {
+			r = CACHE_ON;
+		}
+		lua_pushstring(L, lst[r]);
+		return 1;
+	}
+	int t = luaL_checkoption(L, 1, "OFF" , lst);
+	lua_pushinteger(L, t);
+	lua_rawsetp(L, LUA_REGISTRYINDEX, &cache_key);
+	return 0;
+}
+
 LUALIB_API int luaL_loadfilex (lua_State *L, const char *filename,
                                              const char *mode) {
+  int level = cache_level(L);
+  if (level == CACHE_OFF) {
+    return luaL_loadfilex_(L, filename, mode);
+  }
   const void * proto = load(filename);
   if (proto) {
     lua_clonefunction(L, proto);
     return LUA_OK;
   }
+  if (level == CACHE_EXIST) {
+    return luaL_loadfilex_(L, filename, mode);
+  }
   lua_State * eL = luaL_newstate();
   if (eL == NULL) {
     lua_pushliteral(L, "New state failed");
@@ -1083,6 +1131,7 @@ cache_clear(lua_State *L) {
 LUAMOD_API int luaopen_cache(lua_State *L) {
 	luaL_Reg l[] = {
 		{ "clear", cache_clear },
+		{ "mode", cache_mode },
 		{ NULL, NULL },
 	};
 	luaL_newlib(L,l);

3rd/lua/lfunc.c

@@ -135,6 +135,7 @@ static void freesharedproto (lua_State *L, SharedProto *f) {
   luaM_freearray(L, f->lineinfo, f->sizelineinfo);
   luaM_freearray(L, f->locvars, f->sizelocvars);
   luaM_freearray(L, f->upvalues, f->sizeupvalues);
+  luaM_free(L, f);
 }
 
 void luaF_freeproto (lua_State *L, Proto *f) {

3rd/lua/lgc.c

@@ -188,7 +188,8 @@ void luaC_upvalbarrier_ (lua_State *L, UpVal *uv) {
 
 void luaC_fix (lua_State *L, GCObject *o) {
   global_State *g = G(L);
-  lua_assert(g->allgc == o);  /* object must be 1st in 'allgc' list! */
+  if (g->allgc != o)
+	  return;  /* if object is not 1st in 'allgc' list, it is in global short string table */
   white2gray(o);  /* they will be gray forever */
   g->allgc = o->next;  /* remove object from 'allgc' list */
   o->next = g->fixedgc;  /* link it to 'fixedgc' list */

3rd/lua/lstring.c

@@ -148,10 +148,9 @@ void luaS_remove (lua_State *L, TString *ts) {
 /*
 ** checks whether short string exists and reuses it or creates a new one
 */
-static TString *internshrstr (lua_State *L, const char *str, size_t l) {
+static TString *queryshrstr (lua_State *L, const char *str, size_t l, unsigned int h) {
   TString *ts;
   global_State *g = G(L);
-  unsigned int h = luaS_hash(str, l, g->seed);
   TString **list = &g->strt.hash[lmod(h, g->strt.size)];
   for (ts = *list; ts != NULL; ts = ts->u.hnext) {
     if (l == ts->shrlen &&
@@ -162,6 +161,13 @@ static TString *internshrstr (lua_State *L, const char *str, size_t l) {
       return ts;
     }
   }
+  return NULL;
+}
+
+static TString *addshrstr (lua_State *L, const char *str, size_t l, unsigned int h) {
+  TString *ts;
+  global_State *g = G(L);
+  TString **list = &g->strt.hash[lmod(h, g->strt.size)];
   if (g->strt.nuse >= g->strt.size && g->strt.size <= MAX_INT/2) {
     luaS_resize(L, g->strt.size * 2);
     list = &g->strt.hash[lmod(h, g->strt.size)];  /* recompute with new size */
@@ -174,6 +180,7 @@ static TString *internshrstr (lua_State *L, const char *str, size_t l) {
   return ts;
 }
 
+static TString *internshrstr (lua_State *L, const char *str, size_t l);
 
 /*
 ** new string (with explicit length)
@@ -224,3 +231,213 @@ Udata *luaS_newudata (lua_State *L, size_t s) {
   return u;
 }
 
+/*
+ * global shared table
+ */
+
+#include "rwlock.h"
+#include "atomic.h"
+#include <stdlib.h>
+
+#define SHRSTR_SLOT 0x10000
+#define HASH_NODE(h) ((h) % SHRSTR_SLOT)
+
+struct shrmap_slot {
+	struct rwlock lock;
+	TString *str;
+};
+
+struct shrmap {
+	struct shrmap_slot h[SHRSTR_SLOT];
+	int n;
+};
+
+static struct shrmap SSM;
+
+LUA_API void
+luaS_initshr() {
+	struct shrmap * s = &SSM;
+	int i;
+	for (i=0;i<SHRSTR_SLOT;i++) {
+		rwlock_init(&s->h[i].lock);
+	}
+}
+
+LUA_API void
+luaS_exitshr() {
+	int i;
+	for (i=0;i<SHRSTR_SLOT;i++) {
+		TString *str = SSM.h[i].str;
+		while (str) {
+			TString * next = str->u.hnext;
+			free(str);
+			str = next;
+		}
+	}
+}
+
+static TString *
+query_string(unsigned int h, const char *str, lu_byte l) {
+	struct shrmap_slot *s = &SSM.h[HASH_NODE(h)];
+	rwlock_rlock(&s->lock);
+	TString *ts = s->str;
+	while (ts) {
+		if (ts->hash == h && 
+			ts->shrlen == l &&
+			memcmp(str, ts+1, l) == 0) {
+			break;
+		}
+		ts = ts->u.hnext;
+	}
+	rwlock_runlock(&s->lock);
+	return ts;
+}
+
+static TString *
+query_ptr(TString *t) {
+	unsigned int h = t->hash;
+	struct shrmap_slot *s = &SSM.h[HASH_NODE(h)];
+	rwlock_rlock(&s->lock);
+	TString *ts = s->str;
+	while (ts) {
+		if (ts == t)
+			break;
+		ts = ts->u.hnext;
+	}
+	rwlock_runlock(&s->lock);
+	return ts;
+}
+
+static TString *
+new_string(unsigned int h, const char *str, lu_byte l) {
+	size_t sz = sizelstring(l);
+	TString *ts = malloc(sz);
+	memset(ts, 0, sz);
+	ts->tt = LUA_TSHRSTR;
+	ts->hash = h;
+	ts->shrlen = l;
+	memcpy(ts+1, str, l);
+	return ts;
+}
+
+static TString *
+add_string(unsigned int h, const char *str, lu_byte l) {
+	TString * tmp = new_string(h, str, l);
+	struct shrmap_slot *s = &SSM.h[HASH_NODE(h)];
+	rwlock_wlock(&s->lock);
+	TString *ts = s->str;
+	while (ts) {
+		if (ts->hash == h && 
+			ts->shrlen == l &&
+			memcmp(str, ts+1, l) == 0) {
+				break;
+		}
+		ts = ts->u.hnext;
+	}
+	if (ts == NULL) {
+		ts = tmp;
+		ts->u.hnext = s->str;
+		s->str = ts;
+		tmp = NULL;
+	}
+	rwlock_wunlock(&s->lock);
+	if (tmp) {
+		// string is create by other thread, so free tmp
+		free(tmp);
+	}
+	return ts;
+}
+
+static TString *
+internshrstr (lua_State *L, const char *str, size_t l) {
+  TString *ts;
+  global_State *g = G(L);
+  unsigned int h = luaS_hash(str, l, g->seed);
+  unsigned int h0;
+  // lookup global state of this L first
+  ts = queryshrstr (L, str, l, h);
+  if (ts)
+    return ts;
+  // lookup SSM again
+  h0 = luaS_hash(str, l, 0);
+  ts = query_string(h0, str, l);
+  if (ts)
+    return ts;
+  // If SSM.n greate than 0, add it to SSM
+  if (SSM.n > 0) {
+    ATOM_DEC(&SSM.n);
+    return add_string(h0, str, l);
+  }
+  // Else add it to global state (local)
+  return addshrstr (L, str, l, h);
+}
+
+LUA_API void
+luaS_expandshr(int n) {
+  ATOM_ADD(&SSM.n, n);
+}
+
+LUAI_FUNC TString *
+luaS_clonestring(lua_State *L, TString *ts) {
+  unsigned int h;
+  int l;
+  const char * str = getaddrstr(ts);
+  global_State *g = G(L);
+  TString *result;
+  if (ts->tt == LUA_TLNGSTR) 
+    return luaS_newlstr(L, str, ts->u.lnglen);
+  // look up global state of this L first
+  l = ts->shrlen;
+  h = luaS_hash(str, l, g->seed);
+  result = queryshrstr (L, str, l, h);
+  if (result)
+    return result;
+  // look up SSM by ptr
+  result = query_ptr(ts);
+  if (result)
+    return result;
+  // ts is not in SSM, so recalc hash, and add it to SSM
+  h = luaS_hash(str, l, 0);
+  return add_string(h, str, l);
+}
+
+struct slotinfo {
+	int len;
+	int size;
+};
+
+static void
+getslot(struct shrmap_slot *s, struct slotinfo *info) {
+	memset(info, 0, sizeof(*info));
+	rwlock_rlock(&s->lock);
+	TString *ts = s->str;
+	while (ts) {
+		++info->len;
+		info->size += sizelstring(ts->shrlen);
+		ts = ts->u.hnext;
+	}
+	rwlock_runlock(&s->lock);
+}
+
+LUA_API int
+luaS_shrinfo(lua_State *L) {
+	struct slotinfo total;
+	struct slotinfo tmp;
+	memset(&total, 0, sizeof(total));
+	int i;
+	int len = 0;
+	for (i=0;i<SHRSTR_SLOT;i++) {
+		struct shrmap_slot *s = &SSM.h[i];
+		getslot(s, &tmp);
+		len += tmp.len;
+		if (tmp.len > total.len) {
+			total.len = tmp.len;
+		}
+		total.size += tmp.size;
+	}
+	lua_pushinteger(L, len);
+	lua_pushinteger(L, total.size);
+	lua_pushinteger(L, total.len);
+	lua_pushinteger(L, SSM.n);
+	return 4;
+}

3rd/lua/lstring.h

@@ -43,5 +43,12 @@ LUAI_FUNC Udata *luaS_newudata (lua_State *L, size_t s);
 LUAI_FUNC TString *luaS_newlstr (lua_State *L, const char *str, size_t l);
 LUAI_FUNC TString *luaS_new (lua_State *L, const char *str);
 
+#define ENABLE_SHORT_STRING_TABLE
+
+LUA_API void luaS_initshr();
+LUA_API void luaS_exitshr();
+LUA_API void luaS_expandshr(int n);
+LUAI_FUNC TString *luaS_clonestring(lua_State *L, TString *);
+LUA_API int luaS_shrinfo(lua_State *L);
 
 #endif

3rd/lua/lua.c

@@ -18,6 +18,7 @@
 
 #include "lauxlib.h"
 #include "lualib.h"
+#include "lstring.h"
 
 
 #if !defined(LUA_PROMPT)
@@ -596,7 +597,9 @@ static int pmain (lua_State *L) {
 
 int main (int argc, char **argv) {
   int status, result;
-  lua_State *L = luaL_newstate();  /* create state */
+  lua_State *L;
+  luaS_initshr();  /* init global short string table */
+  L = luaL_newstate();  /* create state */
   if (L == NULL) {
     l_message(argv[0], "cannot create state: not enough memory");
     return EXIT_FAILURE;
@@ -608,6 +611,7 @@ int main (int argc, char **argv) {
   result = lua_toboolean(L, -1);  /* get result */
   report(L, status);
   lua_close(L);
+  luaS_exitshr();
   return (result && status == LUA_OK) ? EXIT_SUCCESS : EXIT_FAILURE;
 }
 

3rd/lua/luac.c

@@ -21,6 +21,7 @@
 #include "lobject.h"
 #include "lstate.h"
 #include "lundump.h"
+#include "lstring.h"
 
 static void PrintFunction(const Proto* f, int full);
 #define luaU_print	PrintFunction
@@ -195,6 +196,7 @@ int main(int argc, char* argv[])
  int i=doargs(argc,argv);
  argc-=i; argv+=i;
  if (argc<=0) usage("no input files given");
+ luaS_initshr();
  L=luaL_newstate();
  if (L==NULL) fatal("cannot create state: not enough memory");
  lua_pushcfunction(L,&pmain);

HISTORY.md

@@ -1,10 +1,39 @@
+v1.0.0-beta (2015-11-10)
+-----------
+* Improve and fix bug for sproto
+* Add global short string pool for lua vm
+* Add code cache mode
+* Add a callback for mysql auth
+* Add hmac_md5
+* Sharedata support filename as a string
+* Fix a bug in socket.httpc
+* Fix a lua stack overflow bug in lua bson
+* Fix a socketchannel bug may block the data steam
+* Avoid dead loop when sending message to the service exiting
+* Fix memory leak in netpack
+* Improve DH key exchange implement
+* Minor fix for socket
+* Minor fix for multicast
+* Update jemalloc to 4.0.4
+* Update lpeg to 1.0.0
+
+v1.0.0-alpha10 (2015-8-17)
+-----------
+* Remove the size limit of cluster RPC message.
+* Remove the size limit of local message.
+* Add cluster.query and clsuter.register.
+* Add an option of pthread mutex lock.
+* Add skynet.core.intcommand to optimize skynet.sleep etc.
+* Fix a memory leak bug in lua shared proto.
+* snax.msgserver use string instead of lightuserdata/size.
+* Remove some unused api in netpack.
+* Raise error when skynet.send to 0.
+
 v1.0.0-alpha9 (2015-8-10)
 -----------
 * Improve lua serialization , support pairs metamethod.
 * Bugfix : sproto (See commits log of sproto)
 * Add user log service support (In config)
-* Remove the size limit of cluster RPC message.
-* Remove the size limit of local message.
 * Other minor bugfix (See commits log)
 
 v1.0.0-alpha8 (2015-6-29)

Makefile

@@ -5,7 +5,8 @@ CSERVICE_PATH ?= cservice
 
 SKYNET_BUILD_PATH ?= .
 
-CFLAGS = -g -O2 -Wall -I$(LUA_INC) $(MYCFLAGS) 
+CFLAGS = -g -O2 -Wall -I$(LUA_INC) $(MYCFLAGS)
+# CFLAGS += -DUSE_PTHREAD_LOCK
 
 # lua
 
@@ -23,7 +24,7 @@ JEMALLOC_INC := 3rd/jemalloc/include/jemalloc
 
 all : jemalloc
 	
-.PHONY : jemalloc
+.PHONY : jemalloc update3rd
 
 MALLOC_STATICLIB := $(JEMALLOC_STATICLIB)
 
@@ -38,6 +39,9 @@ $(JEMALLOC_STATICLIB) : 3rd/jemalloc/Makefile
 
 jemalloc : $(MALLOC_STATICLIB)
 
+update3rd :
+	rm -rf 3rd/jemalloc && git submodule update --init
+
 # skynet
 
 CSERVICE = snlua logger gate harbor
@@ -79,7 +83,7 @@ $(LUA_CLIB_PATH)/socketdriver.so : lualib-src/lua-socket.c | $(LUA_CLIB_PATH)
 	$(CC) $(CFLAGS) $(SHARED) $^ -o $@ -Iskynet-src -Iservice-src
 
 $(LUA_CLIB_PATH)/bson.so : lualib-src/lua-bson.c | $(LUA_CLIB_PATH)
-	$(CC) $(CFLAGS) $(SHARED) $^ -o $@ -Iskynet-src
+	$(CC) $(CFLAGS) $(SHARED) -Iskynet-src $^ -o $@ -Iskynet-src
 
 $(LUA_CLIB_PATH)/mongo.so : lualib-src/lua-mongo.c | $(LUA_CLIB_PATH)
 	$(CC) $(CFLAGS) $(SHARED) $^ -o $@ -Iskynet-src
@@ -109,7 +113,7 @@ $(LUA_CLIB_PATH)/crypt.so : lualib-src/lua-crypt.c lualib-src/lsha1.c | $(LUA_CL
 	$(CC) $(CFLAGS) $(SHARED) $^ -o $@ 
 
 $(LUA_CLIB_PATH)/sharedata.so : lualib-src/lua-sharedata.c | $(LUA_CLIB_PATH)
-	$(CC) $(CFLAGS) $(SHARED) $^ -o $@ 
+	$(CC) $(CFLAGS) $(SHARED) -Iskynet-src $^ -o $@ 
 
 $(LUA_CLIB_PATH)/stm.so : lualib-src/lua-stm.c | $(LUA_CLIB_PATH)
 	$(CC) $(CFLAGS) $(SHARED) -Iskynet-src $^ -o $@ 
@@ -124,7 +128,7 @@ $(LUA_CLIB_PATH)/mysqlaux.so : lualib-src/lua-mysqlaux.c | $(LUA_CLIB_PATH)
 	$(CC) $(CFLAGS) $(SHARED) $^ -o $@	
 
 $(LUA_CLIB_PATH)/debugchannel.so : lualib-src/lua-debugchannel.c | $(LUA_CLIB_PATH)
-	$(CC) $(CFLAGS) $(SHARED) $^ -o $@	
+	$(CC) $(CFLAGS) $(SHARED) -Iskynet-src $^ -o $@	
 
 clean :
 	rm -f $(SKYNET_BUILD_PATH)/skynet $(CSERVICE_PATH)/*.so $(LUA_CLIB_PATH)/*.so

examples/cluster1.lua

@@ -1,15 +1,17 @@
 local skynet = require "skynet"
 local cluster = require "cluster"
-require "skynet.manager"	-- import skynet.name
 local snax = require "snax"
 
 skynet.start(function()
 	local sdb = skynet.newservice("simpledb")
-	skynet.name(".simpledb", sdb)
-	print(skynet.call(".simpledb", "lua", "SET", "a", "foobar"))
-	print(skynet.call(".simpledb", "lua", "SET", "b", "foobar2"))
-	print(skynet.call(".simpledb", "lua", "GET", "a"))
-	print(skynet.call(".simpledb", "lua", "GET", "b"))
+	-- register name "sdb" for simpledb, you can use cluster.query() later.
+	-- See cluster2.lua
+	cluster.register("sdb", sdb)
+
+	print(skynet.call(sdb, "lua", "SET", "a", "foobar"))
+	print(skynet.call(sdb, "lua", "SET", "b", "foobar2"))
+	print(skynet.call(sdb, "lua", "GET", "a"))
+	print(skynet.call(sdb, "lua", "GET", "b"))
 	cluster.open "db"
 	cluster.open "db2"
 	-- unique snax service

examples/cluster2.lua

@@ -2,10 +2,18 @@ local skynet = require "skynet"
 local cluster = require "cluster"
 
 skynet.start(function()
-	local proxy = cluster.proxy("db", ".simpledb")
-	print(skynet.call(proxy, "lua", "GET", "a"))
-	print(cluster.call("db", ".simpledb", "GET", "a"))
-	print(cluster.call("db2", ".simpledb", "GET", "b"))
+	-- query name "sdb" of cluster db.
+	local sdb = cluster.query("db", "sdb")
+	print("db.sbd=",sdb)
+	local proxy = cluster.proxy("db", sdb)
+	local largekey = string.rep("X", 128*1024)
+	local largevalue = string.rep("R", 100 * 1024)
+	print(skynet.call(proxy, "lua", "SET", largekey, largevalue))
+	local v = skynet.call(proxy, "lua", "GET", largekey)
+	assert(largevalue == v)
+
+	print(cluster.call("db", sdb, "GET", "a"))
+	print(cluster.call("db2", sdb, "GET", "b"))
 
 	-- test snax service
 	local pingserver = cluster.snax("db", "pingserver")

examples/config.userlog

@@ -3,12 +3,9 @@ thread = 8
 logger = "userlog"
 logservice = "snlua"
 logpath = "."
-harbor = 1
-address = "127.0.0.1:2526"
-master = "127.0.0.1:2013"
+harbor = 0
 start = "main"	-- main script
 bootstrap = "snlua bootstrap"	-- The service for bootstrap
-standalone = "0.0.0.0:2013"
 luaservice = root.."service/?.lua;"..root.."test/?.lua;"..root.."examples/?.lua"
 lualoader = "lualib/loader.lua"
 -- preload = "./examples/preload.lua"	-- run preload.lua before every lua service run

examples/login/gated.lua

@@ -74,9 +74,9 @@ function server.disconnect_handler(username)
 end
 
 -- call by self (when recv a request from client)
-function server.request_handler(username, msg, sz)
+function server.request_handler(username, msg)
 	local u = username_map[username]
-	return skynet.tostring(skynet.rawcall(u.agent, "client", msg, sz))
+	return skynet.tostring(skynet.rawcall(u.agent, "client", msg))
 end
 
 -- call by self (when gate open)

examples/watchdog.lua

@@ -32,6 +32,11 @@ function SOCKET.error(fd, msg)
 	close_agent(fd)
 end
 
+function SOCKET.warning(fd, size)
+	-- size K bytes havn't send out in fd
+	print("socket warning", fd, size)
+end
+
 function SOCKET.data(fd, msg)
 end
 

lualib-src/lua-bson.c

@@ -8,9 +8,12 @@
 #include <stdlib.h>
 #include <string.h>
 #include <stdbool.h>
+#include "atomic.h"
 
 #define DEFAULT_CAP 64
 #define MAX_NUMBER 1024
+// avoid circular reference while encodeing
+#define MAX_DEPTH 128
 
 #define BSON_REAL 1
 #define BSON_STRING 2
@@ -235,7 +238,7 @@ write_double(struct bson *b, lua_Number d) {
 	}
 }
 
-static void pack_dict(lua_State *L, struct bson *b, bool array);
+static void pack_dict(lua_State *L, struct bson *b, bool array, int depth);
 
 static inline void
 append_key(struct bson *bs, int type, const char *key, size_t sz) {
@@ -267,7 +270,7 @@ append_number(struct bson *bs, lua_State *L, const char *key, size_t sz) {
 }
 
 static void
-append_table(struct bson *bs, lua_State *L, const char *key, size_t sz) {
+append_table(struct bson *bs, lua_State *L, const char *key, size_t sz, int depth) {
 	size_t len = lua_rawlen(L, -1);
 	bool isarray = false;
 	if (len > 0) {
@@ -283,7 +286,7 @@ append_table(struct bson *bs, lua_State *L, const char *key, size_t sz) {
 	} else {
 		append_key(bs, BSON_DOCUMENT, key, sz);
 	}
-	pack_dict(L, bs, isarray);
+	pack_dict(L, bs, isarray, depth);
 }
 
 static void
@@ -296,7 +299,7 @@ write_binary(struct bson *b, const void * buffer, size_t sz) {
 }
 
 static void
-append_one(struct bson *bs, lua_State *L, const char *key, size_t sz) {
+append_one(struct bson *bs, lua_State *L, const char *key, size_t sz, int depth) {
 	int vt = lua_type(L,-1);
 	switch(vt) {
 	case LUA_TNUMBER:
@@ -384,7 +387,7 @@ append_one(struct bson *bs, lua_State *L, const char *key, size_t sz) {
 		break;
 	}
 	case LUA_TTABLE:
-		append_table(bs, L, key, sz);
+		append_table(bs, L, key, sz, depth+1);
 		break;
 	case LUA_TBOOLEAN:
 		append_key(bs, BSON_BOOLEAN, key, sz);
@@ -406,7 +409,11 @@ bson_numstr( char *str, unsigned int i ) {
 }
 
 static void
-pack_dict(lua_State *L, struct bson *b, bool isarray) {
+pack_dict(lua_State *L, struct bson *b, bool isarray, int depth) {
+	if (depth > MAX_DEPTH) {
+		luaL_error(L, "Too depth while encoding bson");
+	}
+	luaL_checkstack(L, 16, NULL);	// reserve enough stack space to pack table
 	int length = reserve_length(b);
 	lua_pushnil(L);
 	while(lua_next(L,-2) != 0) {
@@ -422,7 +429,7 @@ pack_dict(lua_State *L, struct bson *b, bool isarray) {
 			sz = bson_numstr(numberkey, (unsigned int)lua_tointeger(L,-2)-1);
 			key = numberkey;
 
-			append_one(b, L, key, sz);
+			append_one(b, L, key, sz, depth);
 			lua_pop(L,1);
 		} else {
 			switch(kt) {
@@ -431,12 +438,12 @@ pack_dict(lua_State *L, struct bson *b, bool isarray) {
 				lua_pushvalue(L,-2);
 				lua_insert(L,-2);
 				key = lua_tolstring(L,-2,&sz);
-				append_one(b, L, key, sz);
+				append_one(b, L, key, sz, depth);
 				lua_pop(L,2);
 				break;
 			case LUA_TSTRING:
 				key = lua_tolstring(L,-2,&sz);
-				append_one(b, L, key, sz);
+				append_one(b, L, key, sz, depth);
 				lua_pop(L,1);
 				break;
 			default:
@@ -450,7 +457,7 @@ pack_dict(lua_State *L, struct bson *b, bool isarray) {
 }
 
 static void
-pack_ordered_dict(lua_State *L, struct bson *b, int n) {
+pack_ordered_dict(lua_State *L, struct bson *b, int n, int depth) {
 	int length = reserve_length(b);
 	int i;
 	for (i=0;i<n;i+=2) {
@@ -460,7 +467,7 @@ pack_ordered_dict(lua_State *L, struct bson *b, int n) {
 			luaL_error(L, "Argument %d need a string", i+1);
 		}
 		lua_pushvalue(L, i+2);
-		append_one(b, L, key, sz);
+		append_one(b, L, key, sz, depth);
 		lua_pop(L,1);
 	}
 	write_byte(b,0);
@@ -494,6 +501,7 @@ make_object(lua_State *L, int type, const void * ptr, size_t len) {
 
 static void
 unpack_dict(lua_State *L, struct bson_reader *br, bool array) {
+	luaL_checkstack(L, 16, NULL);	// reserve enough stack space to unpack table
 	int sz = read_int32(L, br);
 	const void * bytes = read_bytes(L, br, sz-5);
 	struct bson_reader t = { bytes, sz-5 };
@@ -845,7 +853,7 @@ lencode(lua_State *L) {
 	bson_create(&b);
 	lua_settop(L,1);
 	luaL_checktype(L, 1, LUA_TTABLE);
-	pack_dict(L, &b, false);
+	pack_dict(L, &b, false, 0);
 	void * ud = lua_newuserdata(L, b.size);
 	memcpy(ud, b.ptr, b.size);
 	bson_destroy(&b);
@@ -861,7 +869,7 @@ lencode_order(lua_State *L) {
 	if (n%2 != 0) {
 		return luaL_error(L, "Invalid ordered dict");
 	}
-	pack_ordered_dict(L, &b, n);
+	pack_ordered_dict(L, &b, n, 0);
 	lua_settop(L,1);
 	void * ud = lua_newuserdata(L, b.size);
 	memcpy(ud, b.ptr, b.size);
@@ -1140,7 +1148,7 @@ lobjectid(lua_State *L) {
 	} else {
 		time_t ti = time(NULL);
 		// old_counter is a static var, use atom inc.
-		uint32_t id = __sync_fetch_and_add(&oid_counter,1);
+		uint32_t id = ATOM_FINC(&oid_counter);
 
 		oid[2] = (ti>>24) & 0xff;
 		oid[3] = (ti>>16) & 0xff;

lualib-src/lua-clientsocket.c

@@ -128,11 +128,8 @@ lusleep(lua_State *L) {
 
 #define QUEUE_SIZE 1024
 
-#define LOCK(q) while (__sync_lock_test_and_set(&(q)->lock,1)) {}
-#define UNLOCK(q) __sync_lock_release(&(q)->lock);
-
 struct queue {
-	int lock;
+	pthread_mutex_t lock;
 	int head;
 	int tail;
 	char * queue[QUEUE_SIZE];
@@ -153,7 +150,7 @@ readline_stdin(void * arg) {
 		memcpy(str, tmp, n);
 		str[n] = 0;
 
-		LOCK(q);
+		pthread_mutex_lock(&q->lock);
 		q->queue[q->tail] = str;
 
 		if (++q->tail >= QUEUE_SIZE) {
@@ -163,7 +160,7 @@ readline_stdin(void * arg) {
 			// queue overflow
 			exit(1);
 		}
-		UNLOCK(q);
+		pthread_mutex_unlock(&q->lock);
 	}
 	return NULL;
 }
@@ -171,16 +168,16 @@ readline_stdin(void * arg) {
 static int
 lreadstdin(lua_State *L) {
 	struct queue *q = lua_touserdata(L, lua_upvalueindex(1));
-	LOCK(q);
+	pthread_mutex_lock(&q->lock);
 	if (q->head == q->tail) {
-		UNLOCK(q);
+		pthread_mutex_unlock(&q->lock);
 		return 0;
 	}
 	char * str = q->queue[q->head];
 	if (++q->head >= QUEUE_SIZE) {
 		q->head = 0;
 	}
-	UNLOCK(q);
+	pthread_mutex_unlock(&q->lock);
 	lua_pushstring(L, str);
 	free(str);
 	return 1;
@@ -201,6 +198,7 @@ luaopen_clientsocket(lua_State *L) {
 
 	struct queue * q = lua_newuserdata(L, sizeof(*q));
 	memset(q, 0, sizeof(*q));
+	pthread_mutex_init(&q->lock, NULL);
 	lua_pushcclosure(L, lreadstdin, 1);
 	lua_setfield(L, -2, "readstdin");
 

lualib-src/lua-cluster.c

@@ -1,6 +1,7 @@
 #include <lua.h>
 #include <lauxlib.h>
 #include <string.h>
+#include <assert.h>
 
 #include "skynet.h"
 
@@ -15,7 +16,8 @@
 		uint32_t next_session
  */
 
-#define TEMP_LENGTH 0x10007
+#define TEMP_LENGTH 0x8200
+#define MULTI_PART 0x8000
 
 static void
 fill_uint32(uint8_t * buf, uint32_t n) {
@@ -26,30 +28,75 @@ fill_uint32(uint8_t * buf, uint32_t n) {
 }
 
 static void
-fill_header(lua_State *L, uint8_t *buf, int sz, void *msg) {
-	if (sz >= 0x10000) {
-		skynet_free(msg);
-		luaL_error(L, "request message is too long %d", sz);
-	}
+fill_header(lua_State *L, uint8_t *buf, int sz) {
+	assert(sz < 0x10000);
 	buf[0] = (sz >> 8) & 0xff;
 	buf[1] = sz & 0xff;
 }
 
-static void
-packreq_number(lua_State *L, int session, void * msg, size_t sz) {
+/*
+	The request package :
+	size <= 0x8000 (32K) and address is id
+		WORD sz+9
+		BYTE 0
+		DWORD addr
+		DWORD session
+		PADDING msg(sz)
+	size > 0x8000 and address is id
+		DWORD 13
+		BYTE 1	; multireq	
+		DWORD addr
+		DWORD session
+		DWORD sz
+
+	size <= 0x8000 (32K) and address is string
+		WORD sz+6+namelen
+		BYTE 0x80
+		BYTE namelen
+		STRING name
+		DWORD session
+		PADDING msg(sz)
+	size > 0x8000 and address is string
+		DWORD 10 + namelen
+		BYTE 0x81
+		BYTE namelen
+		STRING name
+		DWORD session
+		DWORD sz
+
+	multi req
+		WORD sz + 5
+		BYTE 2/3 ; 2:multipart, 3:multipart end
+		DWORD SESSION
+		PADDING msgpart(sz)
+ */
+static int
+packreq_number(lua_State *L, int session, void * msg, uint32_t sz) {
 	uint32_t addr = (uint32_t)lua_tointeger(L,1);
 	uint8_t buf[TEMP_LENGTH];
-	fill_header(L, buf, sz+9, msg);
-	buf[2] = 0;
-	fill_uint32(buf+3, addr);
-	fill_uint32(buf+7, (uint32_t)session);
-	memcpy(buf+11,msg,sz);
+	if (sz < MULTI_PART) {
+		fill_header(L, buf, sz+9);
+		buf[2] = 0;
+		fill_uint32(buf+3, addr);
+		fill_uint32(buf+7, (uint32_t)session);
+		memcpy(buf+11,msg,sz);
 
-	lua_pushlstring(L, (const char *)buf, sz+11);
+		lua_pushlstring(L, (const char *)buf, sz+11);
+		return 0;
+	} else {
+		int part = (sz - 1) / MULTI_PART + 1;
+		fill_header(L, buf, 13);
+		buf[2] = 1;
+		fill_uint32(buf+3, addr);
+		fill_uint32(buf+7, (uint32_t)session);
+		fill_uint32(buf+11, sz);
+		lua_pushlstring(L, (const char *)buf, 15);
+		return part;
+	}
 }
 
-static void
-packreq_string(lua_State *L, int session, void * msg, size_t sz) {
+static int
+packreq_string(lua_State *L, int session, void * msg, uint32_t sz) {
 	size_t namelen = 0;
 	const char *name = lua_tolstring(L, 1, &namelen);
 	if (name == NULL || namelen < 1 || namelen > 255) {
@@ -58,13 +105,53 @@ packreq_string(lua_State *L, int session, void * msg, size_t sz) {
 	}
 
 	uint8_t buf[TEMP_LENGTH];
-	fill_header(L, buf, sz+5+namelen, msg);
-	buf[2] = (uint8_t)namelen;
-	memcpy(buf+3, name, namelen);
-	fill_uint32(buf+3+namelen, (uint32_t)session);
-	memcpy(buf+7+namelen,msg,sz);
+	if (sz < MULTI_PART) {
+		fill_header(L, buf, sz+6+namelen);
+		buf[2] = 0x80;
+		buf[3] = (uint8_t)namelen;
+		memcpy(buf+4, name, namelen);
+		fill_uint32(buf+4+namelen, (uint32_t)session);
+		memcpy(buf+8+namelen,msg,sz);
 
-	lua_pushlstring(L, (const char *)buf, sz+7+namelen);
+		lua_pushlstring(L, (const char *)buf, sz+8+namelen);
+		return 0;
+	} else {
+		int part = (sz - 1) / MULTI_PART + 1;
+		fill_header(L, buf, 10+namelen);
+		buf[2] = 0x81;
+		buf[3] = (uint8_t)namelen;
+		memcpy(buf+4, name, namelen);
+		fill_uint32(buf+4+namelen, (uint32_t)session);
+		fill_uint32(buf+8+namelen, sz);
+
+		lua_pushlstring(L, (const char *)buf, 12+namelen);
+		return part;
+	}
+}
+
+static void
+packreq_multi(lua_State *L, int session, void * msg, uint32_t sz) {
+	uint8_t buf[TEMP_LENGTH];
+	int part = (sz - 1) / MULTI_PART + 1;
+	int i;
+	char *ptr = msg;
+	for (i=0;i<part;i++) {
+		uint32_t s;
+		if (sz > MULTI_PART) {
+			s = MULTI_PART;
+			buf[2] = 2;
+		} else {
+			s = sz;
+			buf[2] = 3;	// the last multi part
+		}
+		fill_header(L, buf, s+5);
+		fill_uint32(buf+3, (uint32_t)session);
+		memcpy(buf+7, ptr, s);
+		lua_pushlstring(L, (const char *)buf, s+7);
+		lua_rawseti(L, -2, i+1);
+		sz -= s;
+		ptr += s;
+	}
 }
 
 static int
@@ -73,24 +160,33 @@ lpackrequest(lua_State *L) {
 	if (msg == NULL) {
 		return luaL_error(L, "Invalid request message");
 	}
-	size_t sz = (size_t)luaL_checkinteger(L,4);
+	uint32_t sz = (uint32_t)luaL_checkinteger(L,4);
 	int session = luaL_checkinteger(L,2);
 	if (session <= 0) {
 		skynet_free(msg);
 		return luaL_error(L, "Invalid request session %d", session);
 	}
 	int addr_type = lua_type(L,1);
+	int multipak;
 	if (addr_type == LUA_TNUMBER) {
-		packreq_number(L, session, msg, sz);
+		multipak = packreq_number(L, session, msg, sz);
 	} else {
-		packreq_string(L, session, msg, sz);
+		multipak = packreq_string(L, session, msg, sz);
 	}
+	int current_session = session;
 	if (++session < 0) {
 		session = 1;
 	}
-	skynet_free(msg);
 	lua_pushinteger(L, session);
-	return 2;
+	if (multipak) {
+		lua_createtable(L, multipak, 0);
+		packreq_multi(L, current_session, msg, sz);
+		skynet_free(msg);
+		return 3;
+	} else {
+		skynet_free(msg);
+		return 2;
+	}
 }
 
 /*
@@ -99,6 +195,7 @@ lpackrequest(lua_State *L) {
 		uint32_t or string addr
 		int session
 		string msg
+		boolean padding
  */
 
 static inline uint32_t
@@ -107,44 +204,122 @@ unpack_uint32(const uint8_t * buf) {
 }
 
 static int
-unpackreq_number(lua_State *L, const uint8_t * buf, size_t sz) {
+unpackreq_number(lua_State *L, const uint8_t * buf, int sz) {
 	if (sz < 9) {
-		return luaL_error(L, "Invalid cluster message");
+		return luaL_error(L, "Invalid cluster message (size=%d)", sz);
 	}
 	uint32_t address = unpack_uint32(buf+1);
 	uint32_t session = unpack_uint32(buf+5);
-	lua_pushinteger(L, (uint32_t)address);
-	lua_pushinteger(L, (uint32_t)session);
+	lua_pushinteger(L, address);
+	lua_pushinteger(L, session);
 	lua_pushlstring(L, (const char *)buf+9, sz-9);
 
 	return 3;
 }
 
 static int
-unpackreq_string(lua_State *L, const uint8_t * buf, size_t sz) {
-	size_t namesz = buf[0];
-	if (sz < namesz + 5) {
-		return luaL_error(L, "Invalid cluster message");
+unpackmreq_number(lua_State *L, const uint8_t * buf, int sz) {
+	if (sz != 13) {
+		return luaL_error(L, "Invalid cluster message size %d (multi req must be 13)", sz);
 	}
-	lua_pushlstring(L, (const char *)buf+1, namesz);
-	uint32_t session = unpack_uint32(buf + namesz + 1);
+	uint32_t address = unpack_uint32(buf+1);
+	uint32_t session = unpack_uint32(buf+5);
+	uint32_t size = unpack_uint32(buf+9);
+	lua_pushinteger(L, address);
+	lua_pushinteger(L, session);
+	lua_pushinteger(L, size);
+	lua_pushboolean(L, 1);	// padding multi part
+
+	return 4;
+}
+
+static int
+unpackmreq_part(lua_State *L, const uint8_t * buf, int sz) {
+	if (sz < 5) {
+		return luaL_error(L, "Invalid cluster multi part message");
+	}
+	int padding = (buf[0] == 2);
+	uint32_t session = unpack_uint32(buf+1);
+	lua_pushboolean(L, 0);	// no address
+	lua_pushinteger(L, session);
+	lua_pushlstring(L, (const char *)buf+5, sz-5);
+	lua_pushboolean(L, padding);
+
+	return 4;
+}
+
+static int
+unpackreq_string(lua_State *L, const uint8_t * buf, int sz) {
+	if (sz < 2) {
+		return luaL_error(L, "Invalid cluster message (size=%d)", sz);
+	}
+	size_t namesz = buf[1];
+	if (sz < namesz + 6) {
+		return luaL_error(L, "Invalid cluster message (size=%d)", sz);
+	}
+	lua_pushlstring(L, (const char *)buf+2, namesz);
+	uint32_t session = unpack_uint32(buf + namesz + 2);
 	lua_pushinteger(L, (uint32_t)session);
-	lua_pushlstring(L, (const char *)buf+1+namesz+4, sz - namesz - 5);
+	lua_pushlstring(L, (const char *)buf+2+namesz+4, sz - namesz - 6);
 
 	return 3;
 }
 
+static int
+unpackmreq_string(lua_State *L, const uint8_t * buf, int sz) {
+	if (sz < 2) {
+		return luaL_error(L, "Invalid cluster message (size=%d)", sz);
+	}
+	size_t namesz = buf[1];
+	if (sz < namesz + 10) {
+		return luaL_error(L, "Invalid cluster message (size=%d)", sz);
+	}
+	lua_pushlstring(L, (const char *)buf+2, namesz);
+	uint32_t session = unpack_uint32(buf + namesz + 2);
+	uint32_t size = unpack_uint32(buf + namesz + 6);
+	lua_pushinteger(L, session);
+	lua_pushinteger(L, size);
+	lua_pushboolean(L, 1);	// padding multipart
+
+	return 4;
+}
+
 static int
 lunpackrequest(lua_State *L) {
-	size_t sz;
-	const char *msg = luaL_checklstring(L,1,&sz);
-	if (msg[0] == 0) {
+	size_t ssz;
+	const char *msg = luaL_checklstring(L,1,&ssz);
+	int sz = (int)ssz;
+	switch (msg[0]) {
+	case 0:
 		return unpackreq_number(L, (const uint8_t *)msg, sz);
-	} else {
+	case 1:
+		return unpackmreq_number(L, (const uint8_t *)msg, sz);
+	case 2:
+	case 3:
+		return unpackmreq_part(L, (const uint8_t *)msg, sz);
+	case '\x80':
 		return unpackreq_string(L, (const uint8_t *)msg, sz);
+	case '\x81':
+		return unpackmreq_string(L, (const uint8_t *)msg, sz);
+	default:
+		return luaL_error(L, "Invalid req package type %d", msg[0]);
 	}
 }
 
+/*
+	DWORD session
+	BYTE type
+		0: error
+		1: ok
+		2: multi begin
+		3: multi part
+		4: multi end
+	PADDING msg
+		type = 0, error msg
+		type = 1, msg
+		type = 2, DWORD size
+		type = 3/4, msg
+ */
 /*
 	int session
 	boolean ok
@@ -163,16 +338,56 @@ lpackresponse(lua_State *L) {
 	
 	if (lua_type(L,3) == LUA_TSTRING) {
 		msg = (void *)lua_tolstring(L, 3, &sz);
-		if (sz > 0x1000) {
-			sz = 0x1000;
-		}
 	} else {
 		msg = lua_touserdata(L,3);
 		sz = (size_t)luaL_checkinteger(L, 4);
 	}
 
+	if (!ok) {
+		if (sz > MULTI_PART) {
+			// truncate the error msg if too long
+			sz = MULTI_PART;
+		}
+	} else {
+		if (sz > MULTI_PART) {
+			// return 
+			int part = (sz - 1) / MULTI_PART + 1;
+			lua_createtable(L, part+1, 0);
+			uint8_t buf[TEMP_LENGTH];
+
+			// multi part begin
+			fill_header(L, buf, 9);
+			fill_uint32(buf+2, session);
+			buf[6] = 2;
+			fill_uint32(buf+7, (uint32_t)sz);
+			lua_pushlstring(L, (const char *)buf, 11);
+			lua_rawseti(L, -2, 1);
+
+			char * ptr = msg;
+			int i;
+			for (i=0;i<part;i++) {
+				int s;
+				if (sz > MULTI_PART) {
+					s = MULTI_PART;
+					buf[6] = 3;
+				} else {
+					s = sz;
+					buf[6] = 4;
+				}
+				fill_header(L, buf, s+5);
+				fill_uint32(buf+2, session);
+				memcpy(buf+7,ptr,s);
+				lua_pushlstring(L, (const char *)buf, s+7);
+				lua_rawseti(L, -2, i+2);
+				sz -= s;
+				ptr += s;
+			}
+			return 1;
+		}
+	}
+
 	uint8_t buf[TEMP_LENGTH];
-	fill_header(L, buf, sz+5, msg);
+	fill_header(L, buf, sz+5);
 	fill_uint32(buf+2, session);
 	buf[6] = ok;
 	memcpy(buf+7,msg,sz);
@@ -187,6 +402,7 @@ lpackresponse(lua_State *L) {
 	return integer session
 		boolean ok
 		string msg
+		boolean padding
  */
 static int
 lunpackresponse(lua_State *L) {
@@ -197,10 +413,66 @@ lunpackresponse(lua_State *L) {
 	}
 	uint32_t session = unpack_uint32((const uint8_t *)buf);
 	lua_pushinteger(L, (lua_Integer)session);
-	lua_pushboolean(L, buf[4]);
-	lua_pushlstring(L, buf+5, sz-5);
+	switch(buf[4]) {
+	case 0:	// error
+		lua_pushboolean(L, 0);
+		lua_pushlstring(L, buf+5, sz-5);
+		return 3;
+	case 1:	// ok
+	case 4:	// multi end
+		lua_pushboolean(L, 1);
+		lua_pushlstring(L, buf+5, sz-5);
+		return 3;
+	case 2:	// multi begin
+		if (sz != 9) {
+			return 0;
+		}
+		sz = unpack_uint32((const uint8_t *)buf+5);
+		lua_pushboolean(L, 1);
+		lua_pushinteger(L, sz);
+		lua_pushboolean(L, 1);
+		return 4;
+	case 3:	// multi part
+		lua_pushboolean(L, 1);
+		lua_pushlstring(L, buf+5, sz-5);
+		lua_pushboolean(L, 1);
+		return 4;
+	default:
+		return 0;
+	}
+}
 
-	return 3;
+static int
+lconcat(lua_State *L) {
+	if (!lua_istable(L,1))
+		return 0;
+	if (lua_geti(L,1,1) != LUA_TNUMBER)
+		return 0;
+	int sz = lua_tointeger(L,-1);
+	lua_pop(L,1);
+	char * buff = skynet_malloc(sz);
+	int idx = 2;
+	int offset = 0;
+	while(lua_geti(L,1,idx) == LUA_TSTRING) {
+		size_t s;
+		const char * str = lua_tolstring(L, -1, &s);
+		if (s+offset > sz) {
+			skynet_free(buff);
+			return 0;
+		}
+		memcpy(buff+offset, str, s);
+		lua_pop(L,1);
+		offset += s;
+		++idx;
+	}
+	if (offset != sz) {
+		skynet_free(buff);
+		return 0;
+	}
+	// buff/sz will send to other service, See clusterd.lua
+	lua_pushlightuserdata(L, buff);
+	lua_pushinteger(L, sz);
+	return 2;
 }
 
 int
@@ -210,6 +482,7 @@ luaopen_cluster_core(lua_State *L) {
 		{ "unpackrequest", lunpackrequest },
 		{ "packresponse", lpackresponse },
 		{ "unpackresponse", lunpackresponse },
+		{ "concat", lconcat },
 		{ NULL, NULL },
 	};
 	luaL_checkversion(L);

lualib-src/lua-crypt.c

@@ -338,8 +338,13 @@ static int
 lrandomkey(lua_State *L) {
 	char tmp[8];
 	int i;
+	char x = 0;
 	for (i=0;i<8;i++) {
 		tmp[i] = random() & 0xff;
+		x ^= tmp[i];
+	}
+	if (x==0) {
+		tmp[0] |= 1;	// avoid 0
 	}
 	lua_pushlstring(L, tmp, 8);
 	return 1;
@@ -718,8 +723,11 @@ static int
 ldhsecret(lua_State *L) {
 	uint32_t x[2], y[2];
 	read64(L, x, y);
-	uint64_t r = powmodp((uint64_t)x[0] | (uint64_t)x[1]<<32,
-		(uint64_t)y[0] | (uint64_t)y[1]<<32);
+	uint64_t xx = (uint64_t)x[0] | (uint64_t)x[1]<<32;
+	uint64_t yy = (uint64_t)y[0] | (uint64_t)y[1]<<32;
+	if (xx == 0 || yy == 0)
+		return luaL_error(L, "Can't be 0");
+	uint64_t r = powmodp(xx, yy);
 
 	push64(L, r);
 
@@ -739,7 +747,11 @@ ldhexchange(lua_State *L) {
 	xx[0] = x[0] | x[1]<<8 | x[2]<<16 | x[3]<<24;
 	xx[1] = x[4] | x[5]<<8 | x[6]<<16 | x[7]<<24;
 
-	uint64_t r = powmodp(5,	(uint64_t)xx[0] | (uint64_t)xx[1]<<32);
+	uint64_t x64 = (uint64_t)xx[0] | (uint64_t)xx[1]<<32;
+	if (x64 == 0)
+		return luaL_error(L, "Can't be 0");
+
+	uint64_t r = powmodp(5,	x64);
 	push64(L, r);
 	return 1;
 }

lualib-src/lua-debugchannel.c

@@ -4,10 +4,9 @@
 #include <unistd.h>
 #include <stdlib.h>
 #include <string.h>
+#include "spinlock.h"
 
 #define METANAME "debugchannel"
-#define LOCK(q) while (__sync_lock_test_and_set(&(q)->lock,1)) {}
-#define UNLOCK(q) __sync_lock_release(&(q)->lock);
 
 struct command {
 	struct command * next;
@@ -15,7 +14,7 @@ struct command {
 };
 
 struct channel {
-	int lock;
+	struct spinlock lock;
 	int ref;
 	struct command * head;
 	struct command * tail;
@@ -26,6 +25,7 @@ channel_new() {
 	struct channel * c = malloc(sizeof(*c));
 	memset(c, 0 , sizeof(*c));
 	c->ref = 1;
+	SPIN_INIT(c)
 
 	return c;
 }
@@ -33,21 +33,21 @@ channel_new() {
 static struct channel *
 channel_connect(struct channel *c) {
 	struct channel * ret = NULL;
-	LOCK(c)
+	SPIN_LOCK(c)
 	if (c->ref == 1) {
 		++c->ref;
 		ret = c;
 	}
-	UNLOCK(c)
+	SPIN_UNLOCK(c)
 	return ret;
 }
 
 static struct channel *
 channel_release(struct channel *c) {
-	LOCK(c)
+	SPIN_LOCK(c)
 	--c->ref;
 	if (c->ref > 0) {
-		UNLOCK(c)
+		SPIN_UNLOCK(c)
 		return c;
 	}
 	// never unlock while reference is 0
@@ -59,6 +59,8 @@ channel_release(struct channel *c) {
 		free(p);
 		p = next;
 	}
+	SPIN_UNLOCK(c)
+	SPIN_DESTROY(c)
 	free(c);
 	return NULL;
 }
@@ -67,9 +69,9 @@ channel_release(struct channel *c) {
 static struct command *
 channel_read(struct channel *c, double timeout) {
 	struct command * ret = NULL;
-	LOCK(c)
+	SPIN_LOCK(c)
 	if (c->head == NULL) {
-		UNLOCK(c)
+		SPIN_UNLOCK(c)
 		int ti = (int)(timeout * 100000);
 		usleep(ti);
 		return NULL;
@@ -79,7 +81,7 @@ channel_read(struct channel *c, double timeout) {
 	if (c->head == NULL) {
 		c->tail = NULL;
 	}
-	UNLOCK(c)
+	SPIN_UNLOCK(c)
 	
 	return ret;
 }
@@ -90,14 +92,14 @@ channel_write(struct channel *c, const char * s, size_t sz) {
 	cmd->sz = sz;
 	cmd->next = NULL;
 	memcpy(cmd+1, s, sz);
-	LOCK(c)
+	SPIN_LOCK(c)
 	if (c->tail == NULL) {
 		c->head = c->tail = cmd;
 	} else {
 		c->tail->next = cmd;
 		c->tail = cmd;
 	}
-	UNLOCK(c)
+	SPIN_UNLOCK(c)
 }
 
 struct channel_box {

lualib-src/lua-memory.c

@@ -2,6 +2,7 @@
 #include <lauxlib.h>
 
 #include "malloc_hook.h"
+#include "luashrtbl.h"
 
 static int
 ltotal(lua_State *L) {
@@ -33,6 +34,13 @@ ldump(lua_State *L) {
 	return 0;
 }
 
+static int
+lexpandshrtbl(lua_State *L) {
+	int n = luaL_checkinteger(L, 1);
+	luaS_expandshr(n);
+	return 0;
+}
+
 int
 luaopen_memory(lua_State *L) {
 	luaL_checkversion(L);
@@ -43,6 +51,8 @@ luaopen_memory(lua_State *L) {
 		{ "dumpinfo", ldumpinfo },
 		{ "dump", ldump },
 		{ "info", dump_mem_lua },
+		{ "ssinfo", luaS_shrinfo },
+		{ "ssexpand", lexpandshrtbl },
 		{ NULL, NULL },
 	};
 

lualib-src/lua-multicast.c

@@ -5,6 +5,8 @@
 #include <stdint.h>
 #include <string.h>
 
+#include "atomic.h"
+
 struct mc_package {
 	int reference;
 	uint32_t size;
@@ -116,7 +118,7 @@ static int
 mc_closelocal(lua_State *L) {
 	struct mc_package *pack = lua_touserdata(L,1);
 
-	int ref = __sync_sub_and_fetch(&pack->reference, 1);
+	int ref = ATOM_DEC(&pack->reference);
 	if (ref <= 0) {
 		skynet_free(pack->data);
 		skynet_free(pack);

lualib-src/lua-netpack.c

@@ -19,6 +19,7 @@
 #define TYPE_ERROR 3
 #define TYPE_OPEN 4
 #define TYPE_CLOSE 5
+#define TYPE_WARNING 6
 
 /*
 	Each package is uint16 + data , uint16 (serialized in big-endian) is the number of bytes comprising the data .
@@ -48,6 +49,7 @@ struct queue {
 static void
 clear_list(struct uncomplete * uc) {
 	while (uc) {
+		skynet_free(uc->pack.buffer);
 		void * tmp = uc;
 		uc = uc->next;
 		skynet_free(tmp);
@@ -371,6 +373,11 @@ lfilter(lua_State *L) {
 		lua_pushinteger(L, message->id);
 		pushstring(L, buffer, size);
 		return 4;
+	case SKYNET_SOCKET_TYPE_WARNING:
+		lua_pushvalue(L, lua_upvalueindex(TYPE_WARNING));
+		lua_pushinteger(L, message->id);
+		lua_pushinteger(L, message->ud);
+		return 4;
 	default:
 		// never get here
 		return 1;
@@ -442,57 +449,6 @@ lpack(lua_State *L) {
 	return 2;
 }
 
-static int
-lpack_string(lua_State *L) {
-	uint8_t tmp[SMALLSTRING+2];
-	size_t len;
-	uint8_t *buffer;
-	const char * ptr = tolstring(L, &len, 1);
-	if (len > 0x10000) {
-		return luaL_error(L, "Invalid size (too long) of data : %d", (int)len);
-	}
-
-	if (len <= SMALLSTRING) {
-		buffer = tmp;
-	} else {
-		buffer = lua_newuserdata(L, len + 2);
-	}
-
-	write_size(buffer, len);
-	memcpy(buffer+2, ptr, len);
-	lua_pushlstring(L, (const char *)buffer, len+2);
-
-	return 1;
-}
-
-static int
-lpack_padding(lua_State *L) {
-	uint8_t tmp[SMALLSTRING+2];
-	size_t content_sz;
-	uint8_t *buffer;
-	const char * ptr = tolstring(L, &content_sz, 2);
-	size_t cookie_sz = 0;
-	const char * cookie = luaL_checklstring(L,1,&cookie_sz);
-	size_t len = cookie_sz + content_sz;
-
-	if (len > 0x10000) {
-		return luaL_error(L, "Invalid size (too long) of data : %d", (int)len);
-	}
-
-	if (len <= SMALLSTRING) {
-		buffer = tmp;
-	} else {
-		buffer = lua_newuserdata(L, len + 2);
-	}
-
-	write_size(buffer, len);
-	memcpy(buffer+2, ptr, content_sz);
-	memcpy(buffer+2+content_sz, cookie, cookie_sz);
-	lua_pushlstring(L, (const char *)buffer, len+2);
-
-	return 1;
-}
-
 static int
 ltostring(lua_State *L) {
 	void * ptr = lua_touserdata(L, 1);
@@ -500,19 +456,8 @@ ltostring(lua_State *L) {
 	if (ptr == NULL) {
 		lua_pushliteral(L, "");
 	} else {
-		if (lua_isnumber(L, 3)) {
-			int offset = lua_tointeger(L, 3);
-			if (offset < 0) {
-				return luaL_error(L, "Invalid offset %d", offset);
-			}
-			if (offset > size) {
-				offset = size;
-			}
-			lua_pushlstring(L, (const char *)ptr + offset, size-offset);
-		} else {
-			lua_pushlstring(L, (const char *)ptr, size);
-			skynet_free(ptr);
-		}
+		lua_pushlstring(L, (const char *)ptr, size);
+		skynet_free(ptr);
 	}
 	return 1;
 }
@@ -523,8 +468,6 @@ luaopen_netpack(lua_State *L) {
 	luaL_Reg l[] = {
 		{ "pop", lpop },
 		{ "pack", lpack },
-		{ "pack_string", lpack_string },
-		{ "pack_padding", lpack_padding },
 		{ "clear", lclear },
 		{ "tostring", ltostring },
 		{ NULL, NULL },
@@ -537,8 +480,9 @@ luaopen_netpack(lua_State *L) {
 	lua_pushliteral(L, "error");
 	lua_pushliteral(L, "open");
 	lua_pushliteral(L, "close");
+	lua_pushliteral(L, "warning");
 
-	lua_pushcclosure(L, lfilter, 5);
+	lua_pushcclosure(L, lfilter, 6);
 	lua_setfield(L, -2, "filter");
 
 	return 1;

lualib-src/lua-sharedata.c

@@ -4,6 +4,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
+#include "atomic.h"
 
 #define KEYTYPE_INTEGER 0
 #define KEYTYPE_STRING 1
@@ -35,8 +36,6 @@ struct node {
 	uint8_t nocolliding;	// 0 means colliding slot
 };
 
-struct table;
-
 struct state {
 	int dirty;
 	int ref;
@@ -663,7 +662,7 @@ releaseobj(lua_State *L) {
 	struct ctrl *c = lua_touserdata(L, 1);
 	struct table *tbl = c->root;
 	struct state *s = lua_touserdata(tbl->L, 1);
-	__sync_fetch_and_sub(&s->ref, 1);
+	ATOM_DEC(&s->ref);
 	c->root = NULL;
 	c->update = NULL;
 
@@ -674,7 +673,7 @@ static int
 lboxconf(lua_State *L) {
 	struct table * tbl = get_table(L,1);	
 	struct state * s = lua_touserdata(tbl->L, 1);
-	__sync_fetch_and_add(&s->ref, 1);
+	ATOM_INC(&s->ref);
 
 	struct ctrl * c = lua_newuserdata(L, sizeof(*c));
 	c->root = tbl;
@@ -719,7 +718,7 @@ static int
 lincref(lua_State *L) {
 	struct table *tbl = get_table(L,1);
 	struct state * s = lua_touserdata(tbl->L, 1);
-	int ref = __sync_add_and_fetch(&s->ref, 1);
+	int ref = ATOM_INC(&s->ref);
 	lua_pushinteger(L , ref);
 
 	return 1;
@@ -729,7 +728,7 @@ static int
 ldecref(lua_State *L) {
 	struct table *tbl = get_table(L,1);
 	struct state * s = lua_touserdata(tbl->L, 1);
-	int ref = __sync_sub_and_fetch(&s->ref, 1);
+	int ref = ATOM_DEC(&s->ref);
 	lua_pushinteger(L , ref);
 
 	return 1;

lualib-src/lua-skynet.c

@@ -118,6 +118,28 @@ _command(lua_State *L) {
 	return 0;
 }
 
+static int
+_intcommand(lua_State *L) {
+	struct skynet_context * context = lua_touserdata(L, lua_upvalueindex(1));
+	const char * cmd = luaL_checkstring(L,1);
+	const char * result;
+	const char * parm = NULL;
+	char tmp[64];	// for integer parm
+	if (lua_gettop(L) == 2) {
+		int32_t n = (int32_t)luaL_checkinteger(L,2);
+		sprintf(tmp, "%d", n);
+		parm = tmp;
+	}
+
+	result = skynet_command(context, cmd, parm);
+	if (result) {
+		lua_Integer r = strtoll(result, NULL, 0);
+		lua_pushinteger(L, r);
+		return 1;
+	}
+	return 0;
+}
+
 static int
 _genid(lua_State *L) {
 	struct skynet_context * context = lua_touserdata(L, lua_upvalueindex(1));
@@ -150,6 +172,9 @@ _send(lua_State *L) {
 	uint32_t dest = (uint32_t)lua_tointeger(L, 1);
 	const char * dest_string = NULL;
 	if (dest == 0) {
+		if (lua_type(L,1) == LUA_TNUMBER) {
+			return luaL_error(L, "Invalid service address 0");
+		}
 		dest_string = get_dest_string(L, 1);
 	}
 
@@ -310,6 +335,7 @@ luaopen_skynet_core(lua_State *L) {
 		{ "genid", _genid },
 		{ "redirect", _redirect },
 		{ "command" , _command },
+		{ "intcommand", _intcommand },
 		{ "error", _error },
 		{ "tostring", _tostring },
 		{ "harbor", _harbor },

lualib-src/lua-stm.c

@@ -7,6 +7,7 @@
 
 #include "rwlock.h"
 #include "skynet_malloc.h"
+#include "atomic.h"
 
 struct stm_object {
 	struct rwlock lock;
@@ -45,7 +46,7 @@ static void
 stm_releasecopy(struct stm_copy *copy) {
 	if (copy == NULL)
 		return;
-	if (__sync_sub_and_fetch(&copy->reference, 1) == 0) {
+	if (ATOM_DEC(&copy->reference) == 0) {
 		skynet_free(copy->msg);
 		skynet_free(copy);
 	}
@@ -70,7 +71,7 @@ stm_release(struct stm_object *obj) {
 static void
 stm_releasereader(struct stm_object *obj) {
 	rwlock_rlock(&obj->lock);
-	if (__sync_sub_and_fetch(&obj->reference,1) == 0) {
+	if (ATOM_DEC(&obj->reference) == 0) {
 		// last reader, no writer. so no need to unlock
 		assert(obj->copy == NULL);
 		skynet_free(obj);
@@ -82,7 +83,7 @@ stm_releasereader(struct stm_object *obj) {
 static void
 stm_grab(struct stm_object *obj) {
 	rwlock_rlock(&obj->lock);
-	int ref = __sync_fetch_and_add(&obj->reference,1);
+	int ref = ATOM_FINC(&obj->reference);
 	rwlock_runlock(&obj->lock);
 	assert(ref > 0);
 }
@@ -92,7 +93,7 @@ stm_copy(struct stm_object *obj) {
 	rwlock_rlock(&obj->lock);
 	struct stm_copy * ret = obj->copy;
 	if (ret) {
-		int ref = __sync_fetch_and_add(&ret->reference,1);
+		int ref = ATOM_FINC(&ret->reference);
 		assert(ref > 0);
 	}
 	rwlock_runlock(&obj->lock);

lualib-src/sproto/lsproto.c

@@ -41,8 +41,18 @@ LUALIB_API void luaL_setfuncs (lua_State *L, const luaL_Reg *l, int nup) {
 
 #if LUA_VERSION_NUM < 503
 
-// lua_isinteger is lua 5.3 api
-#define lua_isinteger lua_isnumber
+#if LUA_VERSION_NUM < 502
+static lua_Integer lua_tointegerx(lua_State *L, int idx, int *isnum) {
+	if (lua_isnumber(L, idx)) {
+		if (isnum) *isnum = 1;
+		return lua_tointeger(L, idx);
+	}
+	else {
+		if (isnum) *isnum = 0;
+		return 0;
+	}
+}
+#endif
 
 // work around , use push & lua_gettable may be better
 #define lua_geti lua_rawgeti
@@ -156,11 +166,11 @@ encode(const struct sproto_arg *args) {
 	case SPROTO_TINTEGER: {
 		lua_Integer v;
 		lua_Integer vh;
-		if (!lua_isinteger(L, -1)) {
+		int isnum;
+		v = lua_tointegerx(L, -1, &isnum);
+		if(!isnum) {
 			return luaL_error(L, ".%s[%d] is not an integer (Is a %s)", 
 				args->tagname, args->index, lua_typename(L, lua_type(L, -1)));
-		} else {
-			v = lua_tointeger(L, -1);
 		}
 		lua_pop(L,1);
 		// notice: in lua 5.2, lua_Integer maybe 52bit

lualib-src/sproto/sproto.c

@@ -682,6 +682,7 @@ encode_object(sproto_callback cb, struct sproto_arg *args, uint8_t *data, int si
 	if (args->type == SPROTO_TSTRING) {
 		--sz;	// the length of null string is 1
 	}
+	assert(sz <= size-SIZEOF_LENGTH);	// verify buffer overflow
 	return fill_size(data, sz);
 }
 

lualib/cluster.lua

@@ -33,6 +33,16 @@ function cluster.snax(node, name, address)
 	return snax.bind(handle, name)
 end
 
+function cluster.register(name, addr)
+	assert(type(name) == "string")
+	assert(addr == nil or type(addr) == "number")
+	return skynet.call(clusterd, "lua", "register", name, addr)
+end
+
+function cluster.query(node, name)
+	return skynet.call(clusterd, "lua", "req", node, 0, skynet.pack(name))
+end
+
 skynet.init(function()
 	clusterd = skynet.uniqueservice("clusterd")
 end)

lualib/dns.lua

@@ -265,7 +265,7 @@ local function suspend(tid, name, qtype)
 		co = coroutine.running(),
 	}
 	request_pool[tid] = req
-	skynet.wait()
+	skynet.wait(req.co)
 	local answers = request_pool[tid].answers
 	request_pool[tid] = nil
 	assert(answers, "no ip")

lualib/http/httpc.lua

@@ -22,8 +22,9 @@ local function request(fd, method, host, url, recvheader, header, content)
 	end
 
 	if content then
-		local data = string.format("%s %s HTTP/1.1\r\n%scontent-length:%d\r\n\r\n%s", method, url, header_content, #content, content)
+		local data = string.format("%s %s HTTP/1.1\r\n%scontent-length:%d\r\n\r\n", method, url, header_content, #content)
 		write(data)
+		write(content)
 	else
 		local request_header = string.format("%s %s HTTP/1.1\r\n%scontent-length:0\r\n\r\n", method, url, header_content)
 		write(request_header)

lualib/md5.lua

@@ -15,4 +15,24 @@ function core.sumhexa (k)
 		 end))
 end
 
-return core
+local function get_ipad(c)
+	return string.char(c:byte() ~ 0x36)
+end
+
+local function get_opad(c)
+	return string.char(c:byte() ~ 0x5c)
+end
+
+function core.hmacmd5(data,key)
+	if #key>64 then
+		key=core.sum(key)
+		key=key:sub(1,16)
+	end
+	local ipad_s=key:gsub(".", get_ipad)..string.rep("6",64-#key)
+	local opad_s=key:gsub(".", get_opad)..string.rep("\\",64-#key)
+	local istr=core.sum(ipad_s..data)
+	local ostr=core.sumhexa(opad_s..istr)
+	return ostr
+end
+
+return core

lualib/mysql.lua

@@ -386,7 +386,8 @@ end
 
 local function _recv_decode_packet_resp(self)
      return function(sock)
-        return true, _recv_packet(self,sock)
+		-- don't return more than 2 results
+        return true, (_recv_packet(self,sock))
     end
 end
 
@@ -417,7 +418,7 @@ local function _recv_auth_resp(self)
 end
 
 
-local function _mysql_login(self,user,password,database)
+local function _mysql_login(self,user,password,database,on_connect)
 
     return function(sockchannel)
           local packet, typ, err =   sockchannel:response( _recv_decode_packet_resp(self) )
@@ -489,7 +490,10 @@ local function _mysql_login(self,user,password,database)
         local packet_len = #req
 
         local authpacket=_compose_packet(self,req,packet_len)
-        return sockchannel:request(authpacket,_recv_auth_resp(self))
+        sockchannel:request(authpacket,_recv_auth_resp(self))
+	if on_connect then
+		on_connect(self)
+	end
     end
 end
 
@@ -625,7 +629,7 @@ local function _query_resp(self)
     end
 end
 
-function _M.connect( opts)
+function _M.connect(opts)
 
     local self = setmetatable( {}, mt)
 
@@ -644,11 +648,11 @@ function _M.connect( opts)
     local channel = socketchannel.channel {
         host = opts.host,
         port = opts.port or 3306,
-        auth = _mysql_login(self,user,password,database ),
+        auth = _mysql_login(self,user,password,database,opts.on_connect),
     }
+    self.sockchannel = channel
     -- try connect first only once
     channel:connect(true)
-    self.sockchannel = channel
 
 
     return self

lualib/skynet.lua

@@ -190,6 +190,7 @@ function suspend(co, result, command, param, size)
 			if ok == "TEST" then
 				if dead_service[co_address] then
 					release_watching(co_address)
+					unresponse[response] = nil
 					f = false
 					return false
 				else
@@ -224,7 +225,7 @@ function suspend(co, result, command, param, size)
 			return ret
 		end
 		watching_service[co_address] = watching_service[co_address] + 1
-		session_response[co] = response
+		session_response[co] = true
 		unresponse[response] = true
 		return suspend(co, coroutine.resume(co, response))
 	elseif command == "EXIT" then
@@ -248,18 +249,16 @@ function suspend(co, result, command, param, size)
 end
 
 function skynet.timeout(ti, func)
-	local session = c.command("TIMEOUT",tostring(ti))
+	local session = c.intcommand("TIMEOUT",ti)
 	assert(session)
-	session = tonumber(session)
 	local co = co_create(func)
 	assert(session_id_coroutine[session] == nil)
 	session_id_coroutine[session] = co
 end
 
 function skynet.sleep(ti)
-	local session = c.command("TIMEOUT",tostring(ti))
+	local session = c.intcommand("TIMEOUT",ti)
 	assert(session)
-	session = tonumber(session)
 	local succ, ret = coroutine_yield("SLEEP", session)
 	sleep_session[coroutine.running()] = nil
 	if succ then
@@ -273,13 +272,13 @@ function skynet.sleep(ti)
 end
 
 function skynet.yield()
-	return skynet.sleep("0")
+	return skynet.sleep(0)
 end
 
-function skynet.wait()
+function skynet.wait(co)
 	local session = c.genid()
 	local ret, msg = coroutine_yield("SLEEP", session)
-	local co = coroutine.running()
+	co = co or coroutine.running()
 	sleep_session[co] = nil
 	session_id_coroutine[session] = nil
 end
@@ -301,11 +300,11 @@ function skynet.localname(name)
 end
 
 function skynet.now()
-	return tonumber(c.command("NOW"))
+	return c.intcommand("NOW")
 end
 
 function skynet.starttime()
-	return tonumber(c.command("STARTTIME"))
+	return c.intcommand("STARTTIME")
 end
 
 function skynet.time()
@@ -609,11 +608,15 @@ local function init_all()
 	end
 end
 
+local function ret(f, ...)
+	f()
+	return ...
+end
+
 local function init_template(start)
 	init_all()
 	init_func = {}
-	start()
-	init_all()
+	return ret(init_all, start())
 end
 
 function skynet.pcall(start)
@@ -643,7 +646,7 @@ function skynet.endless()
 end
 
 function skynet.mqlen()
-	return tonumber(c.command "MQLEN")
+	return c.intcommand "MQLEN"
 end
 
 function skynet.task(ret)

lualib/snax/gateserver.lua

@@ -48,7 +48,6 @@ function gateserver.start(handler)
 	function CMD.close()
 		assert(socket)
 		socketdriver.close(socket)
-		socket = nil
 	end
 
 	local MSG = {}
@@ -101,17 +100,32 @@ function gateserver.start(handler)
 	end
 
 	function MSG.close(fd)
-		if handler.disconnect then
-			handler.disconnect(fd)
+		if fd ~= socket then
+			if handler.disconnect then
+				handler.disconnect(fd)
+			end
+			close_fd(fd)
+		else
+			socket = nil
 		end
-		close_fd(fd)
 	end
 
 	function MSG.error(fd, msg)
-		if handler.error then
-			handler.error(fd, msg)
+		if fd == socket then
+			socketdriver.close(fd)
+			skynet.error(msg)
+		else
+			if handler.error then
+				handler.error(fd, msg)
+			end
+			close_fd(fd)
+		end
+	end
+
+	function MSG.warning(fd, size)
+		if handler.warning then
+			handler.warning(fd, size)
 		end
-		close_fd(fd)
 	end
 
 	skynet.register_protocol {

lualib/snax/interface.lua

@@ -59,8 +59,6 @@ return function (name , G, loader)
 		end
 	end
 
-	setmetatable(G,	{ __index = env , __newindex = init_system })
-
 	local pattern
 
 	do
@@ -85,9 +83,10 @@ return function (name , G, loader)
 		end
 	end
 
-	mainfunc()
-
+	setmetatable(G,	{ __index = env , __newindex = init_system })
+	local ok, err = pcall(mainfunc)
 	setmetatable(G, nil)
+	assert(ok,err)
 
 	for k,v in pairs(temp_global) do
 		G[k] = v

lualib/snax/loginserver.lua

@@ -49,7 +49,6 @@ end
 
 local function launch_slave(auth_handler)
 	local function auth(fd, addr)
-		fd = assert(tonumber(fd))
 		skynet.error(string.format("connect from %s (fd = %d)", addr, fd))
 		socket.start(fd)
 
@@ -84,11 +83,11 @@ local function launch_slave(auth_handler)
 
 		local ok, server, uid =  pcall(auth_handler,token)
 
-		socket.abandon(fd)
 		return ok, server, uid, secret
 	end
 
-	local function ret_pack(ok, err, ...)
+	local function ret_pack(fd, ok, err, ...)
+		socket.abandon(fd)
 		if ok then
 			skynet.ret(skynet.pack(err, ...))
 		else
@@ -100,8 +99,12 @@ local function launch_slave(auth_handler)
 		end
 	end
 
-	skynet.dispatch("lua", function(_,_,...)
-		ret_pack(pcall(auth, ...))
+	skynet.dispatch("lua", function(_,_,fd,...)
+		if type(fd) ~= "number" then
+			skynet.ret(skynet.pack(false, "invalid fd type"))
+		else
+			ret_pack(fd,pcall(auth, fd, ...))
+		end
 	end)
 end
 

lualib/snax/msgserver.lua

@@ -70,7 +70,7 @@ Config for server.start:
 	conf.login_handler(uid, secret) -> subid : the function when a new user login, alloc a subid for it. (may call by login server)
 	conf.logout_handler(uid, subid) : the functon when a user logout. (may call by agent)
 	conf.kick_handler(uid, subid) : the functon when a user logout. (may call by login server)
-	conf.request_handler(username, session, msg, sz) : the function when recv a new request.
+	conf.request_handler(username, session, msg) : the function when recv a new request.
 	conf.register_handler(servername) : call when gate open
 	conf.disconnect_handler(username) : call when a connection disconnect (afk)
 ]]
@@ -234,10 +234,10 @@ function server.start(conf)
 		end
 	end
 
-	local function do_request(fd, msg, sz)
+	local function do_request(fd, message)
 		local u = assert(connection[fd], "invalid fd")
-		local msg_sz = sz - 4
-		local session = netpack.tostring(msg, sz, msg_sz)
+		local session = string.unpack(">I4", message, -4)
+		message = message:sub(1,-5)
 		local p = u.response[session]
 		if p then
 			-- session can be reuse in the same connection
@@ -256,21 +256,20 @@ function server.start(conf)
 		if p == nil then
 			p = { fd }
 			u.response[session] = p
-			local ok, result = pcall(conf.request_handler, u.username, msg, msg_sz)
-			result = result or ""
+			local ok, result = pcall(conf.request_handler, u.username, message)
 			-- NOTICE: YIELD here, socket may close.
+			result = result or ""
 			if not ok then
 				skynet.error(result)
-				result = "\0" .. session
+				result = string.pack(">BI4", 0, session)
 			else
-				result = result .. '\1' .. session
+				result = result .. string.pack(">BI4", 1, session)
 			end
 
-			p[2] = netpack.pack_string(result)
+			p[2] = string.pack(">s2",result)
 			p[3] = u.version
 			p[4] = u.index
 		else
-			netpack.tostring(msg, sz) -- request before, so free msg
 			-- update version/index, change return fd.
 			-- resend response.
 			p[1] = fd
@@ -292,10 +291,11 @@ function server.start(conf)
 	end
 
 	local function request(fd, msg, sz)
-		local ok, err = pcall(do_request, fd, msg, sz)
+		local message = netpack.tostring(msg, sz)
+		local ok, err = pcall(do_request, fd, message)
 		-- not atomic, may yield
 		if not ok then
-			skynet.error(string.format("Invalid package %s : %s", err, netpack.tostring(msg, sz)))
+			skynet.error(string.format("Invalid package %s : %s", err, message))
 			if connection[fd] then
 				gateserver.closeclient(fd)
 			end

lualib/socket.lua

@@ -30,7 +30,7 @@ end
 local function suspend(s)
 	assert(not s.co)
 	s.co = coroutine.running()
-	skynet.wait()
+	skynet.wait(s.co)
 	-- wakeup closing corouting every time suspend,
 	-- because socket.close() will wait last socket buffer operation before clear the buffer.
 	if s.closing then
@@ -106,16 +106,19 @@ socket_message[4] = function(id, newid, addr)
 end
 
 -- SKYNET_SOCKET_TYPE_ERROR = 5
-socket_message[5] = function(id)
+socket_message[5] = function(id, _, err)
 	local s = socket_pool[id]
 	if s == nil then
-		skynet.error("socket: error on unknown", id)
+		skynet.error("socket: error on unknown", id, err)
 		return
 	end
 	if s.connected then
-		skynet.error("socket: error on", id)
+		skynet.error("socket: error on", id, err)
+	elseif s.connecting then
+		s.connecting = err
 	end
 	s.connected = false
+	driver.close(id)
 
 	wakeup(s)
 end
@@ -133,6 +136,25 @@ socket_message[6] = function(id, size, data, address)
 	s.callback(str, address)
 end
 
+local function default_warning(id, size)
+	local s = socket_pool[id]
+		local last = s.warningsize or 0
+		if last + 64 < size then	-- if size increase 64K
+			s.warningsize = size
+			skynet.error(string.format("WARNING: %d K bytes need to send out (fd = %d)", size, id))
+		end
+		s.warningsize = size
+end
+
+-- SKYNET_SOCKET_TYPE_WARNING
+socket_message[7] = function(id, size)
+	local s = socket_pool[id]
+	if s then
+		local warning = s.warning or default_warning
+		warning(id, size)
+	end
+end
+
 skynet.register_protocol {
 	name = "socket",
 	id = skynet.PTYPE_SOCKET,	-- PTYPE_SOCKET = 6
@@ -151,6 +173,7 @@ local function connect(id, func)
 		id = id,
 		buffer = newbuffer,
 		connected = false,
+		connecting = true,
 		read_required = false,
 		co = false,
 		callback = func,
@@ -158,10 +181,13 @@ local function connect(id, func)
 	}
 	socket_pool[id] = s
 	suspend(s)
+	local err = s.connecting
+	s.connecting = nil
 	if s.connected then
 		return id
 	else
 		socket_pool[id] = nil
+		return nil, err
 	end
 end
 
@@ -202,7 +228,7 @@ function socket.close(id)
 		return
 	end
 	if s.connected then
-		driver.close(s.id)
+		driver.close(id)
 		-- notice: call socket.close in __gc should be carefully,
 		-- because skynet.wait never return in __gc, so driver.clear may not be called
 		if s.co then
@@ -210,7 +236,7 @@ function socket.close(id)
 			-- wait reading coroutine read the buffer.
 			assert(not s.closing)
 			s.closing = coroutine.running()
-			skynet.wait()
+			skynet.wait(s.closing)
 		else
 			suspend(s)
 		end
@@ -339,7 +365,7 @@ function socket.lock(id)
 	else
 		local co = coroutine.running()
 		table.insert(lock_set, co)
-		skynet.wait()
+		skynet.wait(co)
 	end
 end
 
@@ -404,4 +430,10 @@ end
 socket.sendto = assert(driver.udp_send)
 socket.udp_address = assert(driver.udp_address)
 
+function socket.warning(id, callback)
+	local obj = socket_pool[id]
+	assert(obj)
+	obj.warning = callback
+end
+
 return socket

lualib/socketchannel.lua

@@ -81,15 +81,27 @@ local function dispatch_by_session(self)
 	local response = self.__response
 	-- response() return session
 	while self.__sock do
-		local ok , session, result_ok, result_data = pcall(response, self.__sock)
+		local ok , session, result_ok, result_data, padding = pcall(response, self.__sock)
 		if ok and session then
 			local co = self.__thread[session]
-			self.__thread[session] = nil
 			if co then
-				self.__result[co] = result_ok
-				self.__result_data[co] = result_data
-				skynet.wakeup(co)
+				if padding and result_ok then
+					-- If padding is true, append result_data to a table (self.__result_data[co])
+					local result = self.__result_data[co] or {}
+					self.__result_data[co] = result
+					table.insert(result, result_data)
+				else
+					self.__thread[session] = nil
+					self.__result[co] = result_ok
+					if result_ok and self.__result_data[co] then
+						table.insert(self.__result_data[co], result_data)
+					else
+						self.__result_data[co] = result_data
+					end
+					skynet.wakeup(co)
+				end
 			else
+				self.__thread[session] = nil
 				skynet.error("socket: unknown session :", session)
 			end
 		else
@@ -104,7 +116,14 @@ local function dispatch_by_session(self)
 end
 
 local function pop_response(self)
-	return table.remove(self.__request, 1), table.remove(self.__thread, 1)
+	while true do
+		local func,co = table.remove(self.__request, 1), table.remove(self.__thread, 1)
+		if func then
+			return func, co
+		end
+		self.__wait_response = coroutine.running()
+		skynet.wait(self.__wait_response)
+	end
 end
 
 local function push_response(self, response, co)
@@ -115,34 +134,43 @@ local function push_response(self, response, co)
 		-- response is a function, push it to __request
 		table.insert(self.__request, response)
 		table.insert(self.__thread, co)
+		if self.__wait_response then
+			skynet.wakeup(self.__wait_response)
+			self.__wait_response = nil
+		end
 	end
 end
 
 local function dispatch_by_order(self)
 	while self.__sock do
 		local func, co = pop_response(self)
-		if func == nil then
-			if not socket.block(self.__sock[1]) then
-				close_channel_socket(self)
-				wakeup_all(self)
+		local ok, result_ok, result_data, padding = pcall(func, self.__sock)
+		if ok then
+			if padding and result_ok then
+				-- if padding is true, wait for next result_data
+				-- self.__result_data[co] is a table
+				local result = self.__result_data[co] or {}
+				self.__result_data[co] = result
+				table.insert(result, result_data)
+			else
+				self.__result[co] = result_ok
+				if result_ok and self.__result_data[co] then
+					table.insert(self.__result_data[co], result_data)
+				else
+					self.__result_data[co] = result_data
+				end
+				skynet.wakeup(co)
 			end
 		else
-			local ok, result_ok, result_data = pcall(func, self.__sock)
-			if ok then
-				self.__result[co] = result_ok
-				self.__result_data[co] = result_data
-				skynet.wakeup(co)
-			else
-				close_channel_socket(self)
-				local errmsg
-				if result_ok ~= socket_error then
-					errmsg = result_ok
-				end
-				self.__result[co] = socket_error
-				self.__result_data[co] = errmsg
-				skynet.wakeup(co)
-				wakeup_all(self, errmsg)
+			close_channel_socket(self)
+			local errmsg
+			if result_ok ~= socket_error then
+				errmsg = result_ok
 			end
+			self.__result[co] = socket_error
+			self.__result_data[co] = errmsg
+			skynet.wakeup(co)
+			wakeup_all(self, errmsg)
 		end
 	end
 end
@@ -181,11 +209,11 @@ local function connect_once(self)
 		return false
 	end
 	assert(not self.__sock and not self.__authcoroutine)
-	local fd = socket.open(self.__host, self.__port)
+	local fd,err = socket.open(self.__host, self.__port)
 	if not fd then
 		fd = connect_backup(self)
 		if not fd then
-			return false
+			return false, err
 		end
 	end
 	if self.__nodelay then
@@ -219,13 +247,16 @@ end
 local function try_connect(self , once)
 	local t = 0
 	while not self.__closed do
-		if connect_once(self) then
+		local ok, err = connect_once(self)
+		if ok then
 			if not once then
 				skynet.error("socket: connect to", self.__host, self.__port)
 			end
-			return true
+			return
 		elseif once then
-			return false
+			return err
+		else
+			skynet.error("socket: connect", err)
 		end
 		if t > 1000 then
 			skynet.error("socket: try to reconnect", self.__host, self.__port)
@@ -259,15 +290,16 @@ local function block_connect(self, once)
 	if r ~= nil then
 		return r
 	end
+	local err
 
 	if #self.__connecting > 0 then
 		-- connecting in other coroutine
 		local co = coroutine.running()
 		table.insert(self.__connecting, co)
-		skynet.wait()
+		skynet.wait(co)
 	else
 		self.__connecting[1] = true
-		try_connect(self, once)
+		err = try_connect(self, once)
 		self.__connecting[1] = nil
 		for i=2, #self.__connecting do
 			local co = self.__connecting[i]
@@ -278,7 +310,7 @@ local function block_connect(self, once)
 
 	r = check_connection(self)
 	if r == nil then
-		error(string.format("Connect to %s:%d failed", self.__host, self.__port))
+		error(string.format("Connect to %s:%d failed (%s)", self.__host, self.__port, err))
 	else
 		return r
 	end
@@ -295,7 +327,7 @@ end
 local function wait_for_response(self, response)
 	local co = coroutine.running()
 	push_response(self, response, co)
-	skynet.wait()
+	skynet.wait(co)
 
 	local result = self.__result[co]
 	self.__result[co] = nil
@@ -314,13 +346,27 @@ local function wait_for_response(self, response)
 	end
 end
 
-function channel:request(request, response)
-	assert(block_connect(self, true))	-- connect once
+local socket_write = socket.write
+local socket_lwrite = socket.lwrite
 
-	if not socket.write(self.__sock[1], request) then
-		close_channel_socket(self)
-		wakeup_all(self)
-		error(socket_error)
+function channel:request(request, response, padding)
+	assert(block_connect(self, true))	-- connect once
+	local fd = self.__sock[1]
+
+	if padding then
+		-- padding may be a table, to support multi part request
+		-- multi part request use low priority socket write
+		-- socket_lwrite returns nothing
+		socket_lwrite(fd , request)
+		for _,v in ipairs(padding) do
+			socket_lwrite(fd, v)
+		end
+	else
+		if not socket_write(fd , request) then
+			close_channel_socket(self)
+			wakeup_all(self)
+			error(socket_error)
+		end
 	end
 
 	if response == nil then

lualib/sproto.lua

@@ -58,6 +58,15 @@ local function querytype(self, typename)
 	return v
 end
 
+function sproto:exist_type(typename)
+	local v = self.__tcache[typename]
+	if not v then
+		return core.querytype(self.__cobj, typename) ~= nil
+	else
+		return true
+	end
+end
+
 function sproto:encode(typename, tbl)
 	local st = querytype(self, typename)
 	return core.encode(st, tbl)
@@ -99,6 +108,15 @@ local function queryproto(self, pname)
 	return v
 end
 
+function sproto:exist_proto(pname)
+	local v = self.__pcache[pname]
+	if not v then
+		return core.protocol(self.__cobj, pname) ~= nil
+	else
+		return true
+	end
+end
+
 function sproto:request_encode(protoname, tbl)
 	local p = queryproto(self, protoname)
 	local request = p.request

service-src/service_gate.c

@@ -132,7 +132,12 @@ _ctrl(struct gate * g, const void * msg, int sz) {
 		return;
 	}
 	if (memcmp(command,"start",i) == 0) {
-		skynet_socket_start(ctx, g->listen_id);
+		_parm(tmp, sz, i);
+		int uid = strtol(command , NULL, 10);
+		int id = hashid_lookup(&g->hash, uid);
+		if (id>=0) {
+			skynet_socket_start(ctx, uid);
+		}
 		return;
 	}
 	if (memcmp(command, "close", i) == 0) {
@@ -225,10 +230,7 @@ dispatch_socket_message(struct gate *g, const struct skynet_socket_message * mes
 			break;
 		}
 		int id = hashid_lookup(&g->hash, message->id);
-		if (id>=0) {
-			struct connection *c = &g->conn[id];
-			_report(g, "%d open %d %s:0",message->id,message->id,c->remote_name);
-		} else {
+		if (id<0) {
 			skynet_error(ctx, "Close unknown connection %d", message->id);
 			skynet_socket_close(ctx, message->id);
 		}
@@ -259,9 +261,13 @@ dispatch_socket_message(struct gate *g, const struct skynet_socket_message * mes
 			c->id = message->ud;
 			memcpy(c->remote_name, message+1, sz);
 			c->remote_name[sz] = '\0';
-			skynet_socket_start(ctx, message->ud);
+			_report(g, "%d open %d %s:0",c->id, c->id, c->remote_name);
+			skynet_error(ctx, "socket open: %x", c->id);
 		}
 		break;
+	case SKYNET_SOCKET_TYPE_WARNING:
+		skynet_error(ctx, "fd (%d) send buffer (%d)K", message->id, message->ud);
+		break;
 	}
 }
 
@@ -292,7 +298,6 @@ _cb(struct skynet_context * ctx, void * ud, int type, int session, uint32_t sour
 		}
 	}
 	case PTYPE_SOCKET:
-		assert(source == 0);
 		// recv socket message from skynet_socket
 		dispatch_socket_message(g, msg, (int)(sz-sizeof(struct skynet_socket_message)));
 		break;
@@ -325,6 +330,7 @@ start_listen(struct gate *g, char * listen_addr) {
 	if (g->listen_id < 0) {
 		return 1;
 	}
+	skynet_socket_start(ctx, g->listen_id);
 	return 0;
 }
 
@@ -333,13 +339,12 @@ gate_init(struct gate *g , struct skynet_context * ctx, char * parm) {
 	if (parm == NULL)
 		return 1;
 	int max = 0;
-	int buffer = 0;
 	int sz = strlen(parm)+1;
 	char watchdog[sz];
 	char binding[sz];
 	int client_tag = 0;
 	char header;
-	int n = sscanf(parm, "%c %s %s %d %d %d",&header,watchdog, binding,&client_tag , &max,&buffer);
+	int n = sscanf(parm, "%c %s %s %d %d", &header, watchdog, binding, &client_tag, &max);
 	if (n<4) {
 		skynet_error(ctx, "Invalid gate parm %s",parm);
 		return 1;

service-src/service_harbor.c

@@ -1,6 +1,7 @@
 #include "skynet.h"
 #include "skynet_harbor.h"
 #include "skynet_socket.h"
+#include "skynet_handle.h"
 
 /*
 	harbor listen the PTYPE_HARBOR (in text)
@@ -312,20 +313,27 @@ forward_local_messsage(struct harbor *h, void *msg, int sz) {
 	message_to_header((const uint32_t *)cookie, &header);
 
 	uint32_t destination = header.destination;
-	int type = (destination >> HANDLE_REMOTE_SHIFT) | PTYPE_TAG_DONTCOPY;
+	int type = destination >> HANDLE_REMOTE_SHIFT;
 	destination = (destination & HANDLE_MASK) | ((uint32_t)h->id << HANDLE_REMOTE_SHIFT);
 
-	if (skynet_send(h->ctx, header.source, destination, type, (int)header.session, (void *)msg, sz-HEADER_COOKIE_LENGTH) < 0) {
-		skynet_send(h->ctx, destination, header.source , PTYPE_ERROR, (int)header.session, NULL, 0);
-		skynet_error(h->ctx, "Unknown destination :%x from :%x", destination, header.source);
+	if (skynet_send(h->ctx, header.source, destination, type | PTYPE_TAG_DONTCOPY , (int)header.session, (void *)msg, sz-HEADER_COOKIE_LENGTH) < 0) {
+		if (type != PTYPE_ERROR) {
+			// don't need report error when type is error
+			skynet_send(h->ctx, destination, header.source , PTYPE_ERROR, (int)header.session, NULL, 0);
+		}
+		skynet_error(h->ctx, "Unknown destination :%x from :%x type(%d)", destination, header.source, type);
 	}
 }
 
 static void
 send_remote(struct skynet_context * ctx, int fd, const char * buffer, size_t sz, struct remote_message_header * cookie) {
-	uint32_t sz_header = sz+sizeof(*cookie);
+	size_t sz_header = sz+sizeof(*cookie);
+	if (sz_header > UINT32_MAX) {
+		skynet_error(ctx, "remote message from :%08x to :%08x is too large.", cookie->source, cookie->destination);
+		return;
+	}
 	uint8_t * sendbuf = skynet_malloc(sz_header+4);
-	to_bigendian(sendbuf, sz_header);
+	to_bigendian(sendbuf, (uint32_t)sz_header);
 	memcpy(sendbuf+4, buffer, sz);
 	header_to_message(cookie, sendbuf+4+sz);
 
@@ -657,6 +665,13 @@ mainloop(struct skynet_context * context, void * ud, int type, int session, uint
 		case SKYNET_SOCKET_TYPE_CONNECT:
 			// fd forward to this service
 			break;
+		case SKYNET_SOCKET_TYPE_WARNING: {
+			int id = harbor_id(h, message->id);
+			if (id) {
+				skynet_error(context, "message havn't send to Harbor (%d) reach %d K", id, message->ud);
+			}
+			break;
+		}
 		default:
 			skynet_error(context, "recv invalid socket message type %d", type);
 			break;

service-src/service_snlua.c

@@ -30,6 +30,7 @@ static int
 codecache(lua_State *L) {
 	luaL_Reg l[] = {
 		{ "clear", cleardummy },
+		{ "mode", cleardummy },
 		{ NULL, NULL },
 	};
 	luaL_newlib(L,l);

service/bootstrap.lua

@@ -1,8 +1,12 @@
 local skynet = require "skynet"
 local harbor = require "skynet.harbor"
 require "skynet.manager"	-- import skynet.launch, ...
+local memory = require "memory"
 
 skynet.start(function()
+	local sharestring = tonumber(skynet.getenv "sharestring")
+	memory.ssexpand(sharestring or 4096)
+
 	local standalone = skynet.getenv "standalone"
 
 	local launcher = assert(skynet.launch("snlua","launcher"))

service/clusterd.lua

@@ -11,7 +11,7 @@ local command = {}
 local function read_response(sock)
 	local sz = socket.header(sock:read(2))
 	local msg = sock:read(sz)
-	return cluster.unpackresponse(msg)	-- session, ok, data
+	return cluster.unpackresponse(msg)	-- session, ok, data, padding
 end
 
 local function open_channel(t, key)
@@ -64,19 +64,23 @@ end
 local function send_request(source, node, addr, msg, sz)
 	local session = node_session[node] or 1
 	-- msg is a local pointer, cluster.packrequest will free it
-	local request, new_session = cluster.packrequest(addr, session, msg, sz)
+	local request, new_session, padding = cluster.packrequest(addr, session, msg, sz)
 	node_session[node] = new_session
 
 	-- node_channel[node] may yield or throw error
 	local c = node_channel[node]
 
-	return c:request(request, session)
+	return c:request(request, session, padding)
 end
 
 function command.req(...)
 	local ok, msg, sz = pcall(send_request, ...)
 	if ok then
-		skynet.ret(msg, sz)
+		if type(msg) == "table" then
+			skynet.ret(cluster.concat(msg))
+		else
+			skynet.ret(msg)
+		end
 	else
 		skynet.error(msg)
 		skynet.response()(false)
@@ -93,23 +97,78 @@ function command.proxy(source, node, name)
 	skynet.ret(skynet.pack(proxy[fullname]))
 end
 
-local request_fd = {}
+local register_name = {}
+
+function command.register(source, name, addr)
+	assert(register_name[name] == nil)
+	addr = addr or source
+	local old_name = register_name[addr]
+	if old_name then
+		register_name[old_name] = nil
+	end
+	register_name[addr] = name
+	register_name[name] = addr
+	skynet.ret(nil)
+	skynet.error(string.format("Register [%s] :%08x", name, addr))
+end
+
+local large_request = {}
 
 function command.socket(source, subcmd, fd, msg)
 	if subcmd == "data" then
-		local addr, session, msg = cluster.unpackrequest(msg)
-		local ok , msg, sz = pcall(skynet.rawcall, addr, "lua", msg)
-		local response
+		local sz
+		local addr, session, msg, padding = cluster.unpackrequest(msg)
+		if padding then
+			local req = large_request[session] or { addr = addr }
+			large_request[session] = req
+			table.insert(req, msg)
+			return
+		else
+			local req = large_request[session]
+			if req then
+				large_request[session] = nil
+				table.insert(req, msg)
+				msg,sz = cluster.concat(req)
+				addr = req.addr
+			end
+			if not msg then
+				local response = cluster.packresponse(session, false, "Invalid large req")
+				socket.write(fd, response)
+				return
+			end
+		end
+		local ok, response
+		if addr == 0 then
+			local name = skynet.unpack(msg, sz)
+			local addr = register_name[name]
+			if addr then
+				ok = true
+				msg, sz = skynet.pack(addr)
+			else
+				ok = false
+				msg = "name not found"
+			end
+		else
+			ok , msg, sz = pcall(skynet.rawcall, addr, "lua", msg, sz)
+		end
 		if ok then
 			response = cluster.packresponse(session, true, msg, sz)
+			if type(response) == "table" then
+				for _, v in ipairs(response) do
+					socket.lwrite(fd, v)
+				end
+			else
+				socket.write(fd, response)
+			end
 		else
 			response = cluster.packresponse(session, false, msg)
+			socket.write(fd, response)
 		end
-		socket.write(fd, response)
 	elseif subcmd == "open" then
 		skynet.error(string.format("socket accept from %s", msg))
 		skynet.call(source, "lua", "accept", fd)
 	else
+		large_request = {}
 		skynet.error(string.format("socket %s %d : %s", subcmd, fd, msg))
 	end
 end

service/debug_console.lua

@@ -3,6 +3,7 @@ local codecache = require "skynet.codecache"
 local core = require "skynet.core"
 local socket = require "socket"
 local snax = require "snax"
+local memory = require "memory"
 
 local port = tonumber(...)
 local COMMAND = {}
@@ -130,6 +131,8 @@ function COMMAND.help()
 		log = "launch a new lua service with log",
 		debug = "debug address : debug a lua service",
 		signal = "signal address sig",
+		cmem = "Show C memory info",
+		shrtbl = "Show shared short string table info",
 	}
 end
 
@@ -258,3 +261,19 @@ function COMMAND.signal(address, sig)
 		core.command("SIGNAL", address)
 	end
 end
+
+function COMMAND.cmem()
+	local info = memory.info()
+	local tmp = {}
+	for k,v in pairs(info) do
+		tmp[skynet.address(k)] = v
+	end
+	return tmp
+end
+
+function COMMAND.shrtbl()
+	local n, total, longest, space = memory.ssinfo()
+	return { n = n, total = total, longest = longest, space = space }
+end
+
+

service/gate.lua

@@ -63,6 +63,10 @@ function handler.error(fd, msg)
 	skynet.send(watchdog, "lua", "socket", "error", fd, msg)
 end
 
+function handler.warning(fd, size)
+	skynet.send(watchdog, "lua", "socket", "warning", fd, size)
+end
+
 local CMD = {}
 
 function CMD.forward(source, fd, client, address)

service/multicastd.lua

@@ -67,7 +67,7 @@ end
 -- for remote node, call remote_publish. (call mc.unpack and skynet.tostring to convert message pointer to string)
 local function publish(c , source, pack, size)
 	local group = channel[c]
-	if group == nil then
+	if group == nil or next(group) == nil then
 		-- dead channel, delete the pack. mc.bind returns the pointer in pack
 		local pack = mc.bind(pack, 1)
 		mc.close(pack)

service/sharedatad.lua

@@ -1,6 +1,8 @@
 local skynet = require "skynet"
 local sharedata = require "sharedata.corelib"
 local table = table
+local cache = require "skynet.codecache"
+cache.mode "OFF"	-- turn off codecache, because CMD.new may load data file
 
 local NORET = {}
 local pool = {}
@@ -43,9 +45,17 @@ function CMD.new(name, t)
 		value = t
 	elseif dt == "string" then
 		value = setmetatable({}, env_mt)
-		local f = load(t, "=" .. name, "t", value)
-		assert(skynet.pcall(f))
+		local f
+		if t:sub(1,1) == "@" then
+			f = assert(loadfile(t:sub(2),"bt",value))
+		else
+			f = assert(load(t, "=" .. name, "bt", value))
+		end
+		local _, ret = assert(skynet.pcall(f))
 		setmetatable(value, nil)
+		if type(ret) == "table" then
+			value = ret
+		end
 	elseif dt == "nil" then
 		value = {}
 	else
@@ -118,12 +128,12 @@ function CMD.monitor(name, obj)
 		return v.obj
 	end
 
-	local n = pool_count[name].n
-	pool_count[name].n = n + 1
+	local n = pool_count[name].n + 1
 	if n > pool_count[name].threshold then
 		n = n - check_watch(v.watch)
 		pool_count[name].threshold = n * 2
 	end
+	pool_count[name].n = n
 
 	table.insert(v.watch, skynet.response())
 

skynet-src/atomic.h

@@ -0,0 +1,14 @@
+#ifndef SKYNET_ATOMIC_H
+#define SKYNET_ATOMIC_H
+
+#define ATOM_CAS(ptr, oval, nval) __sync_bool_compare_and_swap(ptr, oval, nval)
+#define ATOM_CAS_POINTER(ptr, oval, nval) __sync_bool_compare_and_swap(ptr, oval, nval)
+#define ATOM_INC(ptr) __sync_add_and_fetch(ptr, 1)
+#define ATOM_FINC(ptr) __sync_fetch_and_add(ptr, 1)
+#define ATOM_DEC(ptr) __sync_sub_and_fetch(ptr, 1)
+#define ATOM_FDEC(ptr) __sync_fetch_and_sub(ptr, 1)
+#define ATOM_ADD(ptr,n) __sync_add_and_fetch(ptr, n)
+#define ATOM_SUB(ptr,n) __sync_sub_and_fetch(ptr, n)
+#define ATOM_AND(ptr,n) __sync_and_and_fetch(ptr, n)
+
+#endif

skynet-src/luashrtbl.h

@@ -0,0 +1,16 @@
+#ifndef LUA_SHORT_STRING_TABLE_H
+#define LUA_SHORT_STRING_TABLE_H
+
+#include "lstring.h"
+
+// If you use modified lua, this macro would be defined in lstring.h
+#ifndef ENABLE_SHORT_STRING_TABLE
+
+static inline int luaS_shrinfo(lua_State *L) { return 0; }
+static inline void luaS_initshr() {}
+static inline void luaS_exitshr() {}
+static inline void luaS_expandshr(int n);
+
+#endif
+
+#endif

skynet-src/malloc_hook.c

@@ -6,6 +6,7 @@
 
 #include "malloc_hook.h"
 #include "skynet.h"
+#include "atomic.h"
 
 static size_t _used_memory = 0;
 static size_t _memory_block = 0;
@@ -32,11 +33,11 @@ get_allocated_field(uint32_t handle) {
 	ssize_t old_alloc = data->allocated;
 	if(old_handle == 0 || old_alloc <= 0) {
 		// data->allocated may less than zero, because it may not count at start.
-		if(!__sync_bool_compare_and_swap(&data->handle, old_handle, handle)) {
+		if(!ATOM_CAS(&data->handle, old_handle, handle)) {
 			return 0;
 		}
 		if (old_alloc < 0) {
-			__sync_bool_compare_and_swap(&data->allocated, old_alloc, 0);
+			ATOM_CAS(&data->allocated, old_alloc, 0);
 		}
 	}
 	if(data->handle != handle) {
@@ -47,21 +48,21 @@ get_allocated_field(uint32_t handle) {
 
 inline static void 
 update_xmalloc_stat_alloc(uint32_t handle, size_t __n) {
-	__sync_add_and_fetch(&_used_memory, __n);
-	__sync_add_and_fetch(&_memory_block, 1); 
+	ATOM_ADD(&_used_memory, __n);
+	ATOM_INC(&_memory_block); 
 	ssize_t* allocated = get_allocated_field(handle);
 	if(allocated) {
-		__sync_add_and_fetch(allocated, __n);
+		ATOM_ADD(allocated, __n);
 	}
 }
 
 inline static void
 update_xmalloc_stat_free(uint32_t handle, size_t __n) {
-	__sync_sub_and_fetch(&_used_memory, __n);
-	__sync_sub_and_fetch(&_memory_block, 1);
+	ATOM_SUB(&_used_memory, __n);
+	ATOM_DEC(&_memory_block);
 	ssize_t* allocated = get_allocated_field(handle);
 	if(allocated) {
-		__sync_sub_and_fetch(allocated, __n);
+		ATOM_SUB(allocated, __n);
 	}
 }
 

skynet-src/rwlock.h

@@ -1,6 +1,8 @@
 #ifndef SKYNET_RWLOCK_H
 #define SKYNET_RWLOCK_H
 
+#ifndef USE_PTHREAD_LOCK
+
 struct rwlock {
 	int write;
 	int read;
@@ -45,4 +47,42 @@ rwlock_runlock(struct rwlock *lock) {
 	__sync_sub_and_fetch(&lock->read,1);
 }
 
+#else
+
+#include <pthread.h>
+
+// only for some platform doesn't have __sync_*
+// todo: check the result of pthread api
+
+struct rwlock {
+	pthread_rwlock_t lock;
+};
+
+static inline void
+rwlock_init(struct rwlock *lock) {
+	pthread_rwlock_init(&lock->lock, NULL);
+}
+
+static inline void
+rwlock_rlock(struct rwlock *lock) {
+	 pthread_rwlock_rdlock(&lock->lock);
+}
+
+static inline void
+rwlock_wlock(struct rwlock *lock) {
+	 pthread_rwlock_wrlock(&lock->lock);
+}
+
+static inline void
+rwlock_wunlock(struct rwlock *lock) {
+	pthread_rwlock_unlock(&lock->lock);
+}
+
+static inline void
+rwlock_runlock(struct rwlock *lock) {
+	pthread_rwlock_unlock(&lock->lock);
+}
+
+#endif
+
 #endif

skynet-src/skynet_env.c

@@ -1,5 +1,6 @@
 #include "skynet.h"
 #include "skynet_env.h"
+#include "spinlock.h"
 
 #include <lua.h>
 #include <lauxlib.h>
@@ -8,18 +9,15 @@
 #include <assert.h>
 
 struct skynet_env {
-	int lock;
+	struct spinlock lock;
 	lua_State *L;
 };
 
 static struct skynet_env *E = NULL;
 
-#define LOCK(q) while (__sync_lock_test_and_set(&(q)->lock,1)) {}
-#define UNLOCK(q) __sync_lock_release(&(q)->lock);
-
 const char * 
 skynet_getenv(const char *key) {
-	LOCK(E)
+	SPIN_LOCK(E)
 
 	lua_State *L = E->L;
 	
@@ -27,14 +25,14 @@ skynet_getenv(const char *key) {
 	const char * result = lua_tostring(L, -1);
 	lua_pop(L, 1);
 
-	UNLOCK(E)
+	SPIN_UNLOCK(E)
 
 	return result;
 }
 
 void 
 skynet_setenv(const char *key, const char *value) {
-	LOCK(E)
+	SPIN_LOCK(E)
 	
 	lua_State *L = E->L;
 	lua_getglobal(L, key);
@@ -43,12 +41,12 @@ skynet_setenv(const char *key, const char *value) {
 	lua_pushstring(L,value);
 	lua_setglobal(L,key);
 
-	UNLOCK(E)
+	SPIN_UNLOCK(E)
 }
 
 void
 skynet_env_init() {
 	E = skynet_malloc(sizeof(*E));
-	E->lock = 0;
+	SPIN_INIT(E)
 	E->L = luaL_newstate();
 }

skynet-src/skynet_error.c

@@ -54,7 +54,7 @@ skynet_error(struct skynet_context * context, const char *msg, ...) {
 	}
 	smsg.session = 0;
 	smsg.data = data;
-	smsg.sz = len | (PTYPE_TEXT << HANDLE_REMOTE_SHIFT);
+	smsg.sz = len | ((size_t)PTYPE_TEXT << MESSAGE_TYPE_SHIFT);
 	skynet_context_push(logger, &smsg);
 }
 

skynet-src/skynet_handle.h

@@ -3,7 +3,9 @@
 
 #include <stdint.h>
 
-#include "skynet_harbor.h"
+// reserve high 8 bits for remote id
+#define HANDLE_MASK 0xffffff
+#define HANDLE_REMOTE_SHIFT 24
 
 struct skynet_context;
 

skynet-src/skynet_harbor.c

@@ -1,6 +1,8 @@
 #include "skynet.h"
 #include "skynet_harbor.h"
 #include "skynet_server.h"
+#include "skynet_mq.h"
+#include "skynet_handle.h"
 
 #include <string.h>
 #include <stdio.h>
@@ -11,8 +13,8 @@ static unsigned int HARBOR = ~0;
 
 void 
 skynet_harbor_send(struct remote_message *rmsg, uint32_t source, int session) {
-	int type = rmsg->sz >> HANDLE_REMOTE_SHIFT;
-	rmsg->sz &= HANDLE_MASK;
+	int type = rmsg->sz >> MESSAGE_TYPE_SHIFT;
+	rmsg->sz &= MESSAGE_TYPE_MASK;
 	assert(type != PTYPE_SYSTEM && type != PTYPE_HARBOR && REMOTE);
 	skynet_context_send(REMOTE, rmsg, sizeof(*rmsg) , source, type , session);
 }

skynet-src/skynet_harbor.h

@@ -7,10 +7,6 @@
 #define GLOBALNAME_LENGTH 16
 #define REMOTE_MAX 256
 
-// reserve high 8 bits for remote id
-#define HANDLE_MASK 0xffffff
-#define HANDLE_REMOTE_SHIFT 24
-
 struct remote_name {
 	char name[GLOBALNAME_LENGTH];
 	uint32_t handle;

skynet-src/skynet_main.c

@@ -3,6 +3,7 @@
 #include "skynet_imp.h"
 #include "skynet_env.h"
 #include "skynet_server.h"
+#include "luashrtbl.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -105,6 +106,8 @@ main(int argc, char *argv[]) {
 			"usage: skynet configfilename\n");
 		return 1;
 	}
+
+	luaS_initshr();
 	skynet_globalinit();
 	skynet_env_init();
 
@@ -139,6 +142,7 @@ main(int argc, char *argv[]) {
 
 	skynet_start(&config);
 	skynet_globalexit();
+	luaS_exitshr();
 
 	return 0;
 }

skynet-src/skynet_module.c

@@ -1,6 +1,7 @@
 #include "skynet.h"
 
 #include "skynet_module.h"
+#include "spinlock.h"
 
 #include <assert.h>
 #include <string.h>
@@ -13,7 +14,7 @@
 
 struct modules {
 	int count;
-	int lock;
+	struct spinlock lock;
 	const char * path;
 	struct skynet_module m[MAX_MODULE_TYPE];
 };
@@ -95,7 +96,7 @@ skynet_module_query(const char * name) {
 	if (result)
 		return result;
 
-	while(__sync_lock_test_and_set(&M->lock,1)) {}
+	SPIN_LOCK(M)
 
 	result = _query(name); // double check
 
@@ -114,21 +115,22 @@ skynet_module_query(const char * name) {
 		}
 	}
 
-	__sync_lock_release(&M->lock);
+	SPIN_UNLOCK(M)
 
 	return result;
 }
 
 void 
 skynet_module_insert(struct skynet_module *mod) {
-	while(__sync_lock_test_and_set(&M->lock,1)) {}
+	SPIN_LOCK(M)
 
 	struct skynet_module * m = _query(mod->name);
 	assert(m == NULL && M->count < MAX_MODULE_TYPE);
 	int index = M->count;
 	M->m[index] = *mod;
 	++M->count;
-	__sync_lock_release(&M->lock);
+
+	SPIN_UNLOCK(M)
 }
 
 void * 
@@ -164,7 +166,8 @@ skynet_module_init(const char *path) {
 	struct modules *m = skynet_malloc(sizeof(*m));
 	m->count = 0;
 	m->path = skynet_strdup(path);
-	m->lock = 0;
+
+	SPIN_INIT(m)
 
 	M = m;
 }

skynet-src/skynet_monitor.c

@@ -3,6 +3,7 @@
 #include "skynet_monitor.h"
 #include "skynet_server.h"
 #include "skynet.h"
+#include "atomic.h"
 
 #include <stdlib.h>
 #include <string.h>
@@ -30,7 +31,7 @@ void
 skynet_monitor_trigger(struct skynet_monitor *sm, uint32_t source, uint32_t destination) {
 	sm->source = source;
 	sm->destination = destination;
-	__sync_fetch_and_add(&sm->version , 1);
+	ATOM_INC(&sm->version);
 }
 
 void 

skynet-src/skynet_mq.c

@@ -1,6 +1,7 @@
 #include "skynet.h"
 #include "skynet_mq.h"
 #include "skynet_handle.h"
+#include "spinlock.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -18,11 +19,11 @@
 #define MQ_OVERLOAD 1024
 
 struct message_queue {
+	struct spinlock lock;
 	uint32_t handle;
 	int cap;
 	int head;
 	int tail;
-	int lock;
 	int release;
 	int in_global;
 	int overload;
@@ -34,19 +35,16 @@ struct message_queue {
 struct global_queue {
 	struct message_queue *head;
 	struct message_queue *tail;
-	int lock;
+	struct spinlock lock;
 };
 
 static struct global_queue *Q = NULL;
 
-#define LOCK(q) while (__sync_lock_test_and_set(&(q)->lock,1)) {}
-#define UNLOCK(q) __sync_lock_release(&(q)->lock);
-
 void 
 skynet_globalmq_push(struct message_queue * queue) {
 	struct global_queue *q= Q;
 
-	LOCK(q)
+	SPIN_LOCK(q)
 	assert(queue->next == NULL);
 	if(q->tail) {
 		q->tail->next = queue;
@@ -54,14 +52,14 @@ skynet_globalmq_push(struct message_queue * queue) {
 	} else {
 		q->head = q->tail = queue;
 	}
-	UNLOCK(q)
+	SPIN_UNLOCK(q)
 }
 
 struct message_queue * 
 skynet_globalmq_pop() {
 	struct global_queue *q = Q;
 
-	LOCK(q)
+	SPIN_LOCK(q)
 	struct message_queue *mq = q->head;
 	if(mq) {
 		q->head = mq->next;
@@ -71,7 +69,7 @@ skynet_globalmq_pop() {
 		}
 		mq->next = NULL;
 	}
-	UNLOCK(q)
+	SPIN_UNLOCK(q)
 
 	return mq;
 }
@@ -83,7 +81,7 @@ skynet_mq_create(uint32_t handle) {
 	q->cap = DEFAULT_QUEUE_SIZE;
 	q->head = 0;
 	q->tail = 0;
-	q->lock = 0;
+	SPIN_INIT(q)
 	// When the queue is create (always between service create and service init) ,
 	// set in_global flag to avoid push it to global queue .
 	// If the service init success, skynet_context_new will call skynet_mq_force_push to push it to global queue.
@@ -100,6 +98,7 @@ skynet_mq_create(uint32_t handle) {
 static void 
 _release(struct message_queue *q) {
 	assert(q->next == NULL);
+	SPIN_DESTROY(q)
 	skynet_free(q->queue);
 	skynet_free(q);
 }
@@ -113,11 +112,11 @@ int
 skynet_mq_length(struct message_queue *q) {
 	int head, tail,cap;
 
-	LOCK(q)
+	SPIN_LOCK(q)
 	head = q->head;
 	tail = q->tail;
 	cap = q->cap;
-	UNLOCK(q)
+	SPIN_UNLOCK(q)
 	
 	if (head <= tail) {
 		return tail - head;
@@ -138,7 +137,7 @@ skynet_mq_overload(struct message_queue *q) {
 int
 skynet_mq_pop(struct message_queue *q, struct skynet_message *message) {
 	int ret = 1;
-	LOCK(q)
+	SPIN_LOCK(q)
 
 	if (q->head != q->tail) {
 		*message = q->queue[q->head++];
@@ -167,7 +166,7 @@ skynet_mq_pop(struct message_queue *q, struct skynet_message *message) {
 		q->in_global = 0;
 	}
 	
-	UNLOCK(q)
+	SPIN_UNLOCK(q)
 
 	return ret;
 }
@@ -190,7 +189,7 @@ expand_queue(struct message_queue *q) {
 void 
 skynet_mq_push(struct message_queue *q, struct skynet_message *message) {
 	assert(message);
-	LOCK(q)
+	SPIN_LOCK(q)
 
 	q->queue[q->tail] = *message;
 	if (++ q->tail >= q->cap) {
@@ -206,25 +205,26 @@ skynet_mq_push(struct message_queue *q, struct skynet_message *message) {
 		skynet_globalmq_push(q);
 	}
 	
-	UNLOCK(q)
+	SPIN_UNLOCK(q)
 }
 
 void 
 skynet_mq_init() {
 	struct global_queue *q = skynet_malloc(sizeof(*q));
 	memset(q,0,sizeof(*q));
+	SPIN_INIT(q);
 	Q=q;
 }
 
 void 
 skynet_mq_mark_release(struct message_queue *q) {
-	LOCK(q)
+	SPIN_LOCK(q)
 	assert(q->release == 0);
 	q->release = 1;
 	if (q->in_global != MQ_IN_GLOBAL) {
 		skynet_globalmq_push(q);
 	}
-	UNLOCK(q)
+	SPIN_UNLOCK(q)
 }
 
 static void
@@ -238,13 +238,13 @@ _drop_queue(struct message_queue *q, message_drop drop_func, void *ud) {
 
 void 
 skynet_mq_release(struct message_queue *q, message_drop drop_func, void *ud) {
-	LOCK(q)
+	SPIN_LOCK(q)
 	
 	if (q->release) {
-		UNLOCK(q)
+		SPIN_UNLOCK(q)
 		_drop_queue(q, drop_func, ud);
 	} else {
 		skynet_globalmq_push(q);
-		UNLOCK(q)
+		SPIN_UNLOCK(q)
 	}
 }

skynet-src/skynet_mq.h

@@ -11,6 +11,10 @@ struct skynet_message {
 	size_t sz;
 };
 
+// type is encoding in skynet_message.sz high 8bit
+#define MESSAGE_TYPE_MASK (SIZE_MAX >> 8)
+#define MESSAGE_TYPE_SHIFT ((sizeof(size_t)-1) * 8)
+
 struct message_queue;
 
 void skynet_globalmq_push(struct message_queue * queue);

skynet-src/skynet_server.c

@@ -10,6 +10,8 @@
 #include "skynet_monitor.h"
 #include "skynet_imp.h"
 #include "skynet_log.h"
+#include "spinlock.h"
+#include "atomic.h"
 
 #include <pthread.h>
 
@@ -21,16 +23,18 @@
 
 #ifdef CALLING_CHECK
 
-#define CHECKCALLING_BEGIN(ctx) assert(__sync_lock_test_and_set(&ctx->calling,1) == 0);
-#define CHECKCALLING_END(ctx) __sync_lock_release(&ctx->calling);
-#define CHECKCALLING_INIT(ctx) ctx->calling = 0;
-#define CHECKCALLING_DECL int calling;
+#define CHECKCALLING_BEGIN(ctx) if (!(spinlock_trylock(&ctx->calling))) { assert(0); }
+#define CHECKCALLING_END(ctx) spinlock_unlock(&ctx->calling);
+#define CHECKCALLING_INIT(ctx) spinlock_init(&ctx->calling);
+#define CHECKCALLING_DESTROY(ctx) spinlock_destroy(&ctx->calling);
+#define CHECKCALLING_DECL struct spinlock calling;
 
 #else
 
 #define CHECKCALLING_BEGIN(ctx)
 #define CHECKCALLING_END(ctx)
 #define CHECKCALLING_INIT(ctx)
+#define CHECKCALLING_DESTROY(ctx)
 #define CHECKCALLING_DECL
 
 #endif
@@ -68,12 +72,12 @@ skynet_context_total() {
 
 static void
 context_inc() {
-	__sync_fetch_and_add(&G_NODE.total,1);
+	ATOM_INC(&G_NODE.total);
 }
 
 static void
 context_dec() {
-	__sync_fetch_and_sub(&G_NODE.total,1);
+	ATOM_DEC(&G_NODE.total);
 }
 
 uint32_t 
@@ -179,7 +183,7 @@ skynet_context_newsession(struct skynet_context *ctx) {
 
 void 
 skynet_context_grab(struct skynet_context *ctx) {
-	__sync_add_and_fetch(&ctx->ref,1);
+	ATOM_INC(&ctx->ref);
 }
 
 void
@@ -197,13 +201,14 @@ delete_context(struct skynet_context *ctx) {
 	}
 	skynet_module_instance_release(ctx->mod, ctx->instance);
 	skynet_mq_mark_release(ctx->queue);
+	CHECKCALLING_DESTROY(ctx)
 	skynet_free(ctx);
 	context_dec();
 }
 
 struct skynet_context * 
 skynet_context_release(struct skynet_context *ctx) {
-	if (__sync_sub_and_fetch(&ctx->ref,1) == 0) {
+	if (ATOM_DEC(&ctx->ref) == 0) {
 		delete_context(ctx);
 		return NULL;
 	}
@@ -246,8 +251,8 @@ dispatch_message(struct skynet_context *ctx, struct skynet_message *msg) {
 	assert(ctx->init);
 	CHECKCALLING_BEGIN(ctx)
 	pthread_setspecific(G_NODE.handle_key, (void *)(uintptr_t)(ctx->handle));
-	int type = msg->sz >> HANDLE_REMOTE_SHIFT;
-	size_t sz = msg->sz & HANDLE_MASK;
+	int type = msg->sz >> MESSAGE_TYPE_SHIFT;
+	size_t sz = msg->sz & MESSAGE_TYPE_MASK;
 	if (ctx->logfile) {
 		skynet_log_output(ctx->logfile, msg->source, type, msg->session, msg->data, sz);
 	}
@@ -560,7 +565,7 @@ cmd_logon(struct skynet_context * context, const char * param) {
 	if (lastf == NULL) {
 		f = skynet_log_open(context, handle);
 		if (f) {
-			if (!__sync_bool_compare_and_swap(&ctx->logfile, NULL, f)) {
+			if (!ATOM_CAS_POINTER(&ctx->logfile, NULL, f)) {
 				// logfile opens in other thread, close this one.
 				fclose(f);
 			}
@@ -581,7 +586,7 @@ cmd_logoff(struct skynet_context * context, const char * param) {
 	FILE * f = ctx->logfile;
 	if (f) {
 		// logfile may close in other thread
-		if (__sync_bool_compare_and_swap(&ctx->logfile, f, NULL)) {
+		if (ATOM_CAS_POINTER(&ctx->logfile, f, NULL)) {
 			skynet_log_close(context, f, handle);
 		}
 	}
@@ -662,14 +667,16 @@ _filter_args(struct skynet_context * context, int type, int *session, void ** da
 		*data = msg;
 	}
 
-	*sz |= type << HANDLE_REMOTE_SHIFT;
+	*sz |= (size_t)type << MESSAGE_TYPE_SHIFT;
 }
 
 int
 skynet_send(struct skynet_context * context, uint32_t source, uint32_t destination , int type, int session, void * data, size_t sz) {
-	if ((sz & HANDLE_MASK) != sz) {
-		skynet_error(context, "The message to %x is too large (sz = %lu)", destination, sz);
-		skynet_free(data);
+	if ((sz & MESSAGE_TYPE_MASK) != sz) {
+		skynet_error(context, "The message to %x is too large", destination);
+		if (type & PTYPE_TAG_DONTCOPY) {
+			skynet_free(data);
+		}
 		return -1;
 	}
 	_filter_args(context, type, &session, (void **)&data, &sz);
@@ -751,7 +758,7 @@ skynet_context_send(struct skynet_context * ctx, void * msg, size_t sz, uint32_t
 	smsg.source = source;
 	smsg.session = session;
 	smsg.data = msg;
-	smsg.sz = sz | type << HANDLE_REMOTE_SHIFT;
+	smsg.sz = sz | (size_t)type << MESSAGE_TYPE_SHIFT;
 
 	skynet_mq_push(ctx->queue, &smsg);
 }

skynet-src/skynet_socket.c

@@ -33,10 +33,14 @@ skynet_socket_free() {
 static void
 forward_message(int type, bool padding, struct socket_message * result) {
 	struct skynet_socket_message *sm;
-	int sz = sizeof(*sm);
+	size_t sz = sizeof(*sm);
 	if (padding) {
 		if (result->data) {
-			sz += strlen(result->data);
+			size_t msg_sz = strlen(result->data);
+			if (msg_sz > 128) {
+				msg_sz = 128;
+			}
+			sz += msg_sz;
 		} else {
 			result->data = "";
 		}
@@ -56,7 +60,7 @@ forward_message(int type, bool padding, struct socket_message * result) {
 	message.source = 0;
 	message.session = 0;
 	message.data = sm;
-	message.sz = sz | PTYPE_SOCKET << HANDLE_REMOTE_SHIFT;
+	message.sz = sz | ((size_t)PTYPE_SOCKET << MESSAGE_TYPE_SHIFT);
 	
 	if (skynet_context_push((uint32_t)result->opaque, &message)) {
 		// todo: report somewhere to close socket
@@ -86,7 +90,7 @@ skynet_socket_poll() {
 		forward_message(SKYNET_SOCKET_TYPE_CONNECT, true, &result);
 		break;
 	case SOCKET_ERROR:
-		forward_message(SKYNET_SOCKET_TYPE_ERROR, false, &result);
+		forward_message(SKYNET_SOCKET_TYPE_ERROR, true, &result);
 		break;
 	case SOCKET_ACCEPT:
 		forward_message(SKYNET_SOCKET_TYPE_ACCEPT, true, &result);
@@ -109,10 +113,13 @@ check_wsz(struct skynet_context *ctx, int id, void *buffer, int64_t wsz) {
 	if (wsz < 0) {
 		return -1;
 	} else if (wsz > 1024 * 1024) {
-		int kb4 = wsz / 1024 / 4;
-		if (kb4 % 256 == 0) {
-			skynet_error(ctx, "%d Mb bytes on socket %d need to send out", (int)(wsz / (1024 * 1024)), id);
-		}
+		struct skynet_socket_message tmp;
+		tmp.type = SKYNET_SOCKET_TYPE_WARNING;
+		tmp.id = id;
+		tmp.ud = (int)(wsz / 1024);
+		tmp.buffer = NULL;
+		skynet_send(ctx, 0, skynet_context_handle(ctx), PTYPE_SOCKET, 0 , &tmp, sizeof(tmp));
+//		skynet_error(ctx, "%d Mb bytes on socket %d need to send out", (int)(wsz / (1024 * 1024)), id);
 	}
 	return 0;
 }

skynet-src/skynet_socket.h

@@ -9,6 +9,7 @@ struct skynet_context;
 #define SKYNET_SOCKET_TYPE_ACCEPT 4
 #define SKYNET_SOCKET_TYPE_ERROR 5
 #define SKYNET_SOCKET_TYPE_UDP 6
+#define SKYNET_SOCKET_TYPE_WARNING 7
 
 struct skynet_socket_message {
 	int type;

skynet-src/skynet_start.c

@@ -8,6 +8,7 @@
 #include "skynet_monitor.h"
 #include "skynet_socket.h"
 #include "skynet_daemon.h"
+#include "skynet_harbor.h"
 
 #include <pthread.h>
 #include <unistd.h>

skynet-src/skynet_timer.c

@@ -4,6 +4,7 @@
 #include "skynet_mq.h"
 #include "skynet_server.h"
 #include "skynet_handle.h"
+#include "spinlock.h"
 
 #include <time.h>
 #include <assert.h>
@@ -17,9 +18,6 @@
 
 typedef void (*timer_execute_func)(void *ud,void *arg);
 
-#define LOCK(q) while (__sync_lock_test_and_set(&(q)->lock,1)) {}
-#define UNLOCK(q) __sync_lock_release(&(q)->lock);
-
 #define TIME_NEAR_SHIFT 8
 #define TIME_NEAR (1 << TIME_NEAR_SHIFT)
 #define TIME_LEVEL_SHIFT 6
@@ -45,7 +43,7 @@ struct link_list {
 struct timer {
 	struct link_list near[TIME_NEAR];
 	struct link_list t[4][TIME_LEVEL];
-	int lock;
+	struct spinlock lock;
 	uint32_t time;
 	uint32_t current;
 	uint32_t starttime;
@@ -97,12 +95,12 @@ timer_add(struct timer *T,void *arg,size_t sz,int time) {
 	struct timer_node *node = (struct timer_node *)skynet_malloc(sizeof(*node)+sz);
 	memcpy(node+1,arg,sz);
 
-	LOCK(T);
+	SPIN_LOCK(T);
 
 		node->expire=time+T->time;
 		add_node(T,node);
 
-	UNLOCK(T);
+	SPIN_UNLOCK(T);
 }
 
 static void
@@ -146,7 +144,7 @@ dispatch_list(struct timer_node *current) {
 		message.source = 0;
 		message.session = event->session;
 		message.data = NULL;
-		message.sz = PTYPE_RESPONSE << HANDLE_REMOTE_SHIFT;
+		message.sz = (size_t)PTYPE_RESPONSE << MESSAGE_TYPE_SHIFT;
 
 		skynet_context_push(event->handle, &message);
 		
@@ -162,16 +160,16 @@ timer_execute(struct timer *T) {
 	
 	while (T->near[idx].head.next) {
 		struct timer_node *current = link_clear(&T->near[idx]);
-		UNLOCK(T);
+		SPIN_UNLOCK(T);
 		// dispatch_list don't need lock T
 		dispatch_list(current);
-		LOCK(T);
+		SPIN_LOCK(T);
 	}
 }
 
 static void 
 timer_update(struct timer *T) {
-	LOCK(T);
+	SPIN_LOCK(T);
 
 	// try to dispatch timeout 0 (rare condition)
 	timer_execute(T);
@@ -181,7 +179,7 @@ timer_update(struct timer *T) {
 
 	timer_execute(T);
 
-	UNLOCK(T);
+	SPIN_UNLOCK(T);
 }
 
 static struct timer *
@@ -201,7 +199,8 @@ timer_create_timer() {
 		}
 	}
 
-	r->lock = 0;
+	SPIN_INIT(r)
+
 	r->current = 0;
 
 	return r;
@@ -209,12 +208,12 @@ timer_create_timer() {
 
 int
 skynet_timeout(uint32_t handle, int time, int session) {
-	if (time == 0) {
+	if (time <= 0) {
 		struct skynet_message message;
 		message.source = 0;
 		message.session = session;
 		message.data = NULL;
-		message.sz = PTYPE_RESPONSE << HANDLE_REMOTE_SHIFT;
+		message.sz = (size_t)PTYPE_RESPONSE << MESSAGE_TYPE_SHIFT;
 
 		if (skynet_context_push(handle, &message)) {
 			return -1;

skynet-src/socket_server.c

@@ -2,6 +2,7 @@
 
 #include "socket_server.h"
 #include "socket_poll.h"
+#include "atomic.h"
 
 #include <sys/types.h>
 #include <sys/socket.h>
@@ -237,13 +238,13 @@ static int
 reserve_id(struct socket_server *ss) {
 	int i;
 	for (i=0;i<MAX_SOCKET;i++) {
-		int id = __sync_add_and_fetch(&(ss->alloc_id), 1);
+		int id = ATOM_INC(&(ss->alloc_id));
 		if (id < 0) {
-			id = __sync_and_and_fetch(&(ss->alloc_id), 0x7fffffff);
+			id = ATOM_AND(&(ss->alloc_id), 0x7fffffff);
 		}
 		struct socket *s = &ss->slot[HASH_ID(id)];
 		if (s->type == SOCKET_TYPE_INVALID) {
-			if (__sync_bool_compare_and_swap(&s->type, SOCKET_TYPE_INVALID, SOCKET_TYPE_RESERVE)) {
+			if (ATOM_CAS(&s->type, SOCKET_TYPE_INVALID, SOCKET_TYPE_RESERVE)) {
 				s->id = id;
 				s->fd = -1;
 				return id;
@@ -407,6 +408,7 @@ open_socket(struct socket_server *ss, struct request_open * request, struct sock
 
 	status = getaddrinfo( request->host, port, &ai_hints, &ai_list );
 	if ( status != 0 ) {
+		result->data = (void *)gai_strerror(status);
 		goto _failed;
 	}
 	int sock= -1;
@@ -427,12 +429,14 @@ open_socket(struct socket_server *ss, struct request_open * request, struct sock
 	}
 
 	if (sock < 0) {
+		result->data = strerror(errno);
 		goto _failed;
 	}
 
 	ns = new_fd(ss, id, sock, PROTOCOL_TCP, request->opaque, true);
 	if (ns == NULL) {
 		close(sock);
+		result->data = "reach skynet socket number limit";
 		goto _failed;
 	}
 
@@ -761,7 +765,7 @@ _failed:
 	result->opaque = request->opaque;
 	result->id = id;
 	result->ud = 0;
-	result->data = NULL;
+	result->data = "reach skynet socket number limit";
 	ss->slot[HASH_ID(id)].type = SOCKET_TYPE_INVALID;
 
 	return SOCKET_ERROR;
@@ -802,7 +806,7 @@ bind_socket(struct socket_server *ss, struct request_bind *request, struct socke
 	result->ud = 0;
 	struct socket *s = new_fd(ss, id, request->fd, PROTOCOL_TCP, request->opaque, true);
 	if (s == NULL) {
-		result->data = NULL;
+		result->data = "reach skynet socket number limit";
 		return SOCKET_ERROR;
 	}
 	sp_nonblocking(request->fd);
@@ -820,11 +824,13 @@ start_socket(struct socket_server *ss, struct request_start *request, struct soc
 	result->data = NULL;
 	struct socket *s = &ss->slot[HASH_ID(id)];
 	if (s->type == SOCKET_TYPE_INVALID || s->id !=id) {
+		result->data = "invalid socket";
 		return SOCKET_ERROR;
 	}
 	if (s->type == SOCKET_TYPE_PACCEPT || s->type == SOCKET_TYPE_PLISTEN) {
 		if (sp_add(ss->event_fd, s->fd, s)) {
 			s->type = SOCKET_TYPE_INVALID;
+			result->data = strerror(errno);
 			return SOCKET_ERROR;
 		}
 		s->type = (s->type == SOCKET_TYPE_PACCEPT) ? SOCKET_TYPE_CONNECTED : SOCKET_TYPE_LISTEN;
@@ -912,7 +918,7 @@ set_udp_address(struct socket_server *ss, struct request_setudp *request, struct
 		result->opaque = s->opaque;
 		result->id = s->id;
 		result->ud = 0;
-		result->data = NULL;
+		result->data = "protocol mismatch";
 
 		return SOCKET_ERROR;
 	}
@@ -994,6 +1000,7 @@ forward_message_tcp(struct socket_server *ss, struct socket *s, struct socket_me
 		default:
 			// close when error
 			force_close(ss, s, result);
+			result->data = strerror(errno);
 			return SOCKET_ERROR;
 		}
 		return -1;
@@ -1054,6 +1061,7 @@ forward_message_udp(struct socket_server *ss, struct socket *s, struct socket_me
 		default:
 			// close when error
 			force_close(ss, s, result);
+			result->data = strerror(errno);
 			return SOCKET_ERROR;
 		}
 		return -1;
@@ -1087,6 +1095,10 @@ report_connect(struct socket_server *ss, struct socket *s, struct socket_message
 	int code = getsockopt(s->fd, SOL_SOCKET, SO_ERROR, &error, &len);  
 	if (code < 0 || error) {  
 		force_close(ss,s, result);
+		if (code >= 0)
+			result->data = strerror(error);
+		else
+			result->data = strerror(errno);
 		return SOCKET_ERROR;
 	} else {
 		s->type = SOCKET_TYPE_CONNECTED;
@@ -1110,14 +1122,22 @@ report_connect(struct socket_server *ss, struct socket *s, struct socket_message
 	}
 }
 
-// return 0 when failed
+// return 0 when failed, or -1 when file limit
 static int
 report_accept(struct socket_server *ss, struct socket *s, struct socket_message *result) {
 	union sockaddr_all u;
 	socklen_t len = sizeof(u);
 	int client_fd = accept(s->fd, &u.s, &len);
 	if (client_fd < 0) {
-		return 0;
+		if (errno == EMFILE || errno == ENFILE) {
+			result->opaque = s->opaque;
+			result->id = s->id;
+			result->ud = 0;
+			result->data = strerror(errno);
+			return -1;
+		} else {
+			return 0;
+		}
 	}
 	int id = reserve_id(ss);
 	if (id < 0) {
@@ -1159,6 +1179,7 @@ clear_closed_event(struct socket_server *ss, struct socket_message * result, int
 			if (s) {
 				if (s->type == SOCKET_TYPE_INVALID && s->id == id) {
 					e->s = NULL;
+					break;
 				}
 			}
 		}
@@ -1202,11 +1223,16 @@ socket_server_poll(struct socket_server *ss, struct socket_message * result, int
 		switch (s->type) {
 		case SOCKET_TYPE_CONNECTING:
 			return report_connect(ss, s, result);
-		case SOCKET_TYPE_LISTEN:
-			if (report_accept(ss, s, result)) {
+		case SOCKET_TYPE_LISTEN: {
+			int ok = report_accept(ss, s, result);
+			if (ok > 0) {
 				return SOCKET_ACCEPT;
-			} 
+			} if (ok < 0 ) {
+				return SOCKET_ERROR;
+			}
+			// when ok == 0, retry
 			break;
+		}
 		case SOCKET_TYPE_INVALID:
 			fprintf(stderr, "socket-server: invalid socket\n");
 			break;
@@ -1223,21 +1249,19 @@ socket_server_poll(struct socket_server *ss, struct socket_message * result, int
 						return SOCKET_UDP;
 					}
 				}
-				if (e->write) {
+				if (e->write && type != SOCKET_CLOSE && type != SOCKET_ERROR) {
 					// Try to dispatch write message next step if write flag set.
 					e->read = false;
 					--ss->event_index;
 				}
 				if (type == -1)
-					break;
-				clear_closed_event(ss, result, type);
+					break;				
 				return type;
 			}
 			if (e->write) {
 				int type = send_buffer(ss, s, result);
 				if (type == -1)
 					break;
-				clear_closed_event(ss, result, type);
 				return type;
 			}
 			break;

skynet-src/spinlock.h

@@ -0,0 +1,78 @@
+#ifndef SKYNET_SPINLOCK_H
+#define SKYNET_SPINLOCK_H
+
+#define SPIN_INIT(q) spinlock_init(&(q)->lock);
+#define SPIN_LOCK(q) spinlock_lock(&(q)->lock);
+#define SPIN_UNLOCK(q) spinlock_unlock(&(q)->lock);
+#define SPIN_DESTROY(q) spinlock_destroy(&(q)->lock);
+
+#ifndef USE_PTHREAD_LOCK
+
+struct spinlock {
+	int lock;
+};
+
+static inline void
+spinlock_init(struct spinlock *lock) {
+	lock->lock = 0;
+}
+
+static inline void
+spinlock_lock(struct spinlock *lock) {
+	while (__sync_lock_test_and_set(&lock->lock,1)) {}
+}
+
+static inline int
+spinlock_trylock(struct spinlock *lock) {
+	return __sync_lock_test_and_set(&lock->lock,1) == 0;
+}
+
+static inline void
+spinlock_unlock(struct spinlock *lock) {
+	__sync_lock_release(&lock->lock);
+}
+
+static inline void
+spinlock_destroy(struct spinlock *lock) {
+	(void) lock;
+}
+
+#else
+
+#include <pthread.h>
+
+// we use mutex instead of spinlock for some reason
+// you can also replace to pthread_spinlock
+
+struct spinlock {
+	pthread_mutex_t lock;
+};
+
+static inline void
+spinlock_init(struct spinlock *lock) {
+	pthread_mutex_init(&lock->lock, NULL);
+}
+
+static inline void
+spinlock_lock(struct spinlock *lock) {
+	pthread_mutex_lock(&lock->lock);
+}
+
+static inline int
+spinlock_trylock(struct spinlock *lock) {
+	return pthread_mutex_trylock(&lock->lock) == 0;
+}
+
+static inline void
+spinlock_unlock(struct spinlock *lock) {
+	pthread_mutex_unlock(&lock->lock);
+}
+
+static inline void
+spinlock_destroy(struct spinlock *lock) {
+	pthread_mutex_destroy(&lock->lock);
+}
+
+#endif
+
+#endif

test/testmysql.lua

@@ -70,21 +70,23 @@ local function test3( db)
 end
 skynet.start(function()
 
-	local db=mysql.connect{
+	local function on_connect(db)
+		db:query("set charset utf8");
+	end
+	local db=mysql.connect({
 		host="127.0.0.1",
 		port=3306,
 		database="skynet",
 		user="root",
 		password="1",
-		max_packet_size = 1024 * 1024
-	}
+		max_packet_size = 1024 * 1024,
+		on_connect = on_connect
+	})
 	if not db then
 		print("failed to connect")
 	end
 	print("testmysql success to connect to mysql server")
 
-	db:query("set names utf8")
-
 	local res = db:query("drop table if exists cats")
 	res = db:query("create table cats "
 		               .."(id serial primary key, ".. "name varchar(5))")