upgrade lpeg to 1.1.0 (#1766)

* upgrade lpeg to 1.1.0

* fix lpeg compiler error

3rd/lpeg/HISTORY

@@ -1,4 +1,13 @@
-HISTORY for LPeg 1.0.2
+HISTORY for LPeg 1.1.0
+
+* Changes from version 1.0.2 to 1.1.0
+  ---------------------------------
+  + accumulator capture
+  + UTF-8 ranges
+  + Larger limit for number of rules in a grammar
+  + Larger limit for number of captures in a match
+  + bug fixes
+  + other small improvements
 
 * Changes from version 1.0.1 to 1.0.2
   ---------------------------------

3rd/lpeg/README.md

@@ -0,0 +1,4 @@
+# LPeg - Parsing Expression Grammars For Lua
+
+For more information,
+see [Lpeg](//www.inf.puc-rio.br/~roberto/lpeg/).

3rd/lpeg/lpcap.c

@@ -1,29 +1,41 @@
-/*
-** $Id: lpcap.c $
-** Copyright 2007, Lua.org & PUC-Rio  (see 'lpeg.html' for license)
-*/
 
 #include "lua.h"
 #include "lauxlib.h"
 
 #include "lpcap.h"
+#include "lpprint.h"
 #include "lptypes.h"
 
 
-#define captype(cap)	((cap)->kind)
-
-#define isclosecap(cap)	(captype(cap) == Cclose)
-
-#define closeaddr(c)	((c)->s + (c)->siz - 1)
-
-#define isfullcap(cap)	((cap)->siz != 0)
-
 #define getfromktable(cs,v)	lua_rawgeti((cs)->L, ktableidx((cs)->ptop), v)
 
 #define pushluaval(cs)		getfromktable(cs, (cs)->cap->idx)
 
 
 
+#define skipclose(cs,head)  \
+	if (isopencap(head)) { assert(isclosecap(cs->cap)); cs->cap++; }
+
+
+/*
+** Return the size of capture 'cap'. If it is an open capture, 'close'
+** must be its corresponding close.
+*/
+static Index_t capsize (Capture *cap, Capture *close) {
+  if (isopencap(cap)) {
+    assert(isclosecap(close));
+    return close->index - cap->index;
+  }
+  else
+    return cap->siz - 1;
+}
+
+
+static Index_t closesize (CapState *cs, Capture *head) {
+  return capsize(head, cs->cap);
+}
+
+
 /*
 ** Put at the cache for Lua values the value indexed by 'v' in ktable
 ** of the running pattern (if it is not there yet); returns its index.
@@ -44,35 +56,40 @@ static int pushcapture (CapState *cs);
 
 /*
 ** Goes back in a list of captures looking for an open capture
-** corresponding to a close
+** corresponding to a close one.
 */
 static Capture *findopen (Capture *cap) {
   int n = 0;  /* number of closes waiting an open */
   for (;;) {
     cap--;
     if (isclosecap(cap)) n++;  /* one more open to skip */
-    else if (!isfullcap(cap))
+    else if (isopencap(cap))
       if (n-- == 0) return cap;
   }
 }
 
 
 /*
-** Go to the next capture
+** Go to the next capture at the same level.
 */
 static void nextcap (CapState *cs) {
   Capture *cap = cs->cap;
-  if (!isfullcap(cap)) {  /* not a single capture? */
+  if (isopencap(cap)) {  /* must look for a close? */
     int n = 0;  /* number of opens waiting a close */
     for (;;) {  /* look for corresponding close */
       cap++;
-      if (isclosecap(cap)) {
+      if (isopencap(cap)) n++;
+      else if (isclosecap(cap))
         if (n-- == 0) break;
-      }
-      else if (!isfullcap(cap)) n++;
     }
+    cs->cap = cap + 1;  /* + 1 to skip last close */
+  }
+  else {
+    Capture *next;
+    for (next = cap + 1; capinside(cap, next); next++)
+      ;  /* skip captures inside current one */
+    cs->cap = next;
   }
-  cs->cap = cap + 1;  /* + 1 to skip last close (or entire single capture) */
 }
 
 
@@ -84,22 +101,17 @@ static void nextcap (CapState *cs) {
 ** so the function never returns zero.
 */
 static int pushnestedvalues (CapState *cs, int addextra) {
-  Capture *co = cs->cap;
+  Capture *head = cs->cap++;  /* original capture */
-  if (isfullcap(cs->cap++)) {  /* no nested captures? */
+  int n = 0;  /* number of pushed subvalues */
-    lua_pushlstring(cs->L, co->s, co->siz - 1);  /* push whole match */
+  /* repeat for all nested patterns */
-    return 1;  /* that is it */
+  while (capinside(head, cs->cap))
-  }
+    n += pushcapture(cs);
-  else {
+  if (addextra || n == 0) {  /* need extra? */
-    int n = 0;
+    lua_pushlstring(cs->L, cs->s + head->index, closesize(cs, head));
-    while (!isclosecap(cs->cap))  /* repeat for all nested patterns */
+    n++;
-      n += pushcapture(cs);
-    if (addextra || n == 0) {  /* need extra? */
-      lua_pushlstring(cs->L, co->s, cs->cap->s - co->s);  /* push whole match */
-      n++;
-    }
-    cs->cap++;  /* skip close entry */
-    return n;
   }
+  skipclose(cs, head);
+  return n;
 }
 
 
@@ -114,17 +126,43 @@ static void pushonenestedvalue (CapState *cs) {
 
 
 /*
-** Try to find a named group capture with the name given at the top of
+** Checks whether group 'grp' is visible to 'ref', that is, 'grp' is
-** the stack; goes backward from 'cap'.
+** not nested inside a full capture that does not contain 'ref'.  (We
+** only need to care for full captures because the search at 'findback'
+** skips open-end blocks; so, if 'grp' is nested in a non-full capture,
+** 'ref' is also inside it.)  To check this, we search backward for the
+** inner full capture enclosing 'grp'.  A full capture cannot contain
+** non-full captures, so a close capture means we cannot be inside a
+** full capture anymore.
 */
-static Capture *findback (CapState *cs, Capture *cap) {
+static int capvisible (CapState *cs, Capture *grp, Capture *ref) {
+  Capture *cap = grp;
+  int i = MAXLOP;  /* maximum distance for an 'open' */
+  while (i-- > 0 && cap-- > cs->ocap) {
+    if (isclosecap(cap))
+      return 1;  /* can stop the search */
+    else if (grp->index - cap->index >= UCHAR_MAX)
+      return 1;  /* can stop the search */
+    else if (capinside(cap, grp))  /* is 'grp' inside cap? */
+      return capinside(cap, ref);  /* ok iff cap also contains 'ref' */
+  }
+  return 1;  /* 'grp' is not inside any capture */
+}
+
+
+/*
+** Try to find a named group capture with the name given at the top of
+** the stack; goes backward from 'ref'.
+*/
+static Capture *findback (CapState *cs, Capture *ref) {
   lua_State *L = cs->L;
+  Capture *cap = ref;
   while (cap-- > cs->ocap) {  /* repeat until end of list */
     if (isclosecap(cap))
       cap = findopen(cap);  /* skip nested captures */
-    else if (!isfullcap(cap))
+    else if (capinside(cap, ref))
-      continue; /* opening an enclosing capture: skip and get previous */
+      continue;  /* enclosing captures are not visible to 'ref' */
-    if (captype(cap) == Cgroup) {
+    if (captype(cap) == Cgroup && capvisible(cs, cap, ref)) {
       getfromktable(cs, cap->idx);  /* get group name */
       if (lp_equal(L, -2, -1)) {  /* right group? */
         lua_pop(L, 2);  /* remove reference name and group name */
@@ -158,11 +196,10 @@ static int backrefcap (CapState *cs) {
 */
 static int tablecap (CapState *cs) {
   lua_State *L = cs->L;
+  Capture *head = cs->cap++;
   int n = 0;
   lua_newtable(L);
-  if (isfullcap(cs->cap++))
+  while (capinside(head, cs->cap)) {
-    return 1;  /* table is empty */
-  while (!isclosecap(cs->cap)) {
     if (captype(cs->cap) == Cgroup && cs->cap->idx != 0) {  /* named group? */
       pushluaval(cs);  /* push group name */
       pushonenestedvalue(cs);
@@ -176,7 +213,7 @@ static int tablecap (CapState *cs) {
       n += k;
     }
   }
-  cs->cap++;  /* skip close entry */
+  skipclose(cs, head);
   return 1;  /* number of values pushed (only the table) */
 }
 
@@ -203,20 +240,20 @@ static int querycap (CapState *cs) {
 static int foldcap (CapState *cs) {
   int n;
   lua_State *L = cs->L;
-  int idx = cs->cap->idx;
+  Capture *head = cs->cap++;
-  if (isfullcap(cs->cap++) ||  /* no nested captures? */
+  int idx = head->idx;
-      isclosecap(cs->cap) ||  /* no nested captures (large subject)? */
+  if (isclosecap(cs->cap) ||  /* no nested captures (large subject)? */
       (n = pushcapture(cs)) == 0)  /* nested captures with no values? */
     return luaL_error(L, "no initial value for fold capture");
   if (n > 1)
     lua_pop(L, n - 1);  /* leave only one result for accumulator */
-  while (!isclosecap(cs->cap)) {
+  while (capinside(head, cs->cap)) {
     lua_pushvalue(L, updatecache(cs, idx));  /* get folding function */
     lua_insert(L, -2);  /* put it before accumulator */
     n = pushcapture(cs);  /* get next capture's values */
     lua_call(L, n + 1, 1);  /* call folding function */
   }
-  cs->cap++;  /* skip close entry */
+  skipclose(cs, head);
   return 1;  /* only accumulator left on the stack */
 }
 
@@ -234,6 +271,22 @@ static int functioncap (CapState *cs) {
 }
 
 
+/*
+** Accumulator capture
+*/
+static int accumulatorcap (CapState *cs) {
+  lua_State *L = cs->L;
+  int n;
+  if (lua_gettop(L) < cs->firstcap)
+    luaL_error(L, "no previous value for accumulator capture");
+  pushluaval(cs);  /* push function */
+  lua_insert(L, -2);  /* previous value becomes first argument */
+  n = pushnestedvalues(cs, 0);  /* push nested captures */
+  lua_call(L, n + 1, 1);  /* call function */
+  return 0;  /* did not add any extra value */
+}
+
+
 /*
 ** Select capture
 */
@@ -283,7 +336,7 @@ int runtimecap (CapState *cs, Capture *close, const char *s, int *rem) {
   assert(captype(open) == Cgroup);
   id = finddyncap(open, close);  /* get first dynamic capture argument */
   close->kind = Cclose;  /* closes the group */
-  close->s = s;
+  close->index = s - cs->s;
   cs->cap = open; cs->valuecached = 0;  /* prepare capture state */
   luaL_checkstack(L, 4, "too many runtime captures");
   pushluaval(cs);  /* push function to be called */
@@ -313,8 +366,8 @@ typedef struct StrAux {
   union {
     Capture *cp;  /* if not a string, respective capture */
     struct {  /* if it is a string... */
-      const char *s;  /* ... starts here */
+      Index_t idx;  /* starts here */
-      const char *e;  /* ... ends here */
+      Index_t siz;  /* with this size */
     } s;
   } u;
 } StrAux;
@@ -329,24 +382,23 @@ typedef struct StrAux {
 */
 static int getstrcaps (CapState *cs, StrAux *cps, int n) {
   int k = n++;
+  Capture *head = cs->cap++;
   cps[k].isstring = 1;  /* get string value */
-  cps[k].u.s.s = cs->cap->s;  /* starts here */
+  cps[k].u.s.idx = head->index;  /* starts here */
-  if (!isfullcap(cs->cap++)) {  /* nested captures? */
+  while (capinside(head, cs->cap)) {
-    while (!isclosecap(cs->cap)) {  /* traverse them */
+    if (n >= MAXSTRCAPS)  /* too many captures? */
-      if (n >= MAXSTRCAPS)  /* too many captures? */
+      nextcap(cs);  /* skip extra captures (will not need them) */
-        nextcap(cs);  /* skip extra captures (will not need them) */
+    else if (captype(cs->cap) == Csimple)  /* string? */
-      else if (captype(cs->cap) == Csimple)  /* string? */
+      n = getstrcaps(cs, cps, n);  /* put info. into array */
-        n = getstrcaps(cs, cps, n);  /* put info. into array */
+    else {
-      else {
+      cps[n].isstring = 0;  /* not a string */
-        cps[n].isstring = 0;  /* not a string */
+      cps[n].u.cp = cs->cap;  /* keep original capture */
-        cps[n].u.cp = cs->cap;  /* keep original capture */
+      nextcap(cs);
-        nextcap(cs);
+      n++;
-        n++;
-      }
     }
-    cs->cap++;  /* skip close */
   }
-  cps[k].u.s.e = closeaddr(cs->cap - 1);  /* ends here */
+  cps[k].u.s.siz = closesize(cs, head);
+  skipclose(cs, head);
   return n;
 }
 
@@ -368,7 +420,7 @@ static void stringcap (luaL_Buffer *b, CapState *cs) {
   const char *fmt;  /* format string */
   fmt = lua_tolstring(cs->L, updatecache(cs, cs->cap->idx), &len);
   n = getstrcaps(cs, cps, 0) - 1;  /* collect nested captures */
-  for (i = 0; i < len; i++) {  /* traverse them */
+  for (i = 0; i < len; i++) {  /* traverse format string */
     if (fmt[i] != '%')  /* not an escape? */
       luaL_addchar(b, fmt[i]);  /* add it to buffer */
     else if (fmt[++i] < '0' || fmt[i] > '9')  /* not followed by a digit? */
@@ -378,7 +430,7 @@ static void stringcap (luaL_Buffer *b, CapState *cs) {
       if (l > n)
         luaL_error(cs->L, "invalid capture index (%d)", l);
       else if (cps[l].isstring)
-        luaL_addlstring(b, cps[l].u.s.s, cps[l].u.s.e - cps[l].u.s.s);
+        luaL_addlstring(b, cs->s + cps[l].u.s.idx, cps[l].u.s.siz);
       else {
         Capture *curr = cs->cap;
         cs->cap = cps[l].u.cp;  /* go back to evaluate that nested capture */
@@ -395,22 +447,20 @@ static void stringcap (luaL_Buffer *b, CapState *cs) {
 ** Substitution capture: add result to buffer 'b'
 */
 static void substcap (luaL_Buffer *b, CapState *cs) {
-  const char *curr = cs->cap->s;
+  const char *curr = cs->s + cs->cap->index;
-  if (isfullcap(cs->cap))  /* no nested captures? */
+  Capture *head = cs->cap++;
-    luaL_addlstring(b, curr, cs->cap->siz - 1);  /* keep original text */
+  while (capinside(head, cs->cap)) {
-  else {
+    Capture *cap = cs->cap;
-    cs->cap++;  /* skip open entry */
+    const char *caps = cs->s + cap->index;
-    while (!isclosecap(cs->cap)) {  /* traverse nested captures */
+    luaL_addlstring(b, curr, caps - curr);  /* add text up to capture */
-      const char *next = cs->cap->s;
+    if (addonestring(b, cs, "replacement"))
-      luaL_addlstring(b, curr, next - curr);  /* add text up to capture */
+      curr = caps + capsize(cap, cs->cap - 1);  /* continue after match */
-      if (addonestring(b, cs, "replacement"))
+    else  /* no capture value */
-        curr = closeaddr(cs->cap - 1);  /* continue after match */
+      curr = caps;  /* keep original text in final result */
-      else  /* no capture value */
-        curr = next;  /* keep original text in final result */
-    }
-    luaL_addlstring(b, curr, cs->cap->s - curr);  /* add last piece of text */
   }
-  cs->cap++;  /* go to next capture */
+  /* add last piece of text */
+  luaL_addlstring(b, curr, cs->s + head->index + closesize(cs, head) - curr);
+  skipclose(cs, head);
 }
 
 
@@ -426,13 +476,16 @@ static int addonestring (luaL_Buffer *b, CapState *cs, const char *what) {
     case Csubst:
       substcap(b, cs);  /* add capture directly to buffer */
       return 1;
+    case Cacc:  /* accumulator capture? */
+      return luaL_error(cs->L, "invalid context for an accumulator capture");
     default: {
       lua_State *L = cs->L;
       int n = pushcapture(cs);
       if (n > 0) {
         if (n > 1) lua_pop(L, n - 1);  /* only one result */
         if (!lua_isstring(L, -1))
-          luaL_error(L, "invalid %s value (a %s)", what, luaL_typename(L, -1));
+          return luaL_error(L, "invalid %s value (a %s)",
+                               what, luaL_typename(L, -1));
         luaL_addvalue(b);
       }
       return n;
@@ -458,7 +511,7 @@ static int pushcapture (CapState *cs) {
     return luaL_error(L, "subcapture nesting too deep");
   switch (captype(cs->cap)) {
     case Cposition: {
-      lua_pushinteger(L, cs->cap->s - cs->s + 1);
+      lua_pushinteger(L, cs->cap->index + 1);
       cs->cap++;
       res = 1;
       break;
@@ -516,6 +569,7 @@ static int pushcapture (CapState *cs) {
     case Cbackref: res = backrefcap(cs); break;
     case Ctable: res = tablecap(cs); break;
     case Cfunction: res = functioncap(cs); break;
+    case Cacc: res = accumulatorcap(cs); break;
     case Cnum: res = numcap(cs); break;
     case Cquery: res = querycap(cs); break;
     case Cfold: res = foldcap(cs); break;
@@ -529,7 +583,7 @@ static int pushcapture (CapState *cs) {
 /*
 ** Prepare a CapState structure and traverse the entire list of
 ** captures in the stack pushing its results. 's' is the subject
-** string, 'r' is the final position of the match, and 'ptop' 
+** string, 'r' is the final position of the match, and 'ptop'
 ** the index in the stack where some useful values were pushed.
 ** Returns the number of results pushed. (If the list produces no
 ** results, push the final position of the match.)
@@ -537,13 +591,16 @@ static int pushcapture (CapState *cs) {
 int getcaptures (lua_State *L, const char *s, const char *r, int ptop) {
   Capture *capture = (Capture *)lua_touserdata(L, caplistidx(ptop));
   int n = 0;
+  /* printcaplist(capture); */
   if (!isclosecap(capture)) {  /* is there any capture? */
     CapState cs;
     cs.ocap = cs.cap = capture; cs.L = L; cs.reclevel = 0;
     cs.s = s; cs.valuecached = 0; cs.ptop = ptop;
+    cs.firstcap = lua_gettop(L) + 1;  /* where first value (if any) will go */
     do {  /* collect their values */
       n += pushcapture(&cs);
     } while (!isclosecap(cs.cap));
+    assert(lua_gettop(L) - cs.firstcap == n - 1);
   }
   if (n == 0) {  /* no capture values? */
     lua_pushinteger(L, r - s + 1);  /* return only end position */

3rd/lpeg/lpcap.h

@@ -1,6 +1,3 @@
-/*
-** $Id: lpcap.h $
-*/
 
 #if !defined(lpcap_h)
 #define lpcap_h
@@ -19,6 +16,7 @@ typedef enum CapKind {
   Csimple,  /* next node is pattern */
   Ctable,  /* next node is pattern */
   Cfunction,  /* ktable[key] is function; next node is pattern */
+  Cacc,  /* ktable[key] is function; next node is pattern */
   Cquery,  /* ktable[key] is table; next node is pattern */
   Cstring,  /* ktable[key] is string; next node is pattern */
   Cnum,  /* numbered capture; 'key' is number of value to return */
@@ -29,8 +27,20 @@ typedef enum CapKind {
 } CapKind;
 
 
+/*
+** An unsigned integer large enough to index any subject entirely.
+** It can be size_t, but that will double the size of the array
+** of captures in a 64-bit machine.
+*/
+#if !defined(Index_t)
+typedef uint Index_t;
+#endif
+
+#define MAXINDT		(~(Index_t)0)
+
+
 typedef struct Capture {
-  const char *s;  /* subject position */
+  Index_t index;  /* subject position */
   unsigned short idx;  /* extra info (group name, arg index, etc.) */
   byte kind;  /* kind of capture */
   byte siz;  /* size of full capture + 1 (0 = not a full capture) */
@@ -41,13 +51,32 @@ typedef struct CapState {
   Capture *cap;  /* current capture */
   Capture *ocap;  /* (original) capture list */
   lua_State *L;
-  int ptop;  /* index of last argument to 'match' */
+  int ptop;  /* stack index of last argument to 'match' */
+  int firstcap;  /* stack index of first capture pushed in the stack */
   const char *s;  /* original string */
   int valuecached;  /* value stored in cache slot */
   int reclevel;  /* recursion level */
 } CapState;
 
 
+#define captype(cap)    ((cap)->kind)
+
+#define isclosecap(cap) (captype(cap) == Cclose)
+#define isopencap(cap)  ((cap)->siz == 0)
+
+/* true if c2 is (any number of levels) inside c1 */
+#define capinside(c1,c2)  \
+	(isopencap(c1) ? !isclosecap(c2) \
+                       : (c2)->index < (c1)->index + (c1)->siz - 1)
+
+
+/**
+** Maximum number of captures to visit when looking for an 'open'.
+*/
+#define MAXLOP		20
+
+
+
 int runtimecap (CapState *cs, Capture *close, const char *s, int *rem);
 int getcaptures (lua_State *L, const char *s, const char *r, int ptop);
 int finddyncap (Capture *cap, Capture *last);

3rd/lpeg/lpcode.c

@@ -1,7 +1,3 @@
-/*
-** $Id: lpcode.c $
-** Copyright 2007, Lua.org & PUC-Rio  (see 'lpeg.html' for license)
-*/
 
 #include <limits.h>
 
@@ -11,6 +7,7 @@
 
 #include "lptypes.h"
 #include "lpcode.h"
+#include "lpcset.h"
 
 
 /* signals a "no-instruction */
@@ -26,61 +23,13 @@ static const Charset fullset_ =
 
 static const Charset *fullset = &fullset_;
 
+
 /*
 ** {======================================================
 ** Analysis and some optimizations
 ** =======================================================
 */
 
-/*
-** Check whether a charset is empty (returns IFail), singleton (IChar),
-** full (IAny), or none of those (ISet). When singleton, '*c' returns
-** which character it is. (When generic set, the set was the input,
-** so there is no need to return it.)
-*/
-static Opcode charsettype (const byte *cs, int *c) {
-  int count = 0;  /* number of characters in the set */
-  int i;
-  int candidate = -1;  /* candidate position for the singleton char */
-  for (i = 0; i < CHARSETSIZE; i++) {  /* for each byte */
-    int b = cs[i];
-    if (b == 0) {  /* is byte empty? */
-      if (count > 1)  /* was set neither empty nor singleton? */
-        return ISet;  /* neither full nor empty nor singleton */
-      /* else set is still empty or singleton */
-    }
-    else if (b == 0xFF) {  /* is byte full? */
-      if (count < (i * BITSPERCHAR))  /* was set not full? */
-        return ISet;  /* neither full nor empty nor singleton */
-      else count += BITSPERCHAR;  /* set is still full */
-    }
-    else if ((b & (b - 1)) == 0) {  /* has byte only one bit? */
-      if (count > 0)  /* was set not empty? */
-        return ISet;  /* neither full nor empty nor singleton */
-      else {  /* set has only one char till now; track it */
-        count++;
-        candidate = i;
-      }
-    }
-    else return ISet;  /* byte is neither empty, full, nor singleton */
-  }
-  switch (count) {
-    case 0: return IFail;  /* empty set */
-    case 1: {  /* singleton; find character bit inside byte */
-      int b = cs[candidate];
-      *c = candidate * BITSPERCHAR;
-      if ((b & 0xF0) != 0) { *c += 4; b >>= 4; }
-      if ((b & 0x0C) != 0) { *c += 2; b >>= 2; }
-      if ((b & 0x02) != 0) { *c += 1; }
-      return IChar;
-    }
-    default: {
-       assert(count == CHARSETSIZE * BITSPERCHAR);  /* full set */
-       return IAny;
-    }
-  }
-}
-
 
 /*
 ** A few basic operations on Charsets
@@ -89,42 +38,12 @@ static void cs_complement (Charset *cs) {
   loopset(i, cs->cs[i] = ~cs->cs[i]);
 }
 
-static int cs_equal (const byte *cs1, const byte *cs2) {
-  loopset(i, if (cs1[i] != cs2[i]) return 0);
-  return 1;
-}
-
 static int cs_disjoint (const Charset *cs1, const Charset *cs2) {
   loopset(i, if ((cs1->cs[i] & cs2->cs[i]) != 0) return 0;)
   return 1;
 }
 
 
-/*
-** If 'tree' is a 'char' pattern (TSet, TChar, TAny), convert it into a
-** charset and return 1; else return 0.
-*/
-int tocharset (TTree *tree, Charset *cs) {
-  switch (tree->tag) {
-    case TSet: {  /* copy set */
-      loopset(i, cs->cs[i] = treebuffer(tree)[i]);
-      return 1;
-    }
-    case TChar: {  /* only one char */
-      assert(0 <= tree->u.n && tree->u.n <= UCHAR_MAX);
-      loopset(i, cs->cs[i] = 0);  /* erase all chars */
-      setchar(cs->cs, tree->u.n);  /* add that one */
-      return 1;
-    }
-    case TAny: {
-      loopset(i, cs->cs[i] = 0xFF);  /* add all characters to the set */
-      return 1;
-    }
-    default: return 0;
-  }
-}
-
-
 /*
 ** Visit a TCall node taking care to stop recursion. If node not yet
 ** visited, return 'f(sib2(tree))', otherwise return 'def' (default
@@ -196,7 +115,7 @@ int hascaptures (TTree *tree) {
 int checkaux (TTree *tree, int pred) {
  tailcall:
   switch (tree->tag) {
-    case TChar: case TSet: case TAny:
+    case TChar: case TSet: case TAny: case TUTFR:
     case TFalse: case TOpenCall:
       return 0;  /* not nullable */
     case TRep: case TTrue:
@@ -220,7 +139,7 @@ int checkaux (TTree *tree, int pred) {
       if (checkaux(sib2(tree), pred)) return 1;
       /* else return checkaux(sib1(tree), pred); */
       tree = sib1(tree); goto tailcall;
-    case TCapture: case TGrammar: case TRule:
+    case TCapture: case TGrammar: case TRule: case TXInfo:
       /* return checkaux(sib1(tree), pred); */
       tree = sib1(tree); goto tailcall;
     case TCall:  /* return checkaux(sib2(tree), pred); */
@@ -239,11 +158,13 @@ int fixedlen (TTree *tree) {
   switch (tree->tag) {
     case TChar: case TSet: case TAny:
       return len + 1;
+    case TUTFR:
+      return (tree->cap == sib1(tree)->cap) ? len + tree->cap : -1;
     case TFalse: case TTrue: case TNot: case TAnd: case TBehind:
       return len;
     case TRep: case TRunTime: case TOpenCall:
       return -1;
-    case TCapture: case TRule: case TGrammar:
+    case TCapture: case TRule: case TGrammar: case TXInfo:
       /* return fixedlen(sib1(tree)); */
       tree = sib1(tree); goto tailcall;
     case TCall: {
@@ -294,18 +215,21 @@ int fixedlen (TTree *tree) {
 static int getfirst (TTree *tree, const Charset *follow, Charset *firstset) {
  tailcall:
   switch (tree->tag) {
-    case TChar: case TSet: case TAny: {
+    case TChar: case TSet: case TAny: case TFalse: {
       tocharset(tree, firstset);
       return 0;
     }
+    case TUTFR: {
+      int c;
+      clearset(firstset->cs);  /* erase all chars */
+      for (c = tree->key; c <= sib1(tree)->key; c++)
+        setchar(firstset->cs, c);
+      return 0;
+    }
     case TTrue: {
       loopset(i, firstset->cs[i] = follow->cs[i]);
       return 1;  /* accepts the empty string */
     }
-    case TFalse: {
-      loopset(i, firstset->cs[i] = 0);
-      return 0;
-    }
     case TChoice: {
       Charset csaux;
       int e1 = getfirst(sib1(tree), follow, firstset);
@@ -334,7 +258,7 @@ static int getfirst (TTree *tree, const Charset *follow, Charset *firstset) {
       loopset(i, firstset->cs[i] |= follow->cs[i]);
       return 1;  /* accept the empty string */
     }
-    case TCapture: case TGrammar: case TRule: {
+    case TCapture: case TGrammar: case TRule: case TXInfo: {
       /* return getfirst(sib1(tree), follow, firstset); */
       tree = sib1(tree); goto tailcall;
     }
@@ -356,9 +280,8 @@ static int getfirst (TTree *tree, const Charset *follow, Charset *firstset) {
       if (tocharset(sib1(tree), firstset)) {
         cs_complement(firstset);
         return 1;
-      }
+      }  /* else */
-      /* else go through */
+    }    /* FALLTHROUGH */
-    }
     case TBehind: {  /* instruction gives no new information */
       /* call 'getfirst' only to check for math-time captures */
       int e = getfirst(sib1(tree), follow, firstset);
@@ -380,9 +303,9 @@ static int headfail (TTree *tree) {
     case TChar: case TSet: case TAny: case TFalse:
       return 1;
     case TTrue: case TRep: case TRunTime: case TNot:
-    case TBehind:
+    case TBehind: case TUTFR:
       return 0;
-    case TCapture: case TGrammar: case TRule: case TAnd:
+    case TCapture: case TGrammar: case TRule: case TXInfo: case TAnd:
       tree = sib1(tree); goto tailcall;  /* return headfail(sib1(tree)); */
     case TCall:
       tree = sib2(tree); goto tailcall;  /* return headfail(sib2(tree)); */
@@ -407,7 +330,7 @@ static int headfail (TTree *tree) {
 static int needfollow (TTree *tree) {
  tailcall:
   switch (tree->tag) {
-    case TChar: case TSet: case TAny:
+    case TChar: case TSet: case TAny: case TUTFR:
     case TFalse: case TTrue: case TAnd: case TNot:
     case TRunTime: case TGrammar: case TCall: case TBehind:
       return 0;
@@ -418,7 +341,7 @@ static int needfollow (TTree *tree) {
     case TSeq:
       tree = sib2(tree); goto tailcall;
     default: assert(0); return 0;
-  } 
+  }
 }
 
 /* }====================================================== */
@@ -437,10 +360,11 @@ static int needfollow (TTree *tree) {
 */
 int sizei (const Instruction *i) {
   switch((Opcode)i->i.code) {
-    case ISet: case ISpan: return CHARSETINSTSIZE;
+    case ISet: case ISpan: return 1 + i->i.aux2.set.size;
-    case ITestSet: return CHARSETINSTSIZE + 1;
+    case ITestSet: return 2 + i->i.aux2.set.size;
     case ITestChar: case ITestAny: case IChoice: case IJmp: case ICall:
     case IOpenCall: case ICommit: case IPartialCommit: case IBackCommit:
+    case IUTFR:
       return 2;
     default: return 1;
   }
@@ -468,23 +392,67 @@ static void codegen (CompileState *compst, TTree *tree, int opt, int tt,
                      const Charset *fl);
 
 
-void realloccode (lua_State *L, Pattern *p, int nsize) {
+static void finishrelcode (lua_State *L, Pattern *p, Instruction *block,
-  void *ud;
+                                         int size) {
-  lua_Alloc f = lua_getallocf(L, &ud);
+  if (block == NULL)
-  void *newblock = f(ud, p->code, p->codesize * sizeof(Instruction),
-                                  nsize * sizeof(Instruction));
-  if (newblock == NULL && nsize > 0)
     luaL_error(L, "not enough memory");
-  p->code = (Instruction *)newblock;
+  block->codesize = size;
-  p->codesize = nsize;
+  p->code = (Instruction *)block + 1;
 }
 
 
-static int nextinstruction (CompileState *compst) {
+/*
-  int size = compst->p->codesize;
+** Initialize array 'p->code'
-  if (compst->ncode >= size)
+*/
-    realloccode(compst->L, compst->p, size * 2);
+static void newcode (lua_State *L, Pattern *p, int size) {
-  return compst->ncode++;
+  void *ud;
+  Instruction *block;
+  lua_Alloc f = lua_getallocf(L, &ud);
+  size++;  /* slot for 'codesize' */
+  block = (Instruction*) f(ud, NULL, 0, size * sizeof(Instruction));
+  finishrelcode(L, p, block, size);
+}
+
+
+void freecode (lua_State *L, Pattern *p) {
+  if (p->code != NULL) {
+    void *ud;
+    lua_Alloc f = lua_getallocf(L, &ud);
+    uint osize = p->code[-1].codesize;
+    f(ud, p->code - 1, osize * sizeof(Instruction), 0);  /* free block */
+  }
+}
+
+
+/*
+** Assume that 'nsize' is not zero and that 'p->code' already exists.
+*/
+static void realloccode (lua_State *L, Pattern *p, int nsize) {
+  void *ud;
+  lua_Alloc f = lua_getallocf(L, &ud);
+  Instruction *block = p->code - 1;
+  uint osize = block->codesize;
+  nsize++;  /* add the 'codesize' slot to size */
+  block = (Instruction*) f(ud, block, osize * sizeof(Instruction),
+                                      nsize * sizeof(Instruction));
+  finishrelcode(L, p, block, nsize);
+}
+
+
+/*
+** Add space for an instruction with 'n' slots and return its index.
+*/
+static int nextinstruction (CompileState *compst, int n) {
+  int size = compst->p->code[-1].codesize - 1;
+  int ncode = compst->ncode;
+  if (ncode > size - n) {
+    uint nsize = size + (size >> 1) + n;
+    if (nsize >= INT_MAX)
+      luaL_error(compst->L, "pattern code too large");
+    realloccode(compst->L, compst->p, nsize);
+  }
+  compst->ncode = ncode + n;
+  return ncode;
 }
 
 
@@ -492,9 +460,9 @@ static int nextinstruction (CompileState *compst) {
 
 
 static int addinstruction (CompileState *compst, Opcode op, int aux) {
-  int i = nextinstruction(compst);
+  int i = nextinstruction(compst, 1);
   getinstr(compst, i).i.code = op;
-  getinstr(compst, i).i.aux = aux;
+  getinstr(compst, i).i.aux1 = aux;
   return i;
 }
 
@@ -518,6 +486,16 @@ static void setoffset (CompileState *compst, int instruction, int offset) {
 }
 
 
+static void codeutfr (CompileState *compst, TTree *tree) {
+  int i = addoffsetinst(compst, IUTFR);
+  int to = sib1(tree)->u.n;
+  assert(sib1(tree)->tag == TXInfo);
+  getinstr(compst, i + 1).offset = tree->u.n;
+  getinstr(compst, i).i.aux1 = to & 0xff;
+  getinstr(compst, i).i.aux2.key = to >> 8;
+}
+
+
 /*
 ** Add a capture instruction:
 ** 'op' is the capture instruction; 'cap' the capture kind;
@@ -527,7 +505,7 @@ static void setoffset (CompileState *compst, int instruction, int offset) {
 static int addinstcap (CompileState *compst, Opcode op, int cap, int key,
                        int aux) {
   int i = addinstruction(compst, op, joinkindoff(cap, aux));
-  getinstr(compst, i).i.key = key;
+  getinstr(compst, i).i.aux2.key = key;
   return i;
 }
 
@@ -560,7 +538,7 @@ static void jumptohere (CompileState *compst, int instruction) {
 */
 static void codechar (CompileState *compst, int c, int tt) {
   if (tt >= 0 && getinstr(compst, tt).i.code == ITestChar &&
-                 getinstr(compst, tt).i.aux == c)
+                 getinstr(compst, tt).i.aux1 == c)
     addinstruction(compst, IAny, 0);
   else
     addinstruction(compst, IChar, c);
@@ -568,45 +546,63 @@ static void codechar (CompileState *compst, int c, int tt) {
 
 
 /*
-** Add a charset posfix to an instruction
+** Add a charset posfix to an instruction.
 */
-static void addcharset (CompileState *compst, const byte *cs) {
+static void addcharset (CompileState *compst, int inst, charsetinfo *info) {
-  int p = gethere(compst);
+  int p;
+  Instruction *I = &getinstr(compst, inst);
+  byte *charset;
+  int isize = instsize(info->size);  /* size in instructions */
   int i;
-  for (i = 0; i < (int)CHARSETINSTSIZE - 1; i++)
+  I->i.aux2.set.offset = info->offset * 8;  /* offset in bits */
-    nextinstruction(compst);  /* space for buffer */
+  I->i.aux2.set.size = isize;
-  /* fill buffer with charset */
+  I->i.aux1 = info->deflt;
-  loopset(j, getinstr(compst, p).buff[j] = cs[j]);
+  p = nextinstruction(compst, isize);  /* space for charset */
+  charset = getinstr(compst, p).buff;  /* charset buffer */
+  for (i = 0; i < isize * (int)sizeof(Instruction); i++)
+    charset[i] = getbytefromcharset(info, i);  /* copy the buffer */
 }
 
 
 /*
-** code a char set, optimizing unit sets for IChar, "complete"
+** Check whether charset 'info' is dominated by instruction 'p'
-** sets for IAny, and empty sets for IFail; also use an IAny
-** when instruction is dominated by an equivalent test.
 */
-static void codecharset (CompileState *compst, const byte *cs, int tt) {
+static int cs_equal (Instruction *p, charsetinfo *info) {
-  int c = 0;  /* (=) to avoid warnings */
+  if (p->i.code != ITestSet)
-  Opcode op = charsettype(cs, &c);
+    return 0;
-  switch (op) {
+  else if (p->i.aux2.set.offset != info->offset * 8 ||
-    case IChar: codechar(compst, c, tt); break;
+           p->i.aux2.set.size != instsize(info->size) ||
-    case ISet: {  /* non-trivial set? */
+           p->i.aux1 != info->deflt)
-      if (tt >= 0 && getinstr(compst, tt).i.code == ITestSet &&
+    return 0;
-          cs_equal(cs, getinstr(compst, tt + 2).buff))
+  else {
-        addinstruction(compst, IAny, 0);
+    int i;
-      else {
+    for (i = 0; i < instsize(info->size) * (int)sizeof(Instruction); i++) {
-        addinstruction(compst, ISet, 0);
+      if ((p + 2)->buff[i] != getbytefromcharset(info, i))
-        addcharset(compst, cs);
+        return 0;
-      }
-      break;
     }
-    default: addinstruction(compst, op, c); break;
+  }
+  return 1;
+}
+
+
+/*
+** Code a char set, using IAny when instruction is dominated by an
+** equivalent test.
+*/
+static void codecharset (CompileState *compst, TTree *tree, int tt) {
+  charsetinfo info;
+  tree2cset(tree, &info);
+  if (tt >= 0 && cs_equal(&getinstr(compst, tt), &info))
+    addinstruction(compst, IAny, 0);
+  else {
+    int i = addinstruction(compst, ISet, 0);
+    addcharset(compst, i, &info);
   }
 }
 
 
 /*
-** code a test set, optimizing unit sets for ITestChar, "complete"
+** Code a test set, optimizing unit sets for ITestChar, "complete"
 ** sets for ITestAny, and empty sets for IJmp (always fails).
 ** 'e' is true iff test should accept the empty string. (Test
 ** instructions in the current VM never accept the empty string.)
@@ -614,22 +610,22 @@ static void codecharset (CompileState *compst, const byte *cs, int tt) {
 static int codetestset (CompileState *compst, Charset *cs, int e) {
   if (e) return NOINST;  /* no test */
   else {
-    int c = 0;
+    charsetinfo info;
-    Opcode op = charsettype(cs->cs, &c);
+    Opcode op = charsettype(cs->cs, &info);
     switch (op) {
       case IFail: return addoffsetinst(compst, IJmp);  /* always jump */
       case IAny: return addoffsetinst(compst, ITestAny);
       case IChar: {
         int i = addoffsetinst(compst, ITestChar);
-        getinstr(compst, i).i.aux = c;
+        getinstr(compst, i).i.aux1 = info.offset;
         return i;
       }
-      case ISet: {
+      default: {  /* regular set */
         int i = addoffsetinst(compst, ITestSet);
-        addcharset(compst, cs->cs);
+        addcharset(compst, i, &info);
+        assert(op == ISet);
         return i;
       }
-      default: assert(0); return 0;
     }
   }
 }
@@ -665,11 +661,11 @@ static void codebehind (CompileState *compst, TTree *tree) {
 
 /*
 ** Choice; optimizations:
-** - when p1 is headfail or
+** - when p1 is headfail or when first(p1) and first(p2) are disjoint,
-** when first(p1) and first(p2) are disjoint, than
+** than a character not in first(p1) cannot go to p1 and a character
-** a character not in first(p1) cannot go to p1, and a character
+** in first(p1) cannot go to p2, either because p1 will accept
-** in first(p1) cannot go to p2 (at it is not in first(p2)).
+** (headfail) or because it is not in first(p2) (disjoint).
-** (The optimization is not valid if p1 accepts the empty string,
+** (The second case is not valid if p1 accepts the empty string,
 ** as then there is no character at all...)
 ** - when p2 is empty and opt is true; a IPartialCommit can reuse
 ** the Choice already active in the stack.
@@ -686,7 +682,7 @@ static void codechoice (CompileState *compst, TTree *p1, TTree *p2, int opt,
     int jmp = NOINST;
     codegen(compst, p1, 0, test, fl);
     if (!emptyp2)
-      jmp = addoffsetinst(compst, IJmp); 
+      jmp = addoffsetinst(compst, IJmp);
     jumptohere(compst, test);
     codegen(compst, p2, opt, NOINST, fl);
     jumptohere(compst, jmp);
@@ -697,7 +693,7 @@ static void codechoice (CompileState *compst, TTree *p1, TTree *p2, int opt,
     codegen(compst, p1, 1, NOINST, fullset);
   }
   else {
-    /* <p1 / p2> == 
+    /* <p1 / p2> ==
         test(first(p1)) -> L1; choice L1; <p1>; commit L2; L1: <p2>; L2: */
     int pcommit;
     int test = codetestset(compst, &cs1, e1);
@@ -764,9 +760,53 @@ static void coderuntime (CompileState *compst, TTree *tree, int tt) {
 }
 
 
+/*
+** Create a jump to 'test' and fix 'test' to jump to next instruction
+*/
+static void closeloop (CompileState *compst, int test) {
+  int jmp = addoffsetinst(compst, IJmp);
+  jumptohere(compst, test);
+  jumptothere(compst, jmp, test);
+}
+
+
+/*
+** Try repetition of charsets:
+** For an empty set, repetition of fail is a no-op;
+** For any or char, code a tight loop;
+** For generic charset, use a span instruction.
+*/
+static int coderepcharset (CompileState *compst, TTree *tree) {
+  switch (tree->tag) {
+    case TFalse: return 1;  /* 'fail*' is a no-op */
+    case TAny: {  /* L1: testany -> L2; any; jmp L1; L2: */
+      int test = addoffsetinst(compst, ITestAny);
+      addinstruction(compst, IAny, 0);
+      closeloop(compst, test);
+      return 1;
+    }
+    case TChar: {  /* L1: testchar c -> L2; any; jmp L1; L2: */
+      int test = addoffsetinst(compst, ITestChar);
+      getinstr(compst, test).i.aux1 = tree->u.n;
+      addinstruction(compst, IAny, 0);
+      closeloop(compst, test);
+      return 1;
+    }
+    case TSet: {  /* regular set */
+      charsetinfo info;
+      int i = addinstruction(compst, ISpan, 0);
+      tree2cset(tree, &info);
+      addcharset(compst, i, &info);
+      return 1;
+    }
+    default: return 0;  /* not a charset */
+  }
+}
+
+
 /*
 ** Repetion; optimizations:
-** When pattern is a charset, can use special instruction ISpan.
+** When pattern is a charset, use special code.
 ** When pattern is head fail, or if it starts with characters that
 ** are disjoint from what follows the repetions, a simple test
 ** is enough (a fail inside the repetition would backtrack to fail
@@ -776,21 +816,14 @@ static void coderuntime (CompileState *compst, TTree *tree, int tt) {
 */
 static void coderep (CompileState *compst, TTree *tree, int opt,
                      const Charset *fl) {
-  Charset st;
+  if (!coderepcharset(compst, tree)) {
-  if (tocharset(tree, &st)) {
+    Charset st;
-    addinstruction(compst, ISpan, 0);
-    addcharset(compst, st.cs);
-  }
-  else {
     int e1 = getfirst(tree, fullset, &st);
     if (headfail(tree) || (!e1 && cs_disjoint(&st, fl))) {
       /* L1: test (fail(p1)) -> L2; <p>; jmp L1; L2: */
-      int jmp;
       int test = codetestset(compst, &st, 0);
       codegen(compst, tree, 0, test, fullset);
-      jmp = addoffsetinst(compst, IJmp);
+      closeloop(compst, test);
-      jumptohere(compst, test);
-      jumptothere(compst, jmp, test);
     }
     else {
       /* test(fail(p1)) -> L2; choice L2; L1: <p>; partialcommit L1; L2: */
@@ -847,7 +880,7 @@ static void correctcalls (CompileState *compst, int *positions,
   Instruction *code = compst->p->code;
   for (i = from; i < to; i += sizei(&code[i])) {
     if (code[i].i.code == IOpenCall) {
-      int n = code[i].i.key;  /* rule number */
+      int n = code[i].i.aux2.key;  /* rule number */
       int rule = positions[n];  /* rule position */
       assert(rule == from || code[rule - 1].i.code == IRet);
       if (code[finaltarget(code, i + 2)].i.code == IRet)  /* call; ret ? */
@@ -874,8 +907,10 @@ static void codegrammar (CompileState *compst, TTree *grammar) {
   int start = gethere(compst);  /* here starts the initial rule */
   jumptohere(compst, firstcall);
   for (rule = sib1(grammar); rule->tag == TRule; rule = sib2(rule)) {
+    TTree *r = sib1(rule);
+    assert(r->tag == TXInfo);
     positions[rulenumber++] = gethere(compst);  /* save rule position */
-    codegen(compst, sib1(rule), 0, NOINST, fullset);  /* code rule */
+    codegen(compst, sib1(r), 0, NOINST, fullset);  /* code rule */
     addinstruction(compst, IRet, 0);
   }
   assert(rule->tag == TTrue);
@@ -886,8 +921,8 @@ static void codegrammar (CompileState *compst, TTree *grammar) {
 
 static void codecall (CompileState *compst, TTree *call) {
   int c = addoffsetinst(compst, IOpenCall);  /* to be corrected later */
-  getinstr(compst, c).i.key = sib2(call)->cap;  /* rule number */
+  assert(sib1(sib2(call))->tag == TXInfo);
-  assert(sib2(call)->tag == TRule);
+  getinstr(compst, c).i.aux2.key = sib1(sib2(call))->u.n;  /* rule number */
 }
 
 
@@ -922,9 +957,10 @@ static void codegen (CompileState *compst, TTree *tree, int opt, int tt,
   switch (tree->tag) {
     case TChar: codechar(compst, tree->u.n, tt); break;
     case TAny: addinstruction(compst, IAny, 0); break;
-    case TSet: codecharset(compst, treebuffer(tree), tt); break;
+    case TSet: codecharset(compst, tree, tt); break;
     case TTrue: break;
     case TFalse: addinstruction(compst, IFail, 0); break;
+    case TUTFR: codeutfr(compst, tree); break;
     case TChoice: codechoice(compst, sib1(tree), sib2(tree), opt, fl); break;
     case TRep: coderep(compst, sib1(tree), opt, fl); break;
     case TBehind: codebehind(compst, tree); break;
@@ -971,7 +1007,7 @@ static void peephole (CompileState *compst) {
           case IRet: case IFail: case IFailTwice:
           case IEnd: {  /* instructions with unconditional implicit jumps */
             code[i] = code[ft];  /* jump becomes that instruction */
-            code[i + 1].i.code = IAny;  /* 'no-op' for target position */
+            code[i + 1].i.code = IEmpty;  /* 'no-op' for target position */
             break;
           }
           case ICommit: case IPartialCommit:
@@ -996,12 +1032,13 @@ static void peephole (CompileState *compst) {
 
 
 /*
-** Compile a pattern
+** Compile a pattern. 'size' is the size of the pattern's tree,
+** which gives a hint for the size of the final code.
 */
-Instruction *compile (lua_State *L, Pattern *p) {
+Instruction *compile (lua_State *L, Pattern *p, uint size) {
   CompileState compst;
   compst.p = p;  compst.ncode = 0;  compst.L = L;
-  realloccode(L, p, 2);  /* minimum initial size */
+  newcode(L, p, size/2u + 2);  /* set initial size */
   codegen(&compst, p->tree, 0, NOINST, fullset);
   addinstruction(&compst, IEnd, 0);
   realloccode(L, p, compst.ncode);  /* set final size */

3rd/lpeg/lpcode.h

@@ -1,6 +1,3 @@
-/*
-** $Id: lpcode.h $
-*/
 
 #if !defined(lpcode_h)
 #define lpcode_h
@@ -11,13 +8,12 @@
 #include "lptree.h"
 #include "lpvm.h"
 
-int tocharset (TTree *tree, Charset *cs);
 int checkaux (TTree *tree, int pred);
 int fixedlen (TTree *tree);
 int hascaptures (TTree *tree);
 int lp_gc (lua_State *L);
-Instruction *compile (lua_State *L, Pattern *p);
+Instruction *compile (lua_State *L, Pattern *p, uint size);
-void realloccode (lua_State *L, Pattern *p, int nsize);
+void freecode (lua_State *L, Pattern *p);
 int sizei (const Instruction *i);
 
 

3rd/lpeg/lpcset.c

@@ -0,0 +1,110 @@
+
+#include "lptypes.h"
+#include "lpcset.h"
+
+
+/*
+** Add to 'c' the index of the (only) bit set in byte 'b'
+*/
+static int onlybit (int c, int b) {
+  if ((b & 0xF0) != 0) { c += 4; b >>= 4; }
+  if ((b & 0x0C) != 0) { c += 2; b >>= 2; }
+  if ((b & 0x02) != 0) { c += 1; }
+  return c;
+}
+
+
+/*
+** Check whether a charset is empty (returns IFail), singleton (IChar),
+** full (IAny), or none of those (ISet). When singleton, 'info.offset'
+** returns which character it is. When generic set, 'info' returns
+** information about its range.
+*/
+Opcode charsettype (const byte *cs, charsetinfo *info) {
+  int low0, low1, high0, high1;
+  for (low1 = 0; low1 < CHARSETSIZE && cs[low1] == 0; low1++)
+    /* find lowest byte with a 1-bit */;
+  if (low1 == CHARSETSIZE)
+    return IFail;  /* no characters in set */
+  for (high1 = CHARSETSIZE - 1; cs[high1] == 0; high1--)
+    /* find highest byte with a 1-bit; low1 is a sentinel */;
+  if (low1 == high1) {  /* only one byte with 1-bits? */
+    int b = cs[low1];
+    if ((b & (b - 1)) == 0) {  /* does byte has only one 1-bit? */
+      info->offset = onlybit(low1 * BITSPERCHAR, b);  /* get that bit */
+      return IChar;  /* single character */
+    }
+  }
+  for (low0 = 0; low0 < CHARSETSIZE && cs[low0] == 0xFF; low0++)
+    /* find lowest byte with a 0-bit */;
+  if (low0 == CHARSETSIZE)
+    return IAny;  /* set has all bits set */
+  for (high0 = CHARSETSIZE - 1; cs[high0] == 0xFF; high0--)
+    /* find highest byte with a 0-bit; low0 is a sentinel */;
+  if (high1 - low1 <= high0 - low0) {  /* range of 1s smaller than of 0s? */
+    info->offset = low1;
+    info->size = high1 - low1 + 1;
+    info->deflt = 0;  /* all discharged bits were 0 */
+  }
+  else {
+    info->offset = low0;
+    info->size = high0 - low0 + 1;
+    info->deflt = 0xFF;  /* all discharged bits were 1 */
+  }
+  info->cs = cs + info->offset;
+  return ISet;
+}
+
+
+/*
+** Get a byte from a compact charset. If index is inside the charset
+** range, get the byte from the supporting charset (correcting it
+** by the offset). Otherwise, return the default for the set.
+*/
+byte getbytefromcharset (const charsetinfo *info, int index) {
+  if (index < info->size)
+    return info->cs[index];
+  else return info->deflt;
+}
+
+
+/*
+** If 'tree' is a 'char' pattern (TSet, TChar, TAny, TFalse), convert it
+** into a charset and return 1; else return 0.
+*/
+int tocharset (TTree *tree, Charset *cs) {
+  switch (tree->tag) {
+    case TChar: {  /* only one char */
+      assert(0 <= tree->u.n && tree->u.n <= UCHAR_MAX);
+      clearset(cs->cs);  /* erase all chars */
+      setchar(cs->cs, tree->u.n);  /* add that one */
+      return 1;
+    }
+    case TAny: {
+      fillset(cs->cs, 0xFF);  /* add all characters to the set */
+      return 1;
+    }
+    case TFalse: {
+      clearset(cs->cs);  /* empty set */
+      return 1;
+    }
+    case TSet: {  /* fill set */
+      int i;
+      fillset(cs->cs, tree->u.set.deflt);
+      for (i = 0; i < tree->u.set.size; i++)
+        cs->cs[tree->u.set.offset + i] = treebuffer(tree)[i];
+      return 1;
+    }
+    default: return 0;
+  }
+}
+
+
+void tree2cset (TTree *tree, charsetinfo *info) {
+  assert(tree->tag == TSet);
+  info->offset = tree->u.set.offset;
+  info->size = tree->u.set.size;
+  info->deflt = tree->u.set.deflt;
+  info->cs = treebuffer(tree);
+}
+

3rd/lpeg/lpcset.h

@@ -0,0 +1,30 @@
+
+#if !defined(lpset_h)
+#define lpset_h
+
+#include "lpcset.h"
+#include "lpcode.h"
+#include "lptree.h"
+
+
+/*
+** Extra information for the result of 'charsettype'.  When result is
+** IChar, 'offset' is the character.  When result is ISet, 'cs' is the
+** supporting bit array (with offset included), 'offset' is the offset
+** (in bytes), 'size' is the size (in bytes), and 'delt' is the default
+** value for bytes outside the set.
+*/
+typedef struct {
+  const byte *cs;
+  int offset;
+  int size;
+  int deflt;
+} charsetinfo;
+
+
+int tocharset (TTree *tree, Charset *cs);
+Opcode charsettype (const byte *cs, charsetinfo *info);
+byte getbytefromcharset (const charsetinfo *info, int index);
+void tree2cset (TTree *tree, charsetinfo *info);
+
+#endif

3rd/lpeg/lpeg.html

@@ -1,28 +1,27 @@
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
-   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+   "//www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<html xmlns="//www.w3.org/1999/xhtml" xml:lang="en" lang="en">
 <head>
     <title>LPeg - Parsing Expression Grammars For Lua</title>
     <link rel="stylesheet"
-          href="http://www.inf.puc-rio.br/~roberto/lpeg/doc.css"
+          href="//www.inf.puc-rio.br/~roberto/lpeg/doc.css"
           type="text/css"/>
 	<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
 </head>
 <body>
 
-<!-- $Id: lpeg.html $ -->
 
 <div id="container">
 	
 <div id="product">
   <div id="product_logo">
-    <a href="http://www.inf.puc-rio.br/~roberto/lpeg/">
+    <a href="//www.inf.puc-rio.br/~roberto/lpeg/">
     <img alt="LPeg logo" src="lpeg-128.gif"/></a>
     
   </div>
   <div id="product_name"><big><strong>LPeg</strong></big></div>
   <div id="product_description">
-     Parsing Expression Grammars For Lua, version 1.0
+     Parsing Expression Grammars For Lua, version 1.1
   </div>
 </div> <!-- id="product" -->
 
@@ -56,16 +55,16 @@
 <p>
 <em>LPeg</em> is a new pattern-matching library for Lua,
 based on
-<a href="http://pdos.csail.mit.edu/%7Ebaford/packrat/">
+<a href="//bford.info/packrat/">
 Parsing Expression Grammars</a> (PEGs).
 This text is a reference manual for the library.
 For a more formal treatment of LPeg,
 as well as some discussion about its implementation,
 see
-<a href="http://www.inf.puc-rio.br/~roberto/docs/peg.pdf">
+<a href="//www.inf.puc-rio.br/~roberto/docs/peg.pdf">
 A Text Pattern-Matching Tool based on Parsing Expression Grammars</a>.
 (You may also be interested in my
-<a href="http://vimeo.com/1485123">talk about LPeg</a>
+<a href="//vimeo.com/1485123">talk about LPeg</a>
 given at the III Lua Workshop.)
 </p>
 
@@ -107,6 +106,9 @@ for creating patterns:
   <td>Matches any character in <code>string</code> (Set)</td></tr>
 <tr><td><a href="#op-r"><code>lpeg.R("<em>xy</em>")</code></a></td>
   <td>Matches any character between <em>x</em> and <em>y</em> (Range)</td></tr>
+<tr><td><a href="#op-utfR"><code>lpeg.utfR(cp1, cp2)</code></a></td>
+  <td>Matches an UTF-8 code point between <code>cp1</code> and
+  <code>cp2</code></td></tr>
 <tr><td><a href="#op-pow"><code>patt^n</code></a></td>
   <td>Matches at least <code>n</code> repetitions of <code>patt</code></td></tr>
 <tr><td><a href="#op-pow"><code>patt^-n</code></a></td>
@@ -142,7 +144,7 @@ so, it succeeds only at the end of the subject.
 LPeg also offers the <a href="re.html"><code>re</code> module</a>,
 which implements patterns following a regular-expression style
 (e.g., <code>[09]+</code>).
-(This module is 260 lines of Lua code,
+(This module is 270 lines of Lua code,
 and of course it uses LPeg to parse regular expressions and
 translate them to regular LPeg patterns.)
 </p>
@@ -164,7 +166,7 @@ or the <a href="#captures">captured values</a>
 <p>
 An optional numeric argument <code>init</code> makes the match
 start at that position in the subject string.
-As usual in Lua libraries,
+As in the Lua standard libraries,
 a negative value counts from the end.
 </p>
 
@@ -188,9 +190,9 @@ returns the string <code>"pattern"</code>.
 Otherwise returns nil.
 </p>
 
-<h3><a name="f-version"></a><code>lpeg.version ()</code></h3>
+<h3><a name="f-version"></a><code>lpeg.version</code></h3>
 <p>
-Returns a string with the running version of LPeg.
+A string (not a function) with the running version of LPeg.
 </p>
 
 <h3><a name="f-setstack"></a><code>lpeg.setmaxstack (max)</code></h3>
@@ -329,6 +331,15 @@ are patterns that always fail.
 </p>
 
 
+<h3><a name="op-utfR"></a><code>lpeg.utfR (cp1, cp2)</code></h3>
+<p>
+Returns a pattern that matches a valid UTF-8 byte sequence
+representing a code point in the range <code>[cp1, cp2]</code>.
+The range is limited by the natural Unicode limit of 0x10FFFF,
+but may include surrogates.
+</p>
+
+
 <h3><a name="op-v"></a><code>lpeg.V (v)</code></h3>
 <p>
 This operation creates a non-terminal (a <em>variable</em>)
@@ -433,7 +444,8 @@ letter = lower + upper
 
 <h3><a name="op-sub"></a><code>patt1 - patt2</code></h3>
 <p>
-Returns a pattern equivalent to <em>!patt2 patt1</em>.
+Returns a pattern equivalent to <em>!patt2 patt1</em>
+in the origial PEG notation.
 This pattern asserts that the input does not match
 <code>patt2</code> and then matches <code>patt1</code>.
 </p>
@@ -597,17 +609,17 @@ The following table summarizes the basic captures:
 <tr><td><a href="#cap-arg"><code>lpeg.Carg(n)</code></a></td>
     <td>the value of the n<sup>th</sup> extra argument to
         <code>lpeg.match</code> (matches the empty string)</td></tr>
-<tr><td><a href="#cap-b"><code>lpeg.Cb(name)</code></a></td>
+<tr><td><a href="#cap-b"><code>lpeg.Cb(key)</code></a></td>
     <td>the values produced by the previous
-        group capture named <code>name</code>
+        group capture named <code>key</code>
         (matches the empty string)</td></tr>
 <tr><td><a href="#cap-cc"><code>lpeg.Cc(values)</code></a></td>
     <td>the given values (matches the empty string)</td></tr>
 <tr><td><a href="#cap-f"><code>lpeg.Cf(patt, func)</code></a></td>
-  <td>a <em>folding</em> of the captures from <code>patt</code></td></tr>
+  <td>folding capture (<em>deprecated</em>)</td></tr>
-<tr><td><a href="#cap-g"><code>lpeg.Cg(patt [, name])</code></a></td>
+<tr><td><a href="#cap-g"><code>lpeg.Cg(patt [, key])</code></a></td>
     <td>the values produced by <code>patt</code>,
-        optionally tagged with <code>name</code></td></tr>
+        optionally tagged with <code>key</code></td></tr>
 <tr><td><a href="#cap-p"><code>lpeg.Cp()</code></a></td>
     <td>the current position (matches the empty string)</td></tr>
 <tr><td><a href="#cap-s"><code>lpeg.Cs(patt)</code></a></td>
@@ -627,6 +639,11 @@ or no value when <code>number</code> is zero.</td></tr>
 <tr><td><a href="#cap-func"><code>patt / function</code></a></td>
   <td>the returns of <code>function</code> applied to the captures
       of <code>patt</code></td></tr>
+<tr><td><a href="#cap-acc"><code>patt % function</code></a></td>
+  <td>produces no value;
+      it <em>accummulates</em> the captures from <code>patt</code>
+      into the previous capture through <code>function</code>
+      </td></tr>
 <tr><td><a href="#matchtime"><code>lpeg.Cmt(patt, function)</code></a></td>
   <td>the returns of <code>function</code> applied to the captures
       of <code>patt</code>; the application is done at match time</td></tr>
@@ -652,10 +669,10 @@ LPeg does not specify when (and if) it evaluates its captures.
 consider the pattern <code>lpeg.P"a" / func / 0</code>.
 Because the "division" by 0 instructs LPeg to throw away the
 results from the pattern,
-LPeg may or may not call <code>func</code>.)
+it is not specified whether LPeg will call <code>func</code>.)
 Therefore, captures should avoid side effects.
 Moreover,
-most captures cannot affect the way a pattern matches a subject.
+captures cannot affect the way a pattern matches a subject.
 The only exception to this rule is the
 so-called <a href="#matchtime"><em>match-time capture</em></a>.
 When a match-time capture matches,
@@ -684,24 +701,25 @@ argument given in the call to <code>lpeg.match</code>.
 </p>
 
 
-<h3><a name="cap-b"></a><code>lpeg.Cb (name)</code></h3>
+<h3><a name="cap-b"></a><code>lpeg.Cb (key)</code></h3>
 <p>
 Creates a <em>back capture</em>.
 This pattern matches the empty string and
 produces the values produced by the <em>most recent</em>
-<a href="#cap-g">group capture</a> named <code>name</code>
+<a href="#cap-g">group capture</a> named <code>key</code>
-(where <code>name</code> can be any Lua value).
+(where <code>key</code> can be any Lua value).
 </p>
 
 <p>
 <em>Most recent</em> means the last
 <em>complete</em>
 <em>outermost</em>
-group capture with the given name.
+group capture with the given key.
 A <em>Complete</em> capture means that the entire pattern
-corresponding to the capture has matched.
+corresponding to the capture has matched;
+in other words, the back capture is not nested inside the group.
 An <em>Outermost</em> capture means that the capture is not inside
-another complete capture.
+another complete capture that does not contain the back capture itself.
 </p>
 
 <p>
@@ -722,61 +740,21 @@ produces all given values as its captured values.
 <h3><a name="cap-f"></a><code>lpeg.Cf (patt, func)</code></h3>
 <p>
 Creates a <em>fold capture</em>.
-If <code>patt</code> produces a list of captures
+This construction is deprecated;
-<em>C<sub>1</sub> C<sub>2</sub> ... C<sub>n</sub></em>,
+use an <a href="#cap-acc">accumulator pattern</a> instead.
-this capture will produce the value
+In general, a fold like
-<em>func(...func(func(C<sub>1</sub>, C<sub>2</sub>), C<sub>3</sub>)...,
+<code>lpeg.Cf(p1 * p2^0, func)</code>
-         C<sub>n</sub>)</em>,
+can be translated to
-that is, it will <em>fold</em>
+<code>(p1 * (p2 % func)^0)</code>.
-(or <em>accumulate</em>, or <em>reduce</em>)
-the captures from <code>patt</code> using function <code>func</code>.
-</p>
-
-<p>
-This capture assumes that <code>patt</code> should produce
-at least one capture with at least one value (of any type),
-which becomes the initial value of an <em>accumulator</em>.
-(If you need a specific initial value,
-you may prefix a <a href="#cap-cc">constant capture</a> to <code>patt</code>.)
-For each subsequent capture,
-LPeg calls <code>func</code>
-with this accumulator as the first argument and all values produced
-by the capture as extra arguments;
-the first result from this call
-becomes the new value for the accumulator.
-The final value of the accumulator becomes the captured value.
-</p>
-
-<p>
-As an example,
-the following pattern matches a list of numbers separated
-by commas and returns their addition:
-</p>
-<pre class="example">
--- matches a numeral and captures its numerical value
-number = lpeg.R"09"^1 / tonumber
-
--- matches a list of numbers, capturing their values
-list = number * ("," * number)^0
-
--- auxiliary function to add two numbers
-function add (acc, newvalue) return acc + newvalue end
-
--- folds the list of numbers adding them
-sum = lpeg.Cf(list, add)
-
--- example of use
-print(sum:match("10,30,43"))   --&gt; 83
-</pre>
 
 
-<h3><a name="cap-g"></a><code>lpeg.Cg (patt [, name])</code></h3>
+<h3><a name="cap-g"></a><code>lpeg.Cg (patt [, key])</code></h3>
 <p>
 Creates a <em>group capture</em>.
 It groups all values returned by <code>patt</code>
 into a single capture.
-The group may be anonymous (if no name is given)
+The group may be anonymous (if no key is given)
-or named with the given name
+or named with the given key
 (which can be any non-nil Lua value).
 </p>
 
@@ -822,7 +800,7 @@ starting at 1.
 Moreover,
 for each named capture group created by <code>patt</code>,
 the first value of the group is put into the table
-with the group name as its key.
+with the group key as its key.
 The captured value is only the table.
 </p>
 
@@ -878,6 +856,98 @@ there is no captured value.
 </p>
 
 
+<h3><a name="cap-acc"></a><code>patt % function</code></h3>
+<p>
+Creates an <em>accumulator capture</em>.
+This pattern behaves similarly to a
+<a href="#cap-func">function capture</a>,
+with the following differences:
+The last captured value before <code>patt</code>
+is added as a first argument to the call;
+the return of the function is adjusted to one single value;
+that value replaces the last captured value.
+Note that the capture itself produces no values;
+it only changes the value of its previous capture.
+</p>
+
+<p>
+As an example,
+let us consider the problem of adding a list of numbers.
+</p>
+<pre class="example">
+-- matches a numeral and captures its numerical value
+number = lpeg.R"09"^1 / tonumber
+
+-- auxiliary function to add two numbers
+function add (acc, newvalue) return acc + newvalue end
+
+-- matches a list of numbers, adding their values
+sum = number * ("," * number % add)^0
+
+-- example of use
+print(sum:match("10,30,43"))   --&gt; 83
+</pre>
+<p>
+First, the initial <code>number</code> captures a number;
+that first capture will play the role of an accumulator.
+Then, each time the sequence <code>comma-number</code>
+matches inside the loop there is an accumulator capture:
+It calls <code>add</code> with the current value of the
+accumulator&mdash;which is the last captured value, created by the
+first <code>number</code>&mdash; and the value of the new number,
+and the result of the call (the sum of the two numbers)
+replaces the value of the accumulator.
+At the end of the match,
+the accumulator with all sums is the final value.
+</p>
+
+<p>
+As another example,
+consider the following code fragment:
+</p>
+<pre class="example">
+local name = lpeg.C(lpeg.R("az")^1)
+local p = name * (lpeg.P("^") % string.upper)^-1
+print(p:match("count"))    --&gt; count
+print(p:match("count^"))   --&gt; COUNT
+</pre>
+<p>
+In the match against <code>"count"</code>,
+as there is no <code>"^"</code>,
+the optional accumulator capture does not match;
+so, the match results in its sole capture, a name.
+In the match against <code>"count^"</code>,
+the accumulator capture matches,
+so the function <code>string.upper</code>
+is called with the previous captured value (created by <code>name</code>)
+plus the string <code>"^"</code>;
+the function ignores its second argument and returns the first argument
+changed to upper case;
+that value then becomes the first and only
+capture value created by the match.
+</p>
+
+<p>
+Due to the nature of this capture,
+you should avoid using it in places where it is not clear
+what is the "previous" capture,
+such as directly nested in a <a href="#cap-string">string capture</a>
+or a <a href="#cap-num">numbered capture</a>.
+(Note that these captures may not need to evaluate
+all their subcaptures to compute their results.)
+Moreover, due to implementation details,
+you should not use this capture directly nested in a
+<a href="#cap-s">substitution capture</a>.
+You should also avoid a direct nesting of this capture inside
+a <a href="#cap-f">folding capture</a> (deprecated),
+as the folding will try to fold each individual accumulator capture.
+A simple and effective way to avoid all these issues is
+to enclose the whole accumulation composition
+(including the capture that generates the initial value)
+into an anonymous <a href="#cap-g">group capture</a>.
+</p>
+
+
 <h3><a name="matchtime"></a><code>lpeg.Cmt(patt, function)</code></h3>
 <p>
 Creates a <em>match-time capture</em>.
@@ -930,9 +1000,9 @@ local lpeg = require "lpeg"
 -- matches a word followed by end-of-string
 p = lpeg.R"az"^1 * -1
 
-print(p:match("hello"))        --> 6
+print(p:match("hello"))        --&gt; 6
-print(lpeg.match(p, "hello"))  --> 6
+print(lpeg.match(p, "hello"))  --&gt; 6
-print(p:match("1 hello"))      --> nil
+print(p:match("1 hello"))      --&gt; nil
 </pre>
 <p>
 The pattern is simply a sequence of one or more lower-case letters
@@ -957,19 +1027,18 @@ lpeg.locale(lpeg)   -- adds locale entries into 'lpeg' table
 local space = lpeg.space^0
 local name = lpeg.C(lpeg.alpha^1) * space
 local sep = lpeg.S(",;") * space
-local pair = lpeg.Cg(name * "=" * space * name) * sep^-1
+local pair = name * "=" * space * name * sep^-1
-local list = lpeg.Cf(lpeg.Ct("") * pair^0, rawset)
+local list = lpeg.Ct("") * (pair % rawset)^0
-t = list:match("a=b, c = hi; next = pi")  --> { a = "b", c = "hi", next = "pi" }
+t = list:match("a=b, c = hi; next = pi")
+        --&gt; { a = "b", c = "hi", next = "pi" }
 </pre>
 <p>
 Each pair has the format <code>name = name</code> followed by
 an optional separator (a comma or a semicolon).
-The <code>pair</code> pattern encloses the pair in a group pattern,
+The <code>list</code> pattern then <em>folds</em> these captures.
-so that the names become the values of a single capture.
-The <code>list</code> pattern then folds these captures.
 It starts with an empty table,
 created by a table capture matching an empty string;
-then for each capture (a pair of names) it applies <code>rawset</code>
+then for each a pair of names it applies <code>rawset</code>
 over the accumulator (the table) and the capture values (the pair of names).
 <code>rawset</code> returns the table itself,
 so the accumulator is always the table.
@@ -1003,7 +1072,7 @@ by <code>sep</code>.
 If the split results in too many values,
 it may overflow the maximum number of values
 that can be returned by a Lua function.
-In this case,
+To avoid this problem,
 we can collect these values in a table:
 </p>
 <pre class="example">
@@ -1039,7 +1108,7 @@ end
 </pre>
 <p>
 This grammar has a straight reading:
-it matches <code>p</code> or skips one character and tries again.
+its sole rule matches <code>p</code> or skips one character and tries again.
 </p>
 
 <p>
@@ -1048,27 +1117,27 @@ If we want to know where the pattern is in the string
 we can add position captures to the pattern:
 </p>
 <pre class="example">
-local I = lpeg.Cp()
+local Cp = lpeg.Cp()
 function anywhere (p)
-  return lpeg.P{ I * p * I + 1 * lpeg.V(1) }
+  return lpeg.P{ Cp * p * Cp + 1 * lpeg.V(1) }
 end
 
-print(anywhere("world"):match("hello world!"))   -> 7   12
+print(anywhere("world"):match("hello world!"))   --&gt; 7   12
 </pre>
 
 <p>
 Another option for the search is like this:
 </p>
 <pre class="example">
-local I = lpeg.Cp()
+local Cp = lpeg.Cp()
 function anywhere (p)
-  return (1 - lpeg.P(p))^0 * I * p * I
+  return (1 - lpeg.P(p))^0 * Cp * p * Cp
 end
 </pre>
 <p>
 Again the pattern has a straight reading:
 it skips as many characters as possible while not matching <code>p</code>,
-and then matches <code>p</code> (plus appropriate captures).
+and then matches <code>p</code> plus appropriate captures.
 </p>
 
 <p>
@@ -1163,91 +1232,6 @@ local record = lpeg.Ct(field * (',' * field)^0) * (lpeg.P'\n' + -1)
 </pre>
 
 
-<h3>UTF-8 and Latin 1</h3>
-<p>
-It is not difficult to use LPeg to convert a string from
-UTF-8 encoding to Latin 1 (ISO 8859-1):
-</p>
-
-<pre class="example">
--- convert a two-byte UTF-8 sequence to a Latin 1 character
-local function f2 (s)
-  local c1, c2 = string.byte(s, 1, 2)
-  return string.char(c1 * 64 + c2 - 12416)
-end
-
-local utf8 = lpeg.R("\0\127")
-           + lpeg.R("\194\195") * lpeg.R("\128\191") / f2
-
-local decode_pattern = lpeg.Cs(utf8^0) * -1
-</pre>
-<p>
-In this code,
-the definition of UTF-8 is already restricted to the
-Latin 1 range (from 0 to 255).
-Any encoding outside this range (as well as any invalid encoding)
-will not match that pattern.
-</p>
-
-<p>
-As the definition of <code>decode_pattern</code> demands that
-the pattern matches the whole input (because of the -1 at its end),
-any invalid string will simply fail to match,
-without any useful information about the problem.
-We can improve this situation redefining <code>decode_pattern</code>
-as follows:
-</p>
-<pre class="example">
-local function er (_, i) error("invalid encoding at position " .. i) end
-
-local decode_pattern = lpeg.Cs(utf8^0) * (-1 + lpeg.P(er))
-</pre>
-<p>
-Now, if the pattern <code>utf8^0</code> stops
-before the end of the string,
-an appropriate error function is called.
-</p>
-
-
-<h3>UTF-8 and Unicode</h3>
-<p>
-We can extend the previous patterns to handle all Unicode code points.
-Of course,
-we cannot translate them to Latin 1 or any other one-byte encoding.
-Instead, our translation results in a array with the code points
-represented as numbers.
-The full code is here:
-</p>
-<pre class="example">
--- decode a two-byte UTF-8 sequence
-local function f2 (s)
-  local c1, c2 = string.byte(s, 1, 2)
-  return c1 * 64 + c2 - 12416
-end
-
--- decode a three-byte UTF-8 sequence
-local function f3 (s)
-  local c1, c2, c3 = string.byte(s, 1, 3)
-  return (c1 * 64 + c2) * 64 + c3 - 925824
-end
-
--- decode a four-byte UTF-8 sequence
-local function f4 (s)
-  local c1, c2, c3, c4 = string.byte(s, 1, 4)
-  return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168
-end
-
-local cont = lpeg.R("\128\191")   -- continuation byte
-
-local utf8 = lpeg.R("\0\127") / string.byte
-           + lpeg.R("\194\223") * cont / f2
-           + lpeg.R("\224\239") * cont * cont / f3
-           + lpeg.R("\240\244") * cont * cont * cont / f4
-
-local decode_pattern = lpeg.Ct(utf8^0) * -1
-</pre>
-
-
 <h3>Lua's long strings</h3>
 <p>
 A long string in Lua starts with the pattern <code>[=*[</code>
@@ -1347,7 +1331,7 @@ function evalExp (s)
 end
 
 -- small example
-print(evalExp"3 + 5*9 / (1+1) - 12")   --> 13.5
+print(evalExp"3 + 5*9 / (1+1) - 12")   --&gt; 13.5
 </pre>
 
 <p>
@@ -1369,16 +1353,16 @@ end
 -- Grammar
 local V = lpeg.V
 G = lpeg.P{ "Exp",
-  Exp = lpeg.Cf(V"Term" * lpeg.Cg(TermOp * V"Term")^0, eval);
+  Exp = V"Term" * (TermOp * V"Term" % eval)^0;
-  Term = lpeg.Cf(V"Factor" * lpeg.Cg(FactorOp * V"Factor")^0, eval);
+  Term = V"Factor" * (FactorOp * V"Factor" % eval)^0;
   Factor = Number / tonumber + Open * V"Exp" * Close;
 }
 
 -- small example
-print(lpeg.match(G, "3 + 5*9 / (1+1) - 12"))   --> 13.5
+print(lpeg.match(G, "3 + 5*9 / (1+1) - 12"))   --&gt; 13.5
 </pre>
 <p>
-Note the use of the fold (accumulator) capture.
+Note the use of the accumulator capture.
 To compute the value of an expression,
 the accumulator starts with the value of the first term,
 and then applies <code>eval</code> over
@@ -1391,13 +1375,20 @@ and the new term for each repetition.
 <h2><a name="download"></a>Download</h2>
 
 <p>LPeg 
-<a href="http://www.inf.puc-rio.br/~roberto/lpeg/lpeg-1.0.2.tar.gz">source code</a>.</p>
+<a href="//www.inf.puc-rio.br/~roberto/lpeg/lpeg-1.1.0.tar.gz">source code</a>.</p>
+
+<p>
+Probably, the easiest way to install LPeg is with
+<a href="//luarocks.org/">LuaRocks</a>.
+If you have LuaRocks installed,
+the following command is all you need to install LPeg:
+<pre>$ luarocks install lpeg</pre>
 
 
 <h2><a name="license">License</a></h2>
 
 <p>
-Copyright &copy; 2007-2019 Lua.org, PUC-Rio.
+Copyright &copy; 2007-2023 Lua.org, PUC-Rio.
 </p>
 <p>
 Permission is hereby granted, free of charge,

3rd/lpeg/lpprint.c

@@ -1,7 +1,3 @@
-/*
-** $Id: lpprint.c $
-** Copyright 2007, Lua.org & PUC-Rio  (see 'lpeg.html' for license)
-*/
 
 #include <ctype.h>
 #include <limits.h>
@@ -27,7 +23,7 @@ void printcharset (const byte *st) {
   printf("[");
   for (i = 0; i <= UCHAR_MAX; i++) {
     int first = i;
-    while (testchar(st, i) && i <= UCHAR_MAX) i++;
+    while (i <= UCHAR_MAX && testchar(st, i)) i++;
     if (i - 1 == first)  /* unary range? */
       printf("(%02x)", first);
     else if (i - 1 > first)  /* non-empty range? */
@@ -37,10 +33,34 @@ void printcharset (const byte *st) {
 }
 
 
+static void printIcharset (const Instruction *inst, const byte *buff) {
+  byte cs[CHARSETSIZE];
+  int i;
+  printf("(%02x-%d) ", inst->i.aux2.set.offset, inst->i.aux2.set.size);
+  clearset(cs);
+  for (i = 0; i < CHARSETSIZE * 8; i++) {
+    if (charinset(inst, buff, i))
+      setchar(cs, i);
+  }
+  printcharset(cs);
+}
+
+
+static void printTcharset (TTree *tree) {
+  byte cs[CHARSETSIZE];
+  int i;
+  printf("(%02x-%d) ", tree->u.set.offset, tree->u.set.size);
+  fillset(cs, tree->u.set.deflt);
+  for (i = 0; i < tree->u.set.size; i++)
+    cs[tree->u.set.offset + i] = treebuffer(tree)[i];
+  printcharset(cs);
+}
+
+
 static const char *capkind (int kind) {
   const char *const modes[] = {
     "close", "position", "constant", "backref",
-    "argument", "simple", "table", "function",
+    "argument", "simple", "table", "function", "accumulator",
     "query", "string", "num", "substitution", "fold",
     "runtime", "group"};
   return modes[kind];
@@ -56,41 +76,46 @@ void printinst (const Instruction *op, const Instruction *p) {
   const char *const names[] = {
     "any", "char", "set",
     "testany", "testchar", "testset",
-    "span", "behind",
+    "span", "utf-range", "behind",
     "ret", "end",
     "choice", "jmp", "call", "open_call",
     "commit", "partial_commit", "back_commit", "failtwice", "fail", "giveup",
-     "fullcapture", "opencapture", "closecapture", "closeruntime"
+     "fullcapture", "opencapture", "closecapture", "closeruntime",
+     "--"
   };
   printf("%02ld: %s ", (long)(p - op), names[p->i.code]);
   switch ((Opcode)p->i.code) {
     case IChar: {
-      printf("'%c'", p->i.aux);
+      printf("'%c' (%02x)", p->i.aux1, p->i.aux1);
       break;
     }
     case ITestChar: {
-      printf("'%c'", p->i.aux); printjmp(op, p);
+      printf("'%c' (%02x)", p->i.aux1, p->i.aux1); printjmp(op, p);
+      break;
+    }
+    case IUTFR: {
+      printf("%d - %d", p[1].offset, utf_to(p));
       break;
     }
     case IFullCapture: {
       printf("%s (size = %d)  (idx = %d)",
-             capkind(getkind(p)), getoff(p), p->i.key);
+             capkind(getkind(p)), getoff(p), p->i.aux2.key);
       break;
     }
     case IOpenCapture: {
-      printf("%s (idx = %d)", capkind(getkind(p)), p->i.key);
+      printf("%s (idx = %d)", capkind(getkind(p)), p->i.aux2.key);
       break;
     }
     case ISet: {
-      printcharset((p+1)->buff);
+      printIcharset(p, (p+1)->buff);
       break;
     }
     case ITestSet: {
-      printcharset((p+2)->buff); printjmp(op, p);
+      printIcharset(p, (p+2)->buff); printjmp(op, p);
       break;
     }
     case ISpan: {
-      printcharset((p+1)->buff);
+      printIcharset(p, (p+1)->buff);
       break;
     }
     case IOpenCall: {
@@ -98,7 +123,7 @@ void printinst (const Instruction *op, const Instruction *p) {
       break;
     }
     case IBehind: {
-      printf("%d", p->i.aux);
+      printf("%d", p->i.aux1);
       break;
     }
     case IJmp: case ICall: case ICommit: case IChoice:
@@ -112,8 +137,9 @@ void printinst (const Instruction *op, const Instruction *p) {
 }
 
 
-void printpatt (Instruction *p, int n) {
+void printpatt (Instruction *p) {
   Instruction *op = p;
+  uint n = op[-1].codesize - 1;
   while (p < op + n) {
     printinst(op, p);
     p += sizei(p);
@@ -121,20 +147,39 @@ void printpatt (Instruction *p, int n) {
 }
 
 
-#if defined(LPEG_DEBUG)
+static void printcap (Capture *cap, int ident) {
-static void printcap (Capture *cap) {
+  while (ident--) printf(" ");
-  printf("%s (idx: %d - size: %d) -> %p\n",
+  printf("%s (idx: %d - size: %d) -> %lu  (%p)\n",
-         capkind(cap->kind), cap->idx, cap->siz, cap->s);
+         capkind(cap->kind), cap->idx, cap->siz, (long)cap->index, (void*)cap);
 }
 
 
-void printcaplist (Capture *cap, Capture *limit) {
+/*
+** Print a capture and its nested captures
+*/
+static Capture *printcap2close (Capture *cap, int ident) {
+  Capture *head = cap++;
+  printcap(head, ident);  /* print head capture */
+  while (capinside(head, cap))
+    cap = printcap2close(cap, ident + 2);  /* print nested captures */
+  if (isopencap(head)) {
+    assert(isclosecap(cap));
+    printcap(cap++, ident);  /* print and skip close capture */
+  }
+  return cap;
+}
+
+
+void printcaplist (Capture *cap) {
+  {  /* for debugging, print first a raw list of captures */
+    Capture *c = cap;
+    while (c->index != MAXINDT) { printcap(c, 0); c++; }
+  }
   printf(">======\n");
-  for (; cap->s && (limit == NULL || cap < limit); cap++)
+  while (!isclosecap(cap))
-    printcap(cap);
+    cap = printcap2close(cap, 0);
   printf("=======\n");
 }
-#endif
 
 /* }====================================================== */
 
@@ -147,11 +192,11 @@ void printcaplist (Capture *cap, Capture *limit) {
 
 static const char *tagnames[] = {
   "char", "set", "any",
-  "true", "false",
+  "true", "false", "utf8.range",
   "rep",
   "seq", "choice",
   "not", "and",
-  "call", "opencall", "rule", "grammar",
+  "call", "opencall", "rule", "xinfo", "grammar",
   "behind",
   "capture", "run-time"
 };
@@ -159,6 +204,7 @@ static const char *tagnames[] = {
 
 void printtree (TTree *tree, int ident) {
   int i;
+  int sibs = numsiblings[tree->tag];
   for (i = 0; i < ident; i++) printf(" ");
   printf("%s", tagnames[tree->tag]);
   switch (tree->tag) {
@@ -171,29 +217,38 @@ void printtree (TTree *tree, int ident) {
       break;
     }
     case TSet: {
-      printcharset(treebuffer(tree));
+      printTcharset(tree);
       printf("\n");
       break;
     }
+    case TUTFR: {
+      assert(sib1(tree)->tag == TXInfo);
+      printf(" %d (%02x %d) - %d (%02x %d) \n",
+        tree->u.n, tree->key, tree->cap,
+        sib1(tree)->u.n, sib1(tree)->key, sib1(tree)->cap);
+      break;
+    }
     case TOpenCall: case TCall: {
-      assert(sib2(tree)->tag == TRule);
+      assert(sib1(sib2(tree))->tag == TXInfo);
-      printf(" key: %d  (rule: %d)\n", tree->key, sib2(tree)->cap);
+      printf(" key: %d  (rule: %d)\n", tree->key, sib1(sib2(tree))->u.n);
       break;
     }
     case TBehind: {
       printf(" %d\n", tree->u.n);
-        printtree(sib1(tree), ident + 2);
       break;
     }
     case TCapture: {
       printf(" kind: '%s'  key: %d\n", capkind(tree->cap), tree->key);
-      printtree(sib1(tree), ident + 2);
       break;
     }
     case TRule: {
-      printf(" n: %d  key: %d\n", tree->cap, tree->key);
+      printf(" key: %d\n", tree->key);
-      printtree(sib1(tree), ident + 2);
+      sibs = 1;  /* do not print 'sib2' (next rule) as a sibling */
-      break;  /* do not print next rule as a sibling */
+      break;
+    }
+    case TXInfo: {
+      printf(" n: %d\n", tree->u.n);
+      break;
     }
     case TGrammar: {
       TTree *rule = sib1(tree);
@@ -203,18 +258,17 @@ void printtree (TTree *tree, int ident) {
         rule = sib2(rule);
       }
       assert(rule->tag == TTrue);  /* sentinel */
+      sibs = 0;  /* siblings already handled */
       break;
     }
-    default: {
+    default:
-      int sibs = numsiblings[tree->tag];
       printf("\n");
-      if (sibs >= 1) {
-        printtree(sib1(tree), ident + 2);
-        if (sibs >= 2)
-          printtree(sib2(tree), ident + 2);
-      }
       break;
-    }
+  }
+  if (sibs >= 1) {
+    printtree(sib1(tree), ident + 2);
+    if (sibs >= 2)
+      printtree(sib2(tree), ident + 2);
   }
 }
 

3rd/lpeg/lpprint.h

@@ -1,7 +1,3 @@
-/*
-** $Id: lpprint.h $
-*/
-
 
 #if !defined(lpprint_h)
 #define lpprint_h
@@ -13,11 +9,11 @@
 
 #if defined(LPEG_DEBUG)
 
-void printpatt (Instruction *p, int n);
+void printpatt (Instruction *p);
 void printtree (TTree *tree, int ident);
 void printktable (lua_State *L, int idx);
 void printcharset (const byte *st);
-void printcaplist (Capture *cap, Capture *limit);
+void printcaplist (Capture *cap);
 void printinst (const Instruction *op, const Instruction *p);
 
 #else
@@ -26,7 +22,7 @@ void printinst (const Instruction *op, const Instruction *p);
 	luaL_error(L, "function only implemented in debug mode")
 #define printtree(tree,i)  \
 	luaL_error(L, "function only implemented in debug mode")
-#define printpatt(p,n)  \
+#define printpatt(p)  \
 	luaL_error(L, "function only implemented in debug mode")
 
 #endif

3rd/lpeg/lptree.c

@@ -1,7 +1,3 @@
-/*
-** $Id: lptree.c $
-** Copyright 2013, Lua.org & PUC-Rio  (see 'lpeg.html' for license)
-*/
 
 #include <ctype.h>
 #include <limits.h>
@@ -16,16 +12,17 @@
 #include "lpcode.h"
 #include "lpprint.h"
 #include "lptree.h"
+#include "lpcset.h"
 
 
 /* number of siblings for each tree */
 const byte numsiblings[] = {
   0, 0, 0,	/* char, set, any */
-  0, 0,		/* true, false */	
+  0, 0, 0,	/* true, false, utf-range */
-  1,		/* rep */
+  1,		/* acc */
   2, 2,		/* seq, choice */
   1, 1,		/* not, and */
-  0, 0, 2, 1,  /* call, opencall, rule, grammar */
+  0, 0, 2, 1, 1,  /* call, opencall, rule, prerule, grammar */
   1,  /* behind */
   1, 1  /* capture, runtime capture */
 };
@@ -338,7 +335,7 @@ static Pattern *getpattern (lua_State *L, int idx) {
 
 
 static int getsize (lua_State *L, int idx) {
-  return (lua_rawlen(L, idx) - sizeof(Pattern)) / sizeof(TTree) + 1;
+  return (lua_rawlen(L, idx) - offsetof(Pattern, tree)) / sizeof(TTree);
 }
 
 
@@ -351,18 +348,18 @@ static TTree *gettree (lua_State *L, int idx, int *len) {
 
 
 /*
-** create a pattern. Set its uservalue (the 'ktable') equal to its
+** create a pattern followed by a tree with 'len' nodes. Set its
-** metatable. (It could be any empty sequence; the metatable is at
+** uservalue (the 'ktable') equal to its metatable. (It could be any
-** hand here, so we use it.)
+** empty sequence; the metatable is at hand here, so we use it.)
 */
 static TTree *newtree (lua_State *L, int len) {
-  size_t size = (len - 1) * sizeof(TTree) + sizeof(Pattern);
+  size_t size = offsetof(Pattern, tree) + len * sizeof(TTree);
   Pattern *p = (Pattern *)lua_newuserdata(L, size);
   luaL_getmetatable(L, PATTERN_T);
   lua_pushvalue(L, -1);
   lua_setuservalue(L, -3);
   lua_setmetatable(L, -2);
-  p->code = NULL;  p->codesize = 0;
+  p->code = NULL;
   return p->tree;
 }
 
@@ -374,17 +371,44 @@ static TTree *newleaf (lua_State *L, int tag) {
 }
 
 
-static TTree *newcharset (lua_State *L) {
+/*
-  TTree *tree = newtree(L, bytes2slots(CHARSETSIZE) + 1);
+** Create a tree for a charset, optimizing for special cases: empty set,
-  tree->tag = TSet;
+** full set, and singleton set.
-  loopset(i, treebuffer(tree)[i] = 0);
+*/
-  return tree;
+static TTree *newcharset (lua_State *L, byte *cs) {
+  charsetinfo info;
+  Opcode op = charsettype(cs, &info);
+  switch (op) {
+    case IFail: return newleaf(L, TFalse);  /* empty set */
+    case IAny: return newleaf(L, TAny);  /* full set */
+    case IChar: {  /* singleton set */
+      TTree *tree =newleaf(L, TChar);
+      tree->u.n = info.offset;
+      return tree;
+    }
+    default: {  /* regular set */
+      int i;
+      int bsize =  /* tree size in bytes */
+                  (int)offsetof(TTree, u.set.bitmap) + info.size;
+      TTree *tree = newtree(L, bytes2slots(bsize));
+      assert(op == ISet);
+      tree->tag = TSet;
+      tree->u.set.offset = info.offset;
+      tree->u.set.size = info.size;
+      tree->u.set.deflt = info.deflt;
+      for (i = 0; i < info.size; i++) {
+        assert(&treebuffer(tree)[i] < (byte*)tree + bsize);
+        treebuffer(tree)[i] = cs[info.offset + i];
+      }
+      return tree;
+    }
+  }
 }
 
 
 /*
-** add to tree a sequence where first sibling is 'sib' (with size
+** Add to tree a sequence where first sibling is 'sib' (with size
-** 'sibsize'); returns position for second sibling
+** 'sibsize'); return position for second sibling.
 */
 static TTree *seqaux (TTree *tree, TTree *sib, int sibsize) {
   tree->tag = TSeq; tree->u.ps = sibsize + 1;
@@ -553,8 +577,8 @@ static int lp_choice (lua_State *L) {
   TTree *t1 = getpatt(L, 1, NULL);
   TTree *t2 = getpatt(L, 2, NULL);
   if (tocharset(t1, &st1) && tocharset(t2, &st2)) {
-    TTree *t = newcharset(L);
+    loopset(i, st1.cs[i] |= st2.cs[i]);
-    loopset(i, treebuffer(t)[i] = st1.cs[i] | st2.cs[i]);
+    newcharset(L, st1.cs);
   }
   else if (nofail(t1) || t2->tag == TFalse)
     lua_pushvalue(L, 1);  /* true / x => true, x / false => x */
@@ -630,8 +654,8 @@ static int lp_sub (lua_State *L) {
   TTree *t1 = getpatt(L, 1, &s1);
   TTree *t2 = getpatt(L, 2, &s2);
   if (tocharset(t1, &st1) && tocharset(t2, &st2)) {
-    TTree *t = newcharset(L);
+    loopset(i, st1.cs[i] &= ~st2.cs[i]);
-    loopset(i, treebuffer(t)[i] = st1.cs[i] & ~st2.cs[i]);
+    newcharset(L, st1.cs);
   }
   else {
     TTree *tree = newtree(L, 2 + s1 + s2);
@@ -649,11 +673,13 @@ static int lp_sub (lua_State *L) {
 static int lp_set (lua_State *L) {
   size_t l;
   const char *s = luaL_checklstring(L, 1, &l);
-  TTree *tree = newcharset(L);
+  byte buff[CHARSETSIZE];
+  clearset(buff);
   while (l--) {
-    setchar(treebuffer(tree), (byte)(*s));
+    setchar(buff, (byte)(*s));
     s++;
   }
+  newcharset(L, buff);
   return 1;
 }
 
@@ -661,14 +687,68 @@ static int lp_set (lua_State *L) {
 static int lp_range (lua_State *L) {
   int arg;
   int top = lua_gettop(L);
-  TTree *tree = newcharset(L);
+  byte buff[CHARSETSIZE];
+  clearset(buff);
   for (arg = 1; arg <= top; arg++) {
     int c;
     size_t l;
     const char *r = luaL_checklstring(L, arg, &l);
     luaL_argcheck(L, l == 2, arg, "range must have two characters");
     for (c = (byte)r[0]; c <= (byte)r[1]; c++)
-      setchar(treebuffer(tree), c);
+      setchar(buff, c);
+  }
+  newcharset(L, buff);
+  return 1;
+}
+
+
+/*
+** Fills a tree node with basic information about the UTF-8 code point
+** 'cpu': its value in 'n', its length in 'cap', and its first byte in
+** 'key'
+*/
+static void codeutftree (lua_State *L, TTree *t, lua_Unsigned cpu, int arg) {
+  int len, fb, cp;
+  cp = (int)cpu;
+  if (cp <= 0x7f) {  /* one byte? */
+    len = 1;
+    fb = cp;
+  } else if (cp <= 0x7ff) {
+    len = 2;
+    fb = 0xC0 | (cp >> 6);
+  } else if (cp <= 0xffff) {
+    len = 3;
+    fb = 0xE0 | (cp >> 12);
+  }
+  else {
+    luaL_argcheck(L, cpu <= 0x10ffffu, arg, "invalid code point");
+    len = 4;
+    fb = 0xF0 | (cp >> 18);
+  }
+  t->u.n = cp;
+  t->cap = len;
+  t->key = fb;
+}
+
+
+static int lp_utfr (lua_State *L) {
+  lua_Unsigned from = (lua_Unsigned)luaL_checkinteger(L, 1);
+  lua_Unsigned to = (lua_Unsigned)luaL_checkinteger(L, 2);
+  luaL_argcheck(L, from <= to, 2, "empty range");
+  if (to <= 0x7f) {  /* ascii range? */
+    uint f;
+    byte buff[CHARSETSIZE];  /* code it as a regular charset */
+    clearset(buff);
+    for (f = (int)from; f <= to; f++)
+      setchar(buff, f);
+    newcharset(L, buff);
+  }
+  else {  /* multi-byte utf-8 range */
+    TTree *tree = newtree(L, 2);
+    tree->tag = TUTFR;
+    codeutftree(L, tree, from, 1);
+    sib1(tree)->tag = TXInfo;
+    codeutftree(L, sib1(tree), to, 2);
   }
   return 1;
 }
@@ -763,11 +843,18 @@ static int lp_divcapture (lua_State *L) {
       tree->key = n;
       return 1;
     }
-    default: return luaL_argerror(L, 2, "invalid replacement value");
+    default:
+      return luaL_error(L, "unexpected %s as 2nd operand to LPeg '/'",
+                           luaL_typename(L, 2));
   }
 }
 
 
+static int lp_acccapture (lua_State *L) {
+  return capture_aux(L, Cacc, 2);
+}
+
+
 static int lp_substcapture (lua_State *L) {
   return capture_aux(L, Csubst, 0);
 }
@@ -906,7 +993,7 @@ static int collectrules (lua_State *L, int arg, int *totalsize) {
   int size;  /* accumulator for total size */
   lua_newtable(L);  /* create position table */
   getfirstrule(L, arg, postab);
-  size = 2 + getsize(L, postab + 2);  /* TGrammar + TRule + rule */
+  size = 3 + getsize(L, postab + 2);  /* TGrammar + TRule + TXInfo + rule */
   lua_pushnil(L);  /* prepare to traverse grammar table */
   while (lua_next(L, arg) != 0) {
     if (lua_tonumber(L, -2) == 1 ||
@@ -920,11 +1007,11 @@ static int collectrules (lua_State *L, int arg, int *totalsize) {
     lua_pushvalue(L, -2);  /* push key (to insert into position table) */
     lua_pushinteger(L, size);
     lua_settable(L, postab);
-    size += 1 + getsize(L, -1);  /* update size */
+    size += 2 + getsize(L, -1);  /* add 'TRule + TXInfo + rule' to size */
     lua_pushvalue(L, -2);  /* push key (for next lua_next) */
     n++;
   }
-  *totalsize = size + 1;  /* TTrue to finish list of rules */
+  *totalsize = size + 1;  /* space for 'TTrue' finishing list of rules */
   return n;
 }
 
@@ -936,11 +1023,13 @@ static void buildgrammar (lua_State *L, TTree *grammar, int frule, int n) {
     int ridx = frule + 2*i + 1;  /* index of i-th rule */
     int rulesize;
     TTree *rn = gettree(L, ridx, &rulesize);
+    TTree *pr = sib1(nd);  /* points to rule's prerule */
     nd->tag = TRule;
     nd->key = 0;  /* will be fixed when rule is used */
-    nd->cap = i;  /* rule number */
+    pr->tag = TXInfo;
-    nd->u.ps = rulesize + 1;  /* point to next rule */
+    pr->u.n = i;  /* rule number */
-    memcpy(sib1(nd), rn, rulesize * sizeof(TTree));  /* copy rule */
+    nd->u.ps = rulesize + 2;  /* point to next rule */
+    memcpy(sib1(pr), rn, rulesize * sizeof(TTree));  /* copy rule */
     mergektable(L, ridx, sib1(nd));  /* merge its ktable into new one */
     nd = sib2(nd);  /* move to next rule */
   }
@@ -976,7 +1065,7 @@ static int checkloops (TTree *tree) {
 ** twice in 'passed', there is path from it back to itself without
 ** advancing the subject.
 */
-static int verifyerror (lua_State *L, int *passed, int npassed) {
+static int verifyerror (lua_State *L, unsigned short *passed, int npassed) {
   int i, j;
   for (i = npassed - 1; i >= 0; i--) {  /* search for a repetition */
     for (j = i - 1; j >= 0; j--) {
@@ -1001,12 +1090,12 @@ static int verifyerror (lua_State *L, int *passed, int npassed) {
 ** counts the elements in 'passed'.
 ** Assume ktable at the top of the stack.
 */
-static int verifyrule (lua_State *L, TTree *tree, int *passed, int npassed,
+static int verifyrule (lua_State *L, TTree *tree, unsigned short *passed,
-                       int nb) {
+                                     int npassed, int nb) {
  tailcall:
   switch (tree->tag) {
     case TChar: case TSet: case TAny:
-    case TFalse:
+    case TFalse: case TUTFR:
       return nb;  /* cannot pass from here */
     case TTrue:
     case TBehind:  /* look-behind cannot have calls */
@@ -1014,7 +1103,7 @@ static int verifyrule (lua_State *L, TTree *tree, int *passed, int npassed,
     case TNot: case TAnd: case TRep:
       /* return verifyrule(L, sib1(tree), passed, npassed, 1); */
       tree = sib1(tree); nb = 1; goto tailcall;
-    case TCapture: case TRunTime:
+    case TCapture: case TRunTime: case TXInfo:
       /* return verifyrule(L, sib1(tree), passed, npassed, nb); */
       tree = sib1(tree); goto tailcall;
     case TCall:
@@ -1030,10 +1119,10 @@ static int verifyrule (lua_State *L, TTree *tree, int *passed, int npassed,
       /* return verifyrule(L, sib2(tree), passed, npassed, nb); */
       tree = sib2(tree); goto tailcall;
     case TRule:
-      if (npassed >= MAXRULES)
+      if (npassed >= MAXRULES)  /* too many steps? */
-        return verifyerror(L, passed, npassed);
+        return verifyerror(L, passed, npassed);  /* error */
       else {
-        passed[npassed++] = tree->key;
+        passed[npassed++] = tree->key;  /* add rule to path */
         /* return verifyrule(L, sib1(tree), passed, npassed); */
         tree = sib1(tree); goto tailcall;
       }
@@ -1045,7 +1134,7 @@ static int verifyrule (lua_State *L, TTree *tree, int *passed, int npassed,
 
 
 static void verifygrammar (lua_State *L, TTree *grammar) {
-  int passed[MAXRULES];
+  unsigned short passed[MAXRULES];
   TTree *rule;
   /* check left-recursive rules */
   for (rule = sib1(grammar); rule->tag == TRule; rule = sib2(rule)) {
@@ -1105,7 +1194,7 @@ static Instruction *prepcompile (lua_State *L, Pattern *p, int idx) {
   lua_getuservalue(L, idx);  /* push 'ktable' (may be used by 'finalfix') */
   finalfix(L, 0, NULL, p->tree);
   lua_pop(L, 1);  /* remove 'ktable' */
-  return compile(L, p);
+  return compile(L, p, getsize(L, idx));
 }
 
 
@@ -1128,7 +1217,7 @@ static int lp_printcode (lua_State *L) {
   printktable(L, 1);
   if (p->code == NULL)  /* not compiled yet? */
     prepcompile(L, p, 1);
-  printpatt(p->code, p->codesize);
+  printpatt(p->code);
   return 0;
 }
 
@@ -1164,9 +1253,10 @@ static int lp_match (lua_State *L) {
   const char *s = luaL_checklstring(L, SUBJIDX, &l);
   size_t i = initposition(L, l);
   int ptop = lua_gettop(L);
+  luaL_argcheck(L, l < MAXINDT, SUBJIDX, "subject too long");
   lua_pushnil(L);  /* initialize subscache */
   lua_pushlightuserdata(L, capture);  /* initialize caplistidx */
-  lua_getuservalue(L, 1);  /* initialize penvidx */
+  lua_getuservalue(L, 1);  /* initialize ktableidx */
   r = match(L, s, s + i, s + l, code, capture, ptop);
   if (r == NULL) {
     lua_pushnil(L);
@@ -1195,12 +1285,6 @@ static int lp_setmax (lua_State *L) {
 }
 
 
-static int lp_version (lua_State *L) {
-  lua_pushstring(L, VERSION);
-  return 1;
-}
-
-
 static int lp_type (lua_State *L) {
   if (testpattern(L, 1))
     lua_pushliteral(L, "pattern");
@@ -1212,16 +1296,22 @@ static int lp_type (lua_State *L) {
 
 int lp_gc (lua_State *L) {
   Pattern *p = getpattern(L, 1);
-  realloccode(L, p, 0);  /* delete code block */
+  freecode(L, p);  /* delete code block */
   return 0;
 }
 
 
+/*
+** Create a charset representing a category of characters, given by
+** the predicate 'catf'.
+*/
 static void createcat (lua_State *L, const char *catname, int (catf) (int)) {
-  TTree *t = newcharset(L);
+  int c;
-  int i;
+  byte buff[CHARSETSIZE];
-  for (i = 0; i <= UCHAR_MAX; i++)
+  clearset(buff);
-    if (catf(i)) setchar(treebuffer(t), i);
+  for (c = 0; c <= UCHAR_MAX; c++)
+    if (catf(c)) setchar(buff, c);
+  newcharset(L, buff);
   lua_setfield(L, -2, catname);
 }
 
@@ -1269,8 +1359,9 @@ static struct luaL_Reg pattreg[] = {
   {"P", lp_P},
   {"S", lp_set},
   {"R", lp_range},
+  {"utfR", lp_utfr},
   {"locale", lp_locale},
-  {"version", lp_version},
+  {"version", NULL},
   {"setmaxstack", lp_setmax},
   {"type", lp_type},
   {NULL, NULL}
@@ -1284,6 +1375,7 @@ static struct luaL_Reg metareg[] = {
   {"__gc", lp_gc},
   {"__len", lp_and},
   {"__div", lp_divcapture},
+  {"__mod", lp_acccapture},
   {"__unm", lp_not},
   {"__sub", lp_sub},
   {NULL, NULL}
@@ -1299,6 +1391,8 @@ int luaopen_lpeg (lua_State *L) {
   luaL_newlib(L, pattreg);
   lua_pushvalue(L, -1);
   lua_setfield(L, -3, "__index");
+  lua_pushliteral(L, "LPeg " VERSION);
+  lua_setfield(L, -2, "version");
   return 1;
 }
 

3rd/lpeg/lptree.h

@@ -1,12 +1,9 @@
-/*  
-** $Id: lptree.h $
-*/
 
 #if !defined(lptree_h)
 #define lptree_h
 
 
-#include "lptypes.h" 
+#include "lptypes.h"
 
 
 /*
@@ -14,10 +11,13 @@
 */
 typedef enum TTag {
   TChar = 0,  /* 'n' = char */
-  TSet,  /* the set is stored in next CHARSETSIZE bytes */
+  TSet,  /* the set is encoded in 'u.set' and the next 'u.set.size' bytes */
   TAny,
   TTrue,
   TFalse,
+  TUTFR,  /* range of UTF-8 codepoints; 'n' has initial codepoint;
+             'cap' has length; 'key' has first byte;
+             extra info is similar for end codepoint */
   TRep,  /* 'sib1'* */
   TSeq,  /* 'sib1' 'sib2' */
   TChoice,  /* 'sib1' / 'sib2' */
@@ -26,8 +26,9 @@ typedef enum TTag {
   TCall,  /* ktable[key] is rule's key; 'sib2' is rule being called */
   TOpenCall,  /* ktable[key] is rule's key */
   TRule,  /* ktable[key] is rule's key (but key == 0 for unused rules);
-             'sib1' is rule's pattern;
+             'sib1' is rule's pattern pre-rule; 'sib2' is next rule;
-             'sib2' is next rule; 'cap' is rule's sequential number */
+             extra info 'n' is rule's sequential number */
+  TXInfo,  /* extra info */
   TGrammar,  /* 'sib1' is initial (and first) rule */
   TBehind,  /* 'sib1' is pattern, 'n' is how much to go back */
   TCapture,  /* captures: 'cap' is kind of capture (enum 'CapKind');
@@ -51,17 +52,26 @@ typedef struct TTree {
   union {
     int ps;  /* occasional second child */
     int n;  /* occasional counter */
+    struct {
+      byte offset;  /* compact set offset (in bytes) */
+      byte size;  /* compact set size (in bytes) */
+      byte deflt;  /* default value */
+      byte bitmap[1];  /* bitmap (open array) */
+    } set;  /* for compact sets */
   } u;
 } TTree;
 
 
+/* access to charset */
+#define treebuffer(t)      ((t)->u.set.bitmap)
+
+
 /*
 ** A complete pattern has its tree plus, if already compiled,
 ** its corresponding code
 */
 typedef struct Pattern {
   union Instruction *code;
-  int codesize;
   TTree tree[1];
 } Pattern;
 

3rd/lpeg/lptypes.h

@@ -1,7 +1,6 @@
 /*
-** $Id: lptypes.h $
 ** LPeg - PEG pattern matching for Lua
-** Copyright 2007-2019, Lua.org & PUC-Rio  (see 'lpeg.html' for license)
+** Copyright 2007-2023, Lua.org & PUC-Rio  (see 'lpeg.html' for license)
 ** written by Roberto Ierusalimschy
 */
 
@@ -11,11 +10,12 @@
 
 #include <assert.h>
 #include <limits.h>
+#include <string.h>
 
 #include "lua.h"
 
 
-#define VERSION         "1.0.2"
+#define VERSION         "1.1.0"
 
 
 #define PATTERN_T	"lpeg-pattern"
@@ -37,6 +37,8 @@
 #define luaL_setfuncs(L,f,n)	luaL_register(L,NULL,f)
 #define luaL_newlib(L,f)	luaL_register(L,"lpeg",f)
 
+typedef size_t lua_Unsigned;
+
 #endif
 
 
@@ -51,9 +53,9 @@
 #endif
 
 
-/* maximum number of rules in a grammar (limited by 'unsigned char') */
+/* maximum number of rules in a grammar (limited by 'unsigned short') */
 #if !defined(MAXRULES)
-#define MAXRULES        250
+#define MAXRULES        1000
 #endif
 
 
@@ -81,6 +83,8 @@
 
 typedef unsigned char byte;
 
+typedef unsigned int uint;
+
 
 #define BITSPERCHAR		8
 
@@ -96,11 +100,11 @@ typedef struct Charset {
 
 #define loopset(v,b)    { int v; for (v = 0; v < CHARSETSIZE; v++) {b;} }
 
-/* access to charset */
+#define fillset(s,c)	memset(s,c,CHARSETSIZE)
-#define treebuffer(t)      ((byte *)((t) + 1))
+#define clearset(s)	fillset(s,0)
 
 /* number of slots needed for 'n' bytes */
-#define bytes2slots(n)  (((n) - 1) / sizeof(TTree) + 1)
+#define bytes2slots(n)  (((n) - 1u) / (uint)sizeof(TTree) + 1u)
 
 /* set 'b' bit in charset 'cs' */
 #define setchar(cs,b)   ((cs)[(b) >> 3] |= (1 << ((b) & 7)))
@@ -110,8 +114,8 @@ typedef struct Charset {
 ** in capture instructions, 'kind' of capture and its offset are
 ** packed in field 'aux', 4 bits for each
 */
-#define getkind(op)		((op)->i.aux & 0xF)
+#define getkind(op)		((op)->i.aux1 & 0xF)
-#define getoff(op)		(((op)->i.aux >> 4) & 0xF)
+#define getoff(op)		(((op)->i.aux1 >> 4) & 0xF)
 #define joinkindoff(k,o)	((k) | ((o) << 4))
 
 #define MAXOFF		0xF
@@ -126,19 +130,20 @@ typedef struct Charset {
 #define MAXPATTSIZE	(SHRT_MAX - 10)
 
 
-/* size (in elements) for an instruction plus extra l bytes */
+/* size (in instructions) for l bytes (l > 0) */
-#define instsize(l)  (((l) + sizeof(Instruction) - 1)/sizeof(Instruction) + 1)
+#define instsize(l) ((int)(((l) + (uint)sizeof(Instruction) - 1u) \
+			/ (uint)sizeof(Instruction)))
 
 
 /* size (in elements) for a ISet instruction */
-#define CHARSETINSTSIZE		instsize(CHARSETSIZE)
+#define CHARSETINSTSIZE		(1 + instsize(CHARSETSIZE))
 
 /* size (in elements) for a IFunc instruction */
 #define funcinstsize(p)		((p)->i.aux + 2)
 
 
 
-#define testchar(st,c)	(((int)(st)[((c) >> 3)] & (1 << ((c) & 7))))
+#define testchar(st,c)	((((uint)(st)[((c) >> 3)]) >> ((c) & 7)) & 1)
 
 
 #endif

3rd/lpeg/lpvm.c

@@ -1,7 +1,3 @@
-/*
-** $Id: lpvm.c $
-** Copyright 2007, Lua.org & PUC-Rio  (see 'lpeg.html' for license)
-*/
 
 #include <limits.h>
 #include <string.h>
@@ -24,7 +20,46 @@
 
 #define getoffset(p)	(((p) + 1)->offset)
 
-static const Instruction giveup = {{IGiveup, 0, 0}};
+static const Instruction giveup = {{IGiveup, 0, {0}}};
+
+
+int charinset (const Instruction *i, const byte *buff, uint c) {
+  c -= i->i.aux2.set.offset;
+  if (c >= ((uint)i->i.aux2.set.size  /* size in instructions... */
+           * (uint)sizeof(Instruction)  /* in bytes... */
+           * 8u))  /* in bits */
+    return i->i.aux1;  /* out of range; return default value */
+  return testchar(buff, c);
+}
+
+
+/*
+** Decode one UTF-8 sequence, returning NULL if byte sequence is invalid.
+*/
+static const char *utf8_decode (const char *o, int *val) {
+  static const uint limits[] = {0xFF, 0x7F, 0x7FF, 0xFFFFu};
+  const unsigned char *s = (const unsigned char *)o;
+  uint c = s[0];  /* first byte */
+  uint res = 0;  /* final result */
+  if (c < 0x80)  /* ascii? */
+    res = c;
+  else {
+    int count = 0;  /* to count number of continuation bytes */
+    while (c & 0x40) {  /* still have continuation bytes? */
+      int cc = s[++count];  /* read next byte */
+      if ((cc & 0xC0) != 0x80)  /* not a continuation byte? */
+        return NULL;  /* invalid byte sequence */
+      res = (res << 6) | (cc & 0x3F);  /* add lower 6 bits from cont. byte */
+      c <<= 1;  /* to test next bit */
+    }
+    res |= (c & 0x7F) << (count * 5);  /* add first byte */
+    if (count > 3 || res > 0x10FFFFu || res <= limits[count])
+      return NULL;  /* invalid byte sequence */
+    s += count;  /* skip continuation bytes read */
+  }
+  *val = res;
+  return (const char *)s + 1;  /* +1 to include first byte */
+}
 
 
 /*
@@ -51,16 +86,25 @@ typedef struct Stack {
 ** array, it is simpler to ensure the array always has at least one free
 ** slot upfront and check its size later.)
 */
+
+/* new size in number of elements cannot overflow integers, and new
+   size in bytes cannot overflow size_t. */
+#define MAXNEWSIZE  \
+    (((size_t)INT_MAX) <= (~(size_t)0 / sizeof(Capture)) ?  \
+     ((size_t)INT_MAX) : (~(size_t)0 / sizeof(Capture)))
+
 static Capture *growcap (lua_State *L, Capture *capture, int *capsize,
                                        int captop, int n, int ptop) {
   if (*capsize - captop > n)
     return capture;  /* no need to grow array */
   else {  /* must grow */
     Capture *newc;
-    int newsize = captop + n + 1;  /* minimum size needed */
+    uint newsize = captop + n + 1;  /* minimum size needed */
-    if (newsize < INT_MAX/((int)sizeof(Capture) * 2))
+    if (newsize < (MAXNEWSIZE / 3) * 2)
-      newsize *= 2;  /* twice that size, if not too big */
+      newsize += newsize / 2;  /* 1.5 that size, if not too big */
-    else if (newsize >= INT_MAX/((int)sizeof(Capture)))
+    else if (newsize < (MAXNEWSIZE / 9) * 8)
+      newsize += newsize / 8;  /* else, try 9/8 that size */
+    else
       luaL_error(L, "too many captures");
     newc = (Capture *)lua_newuserdata(L, newsize * sizeof(Capture));
     memcpy(newc, capture, captop * sizeof(Capture));
@@ -125,7 +169,7 @@ static int resdyncaptures (lua_State *L, int fr, int curr, int limit) {
 ** value, 'n' is the number of values (at least 1). The open group
 ** capture is already in 'capture', before the place for the new entries.
 */
-static void adddyncaptures (const char *s, Capture *capture, int n, int fd) {
+static void adddyncaptures (Index_t index, Capture *capture, int n, int fd) {
   int i;
   assert(capture[-1].kind == Cgroup && capture[-1].siz == 0);
   capture[-1].idx = 0;  /* make group capture an anonymous group */
@@ -133,11 +177,11 @@ static void adddyncaptures (const char *s, Capture *capture, int n, int fd) {
     capture[i].kind = Cruntime;
     capture[i].siz = 1;  /* mark it as closed */
     capture[i].idx = fd + i;  /* stack index of capture value */
-    capture[i].s = s;
+    capture[i].index = index;
   }
   capture[n].kind = Cclose;  /* close group */
   capture[n].siz = 1;
-  capture[n].s = s;
+  capture[n].index = index;
 }
 
 
@@ -154,6 +198,32 @@ static int removedyncap (lua_State *L, Capture *capture,
 }
 
 
+/*
+** Find the corresponding 'open' capture before 'cap', when that capture
+** can become a full capture. If a full capture c1 is followed by an
+** empty capture c2, there is no way to know whether c2 is inside
+** c1. So, full captures can enclose only captures that start *before*
+** its end.
+*/
+static Capture *findopen (Capture *cap, Index_t currindex) {
+  int i;
+  cap--;  /* check last capture */
+  /* Must it be inside current one, but starts where current one ends? */
+  if (!isopencap(cap) && cap->index == currindex)
+    return NULL;  /* current one cannot be a full capture */
+  /* else, look for an 'open' capture */
+  for (i = 0; i < MAXLOP; i++, cap--) {
+    if (currindex - cap->index >= UCHAR_MAX)
+      return NULL;  /* capture too long for a full capture */
+    else if (isopencap(cap))  /* open capture? */
+      return cap;  /* that's the one to be closed */
+    else if (cap->kind == Cclose)
+      return NULL;  /* a full capture should not nest a non-full one */
+  }
+  return NULL;  /* not found within allowed search limit */
+}
+
+
 /*
 ** Opcode interpreter
 */
@@ -181,7 +251,7 @@ const char *match (lua_State *L, const char *o, const char *s, const char *e,
       case IEnd: {
         assert(stack == getstackbase(L, ptop) + 1);
         capture[captop].kind = Cclose;
-        capture[captop].s = NULL;
+        capture[captop].index = MAXINDT;
         return s;
       }
       case IGiveup: {
@@ -198,47 +268,58 @@ const char *match (lua_State *L, const char *o, const char *s, const char *e,
         else goto fail;
         continue;
       }
+      case IUTFR: {
+        int codepoint;
+        if (s >= e)
+          goto fail;
+        s = utf8_decode (s, &codepoint);
+        if (s && p[1].offset <= codepoint && codepoint <= utf_to(p))
+          p += 2;
+        else
+          goto fail;
+        continue;
+      }
       case ITestAny: {
         if (s < e) p += 2;
         else p += getoffset(p);
         continue;
       }
       case IChar: {
-        if ((byte)*s == p->i.aux && s < e) { p++; s++; }
+        if ((byte)*s == p->i.aux1 && s < e) { p++; s++; }
         else goto fail;
         continue;
       }
       case ITestChar: {
-        if ((byte)*s == p->i.aux && s < e) p += 2;
+        if ((byte)*s == p->i.aux1 && s < e) p += 2;
         else p += getoffset(p);
         continue;
       }
       case ISet: {
-        int c = (byte)*s;
+        uint c = (byte)*s;
-        if (testchar((p+1)->buff, c) && s < e)
+        if (charinset(p, (p+1)->buff, c) && s < e)
-          { p += CHARSETINSTSIZE; s++; }
+          { p += 1 + p->i.aux2.set.size; s++; }
         else goto fail;
         continue;
       }
       case ITestSet: {
-        int c = (byte)*s;
+        uint c = (byte)*s;
-        if (testchar((p + 2)->buff, c) && s < e)
+        if (charinset(p, (p + 2)->buff, c) && s < e)
-          p += 1 + CHARSETINSTSIZE;
+          p += 2 + p->i.aux2.set.size;
         else p += getoffset(p);
         continue;
       }
       case IBehind: {
-        int n = p->i.aux;
+        int n = p->i.aux1;
         if (n > s - o) goto fail;
         s -= n; p++;
         continue;
       }
       case ISpan: {
         for (; s < e; s++) {
-          int c = (byte)*s;
+          uint c = (byte)*s;
-          if (!testchar((p+1)->buff, c)) break;
+          if (!charinset(p, (p+1)->buff, c)) break;
         }
-        p += CHARSETINSTSIZE;
+        p += 1 + p->i.aux2.set.size;
         continue;
       }
       case IJmp: {
@@ -280,6 +361,8 @@ const char *match (lua_State *L, const char *o, const char *s, const char *e,
       case IBackCommit: {
         assert(stack > getstackbase(L, ptop) && (stack - 1)->s != NULL);
         s = (--stack)->s;
+        if (ndyncap > 0)  /* are there matchtime captures? */
+          ndyncap -= removedyncap(L, capture, stack->caplevel, captop);
         captop = stack->caplevel;
         p += getoffset(p);
         continue;
@@ -287,7 +370,7 @@ const char *match (lua_State *L, const char *o, const char *s, const char *e,
       case IFailTwice:
         assert(stack > getstackbase(L, ptop));
         stack--;
-        /* go through */
+        /* FALLTHROUGH */
       case IFail:
       fail: { /* pattern failed: try to backtrack */
         do {  /* remove pending calls */
@@ -326,38 +409,36 @@ const char *match (lua_State *L, const char *o, const char *s, const char *e,
             luaL_error(L, "too many results in match-time capture");
           /* add new captures + close group to 'capture' list */
           capture = growcap(L, capture, &capsize, captop, n + 1, ptop);
-          adddyncaptures(s, capture + captop, n, fr);
+          adddyncaptures(s - o, capture + captop, n, fr);
           captop += n + 1;  /* new captures + close group */
         }
         p++;
         continue;
       }
       case ICloseCapture: {
-        const char *s1 = s;
+        Capture *open = findopen(capture + captop, s - o);
         assert(captop > 0);
-        /* if possible, turn capture into a full capture */
+        if (open) {  /* if possible, turn capture into a full capture */
-        if (capture[captop - 1].siz == 0 &&
+          open->siz = (s - o) - open->index + 1;
-            s1 - capture[captop - 1].s < UCHAR_MAX) {
-          capture[captop - 1].siz = s1 - capture[captop - 1].s + 1;
           p++;
           continue;
         }
-        else {
+        else {  /* must create a close capture */
           capture[captop].siz = 1;  /* mark entry as closed */
-          capture[captop].s = s;
+          capture[captop].index = s - o;
           goto pushcapture;
         }
       }
       case IOpenCapture:
         capture[captop].siz = 0;  /* mark entry as open */
-        capture[captop].s = s;
+        capture[captop].index = s - o;
         goto pushcapture;
       case IFullCapture:
         capture[captop].siz = getoff(p) + 1;  /* save capture size */
-        capture[captop].s = s - getoff(p);
+        capture[captop].index = s - o - getoff(p);
         /* goto pushcapture; */
       pushcapture: {
-        capture[captop].idx = p->i.key;
+        capture[captop].idx = p->i.aux2.key;
         capture[captop].kind = getkind(p);
         captop++;
         capture = growcap(L, capture, &capsize, captop, 0, ptop);

3rd/lpeg/lpvm.h

@@ -1,6 +1,3 @@
-/*
-** $Id: lpvm.h $
-*/
 
 #if !defined(lpvm_h)
 #define lpvm_h
@@ -8,16 +5,24 @@
 #include "lpcap.h"
 
 
+/*
+** About Character sets in instructions: a set is a bit map with an
+** initial offset, in bits, and a size, in number of instructions.
+** aux1 has the default value for the bits outsize that range.
+*/
+
+
 /* Virtual Machine's instructions */
 typedef enum Opcode {
   IAny, /* if no char, fail */
-  IChar,  /* if char != aux, fail */
+  IChar,  /* if char != aux1, fail */
-  ISet,  /* if char not in buff, fail */
+  ISet,  /* if char not in set, fail */
   ITestAny,  /* in no char, jump to 'offset' */
-  ITestChar,  /* if char != aux, jump to 'offset' */
+  ITestChar,  /* if char != aux1, jump to 'offset' */
-  ITestSet,  /* if char not in buff, jump to 'offset' */
+  ITestSet,  /* if char not in set, jump to 'offset' */
-  ISpan,  /* read a span of chars in buff */
+  ISpan,  /* read a span of chars in set */
-  IBehind,  /* walk back 'aux' characters (fail if not possible) */
+  IUTFR,  /* if codepoint not in range [offset, utf_to], fail */
+  IBehind,  /* walk back 'aux1' characters (fail if not possible) */
   IRet,  /* return from a rule */
   IEnd,  /* end of pattern */
   IChoice,  /* stack a choice; next fail will jump to 'offset' */
@@ -26,30 +31,46 @@ typedef enum Opcode {
   IOpenCall,  /* call rule number 'key' (must be closed to a ICall) */
   ICommit,  /* pop choice and jump to 'offset' */
   IPartialCommit,  /* update top choice to current position and jump */
-  IBackCommit,  /* "fails" but jump to its own 'offset' */
+  IBackCommit,  /* backtrack like "fail" but jump to its own 'offset' */
   IFailTwice,  /* pop one choice and then fail */
   IFail,  /* go back to saved state on choice and jump to saved offset */
   IGiveup,  /* internal use */
   IFullCapture,  /* complete capture of last 'off' chars */
   IOpenCapture,  /* start a capture */
   ICloseCapture,
-  ICloseRunTime
+  ICloseRunTime,
+  IEmpty  /* to fill empty slots left by optimizations */
 } Opcode;
 
 
-
+/*
+** All array of instructions has a 'codesize' as its first element
+** and is referred by a pointer to its second element, which is the
+** first actual opcode.
+*/
 typedef union Instruction {
   struct Inst {
     byte code;
-    byte aux;
+    byte aux1;
-    short key;
+    union {
+      short key;
+      struct {
+        byte offset;
+        byte size;
+      } set;
+    } aux2;
   } i;
   int offset;
+  uint codesize;
   byte buff[1];
 } Instruction;
 
 
-void printpatt (Instruction *p, int n);
+/* extract 24-bit value from an instruction */
+#define utf_to(inst)	(((inst)->i.aux2.key << 8) | (inst)->i.aux1)
+
+
+int charinset (const Instruction *i, const byte *buff, uint c);
 const char *match (lua_State *L, const char *o, const char *s, const char *e,
                    Instruction *op, Capture *capture, int ptop);
 

3rd/lpeg/makefile

@@ -2,7 +2,7 @@ LIBNAME = lpeg
 LUADIR = ../lua/
 
 COPT = -O2 -DNDEBUG
-# COPT = -g
+# COPT = -O0 -DLPEG_DEBUG -g
 
 CWARNS = -Wall -Wextra -pedantic \
 	-Waggregate-return \
@@ -11,21 +11,24 @@ CWARNS = -Wall -Wextra -pedantic \
 	-Wdisabled-optimization \
 	-Wpointer-arith \
 	-Wshadow \
+	-Wredundant-decls \
 	-Wsign-compare \
 	-Wundef \
 	-Wwrite-strings \
 	-Wbad-function-cast \
 	-Wdeclaration-after-statement \
 	-Wmissing-prototypes \
+	-Wmissing-declarations \
 	-Wnested-externs \
 	-Wstrict-prototypes \
+	-Wc++-compat \
 # -Wunreachable-code \
 
 
 CFLAGS = $(CWARNS) $(COPT) -std=c99 -I$(LUADIR) -fPIC
 CC = gcc
 
-FILES = lpvm.o lpcap.o lptree.o lpcode.o lpprint.o
+FILES = lpvm.o lpcap.o lptree.o lpcode.o lpprint.o lpcset.o
 
 # For Linux
 linux:
@@ -48,8 +51,9 @@ clean:
 
 
 lpcap.o: lpcap.c lpcap.h lptypes.h
-lpcode.o: lpcode.c lptypes.h lpcode.h lptree.h lpvm.h lpcap.h
+lpcode.o: lpcode.c lptypes.h lpcode.h lptree.h lpvm.h lpcap.h lpcset.h
-lpprint.o: lpprint.c lptypes.h lpprint.h lptree.h lpvm.h lpcap.h
+lpcset.o: lpcset.c lptypes.h lpcset.h lpcode.h lptree.h lpvm.h lpcap.h
-lptree.o: lptree.c lptypes.h lpcap.h lpcode.h lptree.h lpvm.h lpprint.h
+lpprint.o: lpprint.c lptypes.h lpprint.h lptree.h lpvm.h lpcap.h lpcode.h
+lptree.o: lptree.c lptypes.h lpcap.h lpcode.h lptree.h lpvm.h lpprint.h \
+ lpcset.h
 lpvm.o: lpvm.c lpcap.h lptypes.h lpvm.h lpprint.h lptree.h
-

3rd/lpeg/re.html

@@ -1,22 +1,21 @@
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
-   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+   "//www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 <html>
 <head>
     <title>LPeg.re - Regex syntax for LPEG</title>
     <link rel="stylesheet"
-          href="http://www.inf.puc-rio.br/~roberto/lpeg/doc.css"
+          href="//www.inf.puc-rio.br/~roberto/lpeg/doc.css"
           type="text/css"/>
 	<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
 </head>
 <body>
 
-<!-- $Id: re.html $ -->
 
 <div id="container">
 	
 <div id="product">
   <div id="product_logo">
-    <a href="http://www.inf.puc-rio.br/~roberto/lpeg/">
+    <a href="//www.inf.puc-rio.br/~roberto/lpeg/">
     <img alt="LPeg logo" src="lpeg-128.gif"/>
     </a>
   </div>
@@ -62,6 +61,20 @@ Constructions are listed in order of decreasing precedence.
 <table border="1">
 <tbody><tr><td><b>Syntax</b></td><td><b>Description</b></td></tr>
 <tr><td><code>( p )</code></td> <td>grouping</td></tr>
+<tr><td><code>&amp; p</code></td> <td>and predicate</td></tr>
+<tr><td><code>! p</code></td> <td>not predicate</td></tr>
+<tr><td><code>p1 p2</code></td> <td>concatenation</td></tr>
+<tr><td><code>p1 / p2</code></td> <td>ordered choice</td></tr>
+<tr><td><code>p ?</code></td> <td>optional match</td></tr>
+<tr><td><code>p *</code></td> <td>zero or more repetitions</td></tr>
+<tr><td><code>p +</code></td> <td>one or more repetitions</td></tr>
+<tr><td><code>p^num</code></td>
+      <td>exactly <code>num</code> repetitions</td></tr>
+<tr><td><code>p^+num</code></td>
+      <td>at least <code>num</code> repetitions</td></tr>
+<tr><td><code>p^-num</code></td>
+      <td>at most <code>num</code> repetitions</td></tr>
+<tr><td>(<code>name &lt;- p</code>)<sup>+</sup></td> <td>grammar</td></tr>
 <tr><td><code>'string'</code></td> <td>literal string</td></tr>
 <tr><td><code>"string"</code></td> <td>literal string</td></tr>
 <tr><td><code>[class]</code></td> <td>character class</td></tr>
@@ -70,22 +83,15 @@ Constructions are listed in order of decreasing precedence.
   <td>pattern <code>defs[name]</code> or a pre-defined pattern</td></tr>
 <tr><td><code>name</code></td><td>non terminal</td></tr>
 <tr><td><code>&lt;name&gt;</code></td><td>non terminal</td></tr>
+
 <tr><td><code>{}</code></td> <td>position capture</td></tr>
 <tr><td><code>{ p }</code></td> <td>simple capture</td></tr>
 <tr><td><code>{: p :}</code></td> <td>anonymous group capture</td></tr>
 <tr><td><code>{:name: p :}</code></td> <td>named group capture</td></tr>
 <tr><td><code>{~ p ~}</code></td> <td>substitution capture</td></tr>
 <tr><td><code>{| p |}</code></td> <td>table capture</td></tr>
-<tr><td><code>=name</code></td> <td>back reference
+<tr><td><code>=name</code></td> <td>back reference</td></tr>
-</td></tr>
+
-<tr><td><code>p ?</code></td> <td>optional match</td></tr>
-<tr><td><code>p *</code></td> <td>zero or more repetitions</td></tr>
-<tr><td><code>p +</code></td> <td>one or more repetitions</td></tr>
-<tr><td><code>p^num</code></td> <td>exactly <code>n</code> repetitions</td></tr>
-<tr><td><code>p^+num</code></td>
-      <td>at least <code>n</code> repetitions</td></tr>
-<tr><td><code>p^-num</code></td>
-      <td>at most <code>n</code> repetitions</td></tr>
 <tr><td><code>p -&gt; 'string'</code></td> <td>string capture</td></tr>
 <tr><td><code>p -&gt; "string"</code></td> <td>string capture</td></tr>
 <tr><td><code>p -&gt; num</code></td> <td>numbered capture</td></tr>
@@ -94,16 +100,13 @@ equivalent to <code>p / defs[name]</code></td></tr>
 <tr><td><code>p =&gt; name</code></td> <td>match-time capture
 equivalent to <code>lpeg.Cmt(p, defs[name])</code></td></tr>
 <tr><td><code>p ~&gt; name</code></td> <td>fold capture
-equivalent to <code>lpeg.Cf(p, defs[name])</code></td></tr>
+(deprecated)</td></tr>
-<tr><td><code>& p</code></td> <td>and predicate</td></tr>
+<tr><td><code>p &gt;&gt; name</code></td> <td>accumulator capture
-<tr><td><code>! p</code></td> <td>not predicate</td></tr>
+equivalent to <code>(p % defs[name])</code></td></tr>
-<tr><td><code>p1 p2</code></td> <td>concatenation</td></tr>
-<tr><td><code>p1 / p2</code></td> <td>ordered choice</td></tr>
-<tr><td>(<code>name &lt;- p</code>)<sup>+</sup></td> <td>grammar</td></tr>
 </tbody></table>
 <p>
 Any space appearing in a syntax description can be
-replaced by zero or more space characters and Lua-style comments
+replaced by zero or more space characters and Lua-style short comments
 (<code>--</code> until end of line).
 </p>
 
@@ -200,9 +203,10 @@ print(re.match("the number 423 is odd", "({%a+} / .)*"))
 --&gt; the    number    is    odd
 
 -- returns the first numeral in a string
-print(re.match("the number 423 is odd", "s <- {%d+} / . s"))
+print(re.match("the number 423 is odd", "s &lt;- {%d+} / . s"))
 --&gt; 423
 
+-- substitutes a dot for each vowel in a string
 print(re.gsub("hello World", "[aeiou]", "."))
 --&gt; h.ll. W.rld
 </pre>
@@ -329,7 +333,7 @@ respecting the indentation:
 <pre class="example">
 p = re.compile[[
   block &lt;- {| {:ident:' '*:} line
-           ((=ident !' ' line) / &(=ident ' ') block)* |}
+           ((=ident !' ' line) / &amp;(=ident ' ') block)* |}
   line &lt;- {[^%nl]*} %nl
 ]]
 </pre>
@@ -416,6 +420,7 @@ prefix          &lt;- '&amp;' S prefix / '!' S prefix / suffix
 suffix          &lt;- primary S (([+*?]
                             / '^' [+-]? num
                             / '-&gt;' S (string / '{}' / name)
+                            / '&gt&gt;' S name
                             / '=&gt;' S name) S)*
 
 primary         &lt;- '(' exp ')' / string / class / defined
@@ -423,6 +428,7 @@ primary         &lt;- '(' exp ')' / string / class / defined
                  / '=' name
                  / '{}'
                  / '{~' exp '~}'
+                 / '{|' exp '|}'
                  / '{' exp '}'
                  / '.'
                  / name S !arrow
@@ -436,7 +442,7 @@ item            &lt;- defined / range / .
 range           &lt;- . '-' [^]]
 
 S               &lt;- (%s / '--' [^%nl]*)*   -- spaces and comments
-name            &lt;- [A-Za-z][A-Za-z0-9_]*
+name            &lt;- [A-Za-z_][A-Za-z0-9_]*
 arrow           &lt;- '&lt;-'
 num             &lt;- [0-9]+
 string          &lt;- '"' [^"]* '"' / "'" [^']* "'"
@@ -452,37 +458,9 @@ print(re.match(p, p))   -- a self description must match itself
 <h2><a name="license">License</a></h2>
 
 <p>
-Copyright &copy; 2008-2015 Lua.org, PUC-Rio.
+This module is part of the <a href="lpeg.html">LPeg</a> package and shares
-</p>
+its <a href="lpeg.html#license">license</a>.
-<p>
-Permission is hereby granted, free of charge,
-to any person obtaining a copy of this software and
-associated documentation files (the "Software"),
-to deal in the Software without restriction,
-including without limitation the rights to use,
-copy, modify, merge, publish, distribute, sublicense,
-and/or sell copies of the Software,
-and to permit persons to whom the Software is
-furnished to do so,
-subject to the following conditions:
-</p>
 
-<p>
-The above copyright notice and this permission notice
-shall be included in all copies or substantial portions of the Software.
-</p>
-
-<p>
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED,
-INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
-DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-</p>
 
 </div> <!-- id="content" -->
 

3rd/lpeg/re.lua

@@ -1,4 +1,7 @@
--- $Id: re.lua $
+--
+-- Copyright 2007-2023, Lua.org & PUC-Rio  (see 'lpeg.html' for license)
+-- written by Roberto Ierusalimschy
+--
 
 -- imported functions and modules
 local tonumber, type, print, error = tonumber, type, print, error
@@ -10,14 +13,14 @@ local m = require"lpeg"
 -- on 'mm'
 local mm = m
 
--- pattern's metatable
+-- patterns' metatable
 local mt = getmetatable(mm.P(0))
 
 
+local version = _VERSION
 
 -- No more global accesses after this point
-local version = _VERSION
+_ENV = nil     -- does no harm in Lua 5.1
-if version == "Lua 5.2" then _ENV = nil end
 
 
 local any = m.P(1)
@@ -142,7 +145,7 @@ local item = (defined + Range + m.C(any)) / m.P
 local Class =
     "["
   * (m.C(m.P"^"^-1))    -- optional complement symbol
-  * m.Cf(item * (item - "]")^0, mt.__add) /
+  * (item * ((item % mt.__add) - "]")^0) /
                           function (c, p) return c == "^" and any - p or p end
   * "]"
 
@@ -168,13 +171,13 @@ end
 
 local exp = m.P{ "Exp",
   Exp = S * ( m.V"Grammar"
-            + m.Cf(m.V"Seq" * ("/" * S * m.V"Seq")^0, mt.__add) );
+            + m.V"Seq" * ("/" * S * m.V"Seq" % mt.__add)^0 );
-  Seq = m.Cf(m.Cc(m.P"") * m.V"Prefix"^0 , mt.__mul)
+  Seq = (m.Cc(m.P"") * (m.V"Prefix" % mt.__mul)^0)
         * (#seq_follow + patt_error);
   Prefix = "&" * S * m.V"Prefix" / mt.__len
          + "!" * S * m.V"Prefix" / mt.__unm
          + m.V"Suffix";
-  Suffix = m.Cf(m.V"Primary" * S *
+  Suffix = m.V"Primary" * S *
           ( ( m.P"+" * m.Cc(1, mt.__pow)
             + m.P"*" * m.Cc(0, mt.__pow)
             + m.P"?" * m.Cc(-1, mt.__pow)
@@ -185,10 +188,11 @@ local exp = m.P{ "Exp",
                          + m.P"{}" * m.Cc(nil, m.Ct)
                          + defwithfunc(mt.__div)
                          )
-            + "=>" * S * defwithfunc(m.Cmt)
+            + "=>" * S * defwithfunc(mm.Cmt)
-            + "~>" * S * defwithfunc(m.Cf)
+            + ">>" * S * defwithfunc(mt.__mod)
-            ) * S
+            + "~>" * S * defwithfunc(mm.Cf)
-          )^0, function (a,b,f) return f(a,b) end );
+            ) % function (a,b,f) return f(a,b) end * S
+          )^0;
   Primary = "(" * m.V"Exp" * ")"
             + String / mm.P
             + Class
@@ -204,8 +208,7 @@ local exp = m.P{ "Exp",
             + (name * -arrow + "<" * name * ">") * m.Cb("G") / NT;
   Definition = name * arrow * m.V"Exp";
   Grammar = m.Cg(m.Cc(true), "G") *
-            m.Cf(m.V"Definition" / firstdef * m.Cg(m.V"Definition")^0,
+            ((m.V"Definition" / firstdef) * (m.V"Definition" % adddef)^0) / mm.P
-              adddef) / mm.P
 }
 
 local pattern = S * m.Cg(m.Cc(false), "G") * exp / mm.P * (-any + patt_error)

3rd/lpeg/test.lua

@@ -1,7 +1,5 @@
 #!/usr/bin/env lua
 
--- $Id: test.lua $
-
 -- require"strict"    -- just to be pedantic
 
 local m = require"lpeg"
@@ -48,8 +46,8 @@ end
 
 print"General tests for LPeg library"
 
-assert(type(m.version()) == "string")
+assert(type(m.version) == "string")
-print("version " .. m.version())
+print(m.version)
 assert(m.type("alo") ~= "pattern")
 assert(m.type(io.input) ~= "pattern")
 assert(m.type(m.P"alo") == "pattern")
@@ -70,6 +68,8 @@ assert(m.match(#m.P(true) * "a", "a") == 2)
 assert(m.match("a" * #m.P(false), "a") == nil)
 assert(m.match("a" * #m.P(true), "a") == 2)
 
+assert(m.match(m.P(1)^0, "abcd") == 5)
+assert(m.match(m.S("")^0, "abcd") == 1)
 
 -- tests for locale
 do
@@ -119,6 +119,8 @@ eqcharset(m.S"\1\0\2", m.R"\0\2")
 eqcharset(m.S"\1\0\2", m.R"\1\2" + "\0")
 eqcharset(m.S"\1\0\2" - "\0", m.R"\1\2")
 
+eqcharset(m.S("\0\255"), m.P"\0" + "\255")   -- charset extremes
+
 local word = alpha^1 * (1 - alpha)^0
 
 assert((word^0 * -1):match"alo alo")
@@ -406,7 +408,7 @@ assert(p:match('abcx') == 5 and p:match('ayzx') == 5 and not p:match'abc')
 
 
 do
-  -- large dynamic Cc
+  print "testing large dynamic Cc"
   local lim = 2^16 - 1
   local c = 0
   local function seq (n) 
@@ -491,8 +493,8 @@ local function f_term (v1, op, v2, d)
 end
 
 G = m.P{ "Exp",
-  Exp = m.Cf(V"Factor" * m.Cg(FactorOp * V"Factor")^0, f_factor);
+  Exp = V"Factor" * (FactorOp * V"Factor" % f_factor)^0;
-  Factor = m.Cf(V"Term" * m.Cg(TermOp * V"Term")^0, f_term);
+  Factor = V"Term" * (TermOp * V"Term" % f_term)^0;
   Term = Number / tonumber  +  Open * V"Exp" * Close;
 }
 
@@ -864,6 +866,7 @@ print"+"
 -- accumulator capture
 function f (x) return x + 1 end
 assert(m.match(m.Cf(m.Cc(0) * m.C(1)^0, f), "alo alo") == 7)
+assert(m.match(m.Cc(0) * (m.C(1) % f)^0, "alo alo") == 7)
 
 t = {m.match(m.Cf(m.Cc(1,2,3), error), "")}
 checkeq(t, {1})
@@ -873,7 +876,7 @@ t = p:match("a=b;c=du;xux=yuy;")
 checkeq(t, {a="b", c="du", xux="yuy"})
 
 
--- errors in accumulator capture
+-- errors in fold capture
 
 -- no initial capture
 checkerr("no initial value", m.match, m.Cf(m.P(5), print), 'aaaaaa')
@@ -881,8 +884,14 @@ checkerr("no initial value", m.match, m.Cf(m.P(5), print), 'aaaaaa')
 checkerr("no initial value", m.match, m.Cf(m.P(500), print),
                                string.rep('a', 600))
 
--- nested capture produces no initial value
+
-checkerr("no initial value", m.match, m.Cf(m.P(1) / {}, print), "alo")
+-- errors in accumulator capture
+
+-- no initial capture
+checkerr("no previous value", m.match, m.P(5) % print, 'aaaaaa')
+-- no initial capture (very long match forces fold to be a pair open-close)
+checkerr("no previous value", m.match, m.P(500) % print,
+                               string.rep('a', 600))
 
 
 -- tests for loop checker
@@ -985,10 +994,10 @@ for i = 1, 10 do
   assert(p:match("aaaaaaaaaaa") == 11 - i + 1)
 end
 
-print"+"
 
 
--- tests for back references
+print "testing back references"
+
 checkerr("back reference 'x' not found", m.match, m.Cb('x'), '')
 checkerr("back reference 'b' not found", m.match, m.Cg(1, 'a') * m.Cb('b'), 'a')
 
@@ -996,6 +1005,35 @@ p = m.Cg(m.C(1) * m.C(1), "k") * m.Ct(m.Cb("k"))
 t = p:match("ab")
 checkeq(t, {"a", "b"})
 
+
+do
+  -- some basic cases
+  assert(m.match(m.Cg(m.Cc(3), "a") * m.Cb("a"), "a") == 3)
+  assert(m.match(m.Cg(m.C(1), 133) * m.Cb(133), "X") == "X")
+
+  -- first reference to 'x' should not see the group enclosing it
+  local p = m.Cg(m.Cb('x'), 'x') * m.Cb('x')
+  checkerr("back reference 'x' not found", m.match, p, '')
+
+  local p = m.Cg(m.Cb('x') * m.C(1), 'x') * m.Cb('x')
+  checkerr("back reference 'x' not found", m.match, p, 'abc')
+
+  -- reference to 'x' should not see the group enclosed in another capture
+  local s = string.rep("a", 30)
+  local p = (m.C(1)^-4 * m.Cg(m.C(1), 'x')) / {} * m.Cb('x')
+  checkerr("back reference 'x' not found", m.match, p, s)
+
+  local p = (m.C(1)^-20 * m.Cg(m.C(1), 'x')) / {} * m.Cb('x')
+  checkerr("back reference 'x' not found", m.match, p, s)
+
+  -- second reference 'k' should refer to 10 and first ref. 'k'
+  p = m.Cg(m.Cc(20), 'k') * m.Cg(m.Cc(10) * m.Cb('k') * m.C(1), 'k')
+      * (m.Cb('k') / function (a,b,c) return a*10 + b + tonumber(c) end)
+  -- 10 * 10 (Cc) + 20 (Cb) + 7 (C) == 127
+  assert(p:match("756") == 127)
+
+end
+
 p = m.P(true)
 for i = 1, 10 do p = p * m.Cg(1, i) end
 for i = 1, 10 do
@@ -1032,6 +1070,17 @@ local function id (s, i, ...)
   return true, ...
 end
 
+do   -- run-time capture in an end predicate (should discard its value)
+  local x = 0
+  function foo (s, i)
+      x = x + 1
+      return true, x
+  end
+
+  local p = #(m.Cmt("", foo) * "xx") * m.Cmt("", foo)
+  assert(p:match("xx") == 2)
+end
+
 assert(m.Cmt(m.Cs((m.Cmt(m.S'abc' / { a = 'x', c = 'y' }, id) +
               m.R'09'^1 /  string.char +
               m.P(1))^0), id):match"acb98+68c" == "xyb\98+\68y")
@@ -1156,7 +1205,7 @@ end
 
 
 -- bug in 1.0: problems with math-times returning too many captures
-do
+if _VERSION >= "Lua 5.2" then
   local lim = 2^11 - 10
   local res = {m.match(manyCmt(lim), "a")}
   assert(#res == lim and res[1] == lim - 1 and res[lim] == 0)
@@ -1171,9 +1220,91 @@ t = {p:match('abacc')}
 checkeq(t, {'a', 'aa', 20, 'a', 'aaa', 'aaa'})
 
 
+do  print"testing large grammars"
+  local lim = 1000    -- number of rules
+  local t = {}
+
+  for i = 3, lim do
+    t[i] = m.V(i - 1)   -- each rule calls previous one
+  end
+  t[1] = m.V(lim)    -- start on last rule
+  t[2] = m.C("alo")  -- final rule
+
+  local P = m.P(t)   -- build grammar
+  assert(P:match("alo") == "alo")
+
+  t[#t + 1] = m.P("x")   -- one more rule...
+  checkerr("too many rules", m.P, t)
+end
+
+
+print "testing UTF-8 ranges"
+
+do   -- a few typical UTF-8 ranges
+  local p = m.utfR(0x410, 0x44f)^1 / "cyr: %0"
+          + m.utfR(0x4e00, 0x9fff)^1 / "cjk: %0"
+          + m.utfR(0x1F600, 0x1F64F)^1 / "emot: %0"
+          + m.utfR(0, 0x7f)^1 / "ascii: %0"
+          + m.utfR(0, 0x10ffff) / "other: %0"
+
+  p = m.Ct(p^0) * -m.P(1)
+
+  local cyr = "ждюя"
+  local emot = "\240\159\152\128\240\159\153\128"   --  😀🙀
+  local cjk = "专举乸"
+  local ascii = "alo"
+  local last = "\244\143\191\191"                -- U+10FFFF
+
+  local s = cyr .. "—" .. emot .. "—" .. cjk .. "—" .. ascii .. last
+  t = (p:match(s))
+
+  assert(t[1] == "cyr: " .. cyr and t[2] == "other: —" and
+         t[3] == "emot: " .. emot and t[4] == "other: —" and
+         t[5] == "cjk: " .. cjk and t[6] == "other: —" and
+         t[7] == "ascii: " .. ascii and t[8] == "other: " .. last and
+         t[9] == nil)
+
+  -- failing UTF-8 matches and borders
+  assert(not m.match(m.utfR(10, 0x2000), "\9"))
+  assert(not m.match(m.utfR(10, 0x2000), "\226\128\129"))
+  assert(m.match(m.utfR(10, 0x2000), "\10") == 2)
+  assert(m.match(m.utfR(10, 0x2000), "\226\128\128") == 4)
+end
+
+
+do   -- valid and invalid code points
+  local p = m.utfR(0, 0x10ffff)^0
+  assert(p:match("汉字\128") == #"汉字" + 1)
+  assert(p:match("\244\159\191") == 1)
+  assert(p:match("\244\159\191\191") == 1)
+  assert(p:match("\255") == 1)
+
+   -- basic errors
+  checkerr("empty range", m.utfR, 1, 0)
+  checkerr("invalid code point", m.utfR, 1, 0x10ffff + 1)
+end
+
+
+do  -- back references (fixed width)
+  -- match a byte after a CJK point
+  local p = m.B(m.utfR(0x4e00, 0x9fff)) * m.C(1)
+  p = m.P{ p + m.P(1) * m.V(1) }   -- search for 'p'
+  assert(p:match("ab д 专X x") == "X")
+
+  -- match a byte after a hebrew point
+  local p = m.B(m.utfR(0x5d0, 0x5ea)) * m.C(1)
+  p = m.P(#"ש") * p
+  assert(p:match("שX") == "X")
+
+  checkerr("fixed length", m.B, m.utfR(0, 0x10ffff))
+end
+
+
+
 -------------------------------------------------------------------
 -- Tests for 're' module
 -------------------------------------------------------------------
+print"testing 're' module"
 
 local re = require "re"
 
@@ -1307,6 +1438,12 @@ e = compile([[
 e = compile("{[0-9]+'.'?[0-9]*} -> sin", math)
 assert(e:match("2.34") == math.sin(2.34))
 
+e = compile("'pi' -> math", _G)
+assert(e:match("pi") == math.pi)
+
+e = compile("[ ]* 'version' -> _VERSION", _G)
+assert(e:match("  version") == _VERSION)
+
 
 function eq (_, _, a, b) return a == b end
 
@@ -1380,6 +1517,13 @@ c = re.compile([[
 ]], {tonumber = tonumber, add = function (a,b) return a + b end})
 assert(c:match("3 401 50") == 3 + 401 + 50)
 
+-- test for accumulator captures
+c = re.compile([[
+  S <- number (%s+ number >> add)*
+  number <- %d+ -> tonumber
+]], {tonumber = tonumber, add = function (a,b) return a + b end})
+assert(c:match("3 401 50") == 3 + 401 + 50)
+
 -- tests for look-ahead captures
 x = {re.match("alo", "&(&{.}) !{'b'} {&(...)} &{..} {...} {!.}")}
 checkeq(x, {"", "alo", ""})

Makefile

@@ -115,7 +115,7 @@ $(LUA_CLIB_PATH)/sproto.so : lualib-src/sproto/sproto.c lualib-src/sproto/lsprot
 $(LUA_CLIB_PATH)/ltls.so : lualib-src/ltls.c | $(LUA_CLIB_PATH)
 	$(CC) $(CFLAGS) $(SHARED) -Iskynet-src -L$(TLS_LIB) -I$(TLS_INC) $^ -o $@ -lssl
 
-$(LUA_CLIB_PATH)/lpeg.so : 3rd/lpeg/lpcap.c 3rd/lpeg/lpcode.c 3rd/lpeg/lpprint.c 3rd/lpeg/lptree.c 3rd/lpeg/lpvm.c | $(LUA_CLIB_PATH)
+$(LUA_CLIB_PATH)/lpeg.so : 3rd/lpeg/lpcap.c 3rd/lpeg/lpcode.c 3rd/lpeg/lpprint.c 3rd/lpeg/lptree.c 3rd/lpeg/lpvm.c 3rd/lpeg/lpcset.c | $(LUA_CLIB_PATH)
 	$(CC) $(CFLAGS) $(SHARED) -I3rd/lpeg $^ -o $@ 
 
 clean :