diff --git a/3rd/lpeg/HISTORY b/3rd/lpeg/HISTORY index 66a8e14d..a2507098 100644 --- a/3rd/lpeg/HISTORY +++ b/3rd/lpeg/HISTORY @@ -1,4 +1,13 @@ -HISTORY for LPeg 1.0.2 +HISTORY for LPeg 1.1.0 + +* Changes from version 1.0.2 to 1.1.0 + --------------------------------- + + accumulator capture + + UTF-8 ranges + + Larger limit for number of rules in a grammar + + Larger limit for number of captures in a match + + bug fixes + + other small improvements * Changes from version 1.0.1 to 1.0.2 --------------------------------- diff --git a/3rd/lpeg/README.md b/3rd/lpeg/README.md new file mode 100644 index 00000000..65ac1eb5 --- /dev/null +++ b/3rd/lpeg/README.md @@ -0,0 +1,4 @@ +# LPeg - Parsing Expression Grammars For Lua + +For more information, +see [Lpeg](//www.inf.puc-rio.br/~roberto/lpeg/). diff --git a/3rd/lpeg/lpcap.c b/3rd/lpeg/lpcap.c index b332fde4..f13ecf4d 100644 --- a/3rd/lpeg/lpcap.c +++ b/3rd/lpeg/lpcap.c @@ -1,29 +1,41 @@ -/* -** $Id: lpcap.c $ -** Copyright 2007, Lua.org & PUC-Rio (see 'lpeg.html' for license) -*/ #include "lua.h" #include "lauxlib.h" #include "lpcap.h" +#include "lpprint.h" #include "lptypes.h" -#define captype(cap) ((cap)->kind) - -#define isclosecap(cap) (captype(cap) == Cclose) - -#define closeaddr(c) ((c)->s + (c)->siz - 1) - -#define isfullcap(cap) ((cap)->siz != 0) - #define getfromktable(cs,v) lua_rawgeti((cs)->L, ktableidx((cs)->ptop), v) #define pushluaval(cs) getfromktable(cs, (cs)->cap->idx) +#define skipclose(cs,head) \ + if (isopencap(head)) { assert(isclosecap(cs->cap)); cs->cap++; } + + +/* +** Return the size of capture 'cap'. If it is an open capture, 'close' +** must be its corresponding close. +*/ +static Index_t capsize (Capture *cap, Capture *close) { + if (isopencap(cap)) { + assert(isclosecap(close)); + return close->index - cap->index; + } + else + return cap->siz - 1; +} + + +static Index_t closesize (CapState *cs, Capture *head) { + return capsize(head, cs->cap); +} + + /* ** Put at the cache for Lua values the value indexed by 'v' in ktable ** of the running pattern (if it is not there yet); returns its index. @@ -44,35 +56,40 @@ static int pushcapture (CapState *cs); /* ** Goes back in a list of captures looking for an open capture -** corresponding to a close +** corresponding to a close one. */ static Capture *findopen (Capture *cap) { int n = 0; /* number of closes waiting an open */ for (;;) { cap--; if (isclosecap(cap)) n++; /* one more open to skip */ - else if (!isfullcap(cap)) + else if (isopencap(cap)) if (n-- == 0) return cap; } } /* -** Go to the next capture +** Go to the next capture at the same level. */ static void nextcap (CapState *cs) { Capture *cap = cs->cap; - if (!isfullcap(cap)) { /* not a single capture? */ + if (isopencap(cap)) { /* must look for a close? */ int n = 0; /* number of opens waiting a close */ for (;;) { /* look for corresponding close */ cap++; - if (isclosecap(cap)) { + if (isopencap(cap)) n++; + else if (isclosecap(cap)) if (n-- == 0) break; - } - else if (!isfullcap(cap)) n++; } + cs->cap = cap + 1; /* + 1 to skip last close */ + } + else { + Capture *next; + for (next = cap + 1; capinside(cap, next); next++) + ; /* skip captures inside current one */ + cs->cap = next; } - cs->cap = cap + 1; /* + 1 to skip last close (or entire single capture) */ } @@ -84,22 +101,17 @@ static void nextcap (CapState *cs) { ** so the function never returns zero. */ static int pushnestedvalues (CapState *cs, int addextra) { - Capture *co = cs->cap; - if (isfullcap(cs->cap++)) { /* no nested captures? */ - lua_pushlstring(cs->L, co->s, co->siz - 1); /* push whole match */ - return 1; /* that is it */ - } - else { - int n = 0; - while (!isclosecap(cs->cap)) /* repeat for all nested patterns */ - n += pushcapture(cs); - if (addextra || n == 0) { /* need extra? */ - lua_pushlstring(cs->L, co->s, cs->cap->s - co->s); /* push whole match */ - n++; - } - cs->cap++; /* skip close entry */ - return n; + Capture *head = cs->cap++; /* original capture */ + int n = 0; /* number of pushed subvalues */ + /* repeat for all nested patterns */ + while (capinside(head, cs->cap)) + n += pushcapture(cs); + if (addextra || n == 0) { /* need extra? */ + lua_pushlstring(cs->L, cs->s + head->index, closesize(cs, head)); + n++; } + skipclose(cs, head); + return n; } @@ -114,17 +126,43 @@ static void pushonenestedvalue (CapState *cs) { /* -** Try to find a named group capture with the name given at the top of -** the stack; goes backward from 'cap'. +** Checks whether group 'grp' is visible to 'ref', that is, 'grp' is +** not nested inside a full capture that does not contain 'ref'. (We +** only need to care for full captures because the search at 'findback' +** skips open-end blocks; so, if 'grp' is nested in a non-full capture, +** 'ref' is also inside it.) To check this, we search backward for the +** inner full capture enclosing 'grp'. A full capture cannot contain +** non-full captures, so a close capture means we cannot be inside a +** full capture anymore. */ -static Capture *findback (CapState *cs, Capture *cap) { +static int capvisible (CapState *cs, Capture *grp, Capture *ref) { + Capture *cap = grp; + int i = MAXLOP; /* maximum distance for an 'open' */ + while (i-- > 0 && cap-- > cs->ocap) { + if (isclosecap(cap)) + return 1; /* can stop the search */ + else if (grp->index - cap->index >= UCHAR_MAX) + return 1; /* can stop the search */ + else if (capinside(cap, grp)) /* is 'grp' inside cap? */ + return capinside(cap, ref); /* ok iff cap also contains 'ref' */ + } + return 1; /* 'grp' is not inside any capture */ +} + + +/* +** Try to find a named group capture with the name given at the top of +** the stack; goes backward from 'ref'. +*/ +static Capture *findback (CapState *cs, Capture *ref) { lua_State *L = cs->L; + Capture *cap = ref; while (cap-- > cs->ocap) { /* repeat until end of list */ if (isclosecap(cap)) cap = findopen(cap); /* skip nested captures */ - else if (!isfullcap(cap)) - continue; /* opening an enclosing capture: skip and get previous */ - if (captype(cap) == Cgroup) { + else if (capinside(cap, ref)) + continue; /* enclosing captures are not visible to 'ref' */ + if (captype(cap) == Cgroup && capvisible(cs, cap, ref)) { getfromktable(cs, cap->idx); /* get group name */ if (lp_equal(L, -2, -1)) { /* right group? */ lua_pop(L, 2); /* remove reference name and group name */ @@ -158,11 +196,10 @@ static int backrefcap (CapState *cs) { */ static int tablecap (CapState *cs) { lua_State *L = cs->L; + Capture *head = cs->cap++; int n = 0; lua_newtable(L); - if (isfullcap(cs->cap++)) - return 1; /* table is empty */ - while (!isclosecap(cs->cap)) { + while (capinside(head, cs->cap)) { if (captype(cs->cap) == Cgroup && cs->cap->idx != 0) { /* named group? */ pushluaval(cs); /* push group name */ pushonenestedvalue(cs); @@ -176,7 +213,7 @@ static int tablecap (CapState *cs) { n += k; } } - cs->cap++; /* skip close entry */ + skipclose(cs, head); return 1; /* number of values pushed (only the table) */ } @@ -203,20 +240,20 @@ static int querycap (CapState *cs) { static int foldcap (CapState *cs) { int n; lua_State *L = cs->L; - int idx = cs->cap->idx; - if (isfullcap(cs->cap++) || /* no nested captures? */ - isclosecap(cs->cap) || /* no nested captures (large subject)? */ + Capture *head = cs->cap++; + int idx = head->idx; + if (isclosecap(cs->cap) || /* no nested captures (large subject)? */ (n = pushcapture(cs)) == 0) /* nested captures with no values? */ return luaL_error(L, "no initial value for fold capture"); if (n > 1) lua_pop(L, n - 1); /* leave only one result for accumulator */ - while (!isclosecap(cs->cap)) { + while (capinside(head, cs->cap)) { lua_pushvalue(L, updatecache(cs, idx)); /* get folding function */ lua_insert(L, -2); /* put it before accumulator */ n = pushcapture(cs); /* get next capture's values */ lua_call(L, n + 1, 1); /* call folding function */ } - cs->cap++; /* skip close entry */ + skipclose(cs, head); return 1; /* only accumulator left on the stack */ } @@ -234,6 +271,22 @@ static int functioncap (CapState *cs) { } +/* +** Accumulator capture +*/ +static int accumulatorcap (CapState *cs) { + lua_State *L = cs->L; + int n; + if (lua_gettop(L) < cs->firstcap) + luaL_error(L, "no previous value for accumulator capture"); + pushluaval(cs); /* push function */ + lua_insert(L, -2); /* previous value becomes first argument */ + n = pushnestedvalues(cs, 0); /* push nested captures */ + lua_call(L, n + 1, 1); /* call function */ + return 0; /* did not add any extra value */ +} + + /* ** Select capture */ @@ -283,7 +336,7 @@ int runtimecap (CapState *cs, Capture *close, const char *s, int *rem) { assert(captype(open) == Cgroup); id = finddyncap(open, close); /* get first dynamic capture argument */ close->kind = Cclose; /* closes the group */ - close->s = s; + close->index = s - cs->s; cs->cap = open; cs->valuecached = 0; /* prepare capture state */ luaL_checkstack(L, 4, "too many runtime captures"); pushluaval(cs); /* push function to be called */ @@ -313,8 +366,8 @@ typedef struct StrAux { union { Capture *cp; /* if not a string, respective capture */ struct { /* if it is a string... */ - const char *s; /* ... starts here */ - const char *e; /* ... ends here */ + Index_t idx; /* starts here */ + Index_t siz; /* with this size */ } s; } u; } StrAux; @@ -329,24 +382,23 @@ typedef struct StrAux { */ static int getstrcaps (CapState *cs, StrAux *cps, int n) { int k = n++; + Capture *head = cs->cap++; cps[k].isstring = 1; /* get string value */ - cps[k].u.s.s = cs->cap->s; /* starts here */ - if (!isfullcap(cs->cap++)) { /* nested captures? */ - while (!isclosecap(cs->cap)) { /* traverse them */ - if (n >= MAXSTRCAPS) /* too many captures? */ - nextcap(cs); /* skip extra captures (will not need them) */ - else if (captype(cs->cap) == Csimple) /* string? */ - n = getstrcaps(cs, cps, n); /* put info. into array */ - else { - cps[n].isstring = 0; /* not a string */ - cps[n].u.cp = cs->cap; /* keep original capture */ - nextcap(cs); - n++; - } + cps[k].u.s.idx = head->index; /* starts here */ + while (capinside(head, cs->cap)) { + if (n >= MAXSTRCAPS) /* too many captures? */ + nextcap(cs); /* skip extra captures (will not need them) */ + else if (captype(cs->cap) == Csimple) /* string? */ + n = getstrcaps(cs, cps, n); /* put info. into array */ + else { + cps[n].isstring = 0; /* not a string */ + cps[n].u.cp = cs->cap; /* keep original capture */ + nextcap(cs); + n++; } - cs->cap++; /* skip close */ } - cps[k].u.s.e = closeaddr(cs->cap - 1); /* ends here */ + cps[k].u.s.siz = closesize(cs, head); + skipclose(cs, head); return n; } @@ -368,7 +420,7 @@ static void stringcap (luaL_Buffer *b, CapState *cs) { const char *fmt; /* format string */ fmt = lua_tolstring(cs->L, updatecache(cs, cs->cap->idx), &len); n = getstrcaps(cs, cps, 0) - 1; /* collect nested captures */ - for (i = 0; i < len; i++) { /* traverse them */ + for (i = 0; i < len; i++) { /* traverse format string */ if (fmt[i] != '%') /* not an escape? */ luaL_addchar(b, fmt[i]); /* add it to buffer */ else if (fmt[++i] < '0' || fmt[i] > '9') /* not followed by a digit? */ @@ -378,7 +430,7 @@ static void stringcap (luaL_Buffer *b, CapState *cs) { if (l > n) luaL_error(cs->L, "invalid capture index (%d)", l); else if (cps[l].isstring) - luaL_addlstring(b, cps[l].u.s.s, cps[l].u.s.e - cps[l].u.s.s); + luaL_addlstring(b, cs->s + cps[l].u.s.idx, cps[l].u.s.siz); else { Capture *curr = cs->cap; cs->cap = cps[l].u.cp; /* go back to evaluate that nested capture */ @@ -395,22 +447,20 @@ static void stringcap (luaL_Buffer *b, CapState *cs) { ** Substitution capture: add result to buffer 'b' */ static void substcap (luaL_Buffer *b, CapState *cs) { - const char *curr = cs->cap->s; - if (isfullcap(cs->cap)) /* no nested captures? */ - luaL_addlstring(b, curr, cs->cap->siz - 1); /* keep original text */ - else { - cs->cap++; /* skip open entry */ - while (!isclosecap(cs->cap)) { /* traverse nested captures */ - const char *next = cs->cap->s; - luaL_addlstring(b, curr, next - curr); /* add text up to capture */ - if (addonestring(b, cs, "replacement")) - curr = closeaddr(cs->cap - 1); /* continue after match */ - else /* no capture value */ - curr = next; /* keep original text in final result */ - } - luaL_addlstring(b, curr, cs->cap->s - curr); /* add last piece of text */ + const char *curr = cs->s + cs->cap->index; + Capture *head = cs->cap++; + while (capinside(head, cs->cap)) { + Capture *cap = cs->cap; + const char *caps = cs->s + cap->index; + luaL_addlstring(b, curr, caps - curr); /* add text up to capture */ + if (addonestring(b, cs, "replacement")) + curr = caps + capsize(cap, cs->cap - 1); /* continue after match */ + else /* no capture value */ + curr = caps; /* keep original text in final result */ } - cs->cap++; /* go to next capture */ + /* add last piece of text */ + luaL_addlstring(b, curr, cs->s + head->index + closesize(cs, head) - curr); + skipclose(cs, head); } @@ -426,13 +476,16 @@ static int addonestring (luaL_Buffer *b, CapState *cs, const char *what) { case Csubst: substcap(b, cs); /* add capture directly to buffer */ return 1; + case Cacc: /* accumulator capture? */ + return luaL_error(cs->L, "invalid context for an accumulator capture"); default: { lua_State *L = cs->L; int n = pushcapture(cs); if (n > 0) { if (n > 1) lua_pop(L, n - 1); /* only one result */ if (!lua_isstring(L, -1)) - luaL_error(L, "invalid %s value (a %s)", what, luaL_typename(L, -1)); + return luaL_error(L, "invalid %s value (a %s)", + what, luaL_typename(L, -1)); luaL_addvalue(b); } return n; @@ -458,7 +511,7 @@ static int pushcapture (CapState *cs) { return luaL_error(L, "subcapture nesting too deep"); switch (captype(cs->cap)) { case Cposition: { - lua_pushinteger(L, cs->cap->s - cs->s + 1); + lua_pushinteger(L, cs->cap->index + 1); cs->cap++; res = 1; break; @@ -516,6 +569,7 @@ static int pushcapture (CapState *cs) { case Cbackref: res = backrefcap(cs); break; case Ctable: res = tablecap(cs); break; case Cfunction: res = functioncap(cs); break; + case Cacc: res = accumulatorcap(cs); break; case Cnum: res = numcap(cs); break; case Cquery: res = querycap(cs); break; case Cfold: res = foldcap(cs); break; @@ -529,7 +583,7 @@ static int pushcapture (CapState *cs) { /* ** Prepare a CapState structure and traverse the entire list of ** captures in the stack pushing its results. 's' is the subject -** string, 'r' is the final position of the match, and 'ptop' +** string, 'r' is the final position of the match, and 'ptop' ** the index in the stack where some useful values were pushed. ** Returns the number of results pushed. (If the list produces no ** results, push the final position of the match.) @@ -537,13 +591,16 @@ static int pushcapture (CapState *cs) { int getcaptures (lua_State *L, const char *s, const char *r, int ptop) { Capture *capture = (Capture *)lua_touserdata(L, caplistidx(ptop)); int n = 0; + /* printcaplist(capture); */ if (!isclosecap(capture)) { /* is there any capture? */ CapState cs; cs.ocap = cs.cap = capture; cs.L = L; cs.reclevel = 0; cs.s = s; cs.valuecached = 0; cs.ptop = ptop; + cs.firstcap = lua_gettop(L) + 1; /* where first value (if any) will go */ do { /* collect their values */ n += pushcapture(&cs); } while (!isclosecap(cs.cap)); + assert(lua_gettop(L) - cs.firstcap == n - 1); } if (n == 0) { /* no capture values? */ lua_pushinteger(L, r - s + 1); /* return only end position */ diff --git a/3rd/lpeg/lpcap.h b/3rd/lpeg/lpcap.h index dc10d696..abbd5537 100644 --- a/3rd/lpeg/lpcap.h +++ b/3rd/lpeg/lpcap.h @@ -1,6 +1,3 @@ -/* -** $Id: lpcap.h $ -*/ #if !defined(lpcap_h) #define lpcap_h @@ -19,6 +16,7 @@ typedef enum CapKind { Csimple, /* next node is pattern */ Ctable, /* next node is pattern */ Cfunction, /* ktable[key] is function; next node is pattern */ + Cacc, /* ktable[key] is function; next node is pattern */ Cquery, /* ktable[key] is table; next node is pattern */ Cstring, /* ktable[key] is string; next node is pattern */ Cnum, /* numbered capture; 'key' is number of value to return */ @@ -29,8 +27,20 @@ typedef enum CapKind { } CapKind; +/* +** An unsigned integer large enough to index any subject entirely. +** It can be size_t, but that will double the size of the array +** of captures in a 64-bit machine. +*/ +#if !defined(Index_t) +typedef uint Index_t; +#endif + +#define MAXINDT (~(Index_t)0) + + typedef struct Capture { - const char *s; /* subject position */ + Index_t index; /* subject position */ unsigned short idx; /* extra info (group name, arg index, etc.) */ byte kind; /* kind of capture */ byte siz; /* size of full capture + 1 (0 = not a full capture) */ @@ -41,13 +51,32 @@ typedef struct CapState { Capture *cap; /* current capture */ Capture *ocap; /* (original) capture list */ lua_State *L; - int ptop; /* index of last argument to 'match' */ + int ptop; /* stack index of last argument to 'match' */ + int firstcap; /* stack index of first capture pushed in the stack */ const char *s; /* original string */ int valuecached; /* value stored in cache slot */ int reclevel; /* recursion level */ } CapState; +#define captype(cap) ((cap)->kind) + +#define isclosecap(cap) (captype(cap) == Cclose) +#define isopencap(cap) ((cap)->siz == 0) + +/* true if c2 is (any number of levels) inside c1 */ +#define capinside(c1,c2) \ + (isopencap(c1) ? !isclosecap(c2) \ + : (c2)->index < (c1)->index + (c1)->siz - 1) + + +/** +** Maximum number of captures to visit when looking for an 'open'. +*/ +#define MAXLOP 20 + + + int runtimecap (CapState *cs, Capture *close, const char *s, int *rem); int getcaptures (lua_State *L, const char *s, const char *r, int ptop); int finddyncap (Capture *cap, Capture *last); diff --git a/3rd/lpeg/lpcode.c b/3rd/lpeg/lpcode.c index 39234597..f3b8ae36 100644 --- a/3rd/lpeg/lpcode.c +++ b/3rd/lpeg/lpcode.c @@ -1,7 +1,3 @@ -/* -** $Id: lpcode.c $ -** Copyright 2007, Lua.org & PUC-Rio (see 'lpeg.html' for license) -*/ #include @@ -11,6 +7,7 @@ #include "lptypes.h" #include "lpcode.h" +#include "lpcset.h" /* signals a "no-instruction */ @@ -26,61 +23,13 @@ static const Charset fullset_ = static const Charset *fullset = &fullset_; + /* ** {====================================================== ** Analysis and some optimizations ** ======================================================= */ -/* -** Check whether a charset is empty (returns IFail), singleton (IChar), -** full (IAny), or none of those (ISet). When singleton, '*c' returns -** which character it is. (When generic set, the set was the input, -** so there is no need to return it.) -*/ -static Opcode charsettype (const byte *cs, int *c) { - int count = 0; /* number of characters in the set */ - int i; - int candidate = -1; /* candidate position for the singleton char */ - for (i = 0; i < CHARSETSIZE; i++) { /* for each byte */ - int b = cs[i]; - if (b == 0) { /* is byte empty? */ - if (count > 1) /* was set neither empty nor singleton? */ - return ISet; /* neither full nor empty nor singleton */ - /* else set is still empty or singleton */ - } - else if (b == 0xFF) { /* is byte full? */ - if (count < (i * BITSPERCHAR)) /* was set not full? */ - return ISet; /* neither full nor empty nor singleton */ - else count += BITSPERCHAR; /* set is still full */ - } - else if ((b & (b - 1)) == 0) { /* has byte only one bit? */ - if (count > 0) /* was set not empty? */ - return ISet; /* neither full nor empty nor singleton */ - else { /* set has only one char till now; track it */ - count++; - candidate = i; - } - } - else return ISet; /* byte is neither empty, full, nor singleton */ - } - switch (count) { - case 0: return IFail; /* empty set */ - case 1: { /* singleton; find character bit inside byte */ - int b = cs[candidate]; - *c = candidate * BITSPERCHAR; - if ((b & 0xF0) != 0) { *c += 4; b >>= 4; } - if ((b & 0x0C) != 0) { *c += 2; b >>= 2; } - if ((b & 0x02) != 0) { *c += 1; } - return IChar; - } - default: { - assert(count == CHARSETSIZE * BITSPERCHAR); /* full set */ - return IAny; - } - } -} - /* ** A few basic operations on Charsets @@ -89,42 +38,12 @@ static void cs_complement (Charset *cs) { loopset(i, cs->cs[i] = ~cs->cs[i]); } -static int cs_equal (const byte *cs1, const byte *cs2) { - loopset(i, if (cs1[i] != cs2[i]) return 0); - return 1; -} - static int cs_disjoint (const Charset *cs1, const Charset *cs2) { loopset(i, if ((cs1->cs[i] & cs2->cs[i]) != 0) return 0;) return 1; } -/* -** If 'tree' is a 'char' pattern (TSet, TChar, TAny), convert it into a -** charset and return 1; else return 0. -*/ -int tocharset (TTree *tree, Charset *cs) { - switch (tree->tag) { - case TSet: { /* copy set */ - loopset(i, cs->cs[i] = treebuffer(tree)[i]); - return 1; - } - case TChar: { /* only one char */ - assert(0 <= tree->u.n && tree->u.n <= UCHAR_MAX); - loopset(i, cs->cs[i] = 0); /* erase all chars */ - setchar(cs->cs, tree->u.n); /* add that one */ - return 1; - } - case TAny: { - loopset(i, cs->cs[i] = 0xFF); /* add all characters to the set */ - return 1; - } - default: return 0; - } -} - - /* ** Visit a TCall node taking care to stop recursion. If node not yet ** visited, return 'f(sib2(tree))', otherwise return 'def' (default @@ -196,7 +115,7 @@ int hascaptures (TTree *tree) { int checkaux (TTree *tree, int pred) { tailcall: switch (tree->tag) { - case TChar: case TSet: case TAny: + case TChar: case TSet: case TAny: case TUTFR: case TFalse: case TOpenCall: return 0; /* not nullable */ case TRep: case TTrue: @@ -220,7 +139,7 @@ int checkaux (TTree *tree, int pred) { if (checkaux(sib2(tree), pred)) return 1; /* else return checkaux(sib1(tree), pred); */ tree = sib1(tree); goto tailcall; - case TCapture: case TGrammar: case TRule: + case TCapture: case TGrammar: case TRule: case TXInfo: /* return checkaux(sib1(tree), pred); */ tree = sib1(tree); goto tailcall; case TCall: /* return checkaux(sib2(tree), pred); */ @@ -239,11 +158,13 @@ int fixedlen (TTree *tree) { switch (tree->tag) { case TChar: case TSet: case TAny: return len + 1; + case TUTFR: + return (tree->cap == sib1(tree)->cap) ? len + tree->cap : -1; case TFalse: case TTrue: case TNot: case TAnd: case TBehind: return len; case TRep: case TRunTime: case TOpenCall: return -1; - case TCapture: case TRule: case TGrammar: + case TCapture: case TRule: case TGrammar: case TXInfo: /* return fixedlen(sib1(tree)); */ tree = sib1(tree); goto tailcall; case TCall: { @@ -294,18 +215,21 @@ int fixedlen (TTree *tree) { static int getfirst (TTree *tree, const Charset *follow, Charset *firstset) { tailcall: switch (tree->tag) { - case TChar: case TSet: case TAny: { + case TChar: case TSet: case TAny: case TFalse: { tocharset(tree, firstset); return 0; } + case TUTFR: { + int c; + clearset(firstset->cs); /* erase all chars */ + for (c = tree->key; c <= sib1(tree)->key; c++) + setchar(firstset->cs, c); + return 0; + } case TTrue: { loopset(i, firstset->cs[i] = follow->cs[i]); return 1; /* accepts the empty string */ } - case TFalse: { - loopset(i, firstset->cs[i] = 0); - return 0; - } case TChoice: { Charset csaux; int e1 = getfirst(sib1(tree), follow, firstset); @@ -334,7 +258,7 @@ static int getfirst (TTree *tree, const Charset *follow, Charset *firstset) { loopset(i, firstset->cs[i] |= follow->cs[i]); return 1; /* accept the empty string */ } - case TCapture: case TGrammar: case TRule: { + case TCapture: case TGrammar: case TRule: case TXInfo: { /* return getfirst(sib1(tree), follow, firstset); */ tree = sib1(tree); goto tailcall; } @@ -356,9 +280,8 @@ static int getfirst (TTree *tree, const Charset *follow, Charset *firstset) { if (tocharset(sib1(tree), firstset)) { cs_complement(firstset); return 1; - } - /* else go through */ - } + } /* else */ + } /* FALLTHROUGH */ case TBehind: { /* instruction gives no new information */ /* call 'getfirst' only to check for math-time captures */ int e = getfirst(sib1(tree), follow, firstset); @@ -380,9 +303,9 @@ static int headfail (TTree *tree) { case TChar: case TSet: case TAny: case TFalse: return 1; case TTrue: case TRep: case TRunTime: case TNot: - case TBehind: + case TBehind: case TUTFR: return 0; - case TCapture: case TGrammar: case TRule: case TAnd: + case TCapture: case TGrammar: case TRule: case TXInfo: case TAnd: tree = sib1(tree); goto tailcall; /* return headfail(sib1(tree)); */ case TCall: tree = sib2(tree); goto tailcall; /* return headfail(sib2(tree)); */ @@ -407,7 +330,7 @@ static int headfail (TTree *tree) { static int needfollow (TTree *tree) { tailcall: switch (tree->tag) { - case TChar: case TSet: case TAny: + case TChar: case TSet: case TAny: case TUTFR: case TFalse: case TTrue: case TAnd: case TNot: case TRunTime: case TGrammar: case TCall: case TBehind: return 0; @@ -418,7 +341,7 @@ static int needfollow (TTree *tree) { case TSeq: tree = sib2(tree); goto tailcall; default: assert(0); return 0; - } + } } /* }====================================================== */ @@ -437,10 +360,11 @@ static int needfollow (TTree *tree) { */ int sizei (const Instruction *i) { switch((Opcode)i->i.code) { - case ISet: case ISpan: return CHARSETINSTSIZE; - case ITestSet: return CHARSETINSTSIZE + 1; + case ISet: case ISpan: return 1 + i->i.aux2.set.size; + case ITestSet: return 2 + i->i.aux2.set.size; case ITestChar: case ITestAny: case IChoice: case IJmp: case ICall: case IOpenCall: case ICommit: case IPartialCommit: case IBackCommit: + case IUTFR: return 2; default: return 1; } @@ -468,23 +392,67 @@ static void codegen (CompileState *compst, TTree *tree, int opt, int tt, const Charset *fl); -void realloccode (lua_State *L, Pattern *p, int nsize) { - void *ud; - lua_Alloc f = lua_getallocf(L, &ud); - void *newblock = f(ud, p->code, p->codesize * sizeof(Instruction), - nsize * sizeof(Instruction)); - if (newblock == NULL && nsize > 0) +static void finishrelcode (lua_State *L, Pattern *p, Instruction *block, + int size) { + if (block == NULL) luaL_error(L, "not enough memory"); - p->code = (Instruction *)newblock; - p->codesize = nsize; + block->codesize = size; + p->code = (Instruction *)block + 1; } -static int nextinstruction (CompileState *compst) { - int size = compst->p->codesize; - if (compst->ncode >= size) - realloccode(compst->L, compst->p, size * 2); - return compst->ncode++; +/* +** Initialize array 'p->code' +*/ +static void newcode (lua_State *L, Pattern *p, int size) { + void *ud; + Instruction *block; + lua_Alloc f = lua_getallocf(L, &ud); + size++; /* slot for 'codesize' */ + block = (Instruction*) f(ud, NULL, 0, size * sizeof(Instruction)); + finishrelcode(L, p, block, size); +} + + +void freecode (lua_State *L, Pattern *p) { + if (p->code != NULL) { + void *ud; + lua_Alloc f = lua_getallocf(L, &ud); + uint osize = p->code[-1].codesize; + f(ud, p->code - 1, osize * sizeof(Instruction), 0); /* free block */ + } +} + + +/* +** Assume that 'nsize' is not zero and that 'p->code' already exists. +*/ +static void realloccode (lua_State *L, Pattern *p, int nsize) { + void *ud; + lua_Alloc f = lua_getallocf(L, &ud); + Instruction *block = p->code - 1; + uint osize = block->codesize; + nsize++; /* add the 'codesize' slot to size */ + block = (Instruction*) f(ud, block, osize * sizeof(Instruction), + nsize * sizeof(Instruction)); + finishrelcode(L, p, block, nsize); +} + + +/* +** Add space for an instruction with 'n' slots and return its index. +*/ +static int nextinstruction (CompileState *compst, int n) { + int size = compst->p->code[-1].codesize - 1; + int ncode = compst->ncode; + if (ncode > size - n) { + uint nsize = size + (size >> 1) + n; + if (nsize >= INT_MAX) + luaL_error(compst->L, "pattern code too large"); + realloccode(compst->L, compst->p, nsize); + } + compst->ncode = ncode + n; + return ncode; } @@ -492,9 +460,9 @@ static int nextinstruction (CompileState *compst) { static int addinstruction (CompileState *compst, Opcode op, int aux) { - int i = nextinstruction(compst); + int i = nextinstruction(compst, 1); getinstr(compst, i).i.code = op; - getinstr(compst, i).i.aux = aux; + getinstr(compst, i).i.aux1 = aux; return i; } @@ -518,6 +486,16 @@ static void setoffset (CompileState *compst, int instruction, int offset) { } +static void codeutfr (CompileState *compst, TTree *tree) { + int i = addoffsetinst(compst, IUTFR); + int to = sib1(tree)->u.n; + assert(sib1(tree)->tag == TXInfo); + getinstr(compst, i + 1).offset = tree->u.n; + getinstr(compst, i).i.aux1 = to & 0xff; + getinstr(compst, i).i.aux2.key = to >> 8; +} + + /* ** Add a capture instruction: ** 'op' is the capture instruction; 'cap' the capture kind; @@ -527,7 +505,7 @@ static void setoffset (CompileState *compst, int instruction, int offset) { static int addinstcap (CompileState *compst, Opcode op, int cap, int key, int aux) { int i = addinstruction(compst, op, joinkindoff(cap, aux)); - getinstr(compst, i).i.key = key; + getinstr(compst, i).i.aux2.key = key; return i; } @@ -560,7 +538,7 @@ static void jumptohere (CompileState *compst, int instruction) { */ static void codechar (CompileState *compst, int c, int tt) { if (tt >= 0 && getinstr(compst, tt).i.code == ITestChar && - getinstr(compst, tt).i.aux == c) + getinstr(compst, tt).i.aux1 == c) addinstruction(compst, IAny, 0); else addinstruction(compst, IChar, c); @@ -568,45 +546,63 @@ static void codechar (CompileState *compst, int c, int tt) { /* -** Add a charset posfix to an instruction +** Add a charset posfix to an instruction. */ -static void addcharset (CompileState *compst, const byte *cs) { - int p = gethere(compst); +static void addcharset (CompileState *compst, int inst, charsetinfo *info) { + int p; + Instruction *I = &getinstr(compst, inst); + byte *charset; + int isize = instsize(info->size); /* size in instructions */ int i; - for (i = 0; i < (int)CHARSETINSTSIZE - 1; i++) - nextinstruction(compst); /* space for buffer */ - /* fill buffer with charset */ - loopset(j, getinstr(compst, p).buff[j] = cs[j]); + I->i.aux2.set.offset = info->offset * 8; /* offset in bits */ + I->i.aux2.set.size = isize; + I->i.aux1 = info->deflt; + p = nextinstruction(compst, isize); /* space for charset */ + charset = getinstr(compst, p).buff; /* charset buffer */ + for (i = 0; i < isize * (int)sizeof(Instruction); i++) + charset[i] = getbytefromcharset(info, i); /* copy the buffer */ } /* -** code a char set, optimizing unit sets for IChar, "complete" -** sets for IAny, and empty sets for IFail; also use an IAny -** when instruction is dominated by an equivalent test. +** Check whether charset 'info' is dominated by instruction 'p' */ -static void codecharset (CompileState *compst, const byte *cs, int tt) { - int c = 0; /* (=) to avoid warnings */ - Opcode op = charsettype(cs, &c); - switch (op) { - case IChar: codechar(compst, c, tt); break; - case ISet: { /* non-trivial set? */ - if (tt >= 0 && getinstr(compst, tt).i.code == ITestSet && - cs_equal(cs, getinstr(compst, tt + 2).buff)) - addinstruction(compst, IAny, 0); - else { - addinstruction(compst, ISet, 0); - addcharset(compst, cs); - } - break; +static int cs_equal (Instruction *p, charsetinfo *info) { + if (p->i.code != ITestSet) + return 0; + else if (p->i.aux2.set.offset != info->offset * 8 || + p->i.aux2.set.size != instsize(info->size) || + p->i.aux1 != info->deflt) + return 0; + else { + int i; + for (i = 0; i < instsize(info->size) * (int)sizeof(Instruction); i++) { + if ((p + 2)->buff[i] != getbytefromcharset(info, i)) + return 0; } - default: addinstruction(compst, op, c); break; + } + return 1; +} + + +/* +** Code a char set, using IAny when instruction is dominated by an +** equivalent test. +*/ +static void codecharset (CompileState *compst, TTree *tree, int tt) { + charsetinfo info; + tree2cset(tree, &info); + if (tt >= 0 && cs_equal(&getinstr(compst, tt), &info)) + addinstruction(compst, IAny, 0); + else { + int i = addinstruction(compst, ISet, 0); + addcharset(compst, i, &info); } } /* -** code a test set, optimizing unit sets for ITestChar, "complete" +** Code a test set, optimizing unit sets for ITestChar, "complete" ** sets for ITestAny, and empty sets for IJmp (always fails). ** 'e' is true iff test should accept the empty string. (Test ** instructions in the current VM never accept the empty string.) @@ -614,22 +610,22 @@ static void codecharset (CompileState *compst, const byte *cs, int tt) { static int codetestset (CompileState *compst, Charset *cs, int e) { if (e) return NOINST; /* no test */ else { - int c = 0; - Opcode op = charsettype(cs->cs, &c); + charsetinfo info; + Opcode op = charsettype(cs->cs, &info); switch (op) { case IFail: return addoffsetinst(compst, IJmp); /* always jump */ case IAny: return addoffsetinst(compst, ITestAny); case IChar: { int i = addoffsetinst(compst, ITestChar); - getinstr(compst, i).i.aux = c; + getinstr(compst, i).i.aux1 = info.offset; return i; } - case ISet: { + default: { /* regular set */ int i = addoffsetinst(compst, ITestSet); - addcharset(compst, cs->cs); + addcharset(compst, i, &info); + assert(op == ISet); return i; } - default: assert(0); return 0; } } } @@ -665,11 +661,11 @@ static void codebehind (CompileState *compst, TTree *tree) { /* ** Choice; optimizations: -** - when p1 is headfail or -** when first(p1) and first(p2) are disjoint, than -** a character not in first(p1) cannot go to p1, and a character -** in first(p1) cannot go to p2 (at it is not in first(p2)). -** (The optimization is not valid if p1 accepts the empty string, +** - when p1 is headfail or when first(p1) and first(p2) are disjoint, +** than a character not in first(p1) cannot go to p1 and a character +** in first(p1) cannot go to p2, either because p1 will accept +** (headfail) or because it is not in first(p2) (disjoint). +** (The second case is not valid if p1 accepts the empty string, ** as then there is no character at all...) ** - when p2 is empty and opt is true; a IPartialCommit can reuse ** the Choice already active in the stack. @@ -686,7 +682,7 @@ static void codechoice (CompileState *compst, TTree *p1, TTree *p2, int opt, int jmp = NOINST; codegen(compst, p1, 0, test, fl); if (!emptyp2) - jmp = addoffsetinst(compst, IJmp); + jmp = addoffsetinst(compst, IJmp); jumptohere(compst, test); codegen(compst, p2, opt, NOINST, fl); jumptohere(compst, jmp); @@ -697,7 +693,7 @@ static void codechoice (CompileState *compst, TTree *p1, TTree *p2, int opt, codegen(compst, p1, 1, NOINST, fullset); } else { - /* == + /* == test(first(p1)) -> L1; choice L1; ; commit L2; L1: ; L2: */ int pcommit; int test = codetestset(compst, &cs1, e1); @@ -764,9 +760,53 @@ static void coderuntime (CompileState *compst, TTree *tree, int tt) { } +/* +** Create a jump to 'test' and fix 'test' to jump to next instruction +*/ +static void closeloop (CompileState *compst, int test) { + int jmp = addoffsetinst(compst, IJmp); + jumptohere(compst, test); + jumptothere(compst, jmp, test); +} + + +/* +** Try repetition of charsets: +** For an empty set, repetition of fail is a no-op; +** For any or char, code a tight loop; +** For generic charset, use a span instruction. +*/ +static int coderepcharset (CompileState *compst, TTree *tree) { + switch (tree->tag) { + case TFalse: return 1; /* 'fail*' is a no-op */ + case TAny: { /* L1: testany -> L2; any; jmp L1; L2: */ + int test = addoffsetinst(compst, ITestAny); + addinstruction(compst, IAny, 0); + closeloop(compst, test); + return 1; + } + case TChar: { /* L1: testchar c -> L2; any; jmp L1; L2: */ + int test = addoffsetinst(compst, ITestChar); + getinstr(compst, test).i.aux1 = tree->u.n; + addinstruction(compst, IAny, 0); + closeloop(compst, test); + return 1; + } + case TSet: { /* regular set */ + charsetinfo info; + int i = addinstruction(compst, ISpan, 0); + tree2cset(tree, &info); + addcharset(compst, i, &info); + return 1; + } + default: return 0; /* not a charset */ + } +} + + /* ** Repetion; optimizations: -** When pattern is a charset, can use special instruction ISpan. +** When pattern is a charset, use special code. ** When pattern is head fail, or if it starts with characters that ** are disjoint from what follows the repetions, a simple test ** is enough (a fail inside the repetition would backtrack to fail @@ -776,21 +816,14 @@ static void coderuntime (CompileState *compst, TTree *tree, int tt) { */ static void coderep (CompileState *compst, TTree *tree, int opt, const Charset *fl) { - Charset st; - if (tocharset(tree, &st)) { - addinstruction(compst, ISpan, 0); - addcharset(compst, st.cs); - } - else { + if (!coderepcharset(compst, tree)) { + Charset st; int e1 = getfirst(tree, fullset, &st); if (headfail(tree) || (!e1 && cs_disjoint(&st, fl))) { /* L1: test (fail(p1)) -> L2;

; jmp L1; L2: */ - int jmp; int test = codetestset(compst, &st, 0); codegen(compst, tree, 0, test, fullset); - jmp = addoffsetinst(compst, IJmp); - jumptohere(compst, test); - jumptothere(compst, jmp, test); + closeloop(compst, test); } else { /* test(fail(p1)) -> L2; choice L2; L1:

; partialcommit L1; L2: */ @@ -847,7 +880,7 @@ static void correctcalls (CompileState *compst, int *positions, Instruction *code = compst->p->code; for (i = from; i < to; i += sizei(&code[i])) { if (code[i].i.code == IOpenCall) { - int n = code[i].i.key; /* rule number */ + int n = code[i].i.aux2.key; /* rule number */ int rule = positions[n]; /* rule position */ assert(rule == from || code[rule - 1].i.code == IRet); if (code[finaltarget(code, i + 2)].i.code == IRet) /* call; ret ? */ @@ -874,8 +907,10 @@ static void codegrammar (CompileState *compst, TTree *grammar) { int start = gethere(compst); /* here starts the initial rule */ jumptohere(compst, firstcall); for (rule = sib1(grammar); rule->tag == TRule; rule = sib2(rule)) { + TTree *r = sib1(rule); + assert(r->tag == TXInfo); positions[rulenumber++] = gethere(compst); /* save rule position */ - codegen(compst, sib1(rule), 0, NOINST, fullset); /* code rule */ + codegen(compst, sib1(r), 0, NOINST, fullset); /* code rule */ addinstruction(compst, IRet, 0); } assert(rule->tag == TTrue); @@ -886,8 +921,8 @@ static void codegrammar (CompileState *compst, TTree *grammar) { static void codecall (CompileState *compst, TTree *call) { int c = addoffsetinst(compst, IOpenCall); /* to be corrected later */ - getinstr(compst, c).i.key = sib2(call)->cap; /* rule number */ - assert(sib2(call)->tag == TRule); + assert(sib1(sib2(call))->tag == TXInfo); + getinstr(compst, c).i.aux2.key = sib1(sib2(call))->u.n; /* rule number */ } @@ -922,9 +957,10 @@ static void codegen (CompileState *compst, TTree *tree, int opt, int tt, switch (tree->tag) { case TChar: codechar(compst, tree->u.n, tt); break; case TAny: addinstruction(compst, IAny, 0); break; - case TSet: codecharset(compst, treebuffer(tree), tt); break; + case TSet: codecharset(compst, tree, tt); break; case TTrue: break; case TFalse: addinstruction(compst, IFail, 0); break; + case TUTFR: codeutfr(compst, tree); break; case TChoice: codechoice(compst, sib1(tree), sib2(tree), opt, fl); break; case TRep: coderep(compst, sib1(tree), opt, fl); break; case TBehind: codebehind(compst, tree); break; @@ -971,7 +1007,7 @@ static void peephole (CompileState *compst) { case IRet: case IFail: case IFailTwice: case IEnd: { /* instructions with unconditional implicit jumps */ code[i] = code[ft]; /* jump becomes that instruction */ - code[i + 1].i.code = IAny; /* 'no-op' for target position */ + code[i + 1].i.code = IEmpty; /* 'no-op' for target position */ break; } case ICommit: case IPartialCommit: @@ -996,12 +1032,13 @@ static void peephole (CompileState *compst) { /* -** Compile a pattern +** Compile a pattern. 'size' is the size of the pattern's tree, +** which gives a hint for the size of the final code. */ -Instruction *compile (lua_State *L, Pattern *p) { +Instruction *compile (lua_State *L, Pattern *p, uint size) { CompileState compst; compst.p = p; compst.ncode = 0; compst.L = L; - realloccode(L, p, 2); /* minimum initial size */ + newcode(L, p, size/2u + 2); /* set initial size */ codegen(&compst, p->tree, 0, NOINST, fullset); addinstruction(&compst, IEnd, 0); realloccode(L, p, compst.ncode); /* set final size */ diff --git a/3rd/lpeg/lpcode.h b/3rd/lpeg/lpcode.h index 34ee2763..10c2ced5 100644 --- a/3rd/lpeg/lpcode.h +++ b/3rd/lpeg/lpcode.h @@ -1,6 +1,3 @@ -/* -** $Id: lpcode.h $ -*/ #if !defined(lpcode_h) #define lpcode_h @@ -11,13 +8,12 @@ #include "lptree.h" #include "lpvm.h" -int tocharset (TTree *tree, Charset *cs); int checkaux (TTree *tree, int pred); int fixedlen (TTree *tree); int hascaptures (TTree *tree); int lp_gc (lua_State *L); -Instruction *compile (lua_State *L, Pattern *p); -void realloccode (lua_State *L, Pattern *p, int nsize); +Instruction *compile (lua_State *L, Pattern *p, uint size); +void freecode (lua_State *L, Pattern *p); int sizei (const Instruction *i); diff --git a/3rd/lpeg/lpcset.c b/3rd/lpeg/lpcset.c new file mode 100644 index 00000000..2dcffd9a --- /dev/null +++ b/3rd/lpeg/lpcset.c @@ -0,0 +1,110 @@ + +#include "lptypes.h" +#include "lpcset.h" + + +/* +** Add to 'c' the index of the (only) bit set in byte 'b' +*/ +static int onlybit (int c, int b) { + if ((b & 0xF0) != 0) { c += 4; b >>= 4; } + if ((b & 0x0C) != 0) { c += 2; b >>= 2; } + if ((b & 0x02) != 0) { c += 1; } + return c; +} + + +/* +** Check whether a charset is empty (returns IFail), singleton (IChar), +** full (IAny), or none of those (ISet). When singleton, 'info.offset' +** returns which character it is. When generic set, 'info' returns +** information about its range. +*/ +Opcode charsettype (const byte *cs, charsetinfo *info) { + int low0, low1, high0, high1; + for (low1 = 0; low1 < CHARSETSIZE && cs[low1] == 0; low1++) + /* find lowest byte with a 1-bit */; + if (low1 == CHARSETSIZE) + return IFail; /* no characters in set */ + for (high1 = CHARSETSIZE - 1; cs[high1] == 0; high1--) + /* find highest byte with a 1-bit; low1 is a sentinel */; + if (low1 == high1) { /* only one byte with 1-bits? */ + int b = cs[low1]; + if ((b & (b - 1)) == 0) { /* does byte has only one 1-bit? */ + info->offset = onlybit(low1 * BITSPERCHAR, b); /* get that bit */ + return IChar; /* single character */ + } + } + for (low0 = 0; low0 < CHARSETSIZE && cs[low0] == 0xFF; low0++) + /* find lowest byte with a 0-bit */; + if (low0 == CHARSETSIZE) + return IAny; /* set has all bits set */ + for (high0 = CHARSETSIZE - 1; cs[high0] == 0xFF; high0--) + /* find highest byte with a 0-bit; low0 is a sentinel */; + if (high1 - low1 <= high0 - low0) { /* range of 1s smaller than of 0s? */ + info->offset = low1; + info->size = high1 - low1 + 1; + info->deflt = 0; /* all discharged bits were 0 */ + } + else { + info->offset = low0; + info->size = high0 - low0 + 1; + info->deflt = 0xFF; /* all discharged bits were 1 */ + } + info->cs = cs + info->offset; + return ISet; +} + + +/* +** Get a byte from a compact charset. If index is inside the charset +** range, get the byte from the supporting charset (correcting it +** by the offset). Otherwise, return the default for the set. +*/ +byte getbytefromcharset (const charsetinfo *info, int index) { + if (index < info->size) + return info->cs[index]; + else return info->deflt; +} + + +/* +** If 'tree' is a 'char' pattern (TSet, TChar, TAny, TFalse), convert it +** into a charset and return 1; else return 0. +*/ +int tocharset (TTree *tree, Charset *cs) { + switch (tree->tag) { + case TChar: { /* only one char */ + assert(0 <= tree->u.n && tree->u.n <= UCHAR_MAX); + clearset(cs->cs); /* erase all chars */ + setchar(cs->cs, tree->u.n); /* add that one */ + return 1; + } + case TAny: { + fillset(cs->cs, 0xFF); /* add all characters to the set */ + return 1; + } + case TFalse: { + clearset(cs->cs); /* empty set */ + return 1; + } + case TSet: { /* fill set */ + int i; + fillset(cs->cs, tree->u.set.deflt); + for (i = 0; i < tree->u.set.size; i++) + cs->cs[tree->u.set.offset + i] = treebuffer(tree)[i]; + return 1; + } + default: return 0; + } +} + + +void tree2cset (TTree *tree, charsetinfo *info) { + assert(tree->tag == TSet); + info->offset = tree->u.set.offset; + info->size = tree->u.set.size; + info->deflt = tree->u.set.deflt; + info->cs = treebuffer(tree); +} + diff --git a/3rd/lpeg/lpcset.h b/3rd/lpeg/lpcset.h new file mode 100644 index 00000000..b69fef90 --- /dev/null +++ b/3rd/lpeg/lpcset.h @@ -0,0 +1,30 @@ + +#if !defined(lpset_h) +#define lpset_h + +#include "lpcset.h" +#include "lpcode.h" +#include "lptree.h" + + +/* +** Extra information for the result of 'charsettype'. When result is +** IChar, 'offset' is the character. When result is ISet, 'cs' is the +** supporting bit array (with offset included), 'offset' is the offset +** (in bytes), 'size' is the size (in bytes), and 'delt' is the default +** value for bytes outside the set. +*/ +typedef struct { + const byte *cs; + int offset; + int size; + int deflt; +} charsetinfo; + + +int tocharset (TTree *tree, Charset *cs); +Opcode charsettype (const byte *cs, charsetinfo *info); +byte getbytefromcharset (const charsetinfo *info, int index); +void tree2cset (TTree *tree, charsetinfo *info); + +#endif diff --git a/3rd/lpeg/lpeg.html b/3rd/lpeg/lpeg.html index 8b9f59c2..b31d5759 100644 --- a/3rd/lpeg/lpeg.html +++ b/3rd/lpeg/lpeg.html @@ -1,28 +1,27 @@ - + "//www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> + LPeg - Parsing Expression Grammars For Lua -

LPeg
- Parsing Expression Grammars For Lua, version 1.0 + Parsing Expression Grammars For Lua, version 1.1
@@ -56,16 +55,16 @@

LPeg is a new pattern-matching library for Lua, based on - + Parsing Expression Grammars (PEGs). This text is a reference manual for the library. For a more formal treatment of LPeg, as well as some discussion about its implementation, see - + A Text Pattern-Matching Tool based on Parsing Expression Grammars. (You may also be interested in my -talk about LPeg +talk about LPeg given at the III Lua Workshop.)

@@ -107,6 +106,9 @@ for creating patterns: Matches any character in string (Set) lpeg.R("xy") Matches any character between x and y (Range) +lpeg.utfR(cp1, cp2) + Matches an UTF-8 code point between cp1 and + cp2 patt^n Matches at least n repetitions of patt patt^-n @@ -142,7 +144,7 @@ so, it succeeds only at the end of the subject. LPeg also offers the re module, which implements patterns following a regular-expression style (e.g., [09]+). -(This module is 260 lines of Lua code, +(This module is 270 lines of Lua code, and of course it uses LPeg to parse regular expressions and translate them to regular LPeg patterns.)

@@ -164,7 +166,7 @@ or the captured values

An optional numeric argument init makes the match start at that position in the subject string. -As usual in Lua libraries, +As in the Lua standard libraries, a negative value counts from the end.

@@ -188,9 +190,9 @@ returns the string "pattern". Otherwise returns nil.

-

lpeg.version ()

+

lpeg.version

-Returns a string with the running version of LPeg. +A string (not a function) with the running version of LPeg.

lpeg.setmaxstack (max)

@@ -329,6 +331,15 @@ are patterns that always fail.

+

lpeg.utfR (cp1, cp2)

+

+Returns a pattern that matches a valid UTF-8 byte sequence +representing a code point in the range [cp1, cp2]. +The range is limited by the natural Unicode limit of 0x10FFFF, +but may include surrogates. +

+ +

lpeg.V (v)

This operation creates a non-terminal (a variable) @@ -433,7 +444,8 @@ letter = lower + upper

patt1 - patt2

-Returns a pattern equivalent to !patt2 patt1. +Returns a pattern equivalent to !patt2 patt1 +in the origial PEG notation. This pattern asserts that the input does not match patt2 and then matches patt1.

@@ -597,17 +609,17 @@ The following table summarizes the basic captures: lpeg.Carg(n) the value of the nth extra argument to lpeg.match (matches the empty string) -lpeg.Cb(name) +lpeg.Cb(key) the values produced by the previous - group capture named name + group capture named key (matches the empty string) lpeg.Cc(values) the given values (matches the empty string) lpeg.Cf(patt, func) - a folding of the captures from patt -lpeg.Cg(patt [, name]) + folding capture (deprecated) +lpeg.Cg(patt [, key]) the values produced by patt, - optionally tagged with name + optionally tagged with key lpeg.Cp() the current position (matches the empty string) lpeg.Cs(patt) @@ -627,6 +639,11 @@ or no value when number is zero. patt / function the returns of function applied to the captures of patt +patt % function + produces no value; + it accummulates the captures from patt + into the previous capture through function + lpeg.Cmt(patt, function) the returns of function applied to the captures of patt; the application is done at match time @@ -652,10 +669,10 @@ LPeg does not specify when (and if) it evaluates its captures. consider the pattern lpeg.P"a" / func / 0. Because the "division" by 0 instructs LPeg to throw away the results from the pattern, -LPeg may or may not call func.) +it is not specified whether LPeg will call func.) Therefore, captures should avoid side effects. Moreover, -most captures cannot affect the way a pattern matches a subject. +captures cannot affect the way a pattern matches a subject. The only exception to this rule is the so-called match-time capture. When a match-time capture matches, @@ -684,24 +701,25 @@ argument given in the call to lpeg.match.

-

lpeg.Cb (name)

+

lpeg.Cb (key)

Creates a back capture. This pattern matches the empty string and produces the values produced by the most recent -group capture named name -(where name can be any Lua value). +group capture named key +(where key can be any Lua value).

Most recent means the last complete outermost -group capture with the given name. +group capture with the given key. A Complete capture means that the entire pattern -corresponding to the capture has matched. +corresponding to the capture has matched; +in other words, the back capture is not nested inside the group. An Outermost capture means that the capture is not inside -another complete capture. +another complete capture that does not contain the back capture itself.

@@ -722,61 +740,21 @@ produces all given values as its captured values.

lpeg.Cf (patt, func)

Creates a fold capture. -If patt produces a list of captures -C1 C2 ... Cn, -this capture will produce the value -func(...func(func(C1, C2), C3)..., - Cn), -that is, it will fold -(or accumulate, or reduce) -the captures from patt using function func. -

- -

-This capture assumes that patt should produce -at least one capture with at least one value (of any type), -which becomes the initial value of an accumulator. -(If you need a specific initial value, -you may prefix a constant capture to patt.) -For each subsequent capture, -LPeg calls func -with this accumulator as the first argument and all values produced -by the capture as extra arguments; -the first result from this call -becomes the new value for the accumulator. -The final value of the accumulator becomes the captured value. -

- -

-As an example, -the following pattern matches a list of numbers separated -by commas and returns their addition: -

-
--- matches a numeral and captures its numerical value
-number = lpeg.R"09"^1 / tonumber
-
--- matches a list of numbers, capturing their values
-list = number * ("," * number)^0
-
--- auxiliary function to add two numbers
-function add (acc, newvalue) return acc + newvalue end
-
--- folds the list of numbers adding them
-sum = lpeg.Cf(list, add)
-
--- example of use
-print(sum:match("10,30,43"))   --> 83
-
+This construction is deprecated; +use an accumulator pattern instead. +In general, a fold like +lpeg.Cf(p1 * p2^0, func) +can be translated to +(p1 * (p2 % func)^0). -

lpeg.Cg (patt [, name])

+

lpeg.Cg (patt [, key])

Creates a group capture. It groups all values returned by patt into a single capture. -The group may be anonymous (if no name is given) -or named with the given name +The group may be anonymous (if no key is given) +or named with the given key (which can be any non-nil Lua value).

@@ -822,7 +800,7 @@ starting at 1. Moreover, for each named capture group created by patt, the first value of the group is put into the table -with the group name as its key. +with the group key as its key. The captured value is only the table.

@@ -878,6 +856,98 @@ there is no captured value.

+

patt % function

+

+Creates an accumulator capture. +This pattern behaves similarly to a +function capture, +with the following differences: +The last captured value before patt +is added as a first argument to the call; +the return of the function is adjusted to one single value; +that value replaces the last captured value. +Note that the capture itself produces no values; +it only changes the value of its previous capture. +

+ +

+As an example, +let us consider the problem of adding a list of numbers. +

+
+-- matches a numeral and captures its numerical value
+number = lpeg.R"09"^1 / tonumber
+
+-- auxiliary function to add two numbers
+function add (acc, newvalue) return acc + newvalue end
+
+-- matches a list of numbers, adding their values
+sum = number * ("," * number % add)^0
+
+-- example of use
+print(sum:match("10,30,43"))   --> 83
+
+

+First, the initial number captures a number; +that first capture will play the role of an accumulator. +Then, each time the sequence comma-number +matches inside the loop there is an accumulator capture: +It calls add with the current value of the +accumulator—which is the last captured value, created by the +first number— and the value of the new number, +and the result of the call (the sum of the two numbers) +replaces the value of the accumulator. +At the end of the match, +the accumulator with all sums is the final value. +

+ +

+As another example, +consider the following code fragment: +

+
+local name = lpeg.C(lpeg.R("az")^1)
+local p = name * (lpeg.P("^") % string.upper)^-1
+print(p:match("count"))    --> count
+print(p:match("count^"))   --> COUNT
+
+

+In the match against "count", +as there is no "^", +the optional accumulator capture does not match; +so, the match results in its sole capture, a name. +In the match against "count^", +the accumulator capture matches, +so the function string.upper +is called with the previous captured value (created by name) +plus the string "^"; +the function ignores its second argument and returns the first argument +changed to upper case; +that value then becomes the first and only +capture value created by the match. +

+ +

+Due to the nature of this capture, +you should avoid using it in places where it is not clear +what is the "previous" capture, +such as directly nested in a string capture +or a numbered capture. +(Note that these captures may not need to evaluate +all their subcaptures to compute their results.) +Moreover, due to implementation details, +you should not use this capture directly nested in a +substitution capture. +You should also avoid a direct nesting of this capture inside +a folding capture (deprecated), +as the folding will try to fold each individual accumulator capture. +A simple and effective way to avoid all these issues is +to enclose the whole accumulation composition +(including the capture that generates the initial value) +into an anonymous group capture. +

+ +

lpeg.Cmt(patt, function)

Creates a match-time capture. @@ -930,9 +1000,9 @@ local lpeg = require "lpeg" -- matches a word followed by end-of-string p = lpeg.R"az"^1 * -1 -print(p:match("hello")) --> 6 -print(lpeg.match(p, "hello")) --> 6 -print(p:match("1 hello")) --> nil +print(p:match("hello")) --> 6 +print(lpeg.match(p, "hello")) --> 6 +print(p:match("1 hello")) --> nil

The pattern is simply a sequence of one or more lower-case letters @@ -957,19 +1027,18 @@ lpeg.locale(lpeg) -- adds locale entries into 'lpeg' table local space = lpeg.space^0 local name = lpeg.C(lpeg.alpha^1) * space local sep = lpeg.S(",;") * space -local pair = lpeg.Cg(name * "=" * space * name) * sep^-1 -local list = lpeg.Cf(lpeg.Ct("") * pair^0, rawset) -t = list:match("a=b, c = hi; next = pi") --> { a = "b", c = "hi", next = "pi" } +local pair = name * "=" * space * name * sep^-1 +local list = lpeg.Ct("") * (pair % rawset)^0 +t = list:match("a=b, c = hi; next = pi") + --> { a = "b", c = "hi", next = "pi" }

Each pair has the format name = name followed by an optional separator (a comma or a semicolon). -The pair pattern encloses the pair in a group pattern, -so that the names become the values of a single capture. -The list pattern then folds these captures. +The list pattern then folds these captures. It starts with an empty table, created by a table capture matching an empty string; -then for each capture (a pair of names) it applies rawset +then for each a pair of names it applies rawset over the accumulator (the table) and the capture values (the pair of names). rawset returns the table itself, so the accumulator is always the table. @@ -1003,7 +1072,7 @@ by sep. If the split results in too many values, it may overflow the maximum number of values that can be returned by a Lua function. -In this case, +To avoid this problem, we can collect these values in a table:

@@ -1039,7 +1108,7 @@ end
 

This grammar has a straight reading: -it matches p or skips one character and tries again. +its sole rule matches p or skips one character and tries again.

@@ -1048,27 +1117,27 @@ If we want to know where the pattern is in the string we can add position captures to the pattern:

-local I = lpeg.Cp()
+local Cp = lpeg.Cp()
 function anywhere (p)
-  return lpeg.P{ I * p * I + 1 * lpeg.V(1) }
+  return lpeg.P{ Cp * p * Cp + 1 * lpeg.V(1) }
 end
 
-print(anywhere("world"):match("hello world!"))   -> 7   12
+print(anywhere("world"):match("hello world!"))   --> 7   12
 

Another option for the search is like this:

-local I = lpeg.Cp()
+local Cp = lpeg.Cp()
 function anywhere (p)
-  return (1 - lpeg.P(p))^0 * I * p * I
+  return (1 - lpeg.P(p))^0 * Cp * p * Cp
 end
 

Again the pattern has a straight reading: it skips as many characters as possible while not matching p, -and then matches p (plus appropriate captures). +and then matches p plus appropriate captures.

@@ -1163,91 +1232,6 @@ local record = lpeg.Ct(field * (',' * field)^0) * (lpeg.P'\n' + -1) -

UTF-8 and Latin 1

-

-It is not difficult to use LPeg to convert a string from -UTF-8 encoding to Latin 1 (ISO 8859-1): -

- -
--- convert a two-byte UTF-8 sequence to a Latin 1 character
-local function f2 (s)
-  local c1, c2 = string.byte(s, 1, 2)
-  return string.char(c1 * 64 + c2 - 12416)
-end
-
-local utf8 = lpeg.R("\0\127")
-           + lpeg.R("\194\195") * lpeg.R("\128\191") / f2
-
-local decode_pattern = lpeg.Cs(utf8^0) * -1
-
-

-In this code, -the definition of UTF-8 is already restricted to the -Latin 1 range (from 0 to 255). -Any encoding outside this range (as well as any invalid encoding) -will not match that pattern. -

- -

-As the definition of decode_pattern demands that -the pattern matches the whole input (because of the -1 at its end), -any invalid string will simply fail to match, -without any useful information about the problem. -We can improve this situation redefining decode_pattern -as follows: -

-
-local function er (_, i) error("invalid encoding at position " .. i) end
-
-local decode_pattern = lpeg.Cs(utf8^0) * (-1 + lpeg.P(er))
-
-

-Now, if the pattern utf8^0 stops -before the end of the string, -an appropriate error function is called. -

- - -

UTF-8 and Unicode

-

-We can extend the previous patterns to handle all Unicode code points. -Of course, -we cannot translate them to Latin 1 or any other one-byte encoding. -Instead, our translation results in a array with the code points -represented as numbers. -The full code is here: -

-
--- decode a two-byte UTF-8 sequence
-local function f2 (s)
-  local c1, c2 = string.byte(s, 1, 2)
-  return c1 * 64 + c2 - 12416
-end
-
--- decode a three-byte UTF-8 sequence
-local function f3 (s)
-  local c1, c2, c3 = string.byte(s, 1, 3)
-  return (c1 * 64 + c2) * 64 + c3 - 925824
-end
-
--- decode a four-byte UTF-8 sequence
-local function f4 (s)
-  local c1, c2, c3, c4 = string.byte(s, 1, 4)
-  return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168
-end
-
-local cont = lpeg.R("\128\191")   -- continuation byte
-
-local utf8 = lpeg.R("\0\127") / string.byte
-           + lpeg.R("\194\223") * cont / f2
-           + lpeg.R("\224\239") * cont * cont / f3
-           + lpeg.R("\240\244") * cont * cont * cont / f4
-
-local decode_pattern = lpeg.Ct(utf8^0) * -1
-
- -

Lua's long strings

A long string in Lua starts with the pattern [=*[ @@ -1347,7 +1331,7 @@ function evalExp (s) end -- small example -print(evalExp"3 + 5*9 / (1+1) - 12") --> 13.5 +print(evalExp"3 + 5*9 / (1+1) - 12") --> 13.5

@@ -1369,16 +1353,16 @@ end -- Grammar local V = lpeg.V G = lpeg.P{ "Exp", - Exp = lpeg.Cf(V"Term" * lpeg.Cg(TermOp * V"Term")^0, eval); - Term = lpeg.Cf(V"Factor" * lpeg.Cg(FactorOp * V"Factor")^0, eval); + Exp = V"Term" * (TermOp * V"Term" % eval)^0; + Term = V"Factor" * (FactorOp * V"Factor" % eval)^0; Factor = Number / tonumber + Open * V"Exp" * Close; } -- small example -print(lpeg.match(G, "3 + 5*9 / (1+1) - 12")) --> 13.5 +print(lpeg.match(G, "3 + 5*9 / (1+1) - 12")) --> 13.5

-Note the use of the fold (accumulator) capture. +Note the use of the accumulator capture. To compute the value of an expression, the accumulator starts with the value of the first term, and then applies eval over @@ -1391,13 +1375,20 @@ and the new term for each repetition.

Download

LPeg -source code.

+source code.

+ +

+Probably, the easiest way to install LPeg is with +LuaRocks. +If you have LuaRocks installed, +the following command is all you need to install LPeg: +

$ luarocks install lpeg

License

-Copyright © 2007-2019 Lua.org, PUC-Rio. +Copyright © 2007-2023 Lua.org, PUC-Rio.

Permission is hereby granted, free of charge, diff --git a/3rd/lpeg/lpprint.c b/3rd/lpeg/lpprint.c index df62cbee..da902e6f 100644 --- a/3rd/lpeg/lpprint.c +++ b/3rd/lpeg/lpprint.c @@ -1,7 +1,3 @@ -/* -** $Id: lpprint.c $ -** Copyright 2007, Lua.org & PUC-Rio (see 'lpeg.html' for license) -*/ #include #include @@ -27,7 +23,7 @@ void printcharset (const byte *st) { printf("["); for (i = 0; i <= UCHAR_MAX; i++) { int first = i; - while (testchar(st, i) && i <= UCHAR_MAX) i++; + while (i <= UCHAR_MAX && testchar(st, i)) i++; if (i - 1 == first) /* unary range? */ printf("(%02x)", first); else if (i - 1 > first) /* non-empty range? */ @@ -37,10 +33,34 @@ void printcharset (const byte *st) { } +static void printIcharset (const Instruction *inst, const byte *buff) { + byte cs[CHARSETSIZE]; + int i; + printf("(%02x-%d) ", inst->i.aux2.set.offset, inst->i.aux2.set.size); + clearset(cs); + for (i = 0; i < CHARSETSIZE * 8; i++) { + if (charinset(inst, buff, i)) + setchar(cs, i); + } + printcharset(cs); +} + + +static void printTcharset (TTree *tree) { + byte cs[CHARSETSIZE]; + int i; + printf("(%02x-%d) ", tree->u.set.offset, tree->u.set.size); + fillset(cs, tree->u.set.deflt); + for (i = 0; i < tree->u.set.size; i++) + cs[tree->u.set.offset + i] = treebuffer(tree)[i]; + printcharset(cs); +} + + static const char *capkind (int kind) { const char *const modes[] = { "close", "position", "constant", "backref", - "argument", "simple", "table", "function", + "argument", "simple", "table", "function", "accumulator", "query", "string", "num", "substitution", "fold", "runtime", "group"}; return modes[kind]; @@ -56,41 +76,46 @@ void printinst (const Instruction *op, const Instruction *p) { const char *const names[] = { "any", "char", "set", "testany", "testchar", "testset", - "span", "behind", + "span", "utf-range", "behind", "ret", "end", "choice", "jmp", "call", "open_call", "commit", "partial_commit", "back_commit", "failtwice", "fail", "giveup", - "fullcapture", "opencapture", "closecapture", "closeruntime" + "fullcapture", "opencapture", "closecapture", "closeruntime", + "--" }; printf("%02ld: %s ", (long)(p - op), names[p->i.code]); switch ((Opcode)p->i.code) { case IChar: { - printf("'%c'", p->i.aux); + printf("'%c' (%02x)", p->i.aux1, p->i.aux1); break; } case ITestChar: { - printf("'%c'", p->i.aux); printjmp(op, p); + printf("'%c' (%02x)", p->i.aux1, p->i.aux1); printjmp(op, p); + break; + } + case IUTFR: { + printf("%d - %d", p[1].offset, utf_to(p)); break; } case IFullCapture: { printf("%s (size = %d) (idx = %d)", - capkind(getkind(p)), getoff(p), p->i.key); + capkind(getkind(p)), getoff(p), p->i.aux2.key); break; } case IOpenCapture: { - printf("%s (idx = %d)", capkind(getkind(p)), p->i.key); + printf("%s (idx = %d)", capkind(getkind(p)), p->i.aux2.key); break; } case ISet: { - printcharset((p+1)->buff); + printIcharset(p, (p+1)->buff); break; } case ITestSet: { - printcharset((p+2)->buff); printjmp(op, p); + printIcharset(p, (p+2)->buff); printjmp(op, p); break; } case ISpan: { - printcharset((p+1)->buff); + printIcharset(p, (p+1)->buff); break; } case IOpenCall: { @@ -98,7 +123,7 @@ void printinst (const Instruction *op, const Instruction *p) { break; } case IBehind: { - printf("%d", p->i.aux); + printf("%d", p->i.aux1); break; } case IJmp: case ICall: case ICommit: case IChoice: @@ -112,8 +137,9 @@ void printinst (const Instruction *op, const Instruction *p) { } -void printpatt (Instruction *p, int n) { +void printpatt (Instruction *p) { Instruction *op = p; + uint n = op[-1].codesize - 1; while (p < op + n) { printinst(op, p); p += sizei(p); @@ -121,20 +147,39 @@ void printpatt (Instruction *p, int n) { } -#if defined(LPEG_DEBUG) -static void printcap (Capture *cap) { - printf("%s (idx: %d - size: %d) -> %p\n", - capkind(cap->kind), cap->idx, cap->siz, cap->s); +static void printcap (Capture *cap, int ident) { + while (ident--) printf(" "); + printf("%s (idx: %d - size: %d) -> %lu (%p)\n", + capkind(cap->kind), cap->idx, cap->siz, (long)cap->index, (void*)cap); } -void printcaplist (Capture *cap, Capture *limit) { +/* +** Print a capture and its nested captures +*/ +static Capture *printcap2close (Capture *cap, int ident) { + Capture *head = cap++; + printcap(head, ident); /* print head capture */ + while (capinside(head, cap)) + cap = printcap2close(cap, ident + 2); /* print nested captures */ + if (isopencap(head)) { + assert(isclosecap(cap)); + printcap(cap++, ident); /* print and skip close capture */ + } + return cap; +} + + +void printcaplist (Capture *cap) { + { /* for debugging, print first a raw list of captures */ + Capture *c = cap; + while (c->index != MAXINDT) { printcap(c, 0); c++; } + } printf(">======\n"); - for (; cap->s && (limit == NULL || cap < limit); cap++) - printcap(cap); + while (!isclosecap(cap)) + cap = printcap2close(cap, 0); printf("=======\n"); } -#endif /* }====================================================== */ @@ -147,11 +192,11 @@ void printcaplist (Capture *cap, Capture *limit) { static const char *tagnames[] = { "char", "set", "any", - "true", "false", + "true", "false", "utf8.range", "rep", "seq", "choice", "not", "and", - "call", "opencall", "rule", "grammar", + "call", "opencall", "rule", "xinfo", "grammar", "behind", "capture", "run-time" }; @@ -159,6 +204,7 @@ static const char *tagnames[] = { void printtree (TTree *tree, int ident) { int i; + int sibs = numsiblings[tree->tag]; for (i = 0; i < ident; i++) printf(" "); printf("%s", tagnames[tree->tag]); switch (tree->tag) { @@ -171,29 +217,38 @@ void printtree (TTree *tree, int ident) { break; } case TSet: { - printcharset(treebuffer(tree)); + printTcharset(tree); printf("\n"); break; } + case TUTFR: { + assert(sib1(tree)->tag == TXInfo); + printf(" %d (%02x %d) - %d (%02x %d) \n", + tree->u.n, tree->key, tree->cap, + sib1(tree)->u.n, sib1(tree)->key, sib1(tree)->cap); + break; + } case TOpenCall: case TCall: { - assert(sib2(tree)->tag == TRule); - printf(" key: %d (rule: %d)\n", tree->key, sib2(tree)->cap); + assert(sib1(sib2(tree))->tag == TXInfo); + printf(" key: %d (rule: %d)\n", tree->key, sib1(sib2(tree))->u.n); break; } case TBehind: { printf(" %d\n", tree->u.n); - printtree(sib1(tree), ident + 2); break; } case TCapture: { printf(" kind: '%s' key: %d\n", capkind(tree->cap), tree->key); - printtree(sib1(tree), ident + 2); break; } case TRule: { - printf(" n: %d key: %d\n", tree->cap, tree->key); - printtree(sib1(tree), ident + 2); - break; /* do not print next rule as a sibling */ + printf(" key: %d\n", tree->key); + sibs = 1; /* do not print 'sib2' (next rule) as a sibling */ + break; + } + case TXInfo: { + printf(" n: %d\n", tree->u.n); + break; } case TGrammar: { TTree *rule = sib1(tree); @@ -203,18 +258,17 @@ void printtree (TTree *tree, int ident) { rule = sib2(rule); } assert(rule->tag == TTrue); /* sentinel */ + sibs = 0; /* siblings already handled */ break; } - default: { - int sibs = numsiblings[tree->tag]; + default: printf("\n"); - if (sibs >= 1) { - printtree(sib1(tree), ident + 2); - if (sibs >= 2) - printtree(sib2(tree), ident + 2); - } break; - } + } + if (sibs >= 1) { + printtree(sib1(tree), ident + 2); + if (sibs >= 2) + printtree(sib2(tree), ident + 2); } } diff --git a/3rd/lpeg/lpprint.h b/3rd/lpeg/lpprint.h index 15ef121d..e8e04e87 100644 --- a/3rd/lpeg/lpprint.h +++ b/3rd/lpeg/lpprint.h @@ -1,7 +1,3 @@ -/* -** $Id: lpprint.h $ -*/ - #if !defined(lpprint_h) #define lpprint_h @@ -13,11 +9,11 @@ #if defined(LPEG_DEBUG) -void printpatt (Instruction *p, int n); +void printpatt (Instruction *p); void printtree (TTree *tree, int ident); void printktable (lua_State *L, int idx); void printcharset (const byte *st); -void printcaplist (Capture *cap, Capture *limit); +void printcaplist (Capture *cap); void printinst (const Instruction *op, const Instruction *p); #else @@ -26,7 +22,7 @@ void printinst (const Instruction *op, const Instruction *p); luaL_error(L, "function only implemented in debug mode") #define printtree(tree,i) \ luaL_error(L, "function only implemented in debug mode") -#define printpatt(p,n) \ +#define printpatt(p) \ luaL_error(L, "function only implemented in debug mode") #endif diff --git a/3rd/lpeg/lptree.c b/3rd/lpeg/lptree.c index 5c8de947..475b0c36 100644 --- a/3rd/lpeg/lptree.c +++ b/3rd/lpeg/lptree.c @@ -1,7 +1,3 @@ -/* -** $Id: lptree.c $ -** Copyright 2013, Lua.org & PUC-Rio (see 'lpeg.html' for license) -*/ #include #include @@ -16,16 +12,17 @@ #include "lpcode.h" #include "lpprint.h" #include "lptree.h" +#include "lpcset.h" /* number of siblings for each tree */ const byte numsiblings[] = { 0, 0, 0, /* char, set, any */ - 0, 0, /* true, false */ - 1, /* rep */ + 0, 0, 0, /* true, false, utf-range */ + 1, /* acc */ 2, 2, /* seq, choice */ 1, 1, /* not, and */ - 0, 0, 2, 1, /* call, opencall, rule, grammar */ + 0, 0, 2, 1, 1, /* call, opencall, rule, prerule, grammar */ 1, /* behind */ 1, 1 /* capture, runtime capture */ }; @@ -338,7 +335,7 @@ static Pattern *getpattern (lua_State *L, int idx) { static int getsize (lua_State *L, int idx) { - return (lua_rawlen(L, idx) - sizeof(Pattern)) / sizeof(TTree) + 1; + return (lua_rawlen(L, idx) - offsetof(Pattern, tree)) / sizeof(TTree); } @@ -351,18 +348,18 @@ static TTree *gettree (lua_State *L, int idx, int *len) { /* -** create a pattern. Set its uservalue (the 'ktable') equal to its -** metatable. (It could be any empty sequence; the metatable is at -** hand here, so we use it.) +** create a pattern followed by a tree with 'len' nodes. Set its +** uservalue (the 'ktable') equal to its metatable. (It could be any +** empty sequence; the metatable is at hand here, so we use it.) */ static TTree *newtree (lua_State *L, int len) { - size_t size = (len - 1) * sizeof(TTree) + sizeof(Pattern); + size_t size = offsetof(Pattern, tree) + len * sizeof(TTree); Pattern *p = (Pattern *)lua_newuserdata(L, size); luaL_getmetatable(L, PATTERN_T); lua_pushvalue(L, -1); lua_setuservalue(L, -3); lua_setmetatable(L, -2); - p->code = NULL; p->codesize = 0; + p->code = NULL; return p->tree; } @@ -374,17 +371,44 @@ static TTree *newleaf (lua_State *L, int tag) { } -static TTree *newcharset (lua_State *L) { - TTree *tree = newtree(L, bytes2slots(CHARSETSIZE) + 1); - tree->tag = TSet; - loopset(i, treebuffer(tree)[i] = 0); - return tree; +/* +** Create a tree for a charset, optimizing for special cases: empty set, +** full set, and singleton set. +*/ +static TTree *newcharset (lua_State *L, byte *cs) { + charsetinfo info; + Opcode op = charsettype(cs, &info); + switch (op) { + case IFail: return newleaf(L, TFalse); /* empty set */ + case IAny: return newleaf(L, TAny); /* full set */ + case IChar: { /* singleton set */ + TTree *tree =newleaf(L, TChar); + tree->u.n = info.offset; + return tree; + } + default: { /* regular set */ + int i; + int bsize = /* tree size in bytes */ + (int)offsetof(TTree, u.set.bitmap) + info.size; + TTree *tree = newtree(L, bytes2slots(bsize)); + assert(op == ISet); + tree->tag = TSet; + tree->u.set.offset = info.offset; + tree->u.set.size = info.size; + tree->u.set.deflt = info.deflt; + for (i = 0; i < info.size; i++) { + assert(&treebuffer(tree)[i] < (byte*)tree + bsize); + treebuffer(tree)[i] = cs[info.offset + i]; + } + return tree; + } + } } /* -** add to tree a sequence where first sibling is 'sib' (with size -** 'sibsize'); returns position for second sibling +** Add to tree a sequence where first sibling is 'sib' (with size +** 'sibsize'); return position for second sibling. */ static TTree *seqaux (TTree *tree, TTree *sib, int sibsize) { tree->tag = TSeq; tree->u.ps = sibsize + 1; @@ -553,8 +577,8 @@ static int lp_choice (lua_State *L) { TTree *t1 = getpatt(L, 1, NULL); TTree *t2 = getpatt(L, 2, NULL); if (tocharset(t1, &st1) && tocharset(t2, &st2)) { - TTree *t = newcharset(L); - loopset(i, treebuffer(t)[i] = st1.cs[i] | st2.cs[i]); + loopset(i, st1.cs[i] |= st2.cs[i]); + newcharset(L, st1.cs); } else if (nofail(t1) || t2->tag == TFalse) lua_pushvalue(L, 1); /* true / x => true, x / false => x */ @@ -630,8 +654,8 @@ static int lp_sub (lua_State *L) { TTree *t1 = getpatt(L, 1, &s1); TTree *t2 = getpatt(L, 2, &s2); if (tocharset(t1, &st1) && tocharset(t2, &st2)) { - TTree *t = newcharset(L); - loopset(i, treebuffer(t)[i] = st1.cs[i] & ~st2.cs[i]); + loopset(i, st1.cs[i] &= ~st2.cs[i]); + newcharset(L, st1.cs); } else { TTree *tree = newtree(L, 2 + s1 + s2); @@ -649,11 +673,13 @@ static int lp_sub (lua_State *L) { static int lp_set (lua_State *L) { size_t l; const char *s = luaL_checklstring(L, 1, &l); - TTree *tree = newcharset(L); + byte buff[CHARSETSIZE]; + clearset(buff); while (l--) { - setchar(treebuffer(tree), (byte)(*s)); + setchar(buff, (byte)(*s)); s++; } + newcharset(L, buff); return 1; } @@ -661,14 +687,68 @@ static int lp_set (lua_State *L) { static int lp_range (lua_State *L) { int arg; int top = lua_gettop(L); - TTree *tree = newcharset(L); + byte buff[CHARSETSIZE]; + clearset(buff); for (arg = 1; arg <= top; arg++) { int c; size_t l; const char *r = luaL_checklstring(L, arg, &l); luaL_argcheck(L, l == 2, arg, "range must have two characters"); for (c = (byte)r[0]; c <= (byte)r[1]; c++) - setchar(treebuffer(tree), c); + setchar(buff, c); + } + newcharset(L, buff); + return 1; +} + + +/* +** Fills a tree node with basic information about the UTF-8 code point +** 'cpu': its value in 'n', its length in 'cap', and its first byte in +** 'key' +*/ +static void codeutftree (lua_State *L, TTree *t, lua_Unsigned cpu, int arg) { + int len, fb, cp; + cp = (int)cpu; + if (cp <= 0x7f) { /* one byte? */ + len = 1; + fb = cp; + } else if (cp <= 0x7ff) { + len = 2; + fb = 0xC0 | (cp >> 6); + } else if (cp <= 0xffff) { + len = 3; + fb = 0xE0 | (cp >> 12); + } + else { + luaL_argcheck(L, cpu <= 0x10ffffu, arg, "invalid code point"); + len = 4; + fb = 0xF0 | (cp >> 18); + } + t->u.n = cp; + t->cap = len; + t->key = fb; +} + + +static int lp_utfr (lua_State *L) { + lua_Unsigned from = (lua_Unsigned)luaL_checkinteger(L, 1); + lua_Unsigned to = (lua_Unsigned)luaL_checkinteger(L, 2); + luaL_argcheck(L, from <= to, 2, "empty range"); + if (to <= 0x7f) { /* ascii range? */ + uint f; + byte buff[CHARSETSIZE]; /* code it as a regular charset */ + clearset(buff); + for (f = (int)from; f <= to; f++) + setchar(buff, f); + newcharset(L, buff); + } + else { /* multi-byte utf-8 range */ + TTree *tree = newtree(L, 2); + tree->tag = TUTFR; + codeutftree(L, tree, from, 1); + sib1(tree)->tag = TXInfo; + codeutftree(L, sib1(tree), to, 2); } return 1; } @@ -763,11 +843,18 @@ static int lp_divcapture (lua_State *L) { tree->key = n; return 1; } - default: return luaL_argerror(L, 2, "invalid replacement value"); + default: + return luaL_error(L, "unexpected %s as 2nd operand to LPeg '/'", + luaL_typename(L, 2)); } } +static int lp_acccapture (lua_State *L) { + return capture_aux(L, Cacc, 2); +} + + static int lp_substcapture (lua_State *L) { return capture_aux(L, Csubst, 0); } @@ -906,7 +993,7 @@ static int collectrules (lua_State *L, int arg, int *totalsize) { int size; /* accumulator for total size */ lua_newtable(L); /* create position table */ getfirstrule(L, arg, postab); - size = 2 + getsize(L, postab + 2); /* TGrammar + TRule + rule */ + size = 3 + getsize(L, postab + 2); /* TGrammar + TRule + TXInfo + rule */ lua_pushnil(L); /* prepare to traverse grammar table */ while (lua_next(L, arg) != 0) { if (lua_tonumber(L, -2) == 1 || @@ -920,11 +1007,11 @@ static int collectrules (lua_State *L, int arg, int *totalsize) { lua_pushvalue(L, -2); /* push key (to insert into position table) */ lua_pushinteger(L, size); lua_settable(L, postab); - size += 1 + getsize(L, -1); /* update size */ + size += 2 + getsize(L, -1); /* add 'TRule + TXInfo + rule' to size */ lua_pushvalue(L, -2); /* push key (for next lua_next) */ n++; } - *totalsize = size + 1; /* TTrue to finish list of rules */ + *totalsize = size + 1; /* space for 'TTrue' finishing list of rules */ return n; } @@ -936,11 +1023,13 @@ static void buildgrammar (lua_State *L, TTree *grammar, int frule, int n) { int ridx = frule + 2*i + 1; /* index of i-th rule */ int rulesize; TTree *rn = gettree(L, ridx, &rulesize); + TTree *pr = sib1(nd); /* points to rule's prerule */ nd->tag = TRule; nd->key = 0; /* will be fixed when rule is used */ - nd->cap = i; /* rule number */ - nd->u.ps = rulesize + 1; /* point to next rule */ - memcpy(sib1(nd), rn, rulesize * sizeof(TTree)); /* copy rule */ + pr->tag = TXInfo; + pr->u.n = i; /* rule number */ + nd->u.ps = rulesize + 2; /* point to next rule */ + memcpy(sib1(pr), rn, rulesize * sizeof(TTree)); /* copy rule */ mergektable(L, ridx, sib1(nd)); /* merge its ktable into new one */ nd = sib2(nd); /* move to next rule */ } @@ -976,7 +1065,7 @@ static int checkloops (TTree *tree) { ** twice in 'passed', there is path from it back to itself without ** advancing the subject. */ -static int verifyerror (lua_State *L, int *passed, int npassed) { +static int verifyerror (lua_State *L, unsigned short *passed, int npassed) { int i, j; for (i = npassed - 1; i >= 0; i--) { /* search for a repetition */ for (j = i - 1; j >= 0; j--) { @@ -1001,12 +1090,12 @@ static int verifyerror (lua_State *L, int *passed, int npassed) { ** counts the elements in 'passed'. ** Assume ktable at the top of the stack. */ -static int verifyrule (lua_State *L, TTree *tree, int *passed, int npassed, - int nb) { +static int verifyrule (lua_State *L, TTree *tree, unsigned short *passed, + int npassed, int nb) { tailcall: switch (tree->tag) { case TChar: case TSet: case TAny: - case TFalse: + case TFalse: case TUTFR: return nb; /* cannot pass from here */ case TTrue: case TBehind: /* look-behind cannot have calls */ @@ -1014,7 +1103,7 @@ static int verifyrule (lua_State *L, TTree *tree, int *passed, int npassed, case TNot: case TAnd: case TRep: /* return verifyrule(L, sib1(tree), passed, npassed, 1); */ tree = sib1(tree); nb = 1; goto tailcall; - case TCapture: case TRunTime: + case TCapture: case TRunTime: case TXInfo: /* return verifyrule(L, sib1(tree), passed, npassed, nb); */ tree = sib1(tree); goto tailcall; case TCall: @@ -1030,10 +1119,10 @@ static int verifyrule (lua_State *L, TTree *tree, int *passed, int npassed, /* return verifyrule(L, sib2(tree), passed, npassed, nb); */ tree = sib2(tree); goto tailcall; case TRule: - if (npassed >= MAXRULES) - return verifyerror(L, passed, npassed); + if (npassed >= MAXRULES) /* too many steps? */ + return verifyerror(L, passed, npassed); /* error */ else { - passed[npassed++] = tree->key; + passed[npassed++] = tree->key; /* add rule to path */ /* return verifyrule(L, sib1(tree), passed, npassed); */ tree = sib1(tree); goto tailcall; } @@ -1045,7 +1134,7 @@ static int verifyrule (lua_State *L, TTree *tree, int *passed, int npassed, static void verifygrammar (lua_State *L, TTree *grammar) { - int passed[MAXRULES]; + unsigned short passed[MAXRULES]; TTree *rule; /* check left-recursive rules */ for (rule = sib1(grammar); rule->tag == TRule; rule = sib2(rule)) { @@ -1105,7 +1194,7 @@ static Instruction *prepcompile (lua_State *L, Pattern *p, int idx) { lua_getuservalue(L, idx); /* push 'ktable' (may be used by 'finalfix') */ finalfix(L, 0, NULL, p->tree); lua_pop(L, 1); /* remove 'ktable' */ - return compile(L, p); + return compile(L, p, getsize(L, idx)); } @@ -1128,7 +1217,7 @@ static int lp_printcode (lua_State *L) { printktable(L, 1); if (p->code == NULL) /* not compiled yet? */ prepcompile(L, p, 1); - printpatt(p->code, p->codesize); + printpatt(p->code); return 0; } @@ -1164,9 +1253,10 @@ static int lp_match (lua_State *L) { const char *s = luaL_checklstring(L, SUBJIDX, &l); size_t i = initposition(L, l); int ptop = lua_gettop(L); + luaL_argcheck(L, l < MAXINDT, SUBJIDX, "subject too long"); lua_pushnil(L); /* initialize subscache */ lua_pushlightuserdata(L, capture); /* initialize caplistidx */ - lua_getuservalue(L, 1); /* initialize penvidx */ + lua_getuservalue(L, 1); /* initialize ktableidx */ r = match(L, s, s + i, s + l, code, capture, ptop); if (r == NULL) { lua_pushnil(L); @@ -1195,12 +1285,6 @@ static int lp_setmax (lua_State *L) { } -static int lp_version (lua_State *L) { - lua_pushstring(L, VERSION); - return 1; -} - - static int lp_type (lua_State *L) { if (testpattern(L, 1)) lua_pushliteral(L, "pattern"); @@ -1212,16 +1296,22 @@ static int lp_type (lua_State *L) { int lp_gc (lua_State *L) { Pattern *p = getpattern(L, 1); - realloccode(L, p, 0); /* delete code block */ + freecode(L, p); /* delete code block */ return 0; } +/* +** Create a charset representing a category of characters, given by +** the predicate 'catf'. +*/ static void createcat (lua_State *L, const char *catname, int (catf) (int)) { - TTree *t = newcharset(L); - int i; - for (i = 0; i <= UCHAR_MAX; i++) - if (catf(i)) setchar(treebuffer(t), i); + int c; + byte buff[CHARSETSIZE]; + clearset(buff); + for (c = 0; c <= UCHAR_MAX; c++) + if (catf(c)) setchar(buff, c); + newcharset(L, buff); lua_setfield(L, -2, catname); } @@ -1269,8 +1359,9 @@ static struct luaL_Reg pattreg[] = { {"P", lp_P}, {"S", lp_set}, {"R", lp_range}, + {"utfR", lp_utfr}, {"locale", lp_locale}, - {"version", lp_version}, + {"version", NULL}, {"setmaxstack", lp_setmax}, {"type", lp_type}, {NULL, NULL} @@ -1284,6 +1375,7 @@ static struct luaL_Reg metareg[] = { {"__gc", lp_gc}, {"__len", lp_and}, {"__div", lp_divcapture}, + {"__mod", lp_acccapture}, {"__unm", lp_not}, {"__sub", lp_sub}, {NULL, NULL} @@ -1299,6 +1391,8 @@ int luaopen_lpeg (lua_State *L) { luaL_newlib(L, pattreg); lua_pushvalue(L, -1); lua_setfield(L, -3, "__index"); + lua_pushliteral(L, "LPeg " VERSION); + lua_setfield(L, -2, "version"); return 1; } diff --git a/3rd/lpeg/lptree.h b/3rd/lpeg/lptree.h index 25906d5f..c7887412 100644 --- a/3rd/lpeg/lptree.h +++ b/3rd/lpeg/lptree.h @@ -1,12 +1,9 @@ -/* -** $Id: lptree.h $ -*/ #if !defined(lptree_h) #define lptree_h -#include "lptypes.h" +#include "lptypes.h" /* @@ -14,10 +11,13 @@ */ typedef enum TTag { TChar = 0, /* 'n' = char */ - TSet, /* the set is stored in next CHARSETSIZE bytes */ + TSet, /* the set is encoded in 'u.set' and the next 'u.set.size' bytes */ TAny, TTrue, TFalse, + TUTFR, /* range of UTF-8 codepoints; 'n' has initial codepoint; + 'cap' has length; 'key' has first byte; + extra info is similar for end codepoint */ TRep, /* 'sib1'* */ TSeq, /* 'sib1' 'sib2' */ TChoice, /* 'sib1' / 'sib2' */ @@ -26,8 +26,9 @@ typedef enum TTag { TCall, /* ktable[key] is rule's key; 'sib2' is rule being called */ TOpenCall, /* ktable[key] is rule's key */ TRule, /* ktable[key] is rule's key (but key == 0 for unused rules); - 'sib1' is rule's pattern; - 'sib2' is next rule; 'cap' is rule's sequential number */ + 'sib1' is rule's pattern pre-rule; 'sib2' is next rule; + extra info 'n' is rule's sequential number */ + TXInfo, /* extra info */ TGrammar, /* 'sib1' is initial (and first) rule */ TBehind, /* 'sib1' is pattern, 'n' is how much to go back */ TCapture, /* captures: 'cap' is kind of capture (enum 'CapKind'); @@ -51,17 +52,26 @@ typedef struct TTree { union { int ps; /* occasional second child */ int n; /* occasional counter */ + struct { + byte offset; /* compact set offset (in bytes) */ + byte size; /* compact set size (in bytes) */ + byte deflt; /* default value */ + byte bitmap[1]; /* bitmap (open array) */ + } set; /* for compact sets */ } u; } TTree; +/* access to charset */ +#define treebuffer(t) ((t)->u.set.bitmap) + + /* ** A complete pattern has its tree plus, if already compiled, ** its corresponding code */ typedef struct Pattern { union Instruction *code; - int codesize; TTree tree[1]; } Pattern; diff --git a/3rd/lpeg/lptypes.h b/3rd/lpeg/lptypes.h index 1d9d59f6..3f860b97 100644 --- a/3rd/lpeg/lptypes.h +++ b/3rd/lpeg/lptypes.h @@ -1,7 +1,6 @@ /* -** $Id: lptypes.h $ ** LPeg - PEG pattern matching for Lua -** Copyright 2007-2019, Lua.org & PUC-Rio (see 'lpeg.html' for license) +** Copyright 2007-2023, Lua.org & PUC-Rio (see 'lpeg.html' for license) ** written by Roberto Ierusalimschy */ @@ -11,11 +10,12 @@ #include #include +#include #include "lua.h" -#define VERSION "1.0.2" +#define VERSION "1.1.0" #define PATTERN_T "lpeg-pattern" @@ -37,6 +37,8 @@ #define luaL_setfuncs(L,f,n) luaL_register(L,NULL,f) #define luaL_newlib(L,f) luaL_register(L,"lpeg",f) +typedef size_t lua_Unsigned; + #endif @@ -51,9 +53,9 @@ #endif -/* maximum number of rules in a grammar (limited by 'unsigned char') */ +/* maximum number of rules in a grammar (limited by 'unsigned short') */ #if !defined(MAXRULES) -#define MAXRULES 250 +#define MAXRULES 1000 #endif @@ -81,6 +83,8 @@ typedef unsigned char byte; +typedef unsigned int uint; + #define BITSPERCHAR 8 @@ -96,11 +100,11 @@ typedef struct Charset { #define loopset(v,b) { int v; for (v = 0; v < CHARSETSIZE; v++) {b;} } -/* access to charset */ -#define treebuffer(t) ((byte *)((t) + 1)) +#define fillset(s,c) memset(s,c,CHARSETSIZE) +#define clearset(s) fillset(s,0) /* number of slots needed for 'n' bytes */ -#define bytes2slots(n) (((n) - 1) / sizeof(TTree) + 1) +#define bytes2slots(n) (((n) - 1u) / (uint)sizeof(TTree) + 1u) /* set 'b' bit in charset 'cs' */ #define setchar(cs,b) ((cs)[(b) >> 3] |= (1 << ((b) & 7))) @@ -110,8 +114,8 @@ typedef struct Charset { ** in capture instructions, 'kind' of capture and its offset are ** packed in field 'aux', 4 bits for each */ -#define getkind(op) ((op)->i.aux & 0xF) -#define getoff(op) (((op)->i.aux >> 4) & 0xF) +#define getkind(op) ((op)->i.aux1 & 0xF) +#define getoff(op) (((op)->i.aux1 >> 4) & 0xF) #define joinkindoff(k,o) ((k) | ((o) << 4)) #define MAXOFF 0xF @@ -126,19 +130,20 @@ typedef struct Charset { #define MAXPATTSIZE (SHRT_MAX - 10) -/* size (in elements) for an instruction plus extra l bytes */ -#define instsize(l) (((l) + sizeof(Instruction) - 1)/sizeof(Instruction) + 1) +/* size (in instructions) for l bytes (l > 0) */ +#define instsize(l) ((int)(((l) + (uint)sizeof(Instruction) - 1u) \ + / (uint)sizeof(Instruction))) /* size (in elements) for a ISet instruction */ -#define CHARSETINSTSIZE instsize(CHARSETSIZE) +#define CHARSETINSTSIZE (1 + instsize(CHARSETSIZE)) /* size (in elements) for a IFunc instruction */ #define funcinstsize(p) ((p)->i.aux + 2) -#define testchar(st,c) (((int)(st)[((c) >> 3)] & (1 << ((c) & 7)))) +#define testchar(st,c) ((((uint)(st)[((c) >> 3)]) >> ((c) & 7)) & 1) #endif diff --git a/3rd/lpeg/lpvm.c b/3rd/lpeg/lpvm.c index 737418c4..0a2fde4f 100644 --- a/3rd/lpeg/lpvm.c +++ b/3rd/lpeg/lpvm.c @@ -1,7 +1,3 @@ -/* -** $Id: lpvm.c $ -** Copyright 2007, Lua.org & PUC-Rio (see 'lpeg.html' for license) -*/ #include #include @@ -24,7 +20,46 @@ #define getoffset(p) (((p) + 1)->offset) -static const Instruction giveup = {{IGiveup, 0, 0}}; +static const Instruction giveup = {{IGiveup, 0, {0}}}; + + +int charinset (const Instruction *i, const byte *buff, uint c) { + c -= i->i.aux2.set.offset; + if (c >= ((uint)i->i.aux2.set.size /* size in instructions... */ + * (uint)sizeof(Instruction) /* in bytes... */ + * 8u)) /* in bits */ + return i->i.aux1; /* out of range; return default value */ + return testchar(buff, c); +} + + +/* +** Decode one UTF-8 sequence, returning NULL if byte sequence is invalid. +*/ +static const char *utf8_decode (const char *o, int *val) { + static const uint limits[] = {0xFF, 0x7F, 0x7FF, 0xFFFFu}; + const unsigned char *s = (const unsigned char *)o; + uint c = s[0]; /* first byte */ + uint res = 0; /* final result */ + if (c < 0x80) /* ascii? */ + res = c; + else { + int count = 0; /* to count number of continuation bytes */ + while (c & 0x40) { /* still have continuation bytes? */ + int cc = s[++count]; /* read next byte */ + if ((cc & 0xC0) != 0x80) /* not a continuation byte? */ + return NULL; /* invalid byte sequence */ + res = (res << 6) | (cc & 0x3F); /* add lower 6 bits from cont. byte */ + c <<= 1; /* to test next bit */ + } + res |= (c & 0x7F) << (count * 5); /* add first byte */ + if (count > 3 || res > 0x10FFFFu || res <= limits[count]) + return NULL; /* invalid byte sequence */ + s += count; /* skip continuation bytes read */ + } + *val = res; + return (const char *)s + 1; /* +1 to include first byte */ +} /* @@ -51,16 +86,25 @@ typedef struct Stack { ** array, it is simpler to ensure the array always has at least one free ** slot upfront and check its size later.) */ + +/* new size in number of elements cannot overflow integers, and new + size in bytes cannot overflow size_t. */ +#define MAXNEWSIZE \ + (((size_t)INT_MAX) <= (~(size_t)0 / sizeof(Capture)) ? \ + ((size_t)INT_MAX) : (~(size_t)0 / sizeof(Capture))) + static Capture *growcap (lua_State *L, Capture *capture, int *capsize, int captop, int n, int ptop) { if (*capsize - captop > n) return capture; /* no need to grow array */ else { /* must grow */ Capture *newc; - int newsize = captop + n + 1; /* minimum size needed */ - if (newsize < INT_MAX/((int)sizeof(Capture) * 2)) - newsize *= 2; /* twice that size, if not too big */ - else if (newsize >= INT_MAX/((int)sizeof(Capture))) + uint newsize = captop + n + 1; /* minimum size needed */ + if (newsize < (MAXNEWSIZE / 3) * 2) + newsize += newsize / 2; /* 1.5 that size, if not too big */ + else if (newsize < (MAXNEWSIZE / 9) * 8) + newsize += newsize / 8; /* else, try 9/8 that size */ + else luaL_error(L, "too many captures"); newc = (Capture *)lua_newuserdata(L, newsize * sizeof(Capture)); memcpy(newc, capture, captop * sizeof(Capture)); @@ -125,7 +169,7 @@ static int resdyncaptures (lua_State *L, int fr, int curr, int limit) { ** value, 'n' is the number of values (at least 1). The open group ** capture is already in 'capture', before the place for the new entries. */ -static void adddyncaptures (const char *s, Capture *capture, int n, int fd) { +static void adddyncaptures (Index_t index, Capture *capture, int n, int fd) { int i; assert(capture[-1].kind == Cgroup && capture[-1].siz == 0); capture[-1].idx = 0; /* make group capture an anonymous group */ @@ -133,11 +177,11 @@ static void adddyncaptures (const char *s, Capture *capture, int n, int fd) { capture[i].kind = Cruntime; capture[i].siz = 1; /* mark it as closed */ capture[i].idx = fd + i; /* stack index of capture value */ - capture[i].s = s; + capture[i].index = index; } capture[n].kind = Cclose; /* close group */ capture[n].siz = 1; - capture[n].s = s; + capture[n].index = index; } @@ -154,6 +198,32 @@ static int removedyncap (lua_State *L, Capture *capture, } +/* +** Find the corresponding 'open' capture before 'cap', when that capture +** can become a full capture. If a full capture c1 is followed by an +** empty capture c2, there is no way to know whether c2 is inside +** c1. So, full captures can enclose only captures that start *before* +** its end. +*/ +static Capture *findopen (Capture *cap, Index_t currindex) { + int i; + cap--; /* check last capture */ + /* Must it be inside current one, but starts where current one ends? */ + if (!isopencap(cap) && cap->index == currindex) + return NULL; /* current one cannot be a full capture */ + /* else, look for an 'open' capture */ + for (i = 0; i < MAXLOP; i++, cap--) { + if (currindex - cap->index >= UCHAR_MAX) + return NULL; /* capture too long for a full capture */ + else if (isopencap(cap)) /* open capture? */ + return cap; /* that's the one to be closed */ + else if (cap->kind == Cclose) + return NULL; /* a full capture should not nest a non-full one */ + } + return NULL; /* not found within allowed search limit */ +} + + /* ** Opcode interpreter */ @@ -181,7 +251,7 @@ const char *match (lua_State *L, const char *o, const char *s, const char *e, case IEnd: { assert(stack == getstackbase(L, ptop) + 1); capture[captop].kind = Cclose; - capture[captop].s = NULL; + capture[captop].index = MAXINDT; return s; } case IGiveup: { @@ -198,47 +268,58 @@ const char *match (lua_State *L, const char *o, const char *s, const char *e, else goto fail; continue; } + case IUTFR: { + int codepoint; + if (s >= e) + goto fail; + s = utf8_decode (s, &codepoint); + if (s && p[1].offset <= codepoint && codepoint <= utf_to(p)) + p += 2; + else + goto fail; + continue; + } case ITestAny: { if (s < e) p += 2; else p += getoffset(p); continue; } case IChar: { - if ((byte)*s == p->i.aux && s < e) { p++; s++; } + if ((byte)*s == p->i.aux1 && s < e) { p++; s++; } else goto fail; continue; } case ITestChar: { - if ((byte)*s == p->i.aux && s < e) p += 2; + if ((byte)*s == p->i.aux1 && s < e) p += 2; else p += getoffset(p); continue; } case ISet: { - int c = (byte)*s; - if (testchar((p+1)->buff, c) && s < e) - { p += CHARSETINSTSIZE; s++; } + uint c = (byte)*s; + if (charinset(p, (p+1)->buff, c) && s < e) + { p += 1 + p->i.aux2.set.size; s++; } else goto fail; continue; } case ITestSet: { - int c = (byte)*s; - if (testchar((p + 2)->buff, c) && s < e) - p += 1 + CHARSETINSTSIZE; + uint c = (byte)*s; + if (charinset(p, (p + 2)->buff, c) && s < e) + p += 2 + p->i.aux2.set.size; else p += getoffset(p); continue; } case IBehind: { - int n = p->i.aux; + int n = p->i.aux1; if (n > s - o) goto fail; s -= n; p++; continue; } case ISpan: { for (; s < e; s++) { - int c = (byte)*s; - if (!testchar((p+1)->buff, c)) break; + uint c = (byte)*s; + if (!charinset(p, (p+1)->buff, c)) break; } - p += CHARSETINSTSIZE; + p += 1 + p->i.aux2.set.size; continue; } case IJmp: { @@ -280,6 +361,8 @@ const char *match (lua_State *L, const char *o, const char *s, const char *e, case IBackCommit: { assert(stack > getstackbase(L, ptop) && (stack - 1)->s != NULL); s = (--stack)->s; + if (ndyncap > 0) /* are there matchtime captures? */ + ndyncap -= removedyncap(L, capture, stack->caplevel, captop); captop = stack->caplevel; p += getoffset(p); continue; @@ -287,7 +370,7 @@ const char *match (lua_State *L, const char *o, const char *s, const char *e, case IFailTwice: assert(stack > getstackbase(L, ptop)); stack--; - /* go through */ + /* FALLTHROUGH */ case IFail: fail: { /* pattern failed: try to backtrack */ do { /* remove pending calls */ @@ -326,38 +409,36 @@ const char *match (lua_State *L, const char *o, const char *s, const char *e, luaL_error(L, "too many results in match-time capture"); /* add new captures + close group to 'capture' list */ capture = growcap(L, capture, &capsize, captop, n + 1, ptop); - adddyncaptures(s, capture + captop, n, fr); + adddyncaptures(s - o, capture + captop, n, fr); captop += n + 1; /* new captures + close group */ } p++; continue; } case ICloseCapture: { - const char *s1 = s; + Capture *open = findopen(capture + captop, s - o); assert(captop > 0); - /* if possible, turn capture into a full capture */ - if (capture[captop - 1].siz == 0 && - s1 - capture[captop - 1].s < UCHAR_MAX) { - capture[captop - 1].siz = s1 - capture[captop - 1].s + 1; + if (open) { /* if possible, turn capture into a full capture */ + open->siz = (s - o) - open->index + 1; p++; continue; } - else { + else { /* must create a close capture */ capture[captop].siz = 1; /* mark entry as closed */ - capture[captop].s = s; + capture[captop].index = s - o; goto pushcapture; } } case IOpenCapture: capture[captop].siz = 0; /* mark entry as open */ - capture[captop].s = s; + capture[captop].index = s - o; goto pushcapture; case IFullCapture: capture[captop].siz = getoff(p) + 1; /* save capture size */ - capture[captop].s = s - getoff(p); + capture[captop].index = s - o - getoff(p); /* goto pushcapture; */ pushcapture: { - capture[captop].idx = p->i.key; + capture[captop].idx = p->i.aux2.key; capture[captop].kind = getkind(p); captop++; capture = growcap(L, capture, &capsize, captop, 0, ptop); diff --git a/3rd/lpeg/lpvm.h b/3rd/lpeg/lpvm.h index 69ec33dc..684f0c9a 100644 --- a/3rd/lpeg/lpvm.h +++ b/3rd/lpeg/lpvm.h @@ -1,6 +1,3 @@ -/* -** $Id: lpvm.h $ -*/ #if !defined(lpvm_h) #define lpvm_h @@ -8,16 +5,24 @@ #include "lpcap.h" +/* +** About Character sets in instructions: a set is a bit map with an +** initial offset, in bits, and a size, in number of instructions. +** aux1 has the default value for the bits outsize that range. +*/ + + /* Virtual Machine's instructions */ typedef enum Opcode { IAny, /* if no char, fail */ - IChar, /* if char != aux, fail */ - ISet, /* if char not in buff, fail */ + IChar, /* if char != aux1, fail */ + ISet, /* if char not in set, fail */ ITestAny, /* in no char, jump to 'offset' */ - ITestChar, /* if char != aux, jump to 'offset' */ - ITestSet, /* if char not in buff, jump to 'offset' */ - ISpan, /* read a span of chars in buff */ - IBehind, /* walk back 'aux' characters (fail if not possible) */ + ITestChar, /* if char != aux1, jump to 'offset' */ + ITestSet, /* if char not in set, jump to 'offset' */ + ISpan, /* read a span of chars in set */ + IUTFR, /* if codepoint not in range [offset, utf_to], fail */ + IBehind, /* walk back 'aux1' characters (fail if not possible) */ IRet, /* return from a rule */ IEnd, /* end of pattern */ IChoice, /* stack a choice; next fail will jump to 'offset' */ @@ -26,30 +31,46 @@ typedef enum Opcode { IOpenCall, /* call rule number 'key' (must be closed to a ICall) */ ICommit, /* pop choice and jump to 'offset' */ IPartialCommit, /* update top choice to current position and jump */ - IBackCommit, /* "fails" but jump to its own 'offset' */ + IBackCommit, /* backtrack like "fail" but jump to its own 'offset' */ IFailTwice, /* pop one choice and then fail */ IFail, /* go back to saved state on choice and jump to saved offset */ IGiveup, /* internal use */ IFullCapture, /* complete capture of last 'off' chars */ IOpenCapture, /* start a capture */ ICloseCapture, - ICloseRunTime + ICloseRunTime, + IEmpty /* to fill empty slots left by optimizations */ } Opcode; - +/* +** All array of instructions has a 'codesize' as its first element +** and is referred by a pointer to its second element, which is the +** first actual opcode. +*/ typedef union Instruction { struct Inst { byte code; - byte aux; - short key; + byte aux1; + union { + short key; + struct { + byte offset; + byte size; + } set; + } aux2; } i; int offset; + uint codesize; byte buff[1]; } Instruction; -void printpatt (Instruction *p, int n); +/* extract 24-bit value from an instruction */ +#define utf_to(inst) (((inst)->i.aux2.key << 8) | (inst)->i.aux1) + + +int charinset (const Instruction *i, const byte *buff, uint c); const char *match (lua_State *L, const char *o, const char *s, const char *e, Instruction *op, Capture *capture, int ptop); diff --git a/3rd/lpeg/makefile b/3rd/lpeg/makefile index 1e32195a..32485a70 100644 --- a/3rd/lpeg/makefile +++ b/3rd/lpeg/makefile @@ -2,7 +2,7 @@ LIBNAME = lpeg LUADIR = ../lua/ COPT = -O2 -DNDEBUG -# COPT = -g +# COPT = -O0 -DLPEG_DEBUG -g CWARNS = -Wall -Wextra -pedantic \ -Waggregate-return \ @@ -11,21 +11,24 @@ CWARNS = -Wall -Wextra -pedantic \ -Wdisabled-optimization \ -Wpointer-arith \ -Wshadow \ + -Wredundant-decls \ -Wsign-compare \ -Wundef \ -Wwrite-strings \ -Wbad-function-cast \ -Wdeclaration-after-statement \ -Wmissing-prototypes \ + -Wmissing-declarations \ -Wnested-externs \ -Wstrict-prototypes \ + -Wc++-compat \ # -Wunreachable-code \ CFLAGS = $(CWARNS) $(COPT) -std=c99 -I$(LUADIR) -fPIC CC = gcc -FILES = lpvm.o lpcap.o lptree.o lpcode.o lpprint.o +FILES = lpvm.o lpcap.o lptree.o lpcode.o lpprint.o lpcset.o # For Linux linux: @@ -48,8 +51,9 @@ clean: lpcap.o: lpcap.c lpcap.h lptypes.h -lpcode.o: lpcode.c lptypes.h lpcode.h lptree.h lpvm.h lpcap.h -lpprint.o: lpprint.c lptypes.h lpprint.h lptree.h lpvm.h lpcap.h -lptree.o: lptree.c lptypes.h lpcap.h lpcode.h lptree.h lpvm.h lpprint.h +lpcode.o: lpcode.c lptypes.h lpcode.h lptree.h lpvm.h lpcap.h lpcset.h +lpcset.o: lpcset.c lptypes.h lpcset.h lpcode.h lptree.h lpvm.h lpcap.h +lpprint.o: lpprint.c lptypes.h lpprint.h lptree.h lpvm.h lpcap.h lpcode.h +lptree.o: lptree.c lptypes.h lpcap.h lpcode.h lptree.h lpvm.h lpprint.h \ + lpcset.h lpvm.o: lpvm.c lpcap.h lptypes.h lpvm.h lpprint.h lptree.h - diff --git a/3rd/lpeg/re.html b/3rd/lpeg/re.html index ad60d509..c8d1bc8d 100644 --- a/3rd/lpeg/re.html +++ b/3rd/lpeg/re.html @@ -1,22 +1,21 @@ + "//www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> LPeg.re - Regex syntax for LPEG -

@@ -62,6 +61,20 @@ Constructions are listed in order of decreasing precedence. + + + + + + + + + + + + + + @@ -70,22 +83,15 @@ Constructions are listed in order of decreasing precedence. + - - - - - - - - - + + @@ -94,16 +100,13 @@ equivalent to p / defs[name] - - - - - +(deprecated) +
SyntaxDescription
( p ) grouping
& p and predicate
! p not predicate
p1 p2 concatenation
p1 / p2 ordered choice
p ? optional match
p * zero or more repetitions
p + one or more repetitions
p^numexactly num repetitions
p^+numat least num repetitions
p^-numat most num repetitions
(name <- p)+ grammar
'string' literal string
"string" literal string
[class] character class
pattern defs[name] or a pre-defined pattern
namenon terminal
<name>non terminal
{} position capture
{ p } simple capture
{: p :} anonymous group capture
{:name: p :} named group capture
{~ p ~} substitution capture
{| p |} table capture
=name back reference -
p ? optional match
p * zero or more repetitions
p + one or more repetitions
p^num exactly n repetitions
p^+numat least n repetitions
p^-numat most n repetitions
=name back reference
p -> 'string' string capture
p -> "string" string capture
p -> num numbered capture
p => name match-time capture equivalent to lpeg.Cmt(p, defs[name])
p ~> name fold capture -equivalent to lpeg.Cf(p, defs[name])
& p and predicate
! p not predicate
p1 p2 concatenation
p1 / p2 ordered choice
(name <- p)+ grammar
p >> name accumulator capture +equivalent to (p % defs[name])

Any space appearing in a syntax description can be -replaced by zero or more space characters and Lua-style comments +replaced by zero or more space characters and Lua-style short comments (-- until end of line).

@@ -200,9 +203,10 @@ print(re.match("the number 423 is odd", "({%a+} / .)*")) --> the number is odd -- returns the first numeral in a string -print(re.match("the number 423 is odd", "s <- {%d+} / . s")) +print(re.match("the number 423 is odd", "s <- {%d+} / . s")) --> 423 +-- substitutes a dot for each vowel in a string print(re.gsub("hello World", "[aeiou]", ".")) --> h.ll. W.rld @@ -329,7 +333,7 @@ respecting the indentation:
 p = re.compile[[
   block <- {| {:ident:' '*:} line
-           ((=ident !' ' line) / &(=ident ' ') block)* |}
+           ((=ident !' ' line) / &(=ident ' ') block)* |}
   line <- {[^%nl]*} %nl
 ]]
 
@@ -416,6 +420,7 @@ prefix <- '&' S prefix / '!' S prefix / suffix suffix <- primary S (([+*?] / '^' [+-]? num / '->' S (string / '{}' / name) + / '>>' S name / '=>' S name) S)* primary <- '(' exp ')' / string / class / defined @@ -423,6 +428,7 @@ primary <- '(' exp ')' / string / class / defined / '=' name / '{}' / '{~' exp '~}' + / '{|' exp '|}' / '{' exp '}' / '.' / name S !arrow @@ -436,7 +442,7 @@ item <- defined / range / . range <- . '-' [^]] S <- (%s / '--' [^%nl]*)* -- spaces and comments -name <- [A-Za-z][A-Za-z0-9_]* +name <- [A-Za-z_][A-Za-z0-9_]* arrow <- '<-' num <- [0-9]+ string <- '"' [^"]* '"' / "'" [^']* "'" @@ -452,37 +458,9 @@ print(re.match(p, p)) -- a self description must match itself

License

-Copyright © 2008-2015 Lua.org, PUC-Rio. -

-

-Permission is hereby granted, free of charge, -to any person obtaining a copy of this software and -associated documentation files (the "Software"), -to deal in the Software without restriction, -including without limitation the rights to use, -copy, modify, merge, publish, distribute, sublicense, -and/or sell copies of the Software, -and to permit persons to whom the Software is -furnished to do so, -subject to the following conditions: -

+This module is part of the LPeg package and shares +its license. -

-The above copyright notice and this permission notice -shall be included in all copies or substantial portions of the Software. -

- -

-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, -INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -

diff --git a/3rd/lpeg/re.lua b/3rd/lpeg/re.lua index 3bb8af7d..1fb9fa9e 100644 --- a/3rd/lpeg/re.lua +++ b/3rd/lpeg/re.lua @@ -1,4 +1,7 @@ --- $Id: re.lua $ +-- +-- Copyright 2007-2023, Lua.org & PUC-Rio (see 'lpeg.html' for license) +-- written by Roberto Ierusalimschy +-- -- imported functions and modules local tonumber, type, print, error = tonumber, type, print, error @@ -10,14 +13,14 @@ local m = require"lpeg" -- on 'mm' local mm = m --- pattern's metatable +-- patterns' metatable local mt = getmetatable(mm.P(0)) +local version = _VERSION -- No more global accesses after this point -local version = _VERSION -if version == "Lua 5.2" then _ENV = nil end +_ENV = nil -- does no harm in Lua 5.1 local any = m.P(1) @@ -142,7 +145,7 @@ local item = (defined + Range + m.C(any)) / m.P local Class = "[" * (m.C(m.P"^"^-1)) -- optional complement symbol - * m.Cf(item * (item - "]")^0, mt.__add) / + * (item * ((item % mt.__add) - "]")^0) / function (c, p) return c == "^" and any - p or p end * "]" @@ -168,13 +171,13 @@ end local exp = m.P{ "Exp", Exp = S * ( m.V"Grammar" - + m.Cf(m.V"Seq" * ("/" * S * m.V"Seq")^0, mt.__add) ); - Seq = m.Cf(m.Cc(m.P"") * m.V"Prefix"^0 , mt.__mul) + + m.V"Seq" * ("/" * S * m.V"Seq" % mt.__add)^0 ); + Seq = (m.Cc(m.P"") * (m.V"Prefix" % mt.__mul)^0) * (#seq_follow + patt_error); Prefix = "&" * S * m.V"Prefix" / mt.__len + "!" * S * m.V"Prefix" / mt.__unm + m.V"Suffix"; - Suffix = m.Cf(m.V"Primary" * S * + Suffix = m.V"Primary" * S * ( ( m.P"+" * m.Cc(1, mt.__pow) + m.P"*" * m.Cc(0, mt.__pow) + m.P"?" * m.Cc(-1, mt.__pow) @@ -185,10 +188,11 @@ local exp = m.P{ "Exp", + m.P"{}" * m.Cc(nil, m.Ct) + defwithfunc(mt.__div) ) - + "=>" * S * defwithfunc(m.Cmt) - + "~>" * S * defwithfunc(m.Cf) - ) * S - )^0, function (a,b,f) return f(a,b) end ); + + "=>" * S * defwithfunc(mm.Cmt) + + ">>" * S * defwithfunc(mt.__mod) + + "~>" * S * defwithfunc(mm.Cf) + ) % function (a,b,f) return f(a,b) end * S + )^0; Primary = "(" * m.V"Exp" * ")" + String / mm.P + Class @@ -204,8 +208,7 @@ local exp = m.P{ "Exp", + (name * -arrow + "<" * name * ">") * m.Cb("G") / NT; Definition = name * arrow * m.V"Exp"; Grammar = m.Cg(m.Cc(true), "G") * - m.Cf(m.V"Definition" / firstdef * m.Cg(m.V"Definition")^0, - adddef) / mm.P + ((m.V"Definition" / firstdef) * (m.V"Definition" % adddef)^0) / mm.P } local pattern = S * m.Cg(m.Cc(false), "G") * exp / mm.P * (-any + patt_error) diff --git a/3rd/lpeg/test.lua b/3rd/lpeg/test.lua index 8f9f5745..d0b82da4 100755 --- a/3rd/lpeg/test.lua +++ b/3rd/lpeg/test.lua @@ -1,7 +1,5 @@ #!/usr/bin/env lua --- $Id: test.lua $ - -- require"strict" -- just to be pedantic local m = require"lpeg" @@ -48,8 +46,8 @@ end print"General tests for LPeg library" -assert(type(m.version()) == "string") -print("version " .. m.version()) +assert(type(m.version) == "string") +print(m.version) assert(m.type("alo") ~= "pattern") assert(m.type(io.input) ~= "pattern") assert(m.type(m.P"alo") == "pattern") @@ -70,6 +68,8 @@ assert(m.match(#m.P(true) * "a", "a") == 2) assert(m.match("a" * #m.P(false), "a") == nil) assert(m.match("a" * #m.P(true), "a") == 2) +assert(m.match(m.P(1)^0, "abcd") == 5) +assert(m.match(m.S("")^0, "abcd") == 1) -- tests for locale do @@ -119,6 +119,8 @@ eqcharset(m.S"\1\0\2", m.R"\0\2") eqcharset(m.S"\1\0\2", m.R"\1\2" + "\0") eqcharset(m.S"\1\0\2" - "\0", m.R"\1\2") +eqcharset(m.S("\0\255"), m.P"\0" + "\255") -- charset extremes + local word = alpha^1 * (1 - alpha)^0 assert((word^0 * -1):match"alo alo") @@ -406,7 +408,7 @@ assert(p:match('abcx') == 5 and p:match('ayzx') == 5 and not p:match'abc') do - -- large dynamic Cc + print "testing large dynamic Cc" local lim = 2^16 - 1 local c = 0 local function seq (n) @@ -491,8 +493,8 @@ local function f_term (v1, op, v2, d) end G = m.P{ "Exp", - Exp = m.Cf(V"Factor" * m.Cg(FactorOp * V"Factor")^0, f_factor); - Factor = m.Cf(V"Term" * m.Cg(TermOp * V"Term")^0, f_term); + Exp = V"Factor" * (FactorOp * V"Factor" % f_factor)^0; + Factor = V"Term" * (TermOp * V"Term" % f_term)^0; Term = Number / tonumber + Open * V"Exp" * Close; } @@ -864,6 +866,7 @@ print"+" -- accumulator capture function f (x) return x + 1 end assert(m.match(m.Cf(m.Cc(0) * m.C(1)^0, f), "alo alo") == 7) +assert(m.match(m.Cc(0) * (m.C(1) % f)^0, "alo alo") == 7) t = {m.match(m.Cf(m.Cc(1,2,3), error), "")} checkeq(t, {1}) @@ -873,7 +876,7 @@ t = p:match("a=b;c=du;xux=yuy;") checkeq(t, {a="b", c="du", xux="yuy"}) --- errors in accumulator capture +-- errors in fold capture -- no initial capture checkerr("no initial value", m.match, m.Cf(m.P(5), print), 'aaaaaa') @@ -881,8 +884,14 @@ checkerr("no initial value", m.match, m.Cf(m.P(5), print), 'aaaaaa') checkerr("no initial value", m.match, m.Cf(m.P(500), print), string.rep('a', 600)) --- nested capture produces no initial value -checkerr("no initial value", m.match, m.Cf(m.P(1) / {}, print), "alo") + +-- errors in accumulator capture + +-- no initial capture +checkerr("no previous value", m.match, m.P(5) % print, 'aaaaaa') +-- no initial capture (very long match forces fold to be a pair open-close) +checkerr("no previous value", m.match, m.P(500) % print, + string.rep('a', 600)) -- tests for loop checker @@ -985,10 +994,10 @@ for i = 1, 10 do assert(p:match("aaaaaaaaaaa") == 11 - i + 1) end -print"+" --- tests for back references +print "testing back references" + checkerr("back reference 'x' not found", m.match, m.Cb('x'), '') checkerr("back reference 'b' not found", m.match, m.Cg(1, 'a') * m.Cb('b'), 'a') @@ -996,6 +1005,35 @@ p = m.Cg(m.C(1) * m.C(1), "k") * m.Ct(m.Cb("k")) t = p:match("ab") checkeq(t, {"a", "b"}) + +do + -- some basic cases + assert(m.match(m.Cg(m.Cc(3), "a") * m.Cb("a"), "a") == 3) + assert(m.match(m.Cg(m.C(1), 133) * m.Cb(133), "X") == "X") + + -- first reference to 'x' should not see the group enclosing it + local p = m.Cg(m.Cb('x'), 'x') * m.Cb('x') + checkerr("back reference 'x' not found", m.match, p, '') + + local p = m.Cg(m.Cb('x') * m.C(1), 'x') * m.Cb('x') + checkerr("back reference 'x' not found", m.match, p, 'abc') + + -- reference to 'x' should not see the group enclosed in another capture + local s = string.rep("a", 30) + local p = (m.C(1)^-4 * m.Cg(m.C(1), 'x')) / {} * m.Cb('x') + checkerr("back reference 'x' not found", m.match, p, s) + + local p = (m.C(1)^-20 * m.Cg(m.C(1), 'x')) / {} * m.Cb('x') + checkerr("back reference 'x' not found", m.match, p, s) + + -- second reference 'k' should refer to 10 and first ref. 'k' + p = m.Cg(m.Cc(20), 'k') * m.Cg(m.Cc(10) * m.Cb('k') * m.C(1), 'k') + * (m.Cb('k') / function (a,b,c) return a*10 + b + tonumber(c) end) + -- 10 * 10 (Cc) + 20 (Cb) + 7 (C) == 127 + assert(p:match("756") == 127) + +end + p = m.P(true) for i = 1, 10 do p = p * m.Cg(1, i) end for i = 1, 10 do @@ -1032,6 +1070,17 @@ local function id (s, i, ...) return true, ... end +do -- run-time capture in an end predicate (should discard its value) + local x = 0 + function foo (s, i) + x = x + 1 + return true, x + end + + local p = #(m.Cmt("", foo) * "xx") * m.Cmt("", foo) + assert(p:match("xx") == 2) +end + assert(m.Cmt(m.Cs((m.Cmt(m.S'abc' / { a = 'x', c = 'y' }, id) + m.R'09'^1 / string.char + m.P(1))^0), id):match"acb98+68c" == "xyb\98+\68y") @@ -1156,7 +1205,7 @@ end -- bug in 1.0: problems with math-times returning too many captures -do +if _VERSION >= "Lua 5.2" then local lim = 2^11 - 10 local res = {m.match(manyCmt(lim), "a")} assert(#res == lim and res[1] == lim - 1 and res[lim] == 0) @@ -1171,9 +1220,91 @@ t = {p:match('abacc')} checkeq(t, {'a', 'aa', 20, 'a', 'aaa', 'aaa'}) +do print"testing large grammars" + local lim = 1000 -- number of rules + local t = {} + + for i = 3, lim do + t[i] = m.V(i - 1) -- each rule calls previous one + end + t[1] = m.V(lim) -- start on last rule + t[2] = m.C("alo") -- final rule + + local P = m.P(t) -- build grammar + assert(P:match("alo") == "alo") + + t[#t + 1] = m.P("x") -- one more rule... + checkerr("too many rules", m.P, t) +end + + +print "testing UTF-8 ranges" + +do -- a few typical UTF-8 ranges + local p = m.utfR(0x410, 0x44f)^1 / "cyr: %0" + + m.utfR(0x4e00, 0x9fff)^1 / "cjk: %0" + + m.utfR(0x1F600, 0x1F64F)^1 / "emot: %0" + + m.utfR(0, 0x7f)^1 / "ascii: %0" + + m.utfR(0, 0x10ffff) / "other: %0" + + p = m.Ct(p^0) * -m.P(1) + + local cyr = "ждюя" + local emot = "\240\159\152\128\240\159\153\128" -- 😀🙀 + local cjk = "专举乸" + local ascii = "alo" + local last = "\244\143\191\191" -- U+10FFFF + + local s = cyr .. "—" .. emot .. "—" .. cjk .. "—" .. ascii .. last + t = (p:match(s)) + + assert(t[1] == "cyr: " .. cyr and t[2] == "other: —" and + t[3] == "emot: " .. emot and t[4] == "other: —" and + t[5] == "cjk: " .. cjk and t[6] == "other: —" and + t[7] == "ascii: " .. ascii and t[8] == "other: " .. last and + t[9] == nil) + + -- failing UTF-8 matches and borders + assert(not m.match(m.utfR(10, 0x2000), "\9")) + assert(not m.match(m.utfR(10, 0x2000), "\226\128\129")) + assert(m.match(m.utfR(10, 0x2000), "\10") == 2) + assert(m.match(m.utfR(10, 0x2000), "\226\128\128") == 4) +end + + +do -- valid and invalid code points + local p = m.utfR(0, 0x10ffff)^0 + assert(p:match("汉字\128") == #"汉字" + 1) + assert(p:match("\244\159\191") == 1) + assert(p:match("\244\159\191\191") == 1) + assert(p:match("\255") == 1) + + -- basic errors + checkerr("empty range", m.utfR, 1, 0) + checkerr("invalid code point", m.utfR, 1, 0x10ffff + 1) +end + + +do -- back references (fixed width) + -- match a byte after a CJK point + local p = m.B(m.utfR(0x4e00, 0x9fff)) * m.C(1) + p = m.P{ p + m.P(1) * m.V(1) } -- search for 'p' + assert(p:match("ab д 专X x") == "X") + + -- match a byte after a hebrew point + local p = m.B(m.utfR(0x5d0, 0x5ea)) * m.C(1) + p = m.P(#"ש") * p + assert(p:match("שX") == "X") + + checkerr("fixed length", m.B, m.utfR(0, 0x10ffff)) +end + + + ------------------------------------------------------------------- -- Tests for 're' module ------------------------------------------------------------------- +print"testing 're' module" local re = require "re" @@ -1307,6 +1438,12 @@ e = compile([[ e = compile("{[0-9]+'.'?[0-9]*} -> sin", math) assert(e:match("2.34") == math.sin(2.34)) +e = compile("'pi' -> math", _G) +assert(e:match("pi") == math.pi) + +e = compile("[ ]* 'version' -> _VERSION", _G) +assert(e:match(" version") == _VERSION) + function eq (_, _, a, b) return a == b end @@ -1380,6 +1517,13 @@ c = re.compile([[ ]], {tonumber = tonumber, add = function (a,b) return a + b end}) assert(c:match("3 401 50") == 3 + 401 + 50) +-- test for accumulator captures +c = re.compile([[ + S <- number (%s+ number >> add)* + number <- %d+ -> tonumber +]], {tonumber = tonumber, add = function (a,b) return a + b end}) +assert(c:match("3 401 50") == 3 + 401 + 50) + -- tests for look-ahead captures x = {re.match("alo", "&(&{.}) !{'b'} {&(...)} &{..} {...} {!.}")} checkeq(x, {"", "alo", ""}) diff --git a/Makefile b/Makefile index 337ed3e6..daab6d1e 100644 --- a/Makefile +++ b/Makefile @@ -115,7 +115,7 @@ $(LUA_CLIB_PATH)/sproto.so : lualib-src/sproto/sproto.c lualib-src/sproto/lsprot $(LUA_CLIB_PATH)/ltls.so : lualib-src/ltls.c | $(LUA_CLIB_PATH) $(CC) $(CFLAGS) $(SHARED) -Iskynet-src -L$(TLS_LIB) -I$(TLS_INC) $^ -o $@ -lssl -$(LUA_CLIB_PATH)/lpeg.so : 3rd/lpeg/lpcap.c 3rd/lpeg/lpcode.c 3rd/lpeg/lpprint.c 3rd/lpeg/lptree.c 3rd/lpeg/lpvm.c | $(LUA_CLIB_PATH) +$(LUA_CLIB_PATH)/lpeg.so : 3rd/lpeg/lpcap.c 3rd/lpeg/lpcode.c 3rd/lpeg/lpprint.c 3rd/lpeg/lptree.c 3rd/lpeg/lpvm.c 3rd/lpeg/lpcset.c | $(LUA_CLIB_PATH) $(CC) $(CFLAGS) $(SHARED) -I3rd/lpeg $^ -o $@ clean :