lihao/skynet
1
0
Fork 0
mirror of https://github.com/cloudwu/skynet.git synced 2026-07-22 02:53:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

upgrade lpeg to 1.1.0 (#1766)

Browse Source 
* upgrade lpeg to 1.1.0

* fix lpeg compiler error
This commit is contained in:
Jin Xiao
2023-06-27 16:28:21 +08:00
committed by GitHub
parent 2f9d0c805a
commit 69420bdfde
21 changed files with 1388 additions and 735 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
3rd/lpeg/HISTORY
View File

@@ -1,4 +1,13 @@
HISTORY for LPeg 1.0.2
HISTORY for LPeg 1.1.0
* Changes from version 1.0.2 to 1.1.0
---------------------------------
+ accumulator capture
+ UTF-8 ranges
+ Larger limit for number of rules in a grammar
+ Larger limit for number of captures in a match
+ bug fixes
+ other small improvements
* Changes from version 1.0.1 to 1.0.2
---------------------------------

4
3rd/lpeg/README.md Normal file
View File

@@ -0,0 +1,4 @@
# LPeg - Parsing Expression Grammars For Lua
For more information,
see [Lpeg](//www.inf.puc-rio.br/~roberto/lpeg/).

233
3rd/lpeg/lpcap.c
View File

@@ -1,29 +1,41 @@
/*
** $Id: lpcap.c $
** Copyright 2007, Lua.org & PUC-Rio (see 'lpeg.html' for license)
*/
#include "lua.h"
#include "lauxlib.h"
#include "lpcap.h"
#include "lpprint.h"
#include "lptypes.h"
#define captype(cap) ((cap)->kind)
#define isclosecap(cap) (captype(cap) == Cclose)
#define closeaddr(c) ((c)->s + (c)->siz - 1)
#define isfullcap(cap) ((cap)->siz != 0)
#define getfromktable(cs,v) lua_rawgeti((cs)->L, ktableidx((cs)->ptop), v)
#define pushluaval(cs) getfromktable(cs, (cs)->cap->idx)
#define skipclose(cs,head) \
if (isopencap(head)) { assert(isclosecap(cs->cap)); cs->cap++; }
/*
** Return the size of capture 'cap'. If it is an open capture, 'close'
** must be its corresponding close.
*/
static Index_t capsize (Capture *cap, Capture *close) {
if (isopencap(cap)) {
assert(isclosecap(close));
return close->index - cap->index;
}
else
return cap->siz - 1;
}
static Index_t closesize (CapState *cs, Capture *head) {
return capsize(head, cs->cap);
}
/*
** Put at the cache for Lua values the value indexed by 'v' in ktable
** of the running pattern (if it is not there yet); returns its index.
@@ -44,35 +56,40 @@ static int pushcapture (CapState *cs);
/*
** Goes back in a list of captures looking for an open capture
** corresponding to a close
** corresponding to a close one.
*/
static Capture *findopen (Capture *cap) {
int n = 0; /* number of closes waiting an open */
for (;;) {
cap--;
if (isclosecap(cap)) n++; /* one more open to skip */
else if (!isfullcap(cap))
else if (isopencap(cap))
if (n-- == 0) return cap;
}
}
/*
** Go to the next capture
** Go to the next capture at the same level.
*/
static void nextcap (CapState *cs) {
Capture *cap = cs->cap;
if (!isfullcap(cap)) { /* not a single capture? */
if (isopencap(cap)) { /* must look for a close? */
int n = 0; /* number of opens waiting a close */
for (;;) { /* look for corresponding close */
cap++;
if (isclosecap(cap)) {
if (isopencap(cap)) n++;
else if (isclosecap(cap))
if (n-- == 0) break;
}
else if (!isfullcap(cap)) n++;
}
cs->cap = cap + 1; /* + 1 to skip last close */
}
else {
Capture *next;
for (next = cap + 1; capinside(cap, next); next++)
; /* skip captures inside current one */
cs->cap = next;
}
cs->cap = cap + 1; /* + 1 to skip last close (or entire single capture) */
}
@@ -84,22 +101,17 @@ static void nextcap (CapState *cs) {
** so the function never returns zero.
*/
static int pushnestedvalues (CapState *cs, int addextra) {
Capture *co = cs->cap;
if (isfullcap(cs->cap++)) { /* no nested captures? */
lua_pushlstring(cs->L, co->s, co->siz - 1); /* push whole match */
return 1; /* that is it */
}
else {
int n = 0;
while (!isclosecap(cs->cap)) /* repeat for all nested patterns */
n += pushcapture(cs);
if (addextra || n == 0) { /* need extra? */
lua_pushlstring(cs->L, co->s, cs->cap->s - co->s); /* push whole match */
n++;
}
cs->cap++; /* skip close entry */
return n;
Capture *head = cs->cap++; /* original capture */
int n = 0; /* number of pushed subvalues */
/* repeat for all nested patterns */
while (capinside(head, cs->cap))
n += pushcapture(cs);
if (addextra || n == 0) { /* need extra? */
lua_pushlstring(cs->L, cs->s + head->index, closesize(cs, head));
n++;
}
skipclose(cs, head);
return n;
}
@@ -114,17 +126,43 @@ static void pushonenestedvalue (CapState *cs) {
/*
** Try to find a named group capture with the name given at the top of
** the stack; goes backward from 'cap'.
** Checks whether group 'grp' is visible to 'ref', that is, 'grp' is
** not nested inside a full capture that does not contain 'ref'. (We
** only need to care for full captures because the search at 'findback'
** skips open-end blocks; so, if 'grp' is nested in a non-full capture,
** 'ref' is also inside it.) To check this, we search backward for the
** inner full capture enclosing 'grp'. A full capture cannot contain
** non-full captures, so a close capture means we cannot be inside a
** full capture anymore.
*/
static Capture *findback (CapState *cs, Capture *cap) {
static int capvisible (CapState *cs, Capture *grp, Capture *ref) {
Capture *cap = grp;
int i = MAXLOP; /* maximum distance for an 'open' */
while (i-- > 0 && cap-- > cs->ocap) {
if (isclosecap(cap))
return 1; /* can stop the search */
else if (grp->index - cap->index >= UCHAR_MAX)
return 1; /* can stop the search */
else if (capinside(cap, grp)) /* is 'grp' inside cap? */
return capinside(cap, ref); /* ok iff cap also contains 'ref' */
}
return 1; /* 'grp' is not inside any capture */
}
/*
** Try to find a named group capture with the name given at the top of
** the stack; goes backward from 'ref'.
*/
static Capture *findback (CapState *cs, Capture *ref) {
lua_State *L = cs->L;
Capture *cap = ref;
while (cap-- > cs->ocap) { /* repeat until end of list */
if (isclosecap(cap))
cap = findopen(cap); /* skip nested captures */
else if (!isfullcap(cap))
continue; /* opening an enclosing capture: skip and get previous */
if (captype(cap) == Cgroup) {
else if (capinside(cap, ref))
continue; /* enclosing captures are not visible to 'ref' */
if (captype(cap) == Cgroup && capvisible(cs, cap, ref)) {
getfromktable(cs, cap->idx); /* get group name */
if (lp_equal(L, -2, -1)) { /* right group? */
lua_pop(L, 2); /* remove reference name and group name */
@@ -158,11 +196,10 @@ static int backrefcap (CapState *cs) {
*/
static int tablecap (CapState *cs) {
lua_State *L = cs->L;
Capture *head = cs->cap++;
int n = 0;
lua_newtable(L);
if (isfullcap(cs->cap++))
return 1; /* table is empty */
while (!isclosecap(cs->cap)) {
while (capinside(head, cs->cap)) {
if (captype(cs->cap) == Cgroup && cs->cap->idx != 0) { /* named group? */
pushluaval(cs); /* push group name */
pushonenestedvalue(cs);
@@ -176,7 +213,7 @@ static int tablecap (CapState *cs) {
n += k;
}
}
cs->cap++; /* skip close entry */
skipclose(cs, head);
return 1; /* number of values pushed (only the table) */
}
@@ -203,20 +240,20 @@ static int querycap (CapState *cs) {
static int foldcap (CapState *cs) {
int n;
lua_State *L = cs->L;
int idx = cs->cap->idx;
if (isfullcap(cs->cap++) || /* no nested captures? */
isclosecap(cs->cap) || /* no nested captures (large subject)? */
Capture *head = cs->cap++;
int idx = head->idx;
if (isclosecap(cs->cap) || /* no nested captures (large subject)? */
(n = pushcapture(cs)) == 0) /* nested captures with no values? */
return luaL_error(L, "no initial value for fold capture");
if (n > 1)
lua_pop(L, n - 1); /* leave only one result for accumulator */
while (!isclosecap(cs->cap)) {
while (capinside(head, cs->cap)) {
lua_pushvalue(L, updatecache(cs, idx)); /* get folding function */
lua_insert(L, -2); /* put it before accumulator */
n = pushcapture(cs); /* get next capture's values */
lua_call(L, n + 1, 1); /* call folding function */
}
cs->cap++; /* skip close entry */
skipclose(cs, head);
return 1; /* only accumulator left on the stack */
}
@@ -234,6 +271,22 @@ static int functioncap (CapState *cs) {
}
/*
** Accumulator capture
*/
static int accumulatorcap (CapState *cs) {
lua_State *L = cs->L;
int n;
if (lua_gettop(L) < cs->firstcap)
luaL_error(L, "no previous value for accumulator capture");
pushluaval(cs); /* push function */
lua_insert(L, -2); /* previous value becomes first argument */
n = pushnestedvalues(cs, 0); /* push nested captures */
lua_call(L, n + 1, 1); /* call function */
return 0; /* did not add any extra value */
}
/*
** Select capture
*/
@@ -283,7 +336,7 @@ int runtimecap (CapState *cs, Capture *close, const char *s, int *rem) {
assert(captype(open) == Cgroup);
id = finddyncap(open, close); /* get first dynamic capture argument */
close->kind = Cclose; /* closes the group */
close->s = s;
close->index = s - cs->s;
cs->cap = open; cs->valuecached = 0; /* prepare capture state */
luaL_checkstack(L, 4, "too many runtime captures");
pushluaval(cs); /* push function to be called */
@@ -313,8 +366,8 @@ typedef struct StrAux {
union {
Capture *cp; /* if not a string, respective capture */
struct { /* if it is a string... */
const char *s; /* ... starts here */
const char *e; /* ... ends here */
Index_t idx; /* starts here */
Index_t siz; /* with this size */
} s;
} u;
} StrAux;
@@ -329,24 +382,23 @@ typedef struct StrAux {
*/
static int getstrcaps (CapState *cs, StrAux *cps, int n) {
int k = n++;
Capture *head = cs->cap++;
cps[k].isstring = 1; /* get string value */
cps[k].u.s.s = cs->cap->s; /* starts here */
if (!isfullcap(cs->cap++)) { /* nested captures? */
while (!isclosecap(cs->cap)) { /* traverse them */
if (n >= MAXSTRCAPS) /* too many captures? */
nextcap(cs); /* skip extra captures (will not need them) */
else if (captype(cs->cap) == Csimple) /* string? */
n = getstrcaps(cs, cps, n); /* put info. into array */
else {
cps[n].isstring = 0; /* not a string */
cps[n].u.cp = cs->cap; /* keep original capture */
nextcap(cs);
n++;
}
cps[k].u.s.idx = head->index; /* starts here */
while (capinside(head, cs->cap)) {
if (n >= MAXSTRCAPS) /* too many captures? */
nextcap(cs); /* skip extra captures (will not need them) */
else if (captype(cs->cap) == Csimple) /* string? */
n = getstrcaps(cs, cps, n); /* put info. into array */
else {
cps[n].isstring = 0; /* not a string */
cps[n].u.cp = cs->cap; /* keep original capture */
nextcap(cs);
n++;
}
cs->cap++; /* skip close */
}
cps[k].u.s.e = closeaddr(cs->cap - 1); /* ends here */
cps[k].u.s.siz = closesize(cs, head);
skipclose(cs, head);
return n;
}
@@ -368,7 +420,7 @@ static void stringcap (luaL_Buffer *b, CapState *cs) {
const char *fmt; /* format string */
fmt = lua_tolstring(cs->L, updatecache(cs, cs->cap->idx), &len);
n = getstrcaps(cs, cps, 0) - 1; /* collect nested captures */
for (i = 0; i < len; i++) { /* traverse them */
for (i = 0; i < len; i++) { /* traverse format string */
if (fmt[i] != '%') /* not an escape? */
luaL_addchar(b, fmt[i]); /* add it to buffer */
else if (fmt[++i] < '0' || fmt[i] > '9') /* not followed by a digit? */
@@ -378,7 +430,7 @@ static void stringcap (luaL_Buffer *b, CapState *cs) {
if (l > n)
luaL_error(cs->L, "invalid capture index (%d)", l);
else if (cps[l].isstring)
luaL_addlstring(b, cps[l].u.s.s, cps[l].u.s.e - cps[l].u.s.s);
luaL_addlstring(b, cs->s + cps[l].u.s.idx, cps[l].u.s.siz);
else {
Capture *curr = cs->cap;
cs->cap = cps[l].u.cp; /* go back to evaluate that nested capture */
@@ -395,22 +447,20 @@ static void stringcap (luaL_Buffer *b, CapState *cs) {
** Substitution capture: add result to buffer 'b'
*/
static void substcap (luaL_Buffer *b, CapState *cs) {
const char *curr = cs->cap->s;
if (isfullcap(cs->cap)) /* no nested captures? */
luaL_addlstring(b, curr, cs->cap->siz - 1); /* keep original text */
else {
cs->cap++; /* skip open entry */
while (!isclosecap(cs->cap)) { /* traverse nested captures */
const char *next = cs->cap->s;
luaL_addlstring(b, curr, next - curr); /* add text up to capture */
if (addonestring(b, cs, "replacement"))
curr = closeaddr(cs->cap - 1); /* continue after match */
else /* no capture value */
curr = next; /* keep original text in final result */
}
luaL_addlstring(b, curr, cs->cap->s - curr); /* add last piece of text */
const char *curr = cs->s + cs->cap->index;
Capture *head = cs->cap++;
while (capinside(head, cs->cap)) {
Capture *cap = cs->cap;
const char *caps = cs->s + cap->index;
luaL_addlstring(b, curr, caps - curr); /* add text up to capture */
if (addonestring(b, cs, "replacement"))
curr = caps + capsize(cap, cs->cap - 1); /* continue after match */
else /* no capture value */
curr = caps; /* keep original text in final result */
}
cs->cap++; /* go to next capture */
/* add last piece of text */
luaL_addlstring(b, curr, cs->s + head->index + closesize(cs, head) - curr);
skipclose(cs, head);
}
@@ -426,13 +476,16 @@ static int addonestring (luaL_Buffer *b, CapState *cs, const char *what) {
case Csubst:
substcap(b, cs); /* add capture directly to buffer */
return 1;
case Cacc: /* accumulator capture? */
return luaL_error(cs->L, "invalid context for an accumulator capture");
default: {
lua_State *L = cs->L;
int n = pushcapture(cs);
if (n > 0) {
if (n > 1) lua_pop(L, n - 1); /* only one result */
if (!lua_isstring(L, -1))
luaL_error(L, "invalid %s value (a %s)", what, luaL_typename(L, -1));
return luaL_error(L, "invalid %s value (a %s)",
what, luaL_typename(L, -1));
luaL_addvalue(b);
}
return n;
@@ -458,7 +511,7 @@ static int pushcapture (CapState *cs) {
return luaL_error(L, "subcapture nesting too deep");
switch (captype(cs->cap)) {
case Cposition: {
lua_pushinteger(L, cs->cap->s - cs->s + 1);
lua_pushinteger(L, cs->cap->index + 1);
cs->cap++;
res = 1;
break;
@@ -516,6 +569,7 @@ static int pushcapture (CapState *cs) {
case Cbackref: res = backrefcap(cs); break;
case Ctable: res = tablecap(cs); break;
case Cfunction: res = functioncap(cs); break;
case Cacc: res = accumulatorcap(cs); break;
case Cnum: res = numcap(cs); break;
case Cquery: res = querycap(cs); break;
case Cfold: res = foldcap(cs); break;
@@ -529,7 +583,7 @@ static int pushcapture (CapState *cs) {
/*
** Prepare a CapState structure and traverse the entire list of
** captures in the stack pushing its results. 's' is the subject
** string, 'r' is the final position of the match, and 'ptop'
** string, 'r' is the final position of the match, and 'ptop'
** the index in the stack where some useful values were pushed.
** Returns the number of results pushed. (If the list produces no
** results, push the final position of the match.)
@@ -537,13 +591,16 @@ static int pushcapture (CapState *cs) {
int getcaptures (lua_State *L, const char *s, const char *r, int ptop) {
Capture *capture = (Capture *)lua_touserdata(L, caplistidx(ptop));
int n = 0;
/* printcaplist(capture); */
if (!isclosecap(capture)) { /* is there any capture? */
CapState cs;
cs.ocap = cs.cap = capture; cs.L = L; cs.reclevel = 0;
cs.s = s; cs.valuecached = 0; cs.ptop = ptop;
cs.firstcap = lua_gettop(L) + 1; /* where first value (if any) will go */
do { /* collect their values */
n += pushcapture(&cs);
} while (!isclosecap(cs.cap));
assert(lua_gettop(L) - cs.firstcap == n - 1);
}
if (n == 0) { /* no capture values? */
lua_pushinteger(L, r - s + 1); /* return only end position */

39
3rd/lpeg/lpcap.h
View File

@@ -1,6 +1,3 @@
/*
** $Id: lpcap.h $
*/
#if !defined(lpcap_h)
#define lpcap_h
@@ -19,6 +16,7 @@ typedef enum CapKind {
Csimple, /* next node is pattern */
Ctable, /* next node is pattern */
Cfunction, /* ktable[key] is function; next node is pattern */
Cacc, /* ktable[key] is function; next node is pattern */
Cquery, /* ktable[key] is table; next node is pattern */
Cstring, /* ktable[key] is string; next node is pattern */
Cnum, /* numbered capture; 'key' is number of value to return */
@@ -29,8 +27,20 @@ typedef enum CapKind {
} CapKind;
/*
** An unsigned integer large enough to index any subject entirely.
** It can be size_t, but that will double the size of the array
** of captures in a 64-bit machine.
*/
#if !defined(Index_t)
typedef uint Index_t;
#endif
#define MAXINDT (~(Index_t)0)
typedef struct Capture {
const char *s; /* subject position */
Index_t index; /* subject position */
unsigned short idx; /* extra info (group name, arg index, etc.) */
byte kind; /* kind of capture */
byte siz; /* size of full capture + 1 (0 = not a full capture) */
@@ -41,13 +51,32 @@ typedef struct CapState {
Capture *cap; /* current capture */
Capture *ocap; /* (original) capture list */
lua_State *L;
int ptop; /* index of last argument to 'match' */
int ptop; /* stack index of last argument to 'match' */
int firstcap; /* stack index of first capture pushed in the stack */
const char *s; /* original string */
int valuecached; /* value stored in cache slot */
int reclevel; /* recursion level */
} CapState;
#define captype(cap) ((cap)->kind)
#define isclosecap(cap) (captype(cap) == Cclose)
#define isopencap(cap) ((cap)->siz == 0)
/* true if c2 is (any number of levels) inside c1 */
#define capinside(c1,c2) \
(isopencap(c1) ? !isclosecap(c2) \
: (c2)->index < (c1)->index + (c1)->siz - 1)
/**
** Maximum number of captures to visit when looking for an 'open'.
*/
#define MAXLOP 20
int runtimecap (CapState *cs, Capture *close, const char *s, int *rem);
int getcaptures (lua_State *L, const char *s, const char *r, int ptop);
int finddyncap (Capture *cap, Capture *last);

391
3rd/lpeg/lpcode.c
View File

@@ -1,7 +1,3 @@
/*
** $Id: lpcode.c $
** Copyright 2007, Lua.org & PUC-Rio (see 'lpeg.html' for license)
*/
#include <limits.h>
@@ -11,6 +7,7 @@
#include "lptypes.h"
#include "lpcode.h"
#include "lpcset.h"
/* signals a "no-instruction */
@@ -26,61 +23,13 @@ static const Charset fullset_ =
static const Charset *fullset = &fullset_;
/*
** {======================================================
** Analysis and some optimizations
** =======================================================
*/
/*
** Check whether a charset is empty (returns IFail), singleton (IChar),
** full (IAny), or none of those (ISet). When singleton, '*c' returns
** which character it is. (When generic set, the set was the input,
** so there is no need to return it.)
*/
static Opcode charsettype (const byte *cs, int *c) {
int count = 0; /* number of characters in the set */
int i;
int candidate = -1; /* candidate position for the singleton char */
for (i = 0; i < CHARSETSIZE; i++) { /* for each byte */
int b = cs[i];
if (b == 0) { /* is byte empty? */
if (count > 1) /* was set neither empty nor singleton? */
return ISet; /* neither full nor empty nor singleton */
/* else set is still empty or singleton */
}
else if (b == 0xFF) { /* is byte full? */
if (count < (i * BITSPERCHAR)) /* was set not full? */
return ISet; /* neither full nor empty nor singleton */
else count += BITSPERCHAR; /* set is still full */
}
else if ((b & (b - 1)) == 0) { /* has byte only one bit? */
if (count > 0) /* was set not empty? */
return ISet; /* neither full nor empty nor singleton */
else { /* set has only one char till now; track it */
count++;
candidate = i;
}
}
else return ISet; /* byte is neither empty, full, nor singleton */
}
switch (count) {
case 0: return IFail; /* empty set */
case 1: { /* singleton; find character bit inside byte */
int b = cs[candidate];
*c = candidate * BITSPERCHAR;
if ((b & 0xF0) != 0) { *c += 4; b >>= 4; }
if ((b & 0x0C) != 0) { *c += 2; b >>= 2; }
if ((b & 0x02) != 0) { *c += 1; }
return IChar;
}
default: {
assert(count == CHARSETSIZE * BITSPERCHAR); /* full set */
return IAny;
}
}
}
/*
** A few basic operations on Charsets
@@ -89,42 +38,12 @@ static void cs_complement (Charset *cs) {
loopset(i, cs->cs[i] = ~cs->cs[i]);
}
static int cs_equal (const byte *cs1, const byte *cs2) {
loopset(i, if (cs1[i] != cs2[i]) return 0);
return 1;
}
static int cs_disjoint (const Charset *cs1, const Charset *cs2) {
loopset(i, if ((cs1->cs[i] & cs2->cs[i]) != 0) return 0;)
return 1;
}
/*
** If 'tree' is a 'char' pattern (TSet, TChar, TAny), convert it into a
** charset and return 1; else return 0.
*/
int tocharset (TTree *tree, Charset *cs) {
switch (tree->tag) {
case TSet: { /* copy set */
loopset(i, cs->cs[i] = treebuffer(tree)[i]);
return 1;
}
case TChar: { /* only one char */
assert(0 <= tree->u.n && tree->u.n <= UCHAR_MAX);
loopset(i, cs->cs[i] = 0); /* erase all chars */
setchar(cs->cs, tree->u.n); /* add that one */
return 1;
}
case TAny: {
loopset(i, cs->cs[i] = 0xFF); /* add all characters to the set */
return 1;
}
default: return 0;
}
}
/*
** Visit a TCall node taking care to stop recursion. If node not yet
** visited, return 'f(sib2(tree))', otherwise return 'def' (default
@@ -196,7 +115,7 @@ int hascaptures (TTree *tree) {
int checkaux (TTree *tree, int pred) {
tailcall:
switch (tree->tag) {
case TChar: case TSet: case TAny:
case TChar: case TSet: case TAny: case TUTFR:
case TFalse: case TOpenCall:
return 0; /* not nullable */
case TRep: case TTrue:
@@ -220,7 +139,7 @@ int checkaux (TTree *tree, int pred) {
if (checkaux(sib2(tree), pred)) return 1;
/* else return checkaux(sib1(tree), pred); */
tree = sib1(tree); goto tailcall;
case TCapture: case TGrammar: case TRule:
case TCapture: case TGrammar: case TRule: case TXInfo:
/* return checkaux(sib1(tree), pred); */
tree = sib1(tree); goto tailcall;
case TCall: /* return checkaux(sib2(tree), pred); */
@@ -239,11 +158,13 @@ int fixedlen (TTree *tree) {
switch (tree->tag) {
case TChar: case TSet: case TAny:
return len + 1;
case TUTFR:
return (tree->cap == sib1(tree)->cap) ? len + tree->cap : -1;
case TFalse: case TTrue: case TNot: case TAnd: case TBehind:
return len;
case TRep: case TRunTime: case TOpenCall:
return -1;
case TCapture: case TRule: case TGrammar:
case TCapture: case TRule: case TGrammar: case TXInfo:
/* return fixedlen(sib1(tree)); */
tree = sib1(tree); goto tailcall;
case TCall: {
@@ -294,18 +215,21 @@ int fixedlen (TTree *tree) {
static int getfirst (TTree *tree, const Charset *follow, Charset *firstset) {
tailcall:
switch (tree->tag) {
case TChar: case TSet: case TAny: {
case TChar: case TSet: case TAny: case TFalse: {
tocharset(tree, firstset);
return 0;
}
case TUTFR: {
int c;
clearset(firstset->cs); /* erase all chars */
for (c = tree->key; c <= sib1(tree)->key; c++)
setchar(firstset->cs, c);
return 0;
}
case TTrue: {
loopset(i, firstset->cs[i] = follow->cs[i]);
return 1; /* accepts the empty string */
}
case TFalse: {
loopset(i, firstset->cs[i] = 0);
return 0;
}
case TChoice: {
Charset csaux;
int e1 = getfirst(sib1(tree), follow, firstset);
@@ -334,7 +258,7 @@ static int getfirst (TTree *tree, const Charset *follow, Charset *firstset) {
loopset(i, firstset->cs[i] |= follow->cs[i]);
return 1; /* accept the empty string */
}
case TCapture: case TGrammar: case TRule: {
case TCapture: case TGrammar: case TRule: case TXInfo: {
/* return getfirst(sib1(tree), follow, firstset); */
tree = sib1(tree); goto tailcall;
}
@@ -356,9 +280,8 @@ static int getfirst (TTree *tree, const Charset *follow, Charset *firstset) {
if (tocharset(sib1(tree), firstset)) {
cs_complement(firstset);
return 1;
}
/* else go through */
}
} /* else */
} /* FALLTHROUGH */
case TBehind: { /* instruction gives no new information */
/* call 'getfirst' only to check for math-time captures */
int e = getfirst(sib1(tree), follow, firstset);
@@ -380,9 +303,9 @@ static int headfail (TTree *tree) {
case TChar: case TSet: case TAny: case TFalse:
return 1;
case TTrue: case TRep: case TRunTime: case TNot:
case TBehind:
case TBehind: case TUTFR:
return 0;
case TCapture: case TGrammar: case TRule: case TAnd:
case TCapture: case TGrammar: case TRule: case TXInfo: case TAnd:
tree = sib1(tree); goto tailcall; /* return headfail(sib1(tree)); */
case TCall:
tree = sib2(tree); goto tailcall; /* return headfail(sib2(tree)); */
@@ -407,7 +330,7 @@ static int headfail (TTree *tree) {
static int needfollow (TTree *tree) {
tailcall:
switch (tree->tag) {
case TChar: case TSet: case TAny:
case TChar: case TSet: case TAny: case TUTFR:
case TFalse: case TTrue: case TAnd: case TNot:
case TRunTime: case TGrammar: case TCall: case TBehind:
return 0;
@@ -418,7 +341,7 @@ static int needfollow (TTree *tree) {
case TSeq:
tree = sib2(tree); goto tailcall;
default: assert(0); return 0;
}
}
}
/* }====================================================== */
@@ -437,10 +360,11 @@ static int needfollow (TTree *tree) {
*/
int sizei (const Instruction *i) {
switch((Opcode)i->i.code) {
case ISet: case ISpan: return CHARSETINSTSIZE;
case ITestSet: return CHARSETINSTSIZE + 1;
case ISet: case ISpan: return 1 + i->i.aux2.set.size;
case ITestSet: return 2 + i->i.aux2.set.size;
case ITestChar: case ITestAny: case IChoice: case IJmp: case ICall:
case IOpenCall: case ICommit: case IPartialCommit: case IBackCommit:
case IUTFR:
return 2;
default: return 1;
}
@@ -468,23 +392,67 @@ static void codegen (CompileState *compst, TTree *tree, int opt, int tt,
const Charset *fl);
void realloccode (lua_State *L, Pattern *p, int nsize) {
void *ud;
lua_Alloc f = lua_getallocf(L, &ud);
void *newblock = f(ud, p->code, p->codesize * sizeof(Instruction),
nsize * sizeof(Instruction));
if (newblock == NULL && nsize > 0)
static void finishrelcode (lua_State *L, Pattern *p, Instruction *block,
int size) {
if (block == NULL)
luaL_error(L, "not enough memory");
p->code = (Instruction *)newblock;
p->codesize = nsize;
block->codesize = size;
p->code = (Instruction *)block + 1;
}
static int nextinstruction (CompileState *compst) {
int size = compst->p->codesize;
if (compst->ncode >= size)
realloccode(compst->L, compst->p, size * 2);
return compst->ncode++;
/*
** Initialize array 'p->code'
*/
static void newcode (lua_State *L, Pattern *p, int size) {
void *ud;
Instruction *block;
lua_Alloc f = lua_getallocf(L, &ud);
size++; /* slot for 'codesize' */
block = (Instruction*) f(ud, NULL, 0, size * sizeof(Instruction));
finishrelcode(L, p, block, size);
}
void freecode (lua_State *L, Pattern *p) {
if (p->code != NULL) {
void *ud;
lua_Alloc f = lua_getallocf(L, &ud);
uint osize = p->code[-1].codesize;
f(ud, p->code - 1, osize * sizeof(Instruction), 0); /* free block */
}
}
/*
** Assume that 'nsize' is not zero and that 'p->code' already exists.
*/
static void realloccode (lua_State *L, Pattern *p, int nsize) {
void *ud;
lua_Alloc f = lua_getallocf(L, &ud);
Instruction *block = p->code - 1;
uint osize = block->codesize;
nsize++; /* add the 'codesize' slot to size */
block = (Instruction*) f(ud, block, osize * sizeof(Instruction),
nsize * sizeof(Instruction));
finishrelcode(L, p, block, nsize);
}
/*
** Add space for an instruction with 'n' slots and return its index.
*/
static int nextinstruction (CompileState *compst, int n) {
int size = compst->p->code[-1].codesize - 1;
int ncode = compst->ncode;
if (ncode > size - n) {
uint nsize = size + (size >> 1) + n;
if (nsize >= INT_MAX)
luaL_error(compst->L, "pattern code too large");
realloccode(compst->L, compst->p, nsize);
}
compst->ncode = ncode + n;
return ncode;
}
@@ -492,9 +460,9 @@ static int nextinstruction (CompileState *compst) {
static int addinstruction (CompileState *compst, Opcode op, int aux) {
int i = nextinstruction(compst);
int i = nextinstruction(compst, 1);
getinstr(compst, i).i.code = op;
getinstr(compst, i).i.aux = aux;
getinstr(compst, i).i.aux1 = aux;
return i;
}
@@ -518,6 +486,16 @@ static void setoffset (CompileState *compst, int instruction, int offset) {
}
static void codeutfr (CompileState *compst, TTree *tree) {
int i = addoffsetinst(compst, IUTFR);
int to = sib1(tree)->u.n;
assert(sib1(tree)->tag == TXInfo);
getinstr(compst, i + 1).offset = tree->u.n;
getinstr(compst, i).i.aux1 = to & 0xff;
getinstr(compst, i).i.aux2.key = to >> 8;
}
/*
** Add a capture instruction:
** 'op' is the capture instruction; 'cap' the capture kind;
@@ -527,7 +505,7 @@ static void setoffset (CompileState *compst, int instruction, int offset) {
static int addinstcap (CompileState *compst, Opcode op, int cap, int key,
int aux) {
int i = addinstruction(compst, op, joinkindoff(cap, aux));
getinstr(compst, i).i.key = key;
getinstr(compst, i).i.aux2.key = key;
return i;
}
@@ -560,7 +538,7 @@ static void jumptohere (CompileState *compst, int instruction) {
*/
static void codechar (CompileState *compst, int c, int tt) {
if (tt >= 0 && getinstr(compst, tt).i.code == ITestChar &&
getinstr(compst, tt).i.aux == c)
getinstr(compst, tt).i.aux1 == c)
addinstruction(compst, IAny, 0);
else
addinstruction(compst, IChar, c);
@@ -568,45 +546,63 @@ static void codechar (CompileState *compst, int c, int tt) {
/*
** Add a charset posfix to an instruction
** Add a charset posfix to an instruction.
*/
static void addcharset (CompileState *compst, const byte *cs) {
int p = gethere(compst);
static void addcharset (CompileState *compst, int inst, charsetinfo *info) {
int p;
Instruction *I = &getinstr(compst, inst);
byte *charset;
int isize = instsize(info->size); /* size in instructions */
int i;
for (i = 0; i < (int)CHARSETINSTSIZE - 1; i++)
nextinstruction(compst); /* space for buffer */
/* fill buffer with charset */
loopset(j, getinstr(compst, p).buff[j] = cs[j]);
I->i.aux2.set.offset = info->offset * 8; /* offset in bits */
I->i.aux2.set.size = isize;
I->i.aux1 = info->deflt;
p = nextinstruction(compst, isize); /* space for charset */
charset = getinstr(compst, p).buff; /* charset buffer */
for (i = 0; i < isize * (int)sizeof(Instruction); i++)
charset[i] = getbytefromcharset(info, i); /* copy the buffer */
}
/*
** code a char set, optimizing unit sets for IChar, "complete"
** sets for IAny, and empty sets for IFail; also use an IAny
** when instruction is dominated by an equivalent test.
** Check whether charset 'info' is dominated by instruction 'p'
*/
static void codecharset (CompileState *compst, const byte *cs, int tt) {
int c = 0; /* (=) to avoid warnings */
Opcode op = charsettype(cs, &c);
switch (op) {
case IChar: codechar(compst, c, tt); break;
case ISet: { /* non-trivial set? */
if (tt >= 0 && getinstr(compst, tt).i.code == ITestSet &&
cs_equal(cs, getinstr(compst, tt + 2).buff))
addinstruction(compst, IAny, 0);
else {
addinstruction(compst, ISet, 0);
addcharset(compst, cs);
}
break;
static int cs_equal (Instruction *p, charsetinfo *info) {
if (p->i.code != ITestSet)
return 0;
else if (p->i.aux2.set.offset != info->offset * 8 ||
p->i.aux2.set.size != instsize(info->size) ||
p->i.aux1 != info->deflt)
return 0;
else {
int i;
for (i = 0; i < instsize(info->size) * (int)sizeof(Instruction); i++) {
if ((p + 2)->buff[i] != getbytefromcharset(info, i))
return 0;
}
default: addinstruction(compst, op, c); break;
}
return 1;
}
/*
** Code a char set, using IAny when instruction is dominated by an
** equivalent test.
*/
static void codecharset (CompileState *compst, TTree *tree, int tt) {
charsetinfo info;
tree2cset(tree, &info);
if (tt >= 0 && cs_equal(&getinstr(compst, tt), &info))
addinstruction(compst, IAny, 0);
else {
int i = addinstruction(compst, ISet, 0);
addcharset(compst, i, &info);
}
}
/*
** code a test set, optimizing unit sets for ITestChar, "complete"
** Code a test set, optimizing unit sets for ITestChar, "complete"
** sets for ITestAny, and empty sets for IJmp (always fails).
** 'e' is true iff test should accept the empty string. (Test
** instructions in the current VM never accept the empty string.)
@@ -614,22 +610,22 @@ static void codecharset (CompileState *compst, const byte *cs, int tt) {
static int codetestset (CompileState *compst, Charset *cs, int e) {
if (e) return NOINST; /* no test */
else {
int c = 0;
Opcode op = charsettype(cs->cs, &c);
charsetinfo info;
Opcode op = charsettype(cs->cs, &info);
switch (op) {
case IFail: return addoffsetinst(compst, IJmp); /* always jump */
case IAny: return addoffsetinst(compst, ITestAny);
case IChar: {
int i = addoffsetinst(compst, ITestChar);
getinstr(compst, i).i.aux = c;
getinstr(compst, i).i.aux1 = info.offset;
return i;
}
case ISet: {
default: { /* regular set */
int i = addoffsetinst(compst, ITestSet);
addcharset(compst, cs->cs);
addcharset(compst, i, &info);
assert(op == ISet);
return i;
}
default: assert(0); return 0;
}
}
}
@@ -665,11 +661,11 @@ static void codebehind (CompileState *compst, TTree *tree) {
/*
** Choice; optimizations:
** - when p1 is headfail or
** when first(p1) and first(p2) are disjoint, than
** a character not in first(p1) cannot go to p1, and a character
** in first(p1) cannot go to p2 (at it is not in first(p2)).
** (The optimization is not valid if p1 accepts the empty string,
** - when p1 is headfail or when first(p1) and first(p2) are disjoint,
** than a character not in first(p1) cannot go to p1 and a character
** in first(p1) cannot go to p2, either because p1 will accept
** (headfail) or because it is not in first(p2) (disjoint).
** (The second case is not valid if p1 accepts the empty string,
** as then there is no character at all...)
** - when p2 is empty and opt is true; a IPartialCommit can reuse
** the Choice already active in the stack.
@@ -686,7 +682,7 @@ static void codechoice (CompileState *compst, TTree *p1, TTree *p2, int opt,
int jmp = NOINST;
codegen(compst, p1, 0, test, fl);
if (!emptyp2)
jmp = addoffsetinst(compst, IJmp);
jmp = addoffsetinst(compst, IJmp);
jumptohere(compst, test);
codegen(compst, p2, opt, NOINST, fl);
jumptohere(compst, jmp);
@@ -697,7 +693,7 @@ static void codechoice (CompileState *compst, TTree *p1, TTree *p2, int opt,
codegen(compst, p1, 1, NOINST, fullset);
}
else {
/* <p1 / p2> ==
/* <p1 / p2> ==
test(first(p1)) -> L1; choice L1; <p1>; commit L2; L1: <p2>; L2: */
int pcommit;
int test = codetestset(compst, &cs1, e1);
@@ -764,9 +760,53 @@ static void coderuntime (CompileState *compst, TTree *tree, int tt) {
}
/*
** Create a jump to 'test' and fix 'test' to jump to next instruction
*/
static void closeloop (CompileState *compst, int test) {
int jmp = addoffsetinst(compst, IJmp);
jumptohere(compst, test);
jumptothere(compst, jmp, test);
}
/*
** Try repetition of charsets:
** For an empty set, repetition of fail is a no-op;
** For any or char, code a tight loop;
** For generic charset, use a span instruction.
*/
static int coderepcharset (CompileState *compst, TTree *tree) {
switch (tree->tag) {
case TFalse: return 1; /* 'fail*' is a no-op */
case TAny: { /* L1: testany -> L2; any; jmp L1; L2: */
int test = addoffsetinst(compst, ITestAny);
addinstruction(compst, IAny, 0);
closeloop(compst, test);
return 1;
}
case TChar: { /* L1: testchar c -> L2; any; jmp L1; L2: */
int test = addoffsetinst(compst, ITestChar);
getinstr(compst, test).i.aux1 = tree->u.n;
addinstruction(compst, IAny, 0);
closeloop(compst, test);
return 1;
}
case TSet: { /* regular set */
charsetinfo info;
int i = addinstruction(compst, ISpan, 0);
tree2cset(tree, &info);
addcharset(compst, i, &info);
return 1;
}
default: return 0; /* not a charset */
}
}
/*
** Repetion; optimizations:
** When pattern is a charset, can use special instruction ISpan.
** When pattern is a charset, use special code.
** When pattern is head fail, or if it starts with characters that
** are disjoint from what follows the repetions, a simple test
** is enough (a fail inside the repetition would backtrack to fail
@@ -776,21 +816,14 @@ static void coderuntime (CompileState *compst, TTree *tree, int tt) {
*/
static void coderep (CompileState *compst, TTree *tree, int opt,
const Charset *fl) {
Charset st;
if (tocharset(tree, &st)) {
addinstruction(compst, ISpan, 0);
addcharset(compst, st.cs);
}
else {
if (!coderepcharset(compst, tree)) {
Charset st;
int e1 = getfirst(tree, fullset, &st);
if (headfail(tree) || (!e1 && cs_disjoint(&st, fl))) {
/* L1: test (fail(p1)) -> L2; <p>; jmp L1; L2: */
int jmp;
int test = codetestset(compst, &st, 0);
codegen(compst, tree, 0, test, fullset);
jmp = addoffsetinst(compst, IJmp);
jumptohere(compst, test);
jumptothere(compst, jmp, test);
closeloop(compst, test);
}
else {
/* test(fail(p1)) -> L2; choice L2; L1: <p>; partialcommit L1; L2: */
@@ -847,7 +880,7 @@ static void correctcalls (CompileState *compst, int *positions,
Instruction *code = compst->p->code;
for (i = from; i < to; i += sizei(&code[i])) {
if (code[i].i.code == IOpenCall) {
int n = code[i].i.key; /* rule number */
int n = code[i].i.aux2.key; /* rule number */
int rule = positions[n]; /* rule position */
assert(rule == from || code[rule - 1].i.code == IRet);
if (code[finaltarget(code, i + 2)].i.code == IRet) /* call; ret ? */
@@ -874,8 +907,10 @@ static void codegrammar (CompileState *compst, TTree *grammar) {
int start = gethere(compst); /* here starts the initial rule */
jumptohere(compst, firstcall);
for (rule = sib1(grammar); rule->tag == TRule; rule = sib2(rule)) {
TTree *r = sib1(rule);
assert(r->tag == TXInfo);
positions[rulenumber++] = gethere(compst); /* save rule position */
codegen(compst, sib1(rule), 0, NOINST, fullset); /* code rule */
codegen(compst, sib1(r), 0, NOINST, fullset); /* code rule */
addinstruction(compst, IRet, 0);
}
assert(rule->tag == TTrue);
@@ -886,8 +921,8 @@ static void codegrammar (CompileState *compst, TTree *grammar) {
static void codecall (CompileState *compst, TTree *call) {
int c = addoffsetinst(compst, IOpenCall); /* to be corrected later */
getinstr(compst, c).i.key = sib2(call)->cap; /* rule number */
assert(sib2(call)->tag == TRule);
assert(sib1(sib2(call))->tag == TXInfo);
getinstr(compst, c).i.aux2.key = sib1(sib2(call))->u.n; /* rule number */
}
@@ -922,9 +957,10 @@ static void codegen (CompileState *compst, TTree *tree, int opt, int tt,
switch (tree->tag) {
case TChar: codechar(compst, tree->u.n, tt); break;
case TAny: addinstruction(compst, IAny, 0); break;
case TSet: codecharset(compst, treebuffer(tree), tt); break;
case TSet: codecharset(compst, tree, tt); break;
case TTrue: break;
case TFalse: addinstruction(compst, IFail, 0); break;
case TUTFR: codeutfr(compst, tree); break;
case TChoice: codechoice(compst, sib1(tree), sib2(tree), opt, fl); break;
case TRep: coderep(compst, sib1(tree), opt, fl); break;
case TBehind: codebehind(compst, tree); break;
@@ -971,7 +1007,7 @@ static void peephole (CompileState *compst) {
case IRet: case IFail: case IFailTwice:
case IEnd: { /* instructions with unconditional implicit jumps */
code[i] = code[ft]; /* jump becomes that instruction */
code[i + 1].i.code = IAny; /* 'no-op' for target position */
code[i + 1].i.code = IEmpty; /* 'no-op' for target position */
break;
}
case ICommit: case IPartialCommit:
@@ -996,12 +1032,13 @@ static void peephole (CompileState *compst) {
/*
** Compile a pattern
** Compile a pattern. 'size' is the size of the pattern's tree,
** which gives a hint for the size of the final code.
*/
Instruction *compile (lua_State *L, Pattern *p) {
Instruction *compile (lua_State *L, Pattern *p, uint size) {
CompileState compst;
compst.p = p; compst.ncode = 0; compst.L = L;
realloccode(L, p, 2); /* minimum initial size */
newcode(L, p, size/2u + 2); /* set initial size */
codegen(&compst, p->tree, 0, NOINST, fullset);
addinstruction(&compst, IEnd, 0);
realloccode(L, p, compst.ncode); /* set final size */

8
3rd/lpeg/lpcode.h
View File

@@ -1,6 +1,3 @@
/*
** $Id: lpcode.h $
*/
#if !defined(lpcode_h)
#define lpcode_h
@@ -11,13 +8,12 @@
#include "lptree.h"
#include "lpvm.h"
int tocharset (TTree *tree, Charset *cs);
int checkaux (TTree *tree, int pred);
int fixedlen (TTree *tree);
int hascaptures (TTree *tree);
int lp_gc (lua_State *L);
Instruction *compile (lua_State *L, Pattern *p);
void realloccode (lua_State *L, Pattern *p, int nsize);
Instruction *compile (lua_State *L, Pattern *p, uint size);
void freecode (lua_State *L, Pattern *p);
int sizei (const Instruction *i);

110
3rd/lpeg/lpcset.c Normal file
View File

@@ -0,0 +1,110 @@
#include "lptypes.h"
#include "lpcset.h"
/*
** Add to 'c' the index of the (only) bit set in byte 'b'
*/
static int onlybit (int c, int b) {
if ((b & 0xF0) != 0) { c += 4; b >>= 4; }
if ((b & 0x0C) != 0) { c += 2; b >>= 2; }
if ((b & 0x02) != 0) { c += 1; }
return c;
}
/*
** Check whether a charset is empty (returns IFail), singleton (IChar),
** full (IAny), or none of those (ISet). When singleton, 'info.offset'
** returns which character it is. When generic set, 'info' returns
** information about its range.
*/
Opcode charsettype (const byte *cs, charsetinfo *info) {
int low0, low1, high0, high1;
for (low1 = 0; low1 < CHARSETSIZE && cs[low1] == 0; low1++)
/* find lowest byte with a 1-bit */;
if (low1 == CHARSETSIZE)
return IFail; /* no characters in set */
for (high1 = CHARSETSIZE - 1; cs[high1] == 0; high1--)
/* find highest byte with a 1-bit; low1 is a sentinel */;
if (low1 == high1) { /* only one byte with 1-bits? */
int b = cs[low1];
if ((b & (b - 1)) == 0) { /* does byte has only one 1-bit? */
info->offset = onlybit(low1 * BITSPERCHAR, b); /* get that bit */
return IChar; /* single character */
}
}
for (low0 = 0; low0 < CHARSETSIZE && cs[low0] == 0xFF; low0++)
/* find lowest byte with a 0-bit */;
if (low0 == CHARSETSIZE)
return IAny; /* set has all bits set */
for (high0 = CHARSETSIZE - 1; cs[high0] == 0xFF; high0--)
/* find highest byte with a 0-bit; low0 is a sentinel */;
if (high1 - low1 <= high0 - low0) { /* range of 1s smaller than of 0s? */
info->offset = low1;
info->size = high1 - low1 + 1;
info->deflt = 0; /* all discharged bits were 0 */
}
else {
info->offset = low0;
info->size = high0 - low0 + 1;
info->deflt = 0xFF; /* all discharged bits were 1 */
}
info->cs = cs + info->offset;
return ISet;
}
/*
** Get a byte from a compact charset. If index is inside the charset
** range, get the byte from the supporting charset (correcting it
** by the offset). Otherwise, return the default for the set.
*/
byte getbytefromcharset (const charsetinfo *info, int index) {
if (index < info->size)
return info->cs[index];
else return info->deflt;
}
/*
** If 'tree' is a 'char' pattern (TSet, TChar, TAny, TFalse), convert it
** into a charset and return 1; else return 0.
*/
int tocharset (TTree *tree, Charset *cs) {
switch (tree->tag) {
case TChar: { /* only one char */
assert(0 <= tree->u.n && tree->u.n <= UCHAR_MAX);
clearset(cs->cs); /* erase all chars */
setchar(cs->cs, tree->u.n); /* add that one */
return 1;
}
case TAny: {
fillset(cs->cs, 0xFF); /* add all characters to the set */
return 1;
}
case TFalse: {
clearset(cs->cs); /* empty set */
return 1;
}
case TSet: { /* fill set */
int i;
fillset(cs->cs, tree->u.set.deflt);
for (i = 0; i < tree->u.set.size; i++)
cs->cs[tree->u.set.offset + i] = treebuffer(tree)[i];
return 1;
}
default: return 0;
}
}
void tree2cset (TTree *tree, charsetinfo *info) {
assert(tree->tag == TSet);
info->offset = tree->u.set.offset;
info->size = tree->u.set.size;
info->deflt = tree->u.set.deflt;
info->cs = treebuffer(tree);
}

30
3rd/lpeg/lpcset.h Normal file
View File

@@ -0,0 +1,30 @@
#if !defined(lpset_h)
#define lpset_h
#include "lpcset.h"
#include "lpcode.h"
#include "lptree.h"
/*
** Extra information for the result of 'charsettype'. When result is
** IChar, 'offset' is the character. When result is ISet, 'cs' is the
** supporting bit array (with offset included), 'offset' is the offset
** (in bytes), 'size' is the size (in bytes), and 'delt' is the default
** value for bytes outside the set.
*/
typedef struct {
const byte *cs;
int offset;
int size;
int deflt;
} charsetinfo;
int tocharset (TTree *tree, Charset *cs);
Opcode charsettype (const byte *cs, charsetinfo *info);
byte getbytefromcharset (const charsetinfo *info, int index);
void tree2cset (TTree *tree, charsetinfo *info);
#endif

365
3rd/lpeg/lpeg.html
View File

@@ -1,28 +1,27 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
"//www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="//www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<title>LPeg - Parsing Expression Grammars For Lua</title>
<link rel="stylesheet"
href="http://www.inf.puc-rio.br/~roberto/lpeg/doc.css"
href="//www.inf.puc-rio.br/~roberto/lpeg/doc.css"
type="text/css"/>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
</head>
<body>
<!-- $Id: lpeg.html $ -->
<div id="container">
<div id="product">
<div id="product_logo">
<a href="http://www.inf.puc-rio.br/~roberto/lpeg/">
<a href="//www.inf.puc-rio.br/~roberto/lpeg/">
<img alt="LPeg logo" src="lpeg-128.gif"/></a>
</div>
<div id="product_name"><big><strong>LPeg</strong></big></div>
<div id="product_description">
Parsing Expression Grammars For Lua, version 1.0
Parsing Expression Grammars For Lua, version 1.1
</div>
</div> <!-- id="product" -->
@@ -56,16 +55,16 @@
<p>
<em>LPeg</em> is a new pattern-matching library for Lua,
based on
<a href="http://pdos.csail.mit.edu/%7Ebaford/packrat/">
<a href="//bford.info/packrat/">
Parsing Expression Grammars</a> (PEGs).
This text is a reference manual for the library.
For a more formal treatment of LPeg,
as well as some discussion about its implementation,
see
<a href="http://www.inf.puc-rio.br/~roberto/docs/peg.pdf">
<a href="//www.inf.puc-rio.br/~roberto/docs/peg.pdf">
A Text Pattern-Matching Tool based on Parsing Expression Grammars</a>.
(You may also be interested in my
<a href="http://vimeo.com/1485123">talk about LPeg</a>
<a href="//vimeo.com/1485123">talk about LPeg</a>
given at the III Lua Workshop.)
</p>
@@ -107,6 +106,9 @@ for creating patterns:
<td>Matches any character in <code>string</code> (Set)</td></tr>
<tr><td><a href="#op-r"><code>lpeg.R("<em>xy</em>")</code></a></td>
<td>Matches any character between <em>x</em> and <em>y</em> (Range)</td></tr>
<tr><td><a href="#op-utfR"><code>lpeg.utfR(cp1, cp2)</code></a></td>
<td>Matches an UTF-8 code point between <code>cp1</code> and
<code>cp2</code></td></tr>
<tr><td><a href="#op-pow"><code>patt^n</code></a></td>
<td>Matches at least <code>n</code> repetitions of <code>patt</code></td></tr>
<tr><td><a href="#op-pow"><code>patt^-n</code></a></td>
@@ -142,7 +144,7 @@ so, it succeeds only at the end of the subject.
LPeg also offers the <a href="re.html"><code>re</code> module</a>,
which implements patterns following a regular-expression style
(e.g., <code>[09]+</code>).
(This module is 260 lines of Lua code,
(This module is 270 lines of Lua code,
and of course it uses LPeg to parse regular expressions and
translate them to regular LPeg patterns.)
</p>
@@ -164,7 +166,7 @@ or the <a href="#captures">captured values</a>
<p>
An optional numeric argument <code>init</code> makes the match
start at that position in the subject string.
As usual in Lua libraries,
As in the Lua standard libraries,
a negative value counts from the end.
</p>
@@ -188,9 +190,9 @@ returns the string <code>"pattern"</code>.
Otherwise returns nil.
</p>
<h3><a name="f-version"></a><code>lpeg.version ()</code></h3>
<h3><a name="f-version"></a><code>lpeg.version</code></h3>
<p>
Returns a string with the running version of LPeg.
A string (not a function) with the running version of LPeg.
</p>
<h3><a name="f-setstack"></a><code>lpeg.setmaxstack (max)</code></h3>
@@ -329,6 +331,15 @@ are patterns that always fail.
</p>
<h3><a name="op-utfR"></a><code>lpeg.utfR (cp1, cp2)</code></h3>
<p>
Returns a pattern that matches a valid UTF-8 byte sequence
representing a code point in the range <code>[cp1, cp2]</code>.
The range is limited by the natural Unicode limit of 0x10FFFF,
but may include surrogates.
</p>
<h3><a name="op-v"></a><code>lpeg.V (v)</code></h3>
<p>
This operation creates a non-terminal (a <em>variable</em>)
@@ -433,7 +444,8 @@ letter = lower + upper
<h3><a name="op-sub"></a><code>patt1 - patt2</code></h3>
<p>
Returns a pattern equivalent to <em>!patt2 patt1</em>.
Returns a pattern equivalent to <em>!patt2 patt1</em>
in the origial PEG notation.
This pattern asserts that the input does not match
<code>patt2</code> and then matches <code>patt1</code>.
</p>
@@ -597,17 +609,17 @@ The following table summarizes the basic captures:
<tr><td><a href="#cap-arg"><code>lpeg.Carg(n)</code></a></td>
<td>the value of the n<sup>th</sup> extra argument to
<code>lpeg.match</code> (matches the empty string)</td></tr>
<tr><td><a href="#cap-b"><code>lpeg.Cb(name)</code></a></td>
<tr><td><a href="#cap-b"><code>lpeg.Cb(key)</code></a></td>
<td>the values produced by the previous
group capture named <code>name</code>
group capture named <code>key</code>
(matches the empty string)</td></tr>
<tr><td><a href="#cap-cc"><code>lpeg.Cc(values)</code></a></td>
<td>the given values (matches the empty string)</td></tr>
<tr><td><a href="#cap-f"><code>lpeg.Cf(patt, func)</code></a></td>
<td>a <em>folding</em> of the captures from <code>patt</code></td></tr>
<tr><td><a href="#cap-g"><code>lpeg.Cg(patt [, name])</code></a></td>
<td>folding capture (<em>deprecated</em>)</td></tr>
<tr><td><a href="#cap-g"><code>lpeg.Cg(patt [, key])</code></a></td>
<td>the values produced by <code>patt</code>,
optionally tagged with <code>name</code></td></tr>
optionally tagged with <code>key</code></td></tr>
<tr><td><a href="#cap-p"><code>lpeg.Cp()</code></a></td>
<td>the current position (matches the empty string)</td></tr>
<tr><td><a href="#cap-s"><code>lpeg.Cs(patt)</code></a></td>
@@ -627,6 +639,11 @@ or no value when <code>number</code> is zero.</td></tr>
<tr><td><a href="#cap-func"><code>patt / function</code></a></td>
<td>the returns of <code>function</code> applied to the captures
of <code>patt</code></td></tr>
<tr><td><a href="#cap-acc"><code>patt % function</code></a></td>
<td>produces no value;
it <em>accummulates</em> the captures from <code>patt</code>
into the previous capture through <code>function</code>
</td></tr>
<tr><td><a href="#matchtime"><code>lpeg.Cmt(patt, function)</code></a></td>
<td>the returns of <code>function</code> applied to the captures
of <code>patt</code>; the application is done at match time</td></tr>
@@ -652,10 +669,10 @@ LPeg does not specify when (and if) it evaluates its captures.
consider the pattern <code>lpeg.P"a" / func / 0</code>.
Because the "division" by 0 instructs LPeg to throw away the
results from the pattern,
LPeg may or may not call <code>func</code>.)
it is not specified whether LPeg will call <code>func</code>.)
Therefore, captures should avoid side effects.
Moreover,
most captures cannot affect the way a pattern matches a subject.
captures cannot affect the way a pattern matches a subject.
The only exception to this rule is the
so-called <a href="#matchtime"><em>match-time capture</em></a>.
When a match-time capture matches,
@@ -684,24 +701,25 @@ argument given in the call to <code>lpeg.match</code>.
</p>
<h3><a name="cap-b"></a><code>lpeg.Cb (name)</code></h3>
<h3><a name="cap-b"></a><code>lpeg.Cb (key)</code></h3>
<p>
Creates a <em>back capture</em>.
This pattern matches the empty string and
produces the values produced by the <em>most recent</em>
<a href="#cap-g">group capture</a> named <code>name</code>
(where <code>name</code> can be any Lua value).
<a href="#cap-g">group capture</a> named <code>key</code>
(where <code>key</code> can be any Lua value).
</p>
<p>
<em>Most recent</em> means the last
<em>complete</em>
<em>outermost</em>
group capture with the given name.
group capture with the given key.
A <em>Complete</em> capture means that the entire pattern
corresponding to the capture has matched.
corresponding to the capture has matched;
in other words, the back capture is not nested inside the group.
An <em>Outermost</em> capture means that the capture is not inside
another complete capture.
another complete capture that does not contain the back capture itself.
</p>
<p>
@@ -722,61 +740,21 @@ produces all given values as its captured values.
<h3><a name="cap-f"></a><code>lpeg.Cf (patt, func)</code></h3>
<p>
Creates a <em>fold capture</em>.
If <code>patt</code> produces a list of captures
<em>C<sub>1</sub> C<sub>2</sub> ... C<sub>n</sub></em>,
this capture will produce the value
<em>func(...func(func(C<sub>1</sub>, C<sub>2</sub>), C<sub>3</sub>)...,
C<sub>n</sub>)</em>,
that is, it will <em>fold</em>
(or <em>accumulate</em>, or <em>reduce</em>)
the captures from <code>patt</code> using function <code>func</code>.
</p>
<p>
This capture assumes that <code>patt</code> should produce
at least one capture with at least one value (of any type),
which becomes the initial value of an <em>accumulator</em>.
(If you need a specific initial value,
you may prefix a <a href="#cap-cc">constant capture</a> to <code>patt</code>.)
For each subsequent capture,
LPeg calls <code>func</code>
with this accumulator as the first argument and all values produced
by the capture as extra arguments;
the first result from this call
becomes the new value for the accumulator.
The final value of the accumulator becomes the captured value.
</p>
<p>
As an example,
the following pattern matches a list of numbers separated
by commas and returns their addition:
</p>
<pre class="example">
-- matches a numeral and captures its numerical value
number = lpeg.R"09"^1 / tonumber
-- matches a list of numbers, capturing their values
list = number * ("," * number)^0
-- auxiliary function to add two numbers
function add (acc, newvalue) return acc + newvalue end
-- folds the list of numbers adding them
sum = lpeg.Cf(list, add)
-- example of use
print(sum:match("10,30,43")) --&gt; 83
</pre>
This construction is deprecated;
use an <a href="#cap-acc">accumulator pattern</a> instead.
In general, a fold like
<code>lpeg.Cf(p1 * p2^0, func)</code>
can be translated to
<code>(p1 * (p2 % func)^0)</code>.
<h3><a name="cap-g"></a><code>lpeg.Cg (patt [, name])</code></h3>
<h3><a name="cap-g"></a><code>lpeg.Cg (patt [, key])</code></h3>
<p>
Creates a <em>group capture</em>.
It groups all values returned by <code>patt</code>
into a single capture.
The group may be anonymous (if no name is given)
or named with the given name
The group may be anonymous (if no key is given)
or named with the given key
(which can be any non-nil Lua value).
</p>
@@ -822,7 +800,7 @@ starting at 1.
Moreover,
for each named capture group created by <code>patt</code>,
the first value of the group is put into the table
with the group name as its key.
with the group key as its key.
The captured value is only the table.
</p>
@@ -878,6 +856,98 @@ there is no captured value.
</p>
<h3><a name="cap-acc"></a><code>patt % function</code></h3>
<p>
Creates an <em>accumulator capture</em>.
This pattern behaves similarly to a
<a href="#cap-func">function capture</a>,
with the following differences:
The last captured value before <code>patt</code>
is added as a first argument to the call;
the return of the function is adjusted to one single value;
that value replaces the last captured value.
Note that the capture itself produces no values;
it only changes the value of its previous capture.
</p>
<p>
As an example,
let us consider the problem of adding a list of numbers.
</p>
<pre class="example">
-- matches a numeral and captures its numerical value
number = lpeg.R"09"^1 / tonumber
-- auxiliary function to add two numbers
function add (acc, newvalue) return acc + newvalue end
-- matches a list of numbers, adding their values
sum = number * ("," * number % add)^0
-- example of use
print(sum:match("10,30,43")) --&gt; 83
</pre>
<p>
First, the initial <code>number</code> captures a number;
that first capture will play the role of an accumulator.
Then, each time the sequence <code>comma-number</code>
matches inside the loop there is an accumulator capture:
It calls <code>add</code> with the current value of the
accumulator&mdash;which is the last captured value, created by the
first <code>number</code>&mdash; and the value of the new number,
and the result of the call (the sum of the two numbers)
replaces the value of the accumulator.
At the end of the match,
the accumulator with all sums is the final value.
</p>
<p>
As another example,
consider the following code fragment:
</p>
<pre class="example">
local name = lpeg.C(lpeg.R("az")^1)
local p = name * (lpeg.P("^") % string.upper)^-1
print(p:match("count")) --&gt; count
print(p:match("count^")) --&gt; COUNT
</pre>
<p>
In the match against <code>"count"</code>,
as there is no <code>"^"</code>,
the optional accumulator capture does not match;
so, the match results in its sole capture, a name.
In the match against <code>"count^"</code>,
the accumulator capture matches,
so the function <code>string.upper</code>
is called with the previous captured value (created by <code>name</code>)
plus the string <code>"^"</code>;
the function ignores its second argument and returns the first argument
changed to upper case;
that value then becomes the first and only
capture value created by the match.
</p>
<p>
Due to the nature of this capture,
you should avoid using it in places where it is not clear
what is the "previous" capture,
such as directly nested in a <a href="#cap-string">string capture</a>
or a <a href="#cap-num">numbered capture</a>.
(Note that these captures may not need to evaluate
all their subcaptures to compute their results.)
Moreover, due to implementation details,
you should not use this capture directly nested in a
<a href="#cap-s">substitution capture</a>.
You should also avoid a direct nesting of this capture inside
a <a href="#cap-f">folding capture</a> (deprecated),
as the folding will try to fold each individual accumulator capture.
A simple and effective way to avoid all these issues is
to enclose the whole accumulation composition
(including the capture that generates the initial value)
into an anonymous <a href="#cap-g">group capture</a>.
</p>
<h3><a name="matchtime"></a><code>lpeg.Cmt(patt, function)</code></h3>
<p>
Creates a <em>match-time capture</em>.
@@ -930,9 +1000,9 @@ local lpeg = require "lpeg"
-- matches a word followed by end-of-string
p = lpeg.R"az"^1 * -1
print(p:match("hello")) --> 6
print(lpeg.match(p, "hello")) --> 6
print(p:match("1 hello")) --> nil
print(p:match("hello")) --&gt; 6
print(lpeg.match(p, "hello")) --&gt; 6
print(p:match("1 hello")) --&gt; nil
</pre>
<p>
The pattern is simply a sequence of one or more lower-case letters
@@ -957,19 +1027,18 @@ lpeg.locale(lpeg) -- adds locale entries into 'lpeg' table
local space = lpeg.space^0
local name = lpeg.C(lpeg.alpha^1) * space
local sep = lpeg.S(",;") * space
local pair = lpeg.Cg(name * "=" * space * name) * sep^-1
local list = lpeg.Cf(lpeg.Ct("") * pair^0, rawset)
t = list:match("a=b, c = hi; next = pi") --> { a = "b", c = "hi", next = "pi" }
local pair = name * "=" * space * name * sep^-1
local list = lpeg.Ct("") * (pair % rawset)^0
t = list:match("a=b, c = hi; next = pi")
--&gt; { a = "b", c = "hi", next = "pi" }
</pre>
<p>
Each pair has the format <code>name = name</code> followed by
an optional separator (a comma or a semicolon).
The <code>pair</code> pattern encloses the pair in a group pattern,
so that the names become the values of a single capture.
The <code>list</code> pattern then folds these captures.
The <code>list</code> pattern then <em>folds</em> these captures.
It starts with an empty table,
created by a table capture matching an empty string;
then for each capture (a pair of names) it applies <code>rawset</code>
then for each a pair of names it applies <code>rawset</code>
over the accumulator (the table) and the capture values (the pair of names).
<code>rawset</code> returns the table itself,
so the accumulator is always the table.
@@ -1003,7 +1072,7 @@ by <code>sep</code>.
If the split results in too many values,
it may overflow the maximum number of values
that can be returned by a Lua function.
In this case,
To avoid this problem,
we can collect these values in a table:
</p>
<pre class="example">
@@ -1039,7 +1108,7 @@ end
</pre>
<p>
This grammar has a straight reading:
it matches <code>p</code> or skips one character and tries again.
its sole rule matches <code>p</code> or skips one character and tries again.
</p>
<p>
@@ -1048,27 +1117,27 @@ If we want to know where the pattern is in the string
we can add position captures to the pattern:
</p>
<pre class="example">
local I = lpeg.Cp()
local Cp = lpeg.Cp()
function anywhere (p)
return lpeg.P{ I * p * I + 1 * lpeg.V(1) }
return lpeg.P{ Cp * p * Cp + 1 * lpeg.V(1) }
end
print(anywhere("world"):match("hello world!")) -> 7 12
print(anywhere("world"):match("hello world!")) --&gt; 7 12
</pre>
<p>
Another option for the search is like this:
</p>
<pre class="example">
local I = lpeg.Cp()
local Cp = lpeg.Cp()
function anywhere (p)
return (1 - lpeg.P(p))^0 * I * p * I
return (1 - lpeg.P(p))^0 * Cp * p * Cp
end
</pre>
<p>
Again the pattern has a straight reading:
it skips as many characters as possible while not matching <code>p</code>,
and then matches <code>p</code> (plus appropriate captures).
and then matches <code>p</code> plus appropriate captures.
</p>
<p>
@@ -1163,91 +1232,6 @@ local record = lpeg.Ct(field * (',' * field)^0) * (lpeg.P'\n' + -1)
</pre>
<h3>UTF-8 and Latin 1</h3>
<p>
It is not difficult to use LPeg to convert a string from
UTF-8 encoding to Latin 1 (ISO 8859-1):
</p>
<pre class="example">
-- convert a two-byte UTF-8 sequence to a Latin 1 character
local function f2 (s)
local c1, c2 = string.byte(s, 1, 2)
return string.char(c1 * 64 + c2 - 12416)
end
local utf8 = lpeg.R("\0\127")
+ lpeg.R("\194\195") * lpeg.R("\128\191") / f2
local decode_pattern = lpeg.Cs(utf8^0) * -1
</pre>
<p>
In this code,
the definition of UTF-8 is already restricted to the
Latin 1 range (from 0 to 255).
Any encoding outside this range (as well as any invalid encoding)
will not match that pattern.
</p>
<p>
As the definition of <code>decode_pattern</code> demands that
the pattern matches the whole input (because of the -1 at its end),
any invalid string will simply fail to match,
without any useful information about the problem.
We can improve this situation redefining <code>decode_pattern</code>
as follows:
</p>
<pre class="example">
local function er (_, i) error("invalid encoding at position " .. i) end
local decode_pattern = lpeg.Cs(utf8^0) * (-1 + lpeg.P(er))
</pre>
<p>
Now, if the pattern <code>utf8^0</code> stops
before the end of the string,
an appropriate error function is called.
</p>
<h3>UTF-8 and Unicode</h3>
<p>
We can extend the previous patterns to handle all Unicode code points.
Of course,
we cannot translate them to Latin 1 or any other one-byte encoding.
Instead, our translation results in a array with the code points
represented as numbers.
The full code is here:
</p>
<pre class="example">
-- decode a two-byte UTF-8 sequence
local function f2 (s)
local c1, c2 = string.byte(s, 1, 2)
return c1 * 64 + c2 - 12416
end
-- decode a three-byte UTF-8 sequence
local function f3 (s)
local c1, c2, c3 = string.byte(s, 1, 3)
return (c1 * 64 + c2) * 64 + c3 - 925824
end
-- decode a four-byte UTF-8 sequence
local function f4 (s)
local c1, c2, c3, c4 = string.byte(s, 1, 4)
return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168
end
local cont = lpeg.R("\128\191") -- continuation byte
local utf8 = lpeg.R("\0\127") / string.byte
+ lpeg.R("\194\223") * cont / f2
+ lpeg.R("\224\239") * cont * cont / f3
+ lpeg.R("\240\244") * cont * cont * cont / f4
local decode_pattern = lpeg.Ct(utf8^0) * -1
</pre>
<h3>Lua's long strings</h3>
<p>
A long string in Lua starts with the pattern <code>[=*[</code>
@@ -1347,7 +1331,7 @@ function evalExp (s)
end
-- small example
print(evalExp"3 + 5*9 / (1+1) - 12") --> 13.5
print(evalExp"3 + 5*9 / (1+1) - 12") --&gt; 13.5
</pre>
<p>
@@ -1369,16 +1353,16 @@ end
-- Grammar
local V = lpeg.V
G = lpeg.P{ "Exp",
Exp = lpeg.Cf(V"Term" * lpeg.Cg(TermOp * V"Term")^0, eval);
Term = lpeg.Cf(V"Factor" * lpeg.Cg(FactorOp * V"Factor")^0, eval);
Exp = V"Term" * (TermOp * V"Term" % eval)^0;
Term = V"Factor" * (FactorOp * V"Factor" % eval)^0;
Factor = Number / tonumber + Open * V"Exp" * Close;
}
-- small example
print(lpeg.match(G, "3 + 5*9 / (1+1) - 12")) --> 13.5
print(lpeg.match(G, "3 + 5*9 / (1+1) - 12")) --&gt; 13.5
</pre>
<p>
Note the use of the fold (accumulator) capture.
Note the use of the accumulator capture.
To compute the value of an expression,
the accumulator starts with the value of the first term,
and then applies <code>eval</code> over
@@ -1391,13 +1375,20 @@ and the new term for each repetition.
<h2><a name="download"></a>Download</h2>
<p>LPeg
<a href="http://www.inf.puc-rio.br/~roberto/lpeg/lpeg-1.0.2.tar.gz">source code</a>.</p>
<a href="//www.inf.puc-rio.br/~roberto/lpeg/lpeg-1.1.0.tar.gz">source code</a>.</p>
<p>
Probably, the easiest way to install LPeg is with
<a href="//luarocks.org/">LuaRocks</a>.
If you have LuaRocks installed,
the following command is all you need to install LPeg:
<pre>$ luarocks install lpeg</pre>
<h2><a name="license">License</a></h2>
<p>
Copyright &copy; 2007-2019 Lua.org, PUC-Rio.
Copyright &copy; 2007-2023 Lua.org, PUC-Rio.
</p>
<p>
Permission is hereby granted, free of charge,

140
3rd/lpeg/lpprint.c
View File

@@ -1,7 +1,3 @@
/*
** $Id: lpprint.c $
** Copyright 2007, Lua.org & PUC-Rio (see 'lpeg.html' for license)
*/
#include <ctype.h>
#include <limits.h>
@@ -27,7 +23,7 @@ void printcharset (const byte *st) {
printf("[");
for (i = 0; i <= UCHAR_MAX; i++) {
int first = i;
while (testchar(st, i) && i <= UCHAR_MAX) i++;
while (i <= UCHAR_MAX && testchar(st, i)) i++;
if (i - 1 == first) /* unary range? */
printf("(%02x)", first);
else if (i - 1 > first) /* non-empty range? */
@@ -37,10 +33,34 @@ void printcharset (const byte *st) {
}
static void printIcharset (const Instruction *inst, const byte *buff) {
byte cs[CHARSETSIZE];
int i;
printf("(%02x-%d) ", inst->i.aux2.set.offset, inst->i.aux2.set.size);
clearset(cs);
for (i = 0; i < CHARSETSIZE * 8; i++) {
if (charinset(inst, buff, i))
setchar(cs, i);
}
printcharset(cs);
}
static void printTcharset (TTree *tree) {
byte cs[CHARSETSIZE];
int i;
printf("(%02x-%d) ", tree->u.set.offset, tree->u.set.size);
fillset(cs, tree->u.set.deflt);
for (i = 0; i < tree->u.set.size; i++)
cs[tree->u.set.offset + i] = treebuffer(tree)[i];
printcharset(cs);
}
static const char *capkind (int kind) {
const char *const modes[] = {
"close", "position", "constant", "backref",
"argument", "simple", "table", "function",
"argument", "simple", "table", "function", "accumulator",
"query", "string", "num", "substitution", "fold",
"runtime", "group"};
return modes[kind];
@@ -56,41 +76,46 @@ void printinst (const Instruction *op, const Instruction *p) {
const char *const names[] = {
"any", "char", "set",
"testany", "testchar", "testset",
"span", "behind",
"span", "utf-range", "behind",
"ret", "end",
"choice", "jmp", "call", "open_call",
"commit", "partial_commit", "back_commit", "failtwice", "fail", "giveup",
"fullcapture", "opencapture", "closecapture", "closeruntime"
"fullcapture", "opencapture", "closecapture", "closeruntime",
"--"
};
printf("%02ld: %s ", (long)(p - op), names[p->i.code]);
switch ((Opcode)p->i.code) {
case IChar: {
printf("'%c'", p->i.aux);
printf("'%c' (%02x)", p->i.aux1, p->i.aux1);
break;
}
case ITestChar: {
printf("'%c'", p->i.aux); printjmp(op, p);
printf("'%c' (%02x)", p->i.aux1, p->i.aux1); printjmp(op, p);
break;
}
case IUTFR: {
printf("%d - %d", p[1].offset, utf_to(p));
break;
}
case IFullCapture: {
printf("%s (size = %d) (idx = %d)",
capkind(getkind(p)), getoff(p), p->i.key);
capkind(getkind(p)), getoff(p), p->i.aux2.key);
break;
}
case IOpenCapture: {
printf("%s (idx = %d)", capkind(getkind(p)), p->i.key);
printf("%s (idx = %d)", capkind(getkind(p)), p->i.aux2.key);
break;
}
case ISet: {
printcharset((p+1)->buff);
printIcharset(p, (p+1)->buff);
break;
}
case ITestSet: {
printcharset((p+2)->buff); printjmp(op, p);
printIcharset(p, (p+2)->buff); printjmp(op, p);
break;
}
case ISpan: {
printcharset((p+1)->buff);
printIcharset(p, (p+1)->buff);
break;
}
case IOpenCall: {
@@ -98,7 +123,7 @@ void printinst (const Instruction *op, const Instruction *p) {
break;
}
case IBehind: {
printf("%d", p->i.aux);
printf("%d", p->i.aux1);
break;
}
case IJmp: case ICall: case ICommit: case IChoice:
@@ -112,8 +137,9 @@ void printinst (const Instruction *op, const Instruction *p) {
}
void printpatt (Instruction *p, int n) {
void printpatt (Instruction *p) {
Instruction *op = p;
uint n = op[-1].codesize - 1;
while (p < op + n) {
printinst(op, p);
p += sizei(p);
@@ -121,20 +147,39 @@ void printpatt (Instruction *p, int n) {
}
#if defined(LPEG_DEBUG)
static void printcap (Capture *cap) {
printf("%s (idx: %d - size: %d) -> %p\n",
capkind(cap->kind), cap->idx, cap->siz, cap->s);
static void printcap (Capture *cap, int ident) {
while (ident--) printf(" ");
printf("%s (idx: %d - size: %d) -> %lu (%p)\n",
capkind(cap->kind), cap->idx, cap->siz, (long)cap->index, (void*)cap);
}
void printcaplist (Capture *cap, Capture *limit) {
/*
** Print a capture and its nested captures
*/
static Capture *printcap2close (Capture *cap, int ident) {
Capture *head = cap++;
printcap(head, ident); /* print head capture */
while (capinside(head, cap))
cap = printcap2close(cap, ident + 2); /* print nested captures */
if (isopencap(head)) {
assert(isclosecap(cap));
printcap(cap++, ident); /* print and skip close capture */
}
return cap;
}
void printcaplist (Capture *cap) {
{ /* for debugging, print first a raw list of captures */
Capture *c = cap;
while (c->index != MAXINDT) { printcap(c, 0); c++; }
}
printf(">======\n");
for (; cap->s && (limit == NULL || cap < limit); cap++)
printcap(cap);
while (!isclosecap(cap))
cap = printcap2close(cap, 0);
printf("=======\n");
}
#endif
/* }====================================================== */
@@ -147,11 +192,11 @@ void printcaplist (Capture *cap, Capture *limit) {
static const char *tagnames[] = {
"char", "set", "any",
"true", "false",
"true", "false", "utf8.range",
"rep",
"seq", "choice",
"not", "and",
"call", "opencall", "rule", "grammar",
"call", "opencall", "rule", "xinfo", "grammar",
"behind",
"capture", "run-time"
};
@@ -159,6 +204,7 @@ static const char *tagnames[] = {
void printtree (TTree *tree, int ident) {
int i;
int sibs = numsiblings[tree->tag];
for (i = 0; i < ident; i++) printf(" ");
printf("%s", tagnames[tree->tag]);
switch (tree->tag) {
@@ -171,29 +217,38 @@ void printtree (TTree *tree, int ident) {
break;
}
case TSet: {
printcharset(treebuffer(tree));
printTcharset(tree);
printf("\n");
break;
}
case TUTFR: {
assert(sib1(tree)->tag == TXInfo);
printf(" %d (%02x %d) - %d (%02x %d) \n",
tree->u.n, tree->key, tree->cap,
sib1(tree)->u.n, sib1(tree)->key, sib1(tree)->cap);
break;
}
case TOpenCall: case TCall: {
assert(sib2(tree)->tag == TRule);
printf(" key: %d (rule: %d)\n", tree->key, sib2(tree)->cap);
assert(sib1(sib2(tree))->tag == TXInfo);
printf(" key: %d (rule: %d)\n", tree->key, sib1(sib2(tree))->u.n);
break;
}
case TBehind: {
printf(" %d\n", tree->u.n);
printtree(sib1(tree), ident + 2);
break;
}
case TCapture: {
printf(" kind: '%s' key: %d\n", capkind(tree->cap), tree->key);
printtree(sib1(tree), ident + 2);
break;
}
case TRule: {
printf(" n: %d key: %d\n", tree->cap, tree->key);
printtree(sib1(tree), ident + 2);
break; /* do not print next rule as a sibling */
printf(" key: %d\n", tree->key);
sibs = 1; /* do not print 'sib2' (next rule) as a sibling */
break;
}
case TXInfo: {
printf(" n: %d\n", tree->u.n);
break;
}
case TGrammar: {
TTree *rule = sib1(tree);
@@ -203,18 +258,17 @@ void printtree (TTree *tree, int ident) {
rule = sib2(rule);
}
assert(rule->tag == TTrue); /* sentinel */
sibs = 0; /* siblings already handled */
break;
}
default: {
int sibs = numsiblings[tree->tag];
default:
printf("\n");
if (sibs >= 1) {
printtree(sib1(tree), ident + 2);
if (sibs >= 2)
printtree(sib2(tree), ident + 2);
}
break;
}
}
if (sibs >= 1) {
printtree(sib1(tree), ident + 2);
if (sibs >= 2)
printtree(sib2(tree), ident + 2);
}
}

10
3rd/lpeg/lpprint.h
View File

@@ -1,7 +1,3 @@
/*
** $Id: lpprint.h $
*/
#if !defined(lpprint_h)
#define lpprint_h
@@ -13,11 +9,11 @@
#if defined(LPEG_DEBUG)
void printpatt (Instruction *p, int n);
void printpatt (Instruction *p);
void printtree (TTree *tree, int ident);
void printktable (lua_State *L, int idx);
void printcharset (const byte *st);
void printcaplist (Capture *cap, Capture *limit);
void printcaplist (Capture *cap);
void printinst (const Instruction *op, const Instruction *p);
#else
@@ -26,7 +22,7 @@ void printinst (const Instruction *op, const Instruction *p);
luaL_error(L, "function only implemented in debug mode")
#define printtree(tree,i) \
luaL_error(L, "function only implemented in debug mode")
#define printpatt(p,n) \
#define printpatt(p) \
luaL_error(L, "function only implemented in debug mode")
#endif

212
3rd/lpeg/lptree.c
View File

@@ -1,7 +1,3 @@
/*
** $Id: lptree.c $
** Copyright 2013, Lua.org & PUC-Rio (see 'lpeg.html' for license)
*/
#include <ctype.h>
#include <limits.h>
@@ -16,16 +12,17 @@
#include "lpcode.h"
#include "lpprint.h"
#include "lptree.h"
#include "lpcset.h"
/* number of siblings for each tree */
const byte numsiblings[] = {
0, 0, 0, /* char, set, any */
0, 0, /* true, false */
1, /* rep */
0, 0, 0, /* true, false, utf-range */
1, /* acc */
2, 2, /* seq, choice */
1, 1, /* not, and */
0, 0, 2, 1, /* call, opencall, rule, grammar */
0, 0, 2, 1, 1, /* call, opencall, rule, prerule, grammar */
1, /* behind */
1, 1 /* capture, runtime capture */
};
@@ -338,7 +335,7 @@ static Pattern *getpattern (lua_State *L, int idx) {
static int getsize (lua_State *L, int idx) {
return (lua_rawlen(L, idx) - sizeof(Pattern)) / sizeof(TTree) + 1;
return (lua_rawlen(L, idx) - offsetof(Pattern, tree)) / sizeof(TTree);
}
@@ -351,18 +348,18 @@ static TTree *gettree (lua_State *L, int idx, int *len) {
/*
** create a pattern. Set its uservalue (the 'ktable') equal to its
** metatable. (It could be any empty sequence; the metatable is at
** hand here, so we use it.)
** create a pattern followed by a tree with 'len' nodes. Set its
** uservalue (the 'ktable') equal to its metatable. (It could be any
** empty sequence; the metatable is at hand here, so we use it.)
*/
static TTree *newtree (lua_State *L, int len) {
size_t size = (len - 1) * sizeof(TTree) + sizeof(Pattern);
size_t size = offsetof(Pattern, tree) + len * sizeof(TTree);
Pattern *p = (Pattern *)lua_newuserdata(L, size);
luaL_getmetatable(L, PATTERN_T);
lua_pushvalue(L, -1);
lua_setuservalue(L, -3);
lua_setmetatable(L, -2);
p->code = NULL; p->codesize = 0;
p->code = NULL;
return p->tree;
}
@@ -374,17 +371,44 @@ static TTree *newleaf (lua_State *L, int tag) {
}
static TTree *newcharset (lua_State *L) {
TTree *tree = newtree(L, bytes2slots(CHARSETSIZE) + 1);
tree->tag = TSet;
loopset(i, treebuffer(tree)[i] = 0);
return tree;
/*
** Create a tree for a charset, optimizing for special cases: empty set,
** full set, and singleton set.
*/
static TTree *newcharset (lua_State *L, byte *cs) {
charsetinfo info;
Opcode op = charsettype(cs, &info);
switch (op) {
case IFail: return newleaf(L, TFalse); /* empty set */
case IAny: return newleaf(L, TAny); /* full set */
case IChar: { /* singleton set */
TTree *tree =newleaf(L, TChar);
tree->u.n = info.offset;
return tree;
}
default: { /* regular set */
int i;
int bsize = /* tree size in bytes */
(int)offsetof(TTree, u.set.bitmap) + info.size;
TTree *tree = newtree(L, bytes2slots(bsize));
assert(op == ISet);
tree->tag = TSet;
tree->u.set.offset = info.offset;
tree->u.set.size = info.size;
tree->u.set.deflt = info.deflt;
for (i = 0; i < info.size; i++) {
assert(&treebuffer(tree)[i] < (byte*)tree + bsize);
treebuffer(tree)[i] = cs[info.offset + i];
}
return tree;
}
}
}
/*
** add to tree a sequence where first sibling is 'sib' (with size
** 'sibsize'); returns position for second sibling
** Add to tree a sequence where first sibling is 'sib' (with size
** 'sibsize'); return position for second sibling.
*/
static TTree *seqaux (TTree *tree, TTree *sib, int sibsize) {
tree->tag = TSeq; tree->u.ps = sibsize + 1;
@@ -553,8 +577,8 @@ static int lp_choice (lua_State *L) {
TTree *t1 = getpatt(L, 1, NULL);
TTree *t2 = getpatt(L, 2, NULL);
if (tocharset(t1, &st1) && tocharset(t2, &st2)) {
TTree *t = newcharset(L);
loopset(i, treebuffer(t)[i] = st1.cs[i] | st2.cs[i]);
loopset(i, st1.cs[i] |= st2.cs[i]);
newcharset(L, st1.cs);
}
else if (nofail(t1) || t2->tag == TFalse)
lua_pushvalue(L, 1); /* true / x => true, x / false => x */
@@ -630,8 +654,8 @@ static int lp_sub (lua_State *L) {
TTree *t1 = getpatt(L, 1, &s1);
TTree *t2 = getpatt(L, 2, &s2);
if (tocharset(t1, &st1) && tocharset(t2, &st2)) {
TTree *t = newcharset(L);
loopset(i, treebuffer(t)[i] = st1.cs[i] & ~st2.cs[i]);
loopset(i, st1.cs[i] &= ~st2.cs[i]);
newcharset(L, st1.cs);
}
else {
TTree *tree = newtree(L, 2 + s1 + s2);
@@ -649,11 +673,13 @@ static int lp_sub (lua_State *L) {
static int lp_set (lua_State *L) {
size_t l;
const char *s = luaL_checklstring(L, 1, &l);
TTree *tree = newcharset(L);
byte buff[CHARSETSIZE];
clearset(buff);
while (l--) {
setchar(treebuffer(tree), (byte)(*s));
setchar(buff, (byte)(*s));
s++;
}
newcharset(L, buff);
return 1;
}
@@ -661,14 +687,68 @@ static int lp_set (lua_State *L) {
static int lp_range (lua_State *L) {
int arg;
int top = lua_gettop(L);
TTree *tree = newcharset(L);
byte buff[CHARSETSIZE];
clearset(buff);
for (arg = 1; arg <= top; arg++) {
int c;
size_t l;
const char *r = luaL_checklstring(L, arg, &l);
luaL_argcheck(L, l == 2, arg, "range must have two characters");
for (c = (byte)r[0]; c <= (byte)r[1]; c++)
setchar(treebuffer(tree), c);
setchar(buff, c);
}
newcharset(L, buff);
return 1;
}
/*
** Fills a tree node with basic information about the UTF-8 code point
** 'cpu': its value in 'n', its length in 'cap', and its first byte in
** 'key'
*/
static void codeutftree (lua_State *L, TTree *t, lua_Unsigned cpu, int arg) {
int len, fb, cp;
cp = (int)cpu;
if (cp <= 0x7f) { /* one byte? */
len = 1;
fb = cp;
} else if (cp <= 0x7ff) {
len = 2;
fb = 0xC0 | (cp >> 6);
} else if (cp <= 0xffff) {
len = 3;
fb = 0xE0 | (cp >> 12);
}
else {
luaL_argcheck(L, cpu <= 0x10ffffu, arg, "invalid code point");
len = 4;
fb = 0xF0 | (cp >> 18);
}
t->u.n = cp;
t->cap = len;
t->key = fb;
}
static int lp_utfr (lua_State *L) {
lua_Unsigned from = (lua_Unsigned)luaL_checkinteger(L, 1);
lua_Unsigned to = (lua_Unsigned)luaL_checkinteger(L, 2);
luaL_argcheck(L, from <= to, 2, "empty range");
if (to <= 0x7f) { /* ascii range? */
uint f;
byte buff[CHARSETSIZE]; /* code it as a regular charset */
clearset(buff);
for (f = (int)from; f <= to; f++)
setchar(buff, f);
newcharset(L, buff);
}
else { /* multi-byte utf-8 range */
TTree *tree = newtree(L, 2);
tree->tag = TUTFR;
codeutftree(L, tree, from, 1);
sib1(tree)->tag = TXInfo;
codeutftree(L, sib1(tree), to, 2);
}
return 1;
}
@@ -763,11 +843,18 @@ static int lp_divcapture (lua_State *L) {
tree->key = n;
return 1;
}
default: return luaL_argerror(L, 2, "invalid replacement value");
default:
return luaL_error(L, "unexpected %s as 2nd operand to LPeg '/'",
luaL_typename(L, 2));
}
}
static int lp_acccapture (lua_State *L) {
return capture_aux(L, Cacc, 2);
}
static int lp_substcapture (lua_State *L) {
return capture_aux(L, Csubst, 0);
}
@@ -906,7 +993,7 @@ static int collectrules (lua_State *L, int arg, int *totalsize) {
int size; /* accumulator for total size */
lua_newtable(L); /* create position table */
getfirstrule(L, arg, postab);
size = 2 + getsize(L, postab + 2); /* TGrammar + TRule + rule */
size = 3 + getsize(L, postab + 2); /* TGrammar + TRule + TXInfo + rule */
lua_pushnil(L); /* prepare to traverse grammar table */
while (lua_next(L, arg) != 0) {
if (lua_tonumber(L, -2) == 1 ||
@@ -920,11 +1007,11 @@ static int collectrules (lua_State *L, int arg, int *totalsize) {
lua_pushvalue(L, -2); /* push key (to insert into position table) */
lua_pushinteger(L, size);
lua_settable(L, postab);
size += 1 + getsize(L, -1); /* update size */
size += 2 + getsize(L, -1); /* add 'TRule + TXInfo + rule' to size */
lua_pushvalue(L, -2); /* push key (for next lua_next) */
n++;
}
*totalsize = size + 1; /* TTrue to finish list of rules */
*totalsize = size + 1; /* space for 'TTrue' finishing list of rules */
return n;
}
@@ -936,11 +1023,13 @@ static void buildgrammar (lua_State *L, TTree *grammar, int frule, int n) {
int ridx = frule + 2*i + 1; /* index of i-th rule */
int rulesize;
TTree *rn = gettree(L, ridx, &rulesize);
TTree *pr = sib1(nd); /* points to rule's prerule */
nd->tag = TRule;
nd->key = 0; /* will be fixed when rule is used */
nd->cap = i; /* rule number */
nd->u.ps = rulesize + 1; /* point to next rule */
memcpy(sib1(nd), rn, rulesize * sizeof(TTree)); /* copy rule */
pr->tag = TXInfo;
pr->u.n = i; /* rule number */
nd->u.ps = rulesize + 2; /* point to next rule */
memcpy(sib1(pr), rn, rulesize * sizeof(TTree)); /* copy rule */
mergektable(L, ridx, sib1(nd)); /* merge its ktable into new one */
nd = sib2(nd); /* move to next rule */
}
@@ -976,7 +1065,7 @@ static int checkloops (TTree *tree) {
** twice in 'passed', there is path from it back to itself without
** advancing the subject.
*/
static int verifyerror (lua_State *L, int *passed, int npassed) {
static int verifyerror (lua_State *L, unsigned short *passed, int npassed) {
int i, j;
for (i = npassed - 1; i >= 0; i--) { /* search for a repetition */
for (j = i - 1; j >= 0; j--) {
@@ -1001,12 +1090,12 @@ static int verifyerror (lua_State *L, int *passed, int npassed) {
** counts the elements in 'passed'.
** Assume ktable at the top of the stack.
*/
static int verifyrule (lua_State *L, TTree *tree, int *passed, int npassed,
int nb) {
static int verifyrule (lua_State *L, TTree *tree, unsigned short *passed,
int npassed, int nb) {
tailcall:
switch (tree->tag) {
case TChar: case TSet: case TAny:
case TFalse:
case TFalse: case TUTFR:
return nb; /* cannot pass from here */
case TTrue:
case TBehind: /* look-behind cannot have calls */
@@ -1014,7 +1103,7 @@ static int verifyrule (lua_State *L, TTree *tree, int *passed, int npassed,
case TNot: case TAnd: case TRep:
/* return verifyrule(L, sib1(tree), passed, npassed, 1); */
tree = sib1(tree); nb = 1; goto tailcall;
case TCapture: case TRunTime:
case TCapture: case TRunTime: case TXInfo:
/* return verifyrule(L, sib1(tree), passed, npassed, nb); */
tree = sib1(tree); goto tailcall;
case TCall:
@@ -1030,10 +1119,10 @@ static int verifyrule (lua_State *L, TTree *tree, int *passed, int npassed,
/* return verifyrule(L, sib2(tree), passed, npassed, nb); */
tree = sib2(tree); goto tailcall;
case TRule:
if (npassed >= MAXRULES)
return verifyerror(L, passed, npassed);
if (npassed >= MAXRULES) /* too many steps? */
return verifyerror(L, passed, npassed); /* error */
else {
passed[npassed++] = tree->key;
passed[npassed++] = tree->key; /* add rule to path */
/* return verifyrule(L, sib1(tree), passed, npassed); */
tree = sib1(tree); goto tailcall;
}
@@ -1045,7 +1134,7 @@ static int verifyrule (lua_State *L, TTree *tree, int *passed, int npassed,
static void verifygrammar (lua_State *L, TTree *grammar) {
int passed[MAXRULES];
unsigned short passed[MAXRULES];
TTree *rule;
/* check left-recursive rules */
for (rule = sib1(grammar); rule->tag == TRule; rule = sib2(rule)) {
@@ -1105,7 +1194,7 @@ static Instruction *prepcompile (lua_State *L, Pattern *p, int idx) {
lua_getuservalue(L, idx); /* push 'ktable' (may be used by 'finalfix') */
finalfix(L, 0, NULL, p->tree);
lua_pop(L, 1); /* remove 'ktable' */
return compile(L, p);
return compile(L, p, getsize(L, idx));
}
@@ -1128,7 +1217,7 @@ static int lp_printcode (lua_State *L) {
printktable(L, 1);
if (p->code == NULL) /* not compiled yet? */
prepcompile(L, p, 1);
printpatt(p->code, p->codesize);
printpatt(p->code);
return 0;
}
@@ -1164,9 +1253,10 @@ static int lp_match (lua_State *L) {
const char *s = luaL_checklstring(L, SUBJIDX, &l);
size_t i = initposition(L, l);
int ptop = lua_gettop(L);
luaL_argcheck(L, l < MAXINDT, SUBJIDX, "subject too long");
lua_pushnil(L); /* initialize subscache */
lua_pushlightuserdata(L, capture); /* initialize caplistidx */
lua_getuservalue(L, 1); /* initialize penvidx */
lua_getuservalue(L, 1); /* initialize ktableidx */
r = match(L, s, s + i, s + l, code, capture, ptop);
if (r == NULL) {
lua_pushnil(L);
@@ -1195,12 +1285,6 @@ static int lp_setmax (lua_State *L) {
}
static int lp_version (lua_State *L) {
lua_pushstring(L, VERSION);
return 1;
}
static int lp_type (lua_State *L) {
if (testpattern(L, 1))
lua_pushliteral(L, "pattern");
@@ -1212,16 +1296,22 @@ static int lp_type (lua_State *L) {
int lp_gc (lua_State *L) {
Pattern *p = getpattern(L, 1);
realloccode(L, p, 0); /* delete code block */
freecode(L, p); /* delete code block */
return 0;
}
/*
** Create a charset representing a category of characters, given by
** the predicate 'catf'.
*/
static void createcat (lua_State *L, const char *catname, int (catf) (int)) {
TTree *t = newcharset(L);
int i;
for (i = 0; i <= UCHAR_MAX; i++)
if (catf(i)) setchar(treebuffer(t), i);
int c;
byte buff[CHARSETSIZE];
clearset(buff);
for (c = 0; c <= UCHAR_MAX; c++)
if (catf(c)) setchar(buff, c);
newcharset(L, buff);
lua_setfield(L, -2, catname);
}
@@ -1269,8 +1359,9 @@ static struct luaL_Reg pattreg[] = {
{"P", lp_P},
{"S", lp_set},
{"R", lp_range},
{"utfR", lp_utfr},
{"locale", lp_locale},
{"version", lp_version},
{"version", NULL},
{"setmaxstack", lp_setmax},
{"type", lp_type},
{NULL, NULL}
@@ -1284,6 +1375,7 @@ static struct luaL_Reg metareg[] = {
{"__gc", lp_gc},
{"__len", lp_and},
{"__div", lp_divcapture},
{"__mod", lp_acccapture},
{"__unm", lp_not},
{"__sub", lp_sub},
{NULL, NULL}
@@ -1299,6 +1391,8 @@ int luaopen_lpeg (lua_State *L) {
luaL_newlib(L, pattreg);
lua_pushvalue(L, -1);
lua_setfield(L, -3, "__index");
lua_pushliteral(L, "LPeg " VERSION);
lua_setfield(L, -2, "version");
return 1;
}

26
3rd/lpeg/lptree.h
View File

@@ -1,12 +1,9 @@
/*
** $Id: lptree.h $
*/
#if !defined(lptree_h)
#define lptree_h
#include "lptypes.h"
#include "lptypes.h"
/*
@@ -14,10 +11,13 @@
*/
typedef enum TTag {
TChar = 0, /* 'n' = char */
TSet, /* the set is stored in next CHARSETSIZE bytes */
TSet, /* the set is encoded in 'u.set' and the next 'u.set.size' bytes */
TAny,
TTrue,
TFalse,
TUTFR, /* range of UTF-8 codepoints; 'n' has initial codepoint;
'cap' has length; 'key' has first byte;
extra info is similar for end codepoint */
TRep, /* 'sib1'* */
TSeq, /* 'sib1' 'sib2' */
TChoice, /* 'sib1' / 'sib2' */
@@ -26,8 +26,9 @@ typedef enum TTag {
TCall, /* ktable[key] is rule's key; 'sib2' is rule being called */
TOpenCall, /* ktable[key] is rule's key */
TRule, /* ktable[key] is rule's key (but key == 0 for unused rules);
'sib1' is rule's pattern;
'sib2' is next rule; 'cap' is rule's sequential number */
'sib1' is rule's pattern pre-rule; 'sib2' is next rule;
extra info 'n' is rule's sequential number */
TXInfo, /* extra info */
TGrammar, /* 'sib1' is initial (and first) rule */
TBehind, /* 'sib1' is pattern, 'n' is how much to go back */
TCapture, /* captures: 'cap' is kind of capture (enum 'CapKind');
@@ -51,17 +52,26 @@ typedef struct TTree {
union {
int ps; /* occasional second child */
int n; /* occasional counter */
struct {
byte offset; /* compact set offset (in bytes) */
byte size; /* compact set size (in bytes) */
byte deflt; /* default value */
byte bitmap[1]; /* bitmap (open array) */
} set; /* for compact sets */
} u;
} TTree;
/* access to charset */
#define treebuffer(t) ((t)->u.set.bitmap)
/*
** A complete pattern has its tree plus, if already compiled,
** its corresponding code
*/
typedef struct Pattern {
union Instruction *code;
int codesize;
TTree tree[1];
} Pattern;

33
3rd/lpeg/lptypes.h
View File

@@ -1,7 +1,6 @@
/*
** $Id: lptypes.h $
** LPeg - PEG pattern matching for Lua
** Copyright 2007-2019, Lua.org & PUC-Rio (see 'lpeg.html' for license)
** Copyright 2007-2023, Lua.org & PUC-Rio (see 'lpeg.html' for license)
** written by Roberto Ierusalimschy
*/
@@ -11,11 +10,12 @@
#include <assert.h>
#include <limits.h>
#include <string.h>
#include "lua.h"
#define VERSION "1.0.2"
#define VERSION "1.1.0"
#define PATTERN_T "lpeg-pattern"
@@ -37,6 +37,8 @@
#define luaL_setfuncs(L,f,n) luaL_register(L,NULL,f)
#define luaL_newlib(L,f) luaL_register(L,"lpeg",f)
typedef size_t lua_Unsigned;
#endif
@@ -51,9 +53,9 @@
#endif
/* maximum number of rules in a grammar (limited by 'unsigned char') */
/* maximum number of rules in a grammar (limited by 'unsigned short') */
#if !defined(MAXRULES)
#define MAXRULES 250
#define MAXRULES 1000
#endif
@@ -81,6 +83,8 @@
typedef unsigned char byte;
typedef unsigned int uint;
#define BITSPERCHAR 8
@@ -96,11 +100,11 @@ typedef struct Charset {
#define loopset(v,b) { int v; for (v = 0; v < CHARSETSIZE; v++) {b;} }
/* access to charset */
#define treebuffer(t) ((byte *)((t) + 1))
#define fillset(s,c) memset(s,c,CHARSETSIZE)
#define clearset(s) fillset(s,0)
/* number of slots needed for 'n' bytes */
#define bytes2slots(n) (((n) - 1) / sizeof(TTree) + 1)
#define bytes2slots(n) (((n) - 1u) / (uint)sizeof(TTree) + 1u)
/* set 'b' bit in charset 'cs' */
#define setchar(cs,b) ((cs)[(b) >> 3] |= (1 << ((b) & 7)))
@@ -110,8 +114,8 @@ typedef struct Charset {
** in capture instructions, 'kind' of capture and its offset are
** packed in field 'aux', 4 bits for each
*/
#define getkind(op) ((op)->i.aux & 0xF)
#define getoff(op) (((op)->i.aux >> 4) & 0xF)
#define getkind(op) ((op)->i.aux1 & 0xF)
#define getoff(op) (((op)->i.aux1 >> 4) & 0xF)
#define joinkindoff(k,o) ((k) | ((o) << 4))
#define MAXOFF 0xF
@@ -126,19 +130,20 @@ typedef struct Charset {
#define MAXPATTSIZE (SHRT_MAX - 10)
/* size (in elements) for an instruction plus extra l bytes */
#define instsize(l) (((l) + sizeof(Instruction) - 1)/sizeof(Instruction) + 1)
/* size (in instructions) for l bytes (l > 0) */
#define instsize(l) ((int)(((l) + (uint)sizeof(Instruction) - 1u) \
/ (uint)sizeof(Instruction)))
/* size (in elements) for a ISet instruction */
#define CHARSETINSTSIZE instsize(CHARSETSIZE)
#define CHARSETINSTSIZE (1 + instsize(CHARSETSIZE))
/* size (in elements) for a IFunc instruction */
#define funcinstsize(p) ((p)->i.aux + 2)
#define testchar(st,c) (((int)(st)[((c) >> 3)] & (1 << ((c) & 7))))
#define testchar(st,c) ((((uint)(st)[((c) >> 3)]) >> ((c) & 7)) & 1)
#endif

155
3rd/lpeg/lpvm.c
View File

@@ -1,7 +1,3 @@
/*
** $Id: lpvm.c $
** Copyright 2007, Lua.org & PUC-Rio (see 'lpeg.html' for license)
*/
#include <limits.h>
#include <string.h>
@@ -24,7 +20,46 @@
#define getoffset(p) (((p) + 1)->offset)
static const Instruction giveup = {{IGiveup, 0, 0}};
static const Instruction giveup = {{IGiveup, 0, {0}}};
int charinset (const Instruction *i, const byte *buff, uint c) {
c -= i->i.aux2.set.offset;
if (c >= ((uint)i->i.aux2.set.size /* size in instructions... */
* (uint)sizeof(Instruction) /* in bytes... */
* 8u)) /* in bits */
return i->i.aux1; /* out of range; return default value */
return testchar(buff, c);
}
/*
** Decode one UTF-8 sequence, returning NULL if byte sequence is invalid.
*/
static const char *utf8_decode (const char *o, int *val) {
static const uint limits[] = {0xFF, 0x7F, 0x7FF, 0xFFFFu};
const unsigned char *s = (const unsigned char *)o;
uint c = s[0]; /* first byte */
uint res = 0; /* final result */
if (c < 0x80) /* ascii? */
res = c;
else {
int count = 0; /* to count number of continuation bytes */
while (c & 0x40) { /* still have continuation bytes? */
int cc = s[++count]; /* read next byte */
if ((cc & 0xC0) != 0x80) /* not a continuation byte? */
return NULL; /* invalid byte sequence */
res = (res << 6) | (cc & 0x3F); /* add lower 6 bits from cont. byte */
c <<= 1; /* to test next bit */
}
res |= (c & 0x7F) << (count * 5); /* add first byte */
if (count > 3 || res > 0x10FFFFu || res <= limits[count])
return NULL; /* invalid byte sequence */
s += count; /* skip continuation bytes read */
}
*val = res;
return (const char *)s + 1; /* +1 to include first byte */
}
/*
@@ -51,16 +86,25 @@ typedef struct Stack {
** array, it is simpler to ensure the array always has at least one free
** slot upfront and check its size later.)
*/
/* new size in number of elements cannot overflow integers, and new
size in bytes cannot overflow size_t. */
#define MAXNEWSIZE \
(((size_t)INT_MAX) <= (~(size_t)0 / sizeof(Capture)) ? \
((size_t)INT_MAX) : (~(size_t)0 / sizeof(Capture)))
static Capture *growcap (lua_State *L, Capture *capture, int *capsize,
int captop, int n, int ptop) {
if (*capsize - captop > n)
return capture; /* no need to grow array */
else { /* must grow */
Capture *newc;
int newsize = captop + n + 1; /* minimum size needed */
if (newsize < INT_MAX/((int)sizeof(Capture) * 2))
newsize *= 2; /* twice that size, if not too big */
else if (newsize >= INT_MAX/((int)sizeof(Capture)))
uint newsize = captop + n + 1; /* minimum size needed */
if (newsize < (MAXNEWSIZE / 3) * 2)
newsize += newsize / 2; /* 1.5 that size, if not too big */
else if (newsize < (MAXNEWSIZE / 9) * 8)
newsize += newsize / 8; /* else, try 9/8 that size */
else
luaL_error(L, "too many captures");
newc = (Capture *)lua_newuserdata(L, newsize * sizeof(Capture));
memcpy(newc, capture, captop * sizeof(Capture));
@@ -125,7 +169,7 @@ static int resdyncaptures (lua_State *L, int fr, int curr, int limit) {
** value, 'n' is the number of values (at least 1). The open group
** capture is already in 'capture', before the place for the new entries.
*/
static void adddyncaptures (const char *s, Capture *capture, int n, int fd) {
static void adddyncaptures (Index_t index, Capture *capture, int n, int fd) {
int i;
assert(capture[-1].kind == Cgroup && capture[-1].siz == 0);
capture[-1].idx = 0; /* make group capture an anonymous group */
@@ -133,11 +177,11 @@ static void adddyncaptures (const char *s, Capture *capture, int n, int fd) {
capture[i].kind = Cruntime;
capture[i].siz = 1; /* mark it as closed */
capture[i].idx = fd + i; /* stack index of capture value */
capture[i].s = s;
capture[i].index = index;
}
capture[n].kind = Cclose; /* close group */
capture[n].siz = 1;
capture[n].s = s;
capture[n].index = index;
}
@@ -154,6 +198,32 @@ static int removedyncap (lua_State *L, Capture *capture,
}
/*
** Find the corresponding 'open' capture before 'cap', when that capture
** can become a full capture. If a full capture c1 is followed by an
** empty capture c2, there is no way to know whether c2 is inside
** c1. So, full captures can enclose only captures that start *before*
** its end.
*/
static Capture *findopen (Capture *cap, Index_t currindex) {
int i;
cap--; /* check last capture */
/* Must it be inside current one, but starts where current one ends? */
if (!isopencap(cap) && cap->index == currindex)
return NULL; /* current one cannot be a full capture */
/* else, look for an 'open' capture */
for (i = 0; i < MAXLOP; i++, cap--) {
if (currindex - cap->index >= UCHAR_MAX)
return NULL; /* capture too long for a full capture */
else if (isopencap(cap)) /* open capture? */
return cap; /* that's the one to be closed */
else if (cap->kind == Cclose)
return NULL; /* a full capture should not nest a non-full one */
}
return NULL; /* not found within allowed search limit */
}
/*
** Opcode interpreter
*/
@@ -181,7 +251,7 @@ const char *match (lua_State *L, const char *o, const char *s, const char *e,
case IEnd: {
assert(stack == getstackbase(L, ptop) + 1);
capture[captop].kind = Cclose;
capture[captop].s = NULL;
capture[captop].index = MAXINDT;
return s;
}
case IGiveup: {
@@ -198,47 +268,58 @@ const char *match (lua_State *L, const char *o, const char *s, const char *e,
else goto fail;
continue;
}
case IUTFR: {
int codepoint;
if (s >= e)
goto fail;
s = utf8_decode (s, &codepoint);
if (s && p[1].offset <= codepoint && codepoint <= utf_to(p))
p += 2;
else
goto fail;
continue;
}
case ITestAny: {
if (s < e) p += 2;
else p += getoffset(p);
continue;
}
case IChar: {
if ((byte)*s == p->i.aux && s < e) { p++; s++; }
if ((byte)*s == p->i.aux1 && s < e) { p++; s++; }
else goto fail;
continue;
}
case ITestChar: {
if ((byte)*s == p->i.aux && s < e) p += 2;
if ((byte)*s == p->i.aux1 && s < e) p += 2;
else p += getoffset(p);
continue;
}
case ISet: {
int c = (byte)*s;
if (testchar((p+1)->buff, c) && s < e)
{ p += CHARSETINSTSIZE; s++; }
uint c = (byte)*s;
if (charinset(p, (p+1)->buff, c) && s < e)
{ p += 1 + p->i.aux2.set.size; s++; }
else goto fail;
continue;
}
case ITestSet: {
int c = (byte)*s;
if (testchar((p + 2)->buff, c) && s < e)
p += 1 + CHARSETINSTSIZE;
uint c = (byte)*s;
if (charinset(p, (p + 2)->buff, c) && s < e)
p += 2 + p->i.aux2.set.size;
else p += getoffset(p);
continue;
}
case IBehind: {
int n = p->i.aux;
int n = p->i.aux1;
if (n > s - o) goto fail;
s -= n; p++;
continue;
}
case ISpan: {
for (; s < e; s++) {
int c = (byte)*s;
if (!testchar((p+1)->buff, c)) break;
uint c = (byte)*s;
if (!charinset(p, (p+1)->buff, c)) break;
}
p += CHARSETINSTSIZE;
p += 1 + p->i.aux2.set.size;
continue;
}
case IJmp: {
@@ -280,6 +361,8 @@ const char *match (lua_State *L, const char *o, const char *s, const char *e,
case IBackCommit: {
assert(stack > getstackbase(L, ptop) && (stack - 1)->s != NULL);
s = (--stack)->s;
if (ndyncap > 0) /* are there matchtime captures? */
ndyncap -= removedyncap(L, capture, stack->caplevel, captop);
captop = stack->caplevel;
p += getoffset(p);
continue;
@@ -287,7 +370,7 @@ const char *match (lua_State *L, const char *o, const char *s, const char *e,
case IFailTwice:
assert(stack > getstackbase(L, ptop));
stack--;
/* go through */
/* FALLTHROUGH */
case IFail:
fail: { /* pattern failed: try to backtrack */
do { /* remove pending calls */
@@ -326,38 +409,36 @@ const char *match (lua_State *L, const char *o, const char *s, const char *e,
luaL_error(L, "too many results in match-time capture");
/* add new captures + close group to 'capture' list */
capture = growcap(L, capture, &capsize, captop, n + 1, ptop);
adddyncaptures(s, capture + captop, n, fr);
adddyncaptures(s - o, capture + captop, n, fr);
captop += n + 1; /* new captures + close group */
}
p++;
continue;
}
case ICloseCapture: {
const char *s1 = s;
Capture *open = findopen(capture + captop, s - o);
assert(captop > 0);
/* if possible, turn capture into a full capture */
if (capture[captop - 1].siz == 0 &&
s1 - capture[captop - 1].s < UCHAR_MAX) {
capture[captop - 1].siz = s1 - capture[captop - 1].s + 1;
if (open) { /* if possible, turn capture into a full capture */
open->siz = (s - o) - open->index + 1;
p++;
continue;
}
else {
else { /* must create a close capture */
capture[captop].siz = 1; /* mark entry as closed */
capture[captop].s = s;
capture[captop].index = s - o;
goto pushcapture;
}
}
case IOpenCapture:
capture[captop].siz = 0; /* mark entry as open */
capture[captop].s = s;
capture[captop].index = s - o;
goto pushcapture;
case IFullCapture:
capture[captop].siz = getoff(p) + 1; /* save capture size */
capture[captop].s = s - getoff(p);
capture[captop].index = s - o - getoff(p);
/* goto pushcapture; */
pushcapture: {
capture[captop].idx = p->i.key;
capture[captop].idx = p->i.aux2.key;
capture[captop].kind = getkind(p);
captop++;
capture = growcap(L, capture, &capsize, captop, 0, ptop);

51
3rd/lpeg/lpvm.h
View File

@@ -1,6 +1,3 @@
/*
** $Id: lpvm.h $
*/
#if !defined(lpvm_h)
#define lpvm_h
@@ -8,16 +5,24 @@
#include "lpcap.h"
/*
** About Character sets in instructions: a set is a bit map with an
** initial offset, in bits, and a size, in number of instructions.
** aux1 has the default value for the bits outsize that range.
*/
/* Virtual Machine's instructions */
typedef enum Opcode {
IAny, /* if no char, fail */
IChar, /* if char != aux, fail */
ISet, /* if char not in buff, fail */
IChar, /* if char != aux1, fail */
ISet, /* if char not in set, fail */
ITestAny, /* in no char, jump to 'offset' */
ITestChar, /* if char != aux, jump to 'offset' */
ITestSet, /* if char not in buff, jump to 'offset' */
ISpan, /* read a span of chars in buff */
IBehind, /* walk back 'aux' characters (fail if not possible) */
ITestChar, /* if char != aux1, jump to 'offset' */
ITestSet, /* if char not in set, jump to 'offset' */
ISpan, /* read a span of chars in set */
IUTFR, /* if codepoint not in range [offset, utf_to], fail */
IBehind, /* walk back 'aux1' characters (fail if not possible) */
IRet, /* return from a rule */
IEnd, /* end of pattern */
IChoice, /* stack a choice; next fail will jump to 'offset' */
@@ -26,30 +31,46 @@ typedef enum Opcode {
IOpenCall, /* call rule number 'key' (must be closed to a ICall) */
ICommit, /* pop choice and jump to 'offset' */
IPartialCommit, /* update top choice to current position and jump */
IBackCommit, /* "fails" but jump to its own 'offset' */
IBackCommit, /* backtrack like "fail" but jump to its own 'offset' */
IFailTwice, /* pop one choice and then fail */
IFail, /* go back to saved state on choice and jump to saved offset */
IGiveup, /* internal use */
IFullCapture, /* complete capture of last 'off' chars */
IOpenCapture, /* start a capture */
ICloseCapture,
ICloseRunTime
ICloseRunTime,
IEmpty /* to fill empty slots left by optimizations */
} Opcode;
/*
** All array of instructions has a 'codesize' as its first element
** and is referred by a pointer to its second element, which is the
** first actual opcode.
*/
typedef union Instruction {
struct Inst {
byte code;
byte aux;
short key;
byte aux1;
union {
short key;
struct {
byte offset;
byte size;
} set;
} aux2;
} i;
int offset;
uint codesize;
byte buff[1];
} Instruction;
void printpatt (Instruction *p, int n);
/* extract 24-bit value from an instruction */
#define utf_to(inst) (((inst)->i.aux2.key << 8) | (inst)->i.aux1)
int charinset (const Instruction *i, const byte *buff, uint c);
const char *match (lua_State *L, const char *o, const char *s, const char *e,
Instruction *op, Capture *capture, int ptop);

16
3rd/lpeg/makefile
View File

@@ -2,7 +2,7 @@ LIBNAME = lpeg
LUADIR = ../lua/
COPT = -O2 -DNDEBUG
# COPT = -g
# COPT = -O0 -DLPEG_DEBUG -g
CWARNS = -Wall -Wextra -pedantic \
-Waggregate-return \
@@ -11,21 +11,24 @@ CWARNS = -Wall -Wextra -pedantic \
-Wdisabled-optimization \
-Wpointer-arith \
-Wshadow \
-Wredundant-decls \
-Wsign-compare \
-Wundef \
-Wwrite-strings \
-Wbad-function-cast \
-Wdeclaration-after-statement \
-Wmissing-prototypes \
-Wmissing-declarations \
-Wnested-externs \
-Wstrict-prototypes \
-Wc++-compat \
# -Wunreachable-code \
CFLAGS = $(CWARNS) $(COPT) -std=c99 -I$(LUADIR) -fPIC
CC = gcc
FILES = lpvm.o lpcap.o lptree.o lpcode.o lpprint.o
FILES = lpvm.o lpcap.o lptree.o lpcode.o lpprint.o lpcset.o
# For Linux
linux:
@@ -48,8 +51,9 @@ clean:
lpcap.o: lpcap.c lpcap.h lptypes.h
lpcode.o: lpcode.c lptypes.h lpcode.h lptree.h lpvm.h lpcap.h
lpprint.o: lpprint.c lptypes.h lpprint.h lptree.h lpvm.h lpcap.h
lptree.o: lptree.c lptypes.h lpcap.h lpcode.h lptree.h lpvm.h lpprint.h
lpcode.o: lpcode.c lptypes.h lpcode.h lptree.h lpvm.h lpcap.h lpcset.h
lpcset.o: lpcset.c lptypes.h lpcset.h lpcode.h lptree.h lpvm.h lpcap.h
lpprint.o: lpprint.c lptypes.h lpprint.h lptree.h lpvm.h lpcap.h lpcode.h
lptree.o: lptree.c lptypes.h lpcap.h lpcode.h lptree.h lpvm.h lpprint.h \
lpcset.h
lpvm.o: lpvm.c lpcap.h lptypes.h lpvm.h lpprint.h lptree.h

86
3rd/lpeg/re.html
View File

@@ -1,22 +1,21 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
"//www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html>
<head>
<title>LPeg.re - Regex syntax for LPEG</title>
<link rel="stylesheet"
href="http://www.inf.puc-rio.br/~roberto/lpeg/doc.css"
href="//www.inf.puc-rio.br/~roberto/lpeg/doc.css"
type="text/css"/>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
</head>
<body>
<!-- $Id: re.html $ -->
<div id="container">
<div id="product">
<div id="product_logo">
<a href="http://www.inf.puc-rio.br/~roberto/lpeg/">
<a href="//www.inf.puc-rio.br/~roberto/lpeg/">
<img alt="LPeg logo" src="lpeg-128.gif"/>
</a>
</div>
@@ -62,6 +61,20 @@ Constructions are listed in order of decreasing precedence.
<table border="1">
<tbody><tr><td><b>Syntax</b></td><td><b>Description</b></td></tr>
<tr><td><code>( p )</code></td> <td>grouping</td></tr>
<tr><td><code>&amp; p</code></td> <td>and predicate</td></tr>
<tr><td><code>! p</code></td> <td>not predicate</td></tr>
<tr><td><code>p1 p2</code></td> <td>concatenation</td></tr>
<tr><td><code>p1 / p2</code></td> <td>ordered choice</td></tr>
<tr><td><code>p ?</code></td> <td>optional match</td></tr>
<tr><td><code>p *</code></td> <td>zero or more repetitions</td></tr>
<tr><td><code>p +</code></td> <td>one or more repetitions</td></tr>
<tr><td><code>p^num</code></td>
<td>exactly <code>num</code> repetitions</td></tr>
<tr><td><code>p^+num</code></td>
<td>at least <code>num</code> repetitions</td></tr>
<tr><td><code>p^-num</code></td>
<td>at most <code>num</code> repetitions</td></tr>
<tr><td>(<code>name &lt;- p</code>)<sup>+</sup></td> <td>grammar</td></tr>
<tr><td><code>'string'</code></td> <td>literal string</td></tr>
<tr><td><code>"string"</code></td> <td>literal string</td></tr>
<tr><td><code>[class]</code></td> <td>character class</td></tr>
@@ -70,22 +83,15 @@ Constructions are listed in order of decreasing precedence.
<td>pattern <code>defs[name]</code> or a pre-defined pattern</td></tr>
<tr><td><code>name</code></td><td>non terminal</td></tr>
<tr><td><code>&lt;name&gt;</code></td><td>non terminal</td></tr>
<tr><td><code>{}</code></td> <td>position capture</td></tr>
<tr><td><code>{ p }</code></td> <td>simple capture</td></tr>
<tr><td><code>{: p :}</code></td> <td>anonymous group capture</td></tr>
<tr><td><code>{:name: p :}</code></td> <td>named group capture</td></tr>
<tr><td><code>{~ p ~}</code></td> <td>substitution capture</td></tr>
<tr><td><code>{| p |}</code></td> <td>table capture</td></tr>
<tr><td><code>=name</code></td> <td>back reference
</td></tr>
<tr><td><code>p ?</code></td> <td>optional match</td></tr>
<tr><td><code>p *</code></td> <td>zero or more repetitions</td></tr>
<tr><td><code>p +</code></td> <td>one or more repetitions</td></tr>
<tr><td><code>p^num</code></td> <td>exactly <code>n</code> repetitions</td></tr>
<tr><td><code>p^+num</code></td>
<td>at least <code>n</code> repetitions</td></tr>
<tr><td><code>p^-num</code></td>
<td>at most <code>n</code> repetitions</td></tr>
<tr><td><code>=name</code></td> <td>back reference</td></tr>
<tr><td><code>p -&gt; 'string'</code></td> <td>string capture</td></tr>
<tr><td><code>p -&gt; "string"</code></td> <td>string capture</td></tr>
<tr><td><code>p -&gt; num</code></td> <td>numbered capture</td></tr>
@@ -94,16 +100,13 @@ equivalent to <code>p / defs[name]</code></td></tr>
<tr><td><code>p =&gt; name</code></td> <td>match-time capture
equivalent to <code>lpeg.Cmt(p, defs[name])</code></td></tr>
<tr><td><code>p ~&gt; name</code></td> <td>fold capture
equivalent to <code>lpeg.Cf(p, defs[name])</code></td></tr>
<tr><td><code>& p</code></td> <td>and predicate</td></tr>
<tr><td><code>! p</code></td> <td>not predicate</td></tr>
<tr><td><code>p1 p2</code></td> <td>concatenation</td></tr>
<tr><td><code>p1 / p2</code></td> <td>ordered choice</td></tr>
<tr><td>(<code>name &lt;- p</code>)<sup>+</sup></td> <td>grammar</td></tr>
(deprecated)</td></tr>
<tr><td><code>p &gt;&gt; name</code></td> <td>accumulator capture
equivalent to <code>(p % defs[name])</code></td></tr>
</tbody></table>
<p>
Any space appearing in a syntax description can be
replaced by zero or more space characters and Lua-style comments
replaced by zero or more space characters and Lua-style short comments
(<code>--</code> until end of line).
</p>
@@ -200,9 +203,10 @@ print(re.match("the number 423 is odd", "({%a+} / .)*"))
--&gt; the number is odd
-- returns the first numeral in a string
print(re.match("the number 423 is odd", "s <- {%d+} / . s"))
print(re.match("the number 423 is odd", "s &lt;- {%d+} / . s"))
--&gt; 423
-- substitutes a dot for each vowel in a string
print(re.gsub("hello World", "[aeiou]", "."))
--&gt; h.ll. W.rld
</pre>
@@ -329,7 +333,7 @@ respecting the indentation:
<pre class="example">
p = re.compile[[
block &lt;- {| {:ident:' '*:} line
((=ident !' ' line) / &(=ident ' ') block)* |}
((=ident !' ' line) / &amp;(=ident ' ') block)* |}
line &lt;- {[^%nl]*} %nl
]]
</pre>
@@ -416,6 +420,7 @@ prefix &lt;- '&amp;' S prefix / '!' S prefix / suffix
suffix &lt;- primary S (([+*?]
/ '^' [+-]? num
/ '-&gt;' S (string / '{}' / name)
/ '&gt&gt;' S name
/ '=&gt;' S name) S)*
primary &lt;- '(' exp ')' / string / class / defined
@@ -423,6 +428,7 @@ primary &lt;- '(' exp ')' / string / class / defined
/ '=' name
/ '{}'
/ '{~' exp '~}'
/ '{|' exp '|}'
/ '{' exp '}'
/ '.'
/ name S !arrow
@@ -436,7 +442,7 @@ item &lt;- defined / range / .
range &lt;- . '-' [^]]
S &lt;- (%s / '--' [^%nl]*)* -- spaces and comments
name &lt;- [A-Za-z][A-Za-z0-9_]*
name &lt;- [A-Za-z_][A-Za-z0-9_]*
arrow &lt;- '&lt;-'
num &lt;- [0-9]+
string &lt;- '"' [^"]* '"' / "'" [^']* "'"
@@ -452,37 +458,9 @@ print(re.match(p, p)) -- a self description must match itself
<h2><a name="license">License</a></h2>
<p>
Copyright &copy; 2008-2015 Lua.org, PUC-Rio.
</p>
<p>
Permission is hereby granted, free of charge,
to any person obtaining a copy of this software and
associated documentation files (the "Software"),
to deal in the Software without restriction,
including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software,
and to permit persons to whom the Software is
furnished to do so,
subject to the following conditions:
</p>
This module is part of the <a href="lpeg.html">LPeg</a> package and shares
its <a href="lpeg.html#license">license</a>.
<p>
The above copyright notice and this permission notice
shall be included in all copies or substantial portions of the Software.
</p>
<p>
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED,
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
</p>
</div> <!-- id="content" -->

31
3rd/lpeg/re.lua
View File

@@ -1,4 +1,7 @@
-- $Id: re.lua $
--
-- Copyright 2007-2023, Lua.org & PUC-Rio (see 'lpeg.html' for license)
-- written by Roberto Ierusalimschy
--
-- imported functions and modules
local tonumber, type, print, error = tonumber, type, print, error
@@ -10,14 +13,14 @@ local m = require"lpeg"
-- on 'mm'
local mm = m
-- pattern's metatable
-- patterns' metatable
local mt = getmetatable(mm.P(0))
local version = _VERSION
-- No more global accesses after this point
local version = _VERSION
if version == "Lua 5.2" then _ENV = nil end
_ENV = nil -- does no harm in Lua 5.1
local any = m.P(1)
@@ -142,7 +145,7 @@ local item = (defined + Range + m.C(any)) / m.P
local Class =
"["
* (m.C(m.P"^"^-1)) -- optional complement symbol
* m.Cf(item * (item - "]")^0, mt.__add) /
* (item * ((item % mt.__add) - "]")^0) /
function (c, p) return c == "^" and any - p or p end
* "]"
@@ -168,13 +171,13 @@ end
local exp = m.P{ "Exp",
Exp = S * ( m.V"Grammar"
+ m.Cf(m.V"Seq" * ("/" * S * m.V"Seq")^0, mt.__add) );
Seq = m.Cf(m.Cc(m.P"") * m.V"Prefix"^0 , mt.__mul)
+ m.V"Seq" * ("/" * S * m.V"Seq" % mt.__add)^0 );
Seq = (m.Cc(m.P"") * (m.V"Prefix" % mt.__mul)^0)
* (#seq_follow + patt_error);
Prefix = "&" * S * m.V"Prefix" / mt.__len
+ "!" * S * m.V"Prefix" / mt.__unm
+ m.V"Suffix";
Suffix = m.Cf(m.V"Primary" * S *
Suffix = m.V"Primary" * S *
( ( m.P"+" * m.Cc(1, mt.__pow)
+ m.P"*" * m.Cc(0, mt.__pow)
+ m.P"?" * m.Cc(-1, mt.__pow)
@@ -185,10 +188,11 @@ local exp = m.P{ "Exp",
+ m.P"{}" * m.Cc(nil, m.Ct)
+ defwithfunc(mt.__div)
)
+ "=>" * S * defwithfunc(m.Cmt)
+ "~>" * S * defwithfunc(m.Cf)
) * S
)^0, function (a,b,f) return f(a,b) end );
+ "=>" * S * defwithfunc(mm.Cmt)
+ ">>" * S * defwithfunc(mt.__mod)
+ "~>" * S * defwithfunc(mm.Cf)
) % function (a,b,f) return f(a,b) end * S
)^0;
Primary = "(" * m.V"Exp" * ")"
+ String / mm.P
+ Class
@@ -204,8 +208,7 @@ local exp = m.P{ "Exp",
+ (name * -arrow + "<" * name * ">") * m.Cb("G") / NT;
Definition = name * arrow * m.V"Exp";
Grammar = m.Cg(m.Cc(true), "G") *
m.Cf(m.V"Definition" / firstdef * m.Cg(m.V"Definition")^0,
adddef) / mm.P
((m.V"Definition" / firstdef) * (m.V"Definition" % adddef)^0) / mm.P
}
local pattern = S * m.Cg(m.Cc(false), "G") * exp / mm.P * (-any + patt_error)

170
3rd/lpeg/test.lua
View File

@@ -1,7 +1,5 @@
#!/usr/bin/env lua
-- $Id: test.lua $
-- require"strict" -- just to be pedantic
local m = require"lpeg"
@@ -48,8 +46,8 @@ end
print"General tests for LPeg library"
assert(type(m.version()) == "string")
print("version " .. m.version())
assert(type(m.version) == "string")
print(m.version)
assert(m.type("alo") ~= "pattern")
assert(m.type(io.input) ~= "pattern")
assert(m.type(m.P"alo") == "pattern")
@@ -70,6 +68,8 @@ assert(m.match(#m.P(true) * "a", "a") == 2)
assert(m.match("a" * #m.P(false), "a") == nil)
assert(m.match("a" * #m.P(true), "a") == 2)
assert(m.match(m.P(1)^0, "abcd") == 5)
assert(m.match(m.S("")^0, "abcd") == 1)
-- tests for locale
do
@@ -119,6 +119,8 @@ eqcharset(m.S"\1\0\2", m.R"\0\2")
eqcharset(m.S"\1\0\2", m.R"\1\2" + "\0")
eqcharset(m.S"\1\0\2" - "\0", m.R"\1\2")
eqcharset(m.S("\0\255"), m.P"\0" + "\255") -- charset extremes
local word = alpha^1 * (1 - alpha)^0
assert((word^0 * -1):match"alo alo")
@@ -406,7 +408,7 @@ assert(p:match('abcx') == 5 and p:match('ayzx') == 5 and not p:match'abc')
do
-- large dynamic Cc
print "testing large dynamic Cc"
local lim = 2^16 - 1
local c = 0
local function seq (n)
@@ -491,8 +493,8 @@ local function f_term (v1, op, v2, d)
end
G = m.P{ "Exp",
Exp = m.Cf(V"Factor" * m.Cg(FactorOp * V"Factor")^0, f_factor);
Factor = m.Cf(V"Term" * m.Cg(TermOp * V"Term")^0, f_term);
Exp = V"Factor" * (FactorOp * V"Factor" % f_factor)^0;
Factor = V"Term" * (TermOp * V"Term" % f_term)^0;
Term = Number / tonumber + Open * V"Exp" * Close;
}
@@ -864,6 +866,7 @@ print"+"
-- accumulator capture
function f (x) return x + 1 end
assert(m.match(m.Cf(m.Cc(0) * m.C(1)^0, f), "alo alo") == 7)
assert(m.match(m.Cc(0) * (m.C(1) % f)^0, "alo alo") == 7)
t = {m.match(m.Cf(m.Cc(1,2,3), error), "")}
checkeq(t, {1})
@@ -873,7 +876,7 @@ t = p:match("a=b;c=du;xux=yuy;")
checkeq(t, {a="b", c="du", xux="yuy"})
-- errors in accumulator capture
-- errors in fold capture
-- no initial capture
checkerr("no initial value", m.match, m.Cf(m.P(5), print), 'aaaaaa')
@@ -881,8 +884,14 @@ checkerr("no initial value", m.match, m.Cf(m.P(5), print), 'aaaaaa')
checkerr("no initial value", m.match, m.Cf(m.P(500), print),
string.rep('a', 600))
-- nested capture produces no initial value
checkerr("no initial value", m.match, m.Cf(m.P(1) / {}, print), "alo")
-- errors in accumulator capture
-- no initial capture
checkerr("no previous value", m.match, m.P(5) % print, 'aaaaaa')
-- no initial capture (very long match forces fold to be a pair open-close)
checkerr("no previous value", m.match, m.P(500) % print,
string.rep('a', 600))
-- tests for loop checker
@@ -985,10 +994,10 @@ for i = 1, 10 do
assert(p:match("aaaaaaaaaaa") == 11 - i + 1)
end
print"+"
-- tests for back references
print "testing back references"
checkerr("back reference 'x' not found", m.match, m.Cb('x'), '')
checkerr("back reference 'b' not found", m.match, m.Cg(1, 'a') * m.Cb('b'), 'a')
@@ -996,6 +1005,35 @@ p = m.Cg(m.C(1) * m.C(1), "k") * m.Ct(m.Cb("k"))
t = p:match("ab")
checkeq(t, {"a", "b"})
do
-- some basic cases
assert(m.match(m.Cg(m.Cc(3), "a") * m.Cb("a"), "a") == 3)
assert(m.match(m.Cg(m.C(1), 133) * m.Cb(133), "X") == "X")
-- first reference to 'x' should not see the group enclosing it
local p = m.Cg(m.Cb('x'), 'x') * m.Cb('x')
checkerr("back reference 'x' not found", m.match, p, '')
local p = m.Cg(m.Cb('x') * m.C(1), 'x') * m.Cb('x')
checkerr("back reference 'x' not found", m.match, p, 'abc')
-- reference to 'x' should not see the group enclosed in another capture
local s = string.rep("a", 30)
local p = (m.C(1)^-4 * m.Cg(m.C(1), 'x')) / {} * m.Cb('x')
checkerr("back reference 'x' not found", m.match, p, s)
local p = (m.C(1)^-20 * m.Cg(m.C(1), 'x')) / {} * m.Cb('x')
checkerr("back reference 'x' not found", m.match, p, s)
-- second reference 'k' should refer to 10 and first ref. 'k'
p = m.Cg(m.Cc(20), 'k') * m.Cg(m.Cc(10) * m.Cb('k') * m.C(1), 'k')
* (m.Cb('k') / function (a,b,c) return a*10 + b + tonumber(c) end)
-- 10 * 10 (Cc) + 20 (Cb) + 7 (C) == 127
assert(p:match("756") == 127)
end
p = m.P(true)
for i = 1, 10 do p = p * m.Cg(1, i) end
for i = 1, 10 do
@@ -1032,6 +1070,17 @@ local function id (s, i, ...)
return true, ...
end
do -- run-time capture in an end predicate (should discard its value)
local x = 0
function foo (s, i)
x = x + 1
return true, x
end
local p = #(m.Cmt("", foo) * "xx") * m.Cmt("", foo)
assert(p:match("xx") == 2)
end
assert(m.Cmt(m.Cs((m.Cmt(m.S'abc' / { a = 'x', c = 'y' }, id) +
m.R'09'^1 / string.char +
m.P(1))^0), id):match"acb98+68c" == "xyb\98+\68y")
@@ -1156,7 +1205,7 @@ end
-- bug in 1.0: problems with math-times returning too many captures
do
if _VERSION >= "Lua 5.2" then
local lim = 2^11 - 10
local res = {m.match(manyCmt(lim), "a")}
assert(#res == lim and res[1] == lim - 1 and res[lim] == 0)
@@ -1171,9 +1220,91 @@ t = {p:match('abacc')}
checkeq(t, {'a', 'aa', 20, 'a', 'aaa', 'aaa'})
do print"testing large grammars"
local lim = 1000 -- number of rules
local t = {}
for i = 3, lim do
t[i] = m.V(i - 1) -- each rule calls previous one
end
t[1] = m.V(lim) -- start on last rule
t[2] = m.C("alo") -- final rule
local P = m.P(t) -- build grammar
assert(P:match("alo") == "alo")
t[#t + 1] = m.P("x") -- one more rule...
checkerr("too many rules", m.P, t)
end
print "testing UTF-8 ranges"
do -- a few typical UTF-8 ranges
local p = m.utfR(0x410, 0x44f)^1 / "cyr: %0"
+ m.utfR(0x4e00, 0x9fff)^1 / "cjk: %0"
+ m.utfR(0x1F600, 0x1F64F)^1 / "emot: %0"
+ m.utfR(0, 0x7f)^1 / "ascii: %0"
+ m.utfR(0, 0x10ffff) / "other: %0"
p = m.Ct(p^0) * -m.P(1)
local cyr = "ждюя"
local emot = "\240\159\152\128\240\159\153\128" -- 😀🙀
local cjk = "专举乸"
local ascii = "alo"
local last = "\244\143\191\191" -- U+10FFFF
local s = cyr .. "—" .. emot .. "—" .. cjk .. "—" .. ascii .. last
t = (p:match(s))
assert(t[1] == "cyr: " .. cyr and t[2] == "other: —" and
t[3] == "emot: " .. emot and t[4] == "other: —" and
t[5] == "cjk: " .. cjk and t[6] == "other: —" and
t[7] == "ascii: " .. ascii and t[8] == "other: " .. last and
t[9] == nil)
-- failing UTF-8 matches and borders
assert(not m.match(m.utfR(10, 0x2000), "\9"))
assert(not m.match(m.utfR(10, 0x2000), "\226\128\129"))
assert(m.match(m.utfR(10, 0x2000), "\10") == 2)
assert(m.match(m.utfR(10, 0x2000), "\226\128\128") == 4)
end
do -- valid and invalid code points
local p = m.utfR(0, 0x10ffff)^0
assert(p:match("汉字\128") == #"汉字" + 1)
assert(p:match("\244\159\191") == 1)
assert(p:match("\244\159\191\191") == 1)
assert(p:match("\255") == 1)
-- basic errors
checkerr("empty range", m.utfR, 1, 0)
checkerr("invalid code point", m.utfR, 1, 0x10ffff + 1)
end
do -- back references (fixed width)
-- match a byte after a CJK point
local p = m.B(m.utfR(0x4e00, 0x9fff)) * m.C(1)
p = m.P{ p + m.P(1) * m.V(1) } -- search for 'p'
assert(p:match("ab д 专X x") == "X")
-- match a byte after a hebrew point
local p = m.B(m.utfR(0x5d0, 0x5ea)) * m.C(1)
p = m.P(#"ש") * p
assert(p:match("שX") == "X")
checkerr("fixed length", m.B, m.utfR(0, 0x10ffff))
end
-------------------------------------------------------------------
-- Tests for 're' module
-------------------------------------------------------------------
print"testing 're' module"
local re = require "re"
@@ -1307,6 +1438,12 @@ e = compile([[
e = compile("{[0-9]+'.'?[0-9]*} -> sin", math)
assert(e:match("2.34") == math.sin(2.34))
e = compile("'pi' -> math", _G)
assert(e:match("pi") == math.pi)
e = compile("[ ]* 'version' -> _VERSION", _G)
assert(e:match(" version") == _VERSION)
function eq (_, _, a, b) return a == b end
@@ -1380,6 +1517,13 @@ c = re.compile([[
]], {tonumber = tonumber, add = function (a,b) return a + b end})
assert(c:match("3 401 50") == 3 + 401 + 50)
-- test for accumulator captures
c = re.compile([[
S <- number (%s+ number >> add)*
number <- %d+ -> tonumber
]], {tonumber = tonumber, add = function (a,b) return a + b end})
assert(c:match("3 401 50") == 3 + 401 + 50)
-- tests for look-ahead captures
x = {re.match("alo", "&(&{.}) !{'b'} {&(...)} &{..} {...} {!.}")}
checkeq(x, {"", "alo", ""})

2
Makefile
View File

@@ -115,7 +115,7 @@ $(LUA_CLIB_PATH)/sproto.so : lualib-src/sproto/sproto.c lualib-src/sproto/lsprot
$(LUA_CLIB_PATH)/ltls.so : lualib-src/ltls.c | $(LUA_CLIB_PATH)
$(CC) $(CFLAGS) $(SHARED) -Iskynet-src -L$(TLS_LIB) -I$(TLS_INC) $^ -o $@ -lssl
$(LUA_CLIB_PATH)/lpeg.so : 3rd/lpeg/lpcap.c 3rd/lpeg/lpcode.c 3rd/lpeg/lpprint.c 3rd/lpeg/lptree.c 3rd/lpeg/lpvm.c | $(LUA_CLIB_PATH)
$(LUA_CLIB_PATH)/lpeg.so : 3rd/lpeg/lpcap.c 3rd/lpeg/lpcode.c 3rd/lpeg/lpprint.c 3rd/lpeg/lptree.c 3rd/lpeg/lpvm.c 3rd/lpeg/lpcset.c | $(LUA_CLIB_PATH)
$(CC) $(CFLAGS) $(SHARED) -I3rd/lpeg $^ -o $@
clean :