From 4e50079706530210f9c93c1541cde56cc4593108 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BA=91=E9=A3=8E?= Date: Thu, 9 Aug 2012 16:48:28 +0800 Subject: [PATCH] add some lua library --- Makefile | 26 + lpeg/HISTORY | 77 + lpeg/lpeg-128.gif | Bin 0 -> 4923 bytes lpeg/lpeg.c | 2405 ++++++++++++++++++++++++++++++++ lpeg/lpeg.h | 38 + lpeg/lpeg.html | 1423 +++++++++++++++++++ lpeg/lpeg.o | Bin 0 -> 48688 bytes lpeg/makefile | 42 + lpeg/re.html | 486 +++++++ lpeg/re.lua | 248 ++++ lpeg/test.lua | 1188 ++++++++++++++++ lua-int64/Makefile | 6 + lua-int64/README | 1 + lua-int64/int64.c | 317 +++++ lua-int64/int64.so | Bin 0 -> 24305 bytes lua-int64/test.lua | 17 + lua-protobuf/Makefile | 14 + lua-protobuf/README | 2 + lua-protobuf/addressbook.proto | 39 + lua-protobuf/alloc.c | 84 ++ lua-protobuf/alloc.h | 25 + lua-protobuf/array.c | 139 ++ lua-protobuf/array.h | 31 + lua-protobuf/bootstrap.c | 303 ++++ lua-protobuf/bootstrap.h | 10 + lua-protobuf/context.c | 278 ++++ lua-protobuf/context.h | 140 ++ lua-protobuf/decode.c | 349 +++++ lua-protobuf/descriptor.pbc.h | 271 ++++ lua-protobuf/map.c | 477 +++++++ lua-protobuf/map.h | 33 + lua-protobuf/pattern.c | 1131 +++++++++++++++ lua-protobuf/pattern.h | 26 + lua-protobuf/pbc-lua.c | 1004 +++++++++++++ lua-protobuf/pbc.h | 104 ++ lua-protobuf/proto.c | 252 ++++ lua-protobuf/proto.h | 64 + lua-protobuf/register.c | 329 +++++ lua-protobuf/rmessage.c | 455 ++++++ lua-protobuf/stringpool.c | 60 + lua-protobuf/stringpool.h | 10 + lua-protobuf/test.lua | 30 + lua-protobuf/varint.c | 110 ++ lua-protobuf/varint.h | 20 + lua-protobuf/wmessage.c | 509 +++++++ lualib/protobuf.lua | 592 ++++++++ lualib/protobuf/parser.lua | 397 ++++++ 47 files changed, 13562 insertions(+) create mode 100644 lpeg/HISTORY create mode 100644 lpeg/lpeg-128.gif create mode 100644 lpeg/lpeg.c create mode 100644 lpeg/lpeg.h create mode 100644 lpeg/lpeg.html create mode 100644 lpeg/lpeg.o create mode 100644 lpeg/makefile create mode 100644 lpeg/re.html create mode 100644 lpeg/re.lua create mode 100755 lpeg/test.lua create mode 100644 lua-int64/Makefile create mode 100644 lua-int64/README create mode 100644 lua-int64/int64.c create mode 100755 lua-int64/int64.so create mode 100644 lua-int64/test.lua create mode 100644 lua-protobuf/Makefile create mode 100644 lua-protobuf/README create mode 100644 lua-protobuf/addressbook.proto create mode 100644 lua-protobuf/alloc.c create mode 100644 lua-protobuf/alloc.h create mode 100644 lua-protobuf/array.c create mode 100644 lua-protobuf/array.h create mode 100644 lua-protobuf/bootstrap.c create mode 100644 lua-protobuf/bootstrap.h create mode 100644 lua-protobuf/context.c create mode 100644 lua-protobuf/context.h create mode 100644 lua-protobuf/decode.c create mode 100644 lua-protobuf/descriptor.pbc.h create mode 100644 lua-protobuf/map.c create mode 100644 lua-protobuf/map.h create mode 100644 lua-protobuf/pattern.c create mode 100644 lua-protobuf/pattern.h create mode 100644 lua-protobuf/pbc-lua.c create mode 100644 lua-protobuf/pbc.h create mode 100644 lua-protobuf/proto.c create mode 100644 lua-protobuf/proto.h create mode 100644 lua-protobuf/register.c create mode 100644 lua-protobuf/rmessage.c create mode 100644 lua-protobuf/stringpool.c create mode 100644 lua-protobuf/stringpool.h create mode 100644 lua-protobuf/test.lua create mode 100644 lua-protobuf/varint.c create mode 100644 lua-protobuf/varint.h create mode 100644 lua-protobuf/wmessage.c create mode 100644 lualib/protobuf.lua create mode 100644 lualib/protobuf/parser.lua diff --git a/Makefile b/Makefile index 64175e4c..997b149d 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,9 @@ service/client.so \ service/connection.so \ service/broker.so \ client \ +lualib/lpeg.so \ +lualib/protobuf.so \ +lualib/int64.so \ skynet-master skynet : \ @@ -45,6 +48,29 @@ service/connection.so : connection/connection.c connection/main.c service/broker.so : service-src/service_broker.c gcc -Wall -g -fPIC --shared $^ -o $@ -Iskynet-src +lualib/lpeg.so : lpeg/lpeg.c + gcc -Wall -O2 -fPIC --shared $^ -o $@ -Ilpeg + +lualib/protobuf.so : \ +lua-protobuf/context.c \ +lua-protobuf/varint.c \ +lua-protobuf/array.c \ +lua-protobuf/pattern.c \ +lua-protobuf/register.c \ +lua-protobuf/proto.c \ +lua-protobuf/map.c \ +lua-protobuf/alloc.c \ +lua-protobuf/rmessage.c \ +lua-protobuf/wmessage.c \ +lua-protobuf/bootstrap.c \ +lua-protobuf/stringpool.c \ +lua-protobuf/decode.c \ +lua-protobuf/pbc-lua.c + gcc -O2 -Wall --shared -fPIC -o $@ $^ + +lualib/int64.so : lua-int64/int64.c + gcc -O2 -Wall --shared -fPIC -o $@ $^ + client : client-src/client.c gcc -Wall -g $^ -o $@ -lpthread diff --git a/lpeg/HISTORY b/lpeg/HISTORY new file mode 100644 index 00000000..0b8b7577 --- /dev/null +++ b/lpeg/HISTORY @@ -0,0 +1,77 @@ +HISTORY for LPeg 0.10 + +* Changes from version 0.9 to 0.10 + ------------------------------- + + backtrack stack has configurable size + + better error messages + + Notation for non-terminals in 're' back to A instead o + + experimental look-behind pattern + + support for external extensions + + works with Lua 5.2 + + consumes less C stack + + - "and" predicates do not keep captures + +* Changes from version 0.8 to 0.9 + ------------------------------- + + The accumulator capture was replaced by a fold capture; + programs that used the old 'lpeg.Ca' will need small changes. + + Some support for character classes from old C locales. + + A new named-group capture. + +* Changes from version 0.7 to 0.8 + ------------------------------- + + New "match-time" capture. + + New "argument capture" that allows passing arguments into the pattern. + + Better documentation for 're'. + + Several small improvements for 're'. + + The 're' module has an incompatibility with previous versions: + now, any use of a non-terminal must be enclosed in angle brackets + (like ). + +* Changes from version 0.6 to 0.7 + ------------------------------- + + Several improvements in module 're': + - better documentation; + - support for most captures (all but accumulator); + - limited repetitions p{n,m}. + + Small improvements in efficiency. + + Several small bugs corrected (special thanks to Hans Hagen + and Taco Hoekwater). + +* Changes from version 0.5 to 0.6 + ------------------------------- + + Support for non-numeric indices in grammars. + + Some bug fixes (thanks to the luatex team). + + Some new optimizations; (thanks to Mike Pall). + + A new page layout (thanks to Andre Carregal). + + Minimal documentation for module 're'. + +* Changes from version 0.4 to 0.5 + ------------------------------- + + Several optimizations. + + lpeg.P now accepts booleans. + + Some new examples. + + A proper license. + + Several small improvements. + +* Changes from version 0.3 to 0.4 + ------------------------------- + + Static check for loops in repetitions and grammars. + + Removed label option in captures. + + The implementation of captures uses less memory. + +* Changes from version 0.2 to 0.3 + ------------------------------- + + User-defined patterns in Lua. + + Several new captures. + +* Changes from version 0.1 to 0.2 + ------------------------------- + + Several small corrections. + + Handles embedded zeros like any other character. + + Capture "name" can be any Lua value. + + Unlimited number of captures. + + Match gets an optional initial position. + +(end of HISTORY) diff --git a/lpeg/lpeg-128.gif b/lpeg/lpeg-128.gif new file mode 100644 index 0000000000000000000000000000000000000000..bbf5e78b88e71044bd341a2476dcec756c252ae4 GIT binary patch literal 4923 zcmV-B6U6LCNk%w1VSoUD0O$Vz004pj0D}Plga8JE00)Es2!#L%h6@FN4hDe`2Z9p_ zgBS~j4iktF6p0fRixn1(7#EEV9grItj}#!2B_EC`A(0;@k|8FOBPWz3D3vEFmoh4o zEH0HaE0iuTmM}1wGBTPiH=Q&zn>I9^I5?R*I-5N^oj*OFK|Y~7LZduGq((%cN=Kwi zNTp0krbkVwPfn#$P^VK-s8Cm}SXil9S*luEt6N*FU0kkTUa(+av0-4cV`8*qWwB*u zvSw$qXK1u&X|-!>xNL2?b8fYCZ?|u9xN>&8et>;?c)EXqet?31e0sZqgMfW|yn=** zetf-yg@S~JgMfa&g@=TJf53)_g@S;>eS^Y?iiduL#EFZDij0Ybg};S{!G?#zjgO3u zkd26m#E+4Vk&}>%jK`9ck&TVXl$Mi{j>MIhl$My4l#s@knULz#>wW(&B@2g=gP|A&Ckos%;V0`%+1Z^ z&(Y4%($CP(=h4&9(a`A9)X~$`($doD)YjA0*VNS1>(Dk-Y+}P^d z-P+yR>)qen-`nlp;N9Qh-r?Qv+~Dxw;@{oj^5Nv*gVX`^Xcp8>g?(2>-FpI>g??I?eFXD?fCBS?e6dS^6&KU^6&HU_3`uY z^7Qfb^Y`=h^7i!j_4oAn_4@ht`}g_w`uF_!`uF?z{rUU&{QCa-{Q39&|NH&=`2PR- z|Ns2|`~Ls@A^sIZa%Ew3Wn>_CX>@2HRA^-&M@dak04x9i004jhfB*mp{s8|897wRB z!Gj1BDqP60p~Hs|BTAe|v7*I`7&B_z$g!ixk03*e97(dI$&)Bks$9vkrAvX8`q*is z=H9?{l}7E#R?lNQWVLv4P~jVh?hctKNEB_v)Q`AA0{He1AFMltl|F;_};VY9{( zl?>#^V#;+GAcx6`MS^o5VWY+-eH4NZB-lh!2s*%^Q%D=Wpb`p8b@@WsJ{k14+ee}#bO3Jby=YfFj5SY4>pSIg$+7yAlZ_3|IG4ZYXE{+9&0^>gij@c?1K+BVN7L+ zBE8Uo%|6)(0!cRW6;xLjK86S4czYfI0f`*plLsohjDgNS_Sl1qC+O^h7nVis14DTp z_UB)a2aE$!UFv{w2Rn||WsjqsJvIKEv42Q>pl%atELe@uN%~I9w1O zV#cKwqF5Br5C<|5?UMqf)N;9JfW`!>inerBLQX{d@Zv-T1n>!#kN^gl*Z>n8VQoY3 z(2(b=$RW((0nJ{E&n0%)GnFrb*aOi!dZ>EwmKPuA-(diH5QjPo?Sn_Ot)`4{2&4u| zi5>X#vc@|Pm2*V{Hk> zUea4{!V6#lG+X(=U@xX82l*yQ_Mq|()&=on3I1y8a!FpoM(dfd;ENeIK_h^&Gs~rB ze+Z%xxoV7eLF3GDu*IvosxqH_3Q1lJ+7QIgKJX+#E#Y5=#eg~BqQnUWDt8{D=Uq;V z^2y3UfJs5~SVO|AMgOh9Et8kT&lkZ@Og7=1v+8-hPU!OwKSbrD&ZN!j>DnnC<>!z* zG)ODE?Sb1ae)*}z&<;WKbYq0zw1-6kG&yU8&KTf0qz(!|(!R3e%v)Z)l0!8SIgKdz zAl)pm2fqVY01xT-NFUTtyn%Ql1cS4g-;P#54oZsx9Dr8riUWWSgl-_DQ%5fFKm!tx zfCDf{!YSZkrq)%Y7fYDXLewFG+4=8p{#C;r5dUL@Q)B}kNUR4m@IZpgsfz)P9XKO zA#YG24~Fam2W9i0hvI+ zxvPeBB zAhiK7fs~m51VyG|reVGVb`)i)4;g1Z07-MK`%smD0KkAzBE*$Xh(}27fd_t)5lAb= zC^LOTii^Mlc7p^!92R1aC+uQV`1oDm{#LVS<>`m4S?pN58{!Sg z3Yf$bC|+;1>5vMx59`A$vQ)O9%0#-0JDTX}~|6we0k1su%wk%kT$H+31jtRN*_Gd{y0P{w};~v8-U2rC7tb7gCRr(~?620NOrIO7IL?U>1y))@n!)idpEHy-C1^JMKXa zE4pJFr&z(+hC72{ZsH9b*@q9@`@D5u+{v`$06C_3G!Cr*i|QQ@19qX0hr`1izpV0j zPP|P@TxeV;(fuIL1lvk9=ffYQ>oWzl|X^V`!%xrtkwFG@+?~+yfjw zHuiELw@pP90wUt@CPM1_JgNae9izK?pQ=oPeC*>MdItauY=R9R(O@8qfB<5`Qh+1~ zgda0Ya?!frJ+@>5I7Q9`)Dr@aLlii6c85oBQ*TBQ)Fm4nUd+AgF;hJ;f+)zCy?_S( z;doLsSY?EmoMwuQ3oA@+AGKJr=LTT)=u`Uue8)#VHhzG6@1ymtIR!uTfraKOMhMfh zIIR*mZ2u(ObWz8~bogYPA~dClKvcHtz?2Q;`qwWr6>2f4&w-sqtZ1;(;GR0eUnM_^=7# zM10|MJB4F+CN+Q*2LOEc58E($#$kU9CPc2m03<*IUocSh;C(z89!;kZ*$@j4;TgrV z55_k@E(m(;CW9{rg*7-p^_PQThbc6G2ea@Ek`x>J@MQyb01$Q%_%I4)0C)Zip%1vA zgi8oCcn5(Qb^sy}g6MEQFn|yD01f>E1m3U@Y?MF((GH~WU0)c6%AsF(B?y;yEuE1M zEJ$^q7kc8uJTw@BWn_Q&w;V~}4+HTGeqaPJkO9P@0|N17pR^bjuu*K01=P_!zeIP4 z0W|IqRo*sPlQ<8R7(_J?g3xd=9$*>BpaL00jPO7Tx1=0jr4ZPFQxl;G`gRbblYyQ0 zc{$i?Bv>{h5`wi<9JZ%}uJLN;z*VV1DL7D!9P}S_h7j@q49!MLJEIW$;0`8pe0DdD zU6(unFp2qK4*e7s10V|ASRT@}Pw@tVv_&!H;d~VoaoZFOO%PHCq5g*RxEk#TSemmK zBa%Q|g)|f(Nv#(i(!>a~6SPm)1`ScBfzkz>&gLs$`YK?#|FHKjxpQPp|th7G4sm!v=nr*M~extB?J z0Njv+>M(I)!2p(k4~?f@hY?$|!U%gp0&jJHwImiv(0d0l4U`FDEa`aG5S1xocbsK< z^?*d82@gdF59y$l0B{5XQ4ilR3vmz!mN0bsa02qj4B~+R&rnMBunm@g2aEs>Q$Y`u z_795F1yLCw3=j@NrUc0#5~WmMmm?%@*bw_b1ph$@cr*`4{zMexzybnLD+o0J9FQ5K zlY=>k572M|3g8Vn7@8Bg7`(R-qre4-C=uPjHw~x)#liyo*`NODpApa{H}DGUG!Alr zSS+vtN~bUYumESEL-?QzMer2^-~#OV59$Cd0n(EYH3t4A64g_d6?J?tM{xb202?3z z6o3E#ggeCs02AN=9l!uDDi#Uw0UW@YV!;Rvi5I)@1UYpP!=MJ2Nf7lQ4$}s6GdYi{ z<%0f4b%#+LsODL$@uDtB0|_!rOH!Y@p$<=gPj)gr`OpnJ*=;1cY7nSWYUdiomRN(u z04c~8T*?-ZUwjM}J67H^OmHO8{QqTiMR`O^^SKnEsRUcs;gzMv3`G&<>E zVI4_6bT~CA1*rc41L`1K@@G?*UU>__s+(~MR1l}8G7$d24&Cs8(JHh4>Q{8t1@GV}i_}V` z&;-B$Q!CL74RIEy_YKAwi5InW3>X$&@C_1y4TdT*EOCEJun7@?5AZ+@Ohjtt%33r- z0(fw&)HAnpiVQD z;$RFcG*N(8HXeru&`=Ne@VT=^s-{vQ$50WT7@A154YR-qX6HyFSpawx2y#yf+&3m>5 zA-ff!3QZ6RZDzhq@dj$(3>ty6M2HT);9~NN7w7;8OP~mLB)>h|zjhJ~9vZ#2V1@>~ zQ1(!*v*`|<=<;JBG7!h4Gs!GHu(kO!Hdc@Q%Vv4j_w z013X}F-EME>aYut@CLcS8Iw>5ia-gT;62^&23w#6N)QQmAu2e0#n9EWO@Ro4kOz0b t87`~}*f6ke{1|$%8+P%%7keDYf;`BCT*!ue$cUWCioD2-9K8Vn06SzG6$t +#include +#include +#include +#include + +#include "lua.h" +#include "lauxlib.h" + +#include "lpeg.h" + + +#define VERSION "0.10" +#define PATTERN_T "lpeg-pattern" +#define MAXSTACKIDX "lpeg-maxstack" + + +/* +** compatibility with Lua 5.2 +*/ +#if (LUA_VERSION_NUM == 502) + +#undef lua_equal +#define lua_equal(L,idx1,idx2) lua_compare(L,(idx1),(idx2),LUA_OPEQ) + +#undef lua_getfenv +#define lua_getfenv lua_getuservalue +#undef lua_setfenv +#define lua_setfenv lua_setuservalue + +#undef lua_objlen +#define lua_objlen lua_rawlen + +#undef luaL_register +#define luaL_register(L,n,f) \ + { if ((n) == NULL) luaL_setfuncs(L,f,0); else luaL_newlib(L,f); } + +#endif + + + +/* initial size for call/backtrack stack */ +#define INITBACK 100 + +/* default maximum size for call/backtrack stack */ +#define MAXBACK INITBACK + +/* size for call/backtrack stack for verifier */ +#define MAXBACKVER 200 + +/* initial size for capture's list */ +#define INITCAPSIZE 32 + + +/* index, on Lua stack, for subject */ +#define SUBJIDX 2 + +/* number of fixed arguments to 'match' (before capture arguments) */ +#define FIXEDARGS 3 + +/* index, on Lua stack, for substitution value cache */ +#define subscache(cs) ((cs)->ptop + 1) + +/* index, on Lua stack, for capture list */ +#define caplistidx(ptop) ((ptop) + 2) + +/* index, on Lua stack, for pattern's fenv */ +#define penvidx(ptop) ((ptop) + 3) + +/* index, on Lua stack, for backtracking stack */ +#define stackidx(ptop) ((ptop) + 4) + + + +typedef unsigned char byte; + + +#define CHARSETSIZE ((UCHAR_MAX/CHAR_BIT) + 1) + + +typedef byte Charset[CHARSETSIZE]; + + +/* Virtual Machine's instructions */ +typedef enum Opcode { + IAny, IChar, ISet, ISpan, + IBack, + IRet, IEnd, + IChoice, IJmp, ICall, IOpenCall, + ICommit, IPartialCommit, IBackCommit, IFailTwice, IFail, IGiveup, + IFunc, + IFullCapture, IEmptyCapture, IEmptyCaptureIdx, + IOpenCapture, ICloseCapture, ICloseRunTime +} Opcode; + + +#define ISJMP 0x1 +#define ISCHECK 0x2 +#define ISFIXCHECK 0x4 +#define ISNOFAIL 0x8 +#define ISCAPTURE 0x10 +#define ISMOVABLE 0x20 +#define ISFENVOFF 0x40 + +static const int opproperties[] = { + /* IAny */ ISCHECK | ISFIXCHECK | ISJMP, + /* IChar */ ISCHECK | ISFIXCHECK | ISJMP, + /* ISet */ ISCHECK | ISFIXCHECK | ISJMP, + /* ISpan */ ISNOFAIL, + /* IBack */ 0, + /* IRet */ 0, + /* IEnd */ 0, + /* IChoice */ ISJMP, + /* IJmp */ ISJMP | ISNOFAIL, + /* ICall */ ISJMP, + /* IOpenCall */ ISFENVOFF, + /* ICommit */ ISJMP, + /* IPartialCommit */ ISJMP, + /* IBackCommit */ ISJMP, + /* IFailTwice */ 0, + /* IFail */ 0, + /* IGiveup */ 0, + /* IFunc */ ISCHECK | ISJMP, + /* IFullCapture */ ISCAPTURE | ISNOFAIL | ISFENVOFF, + /* IEmptyCapture */ ISCAPTURE | ISNOFAIL | ISMOVABLE, + /* IEmptyCaptureIdx */ISCAPTURE | ISNOFAIL | ISMOVABLE | ISFENVOFF, + /* IOpenCapture */ ISCAPTURE | ISNOFAIL | ISMOVABLE | ISFENVOFF, + /* ICloseCapture */ ISCAPTURE | ISNOFAIL | ISMOVABLE | ISFENVOFF, + /* ICloseRunTime */ ISCAPTURE | ISFENVOFF +}; + + +typedef union Instruction { + struct Inst { + byte code; + byte aux; + short offset; + } i; + PattFunc f; + int iv; + byte buff[1]; +} Instruction; + +static const Instruction giveup = {{IGiveup, 0, 0}}; + +#define getkind(op) ((op)->i.aux & 0xF) +#define getoff(op) (((op)->i.aux >> 4) & 0xF) + +#define dest(p,x) ((x) + ((p)+(x))->i.offset) + +#define MAXOFF 0xF +#define MAXAUX 0xFF + +/* maximum size (in elements) for a pattern */ +#define MAXPATTSIZE (SHRT_MAX - 10) + + +#define isprop(op,p) (opproperties[(op)->i.code] & (p)) +#define isjmp(op) (isprop(op, ISJMP) && (op)->i.offset != 0) +#define iscapture(op) isprop(op, ISCAPTURE) +#define ischeck(op) (isprop(op, ISCHECK) && (op)->i.offset == 0) +#define isfixcheck(op) (isprop(op, ISFIXCHECK) && (op)->i.offset == 0) +#define istest(op) (isprop(op, ISCHECK) && (op)->i.offset != 0) +#define isnofail(op) isprop(op, ISNOFAIL) +#define ismovable(op) isprop(op, ISMOVABLE) +#define isfenvoff(op) isprop(op, ISFENVOFF) + + +/* kinds of captures */ +typedef enum CapKind { + Cclose, Cposition, Cconst, Cbackref, Carg, Csimple, Ctable, Cfunction, + Cquery, Cstring, Csubst, Cfold, Cruntime, Cgroup +} CapKind; + +#define iscapnosize(k) ((k) == Cposition || (k) == Cconst) + + +typedef struct Capture { + const char *s; /* position */ + short idx; + byte kind; + byte siz; +} Capture; + + +/* size (in elements) for an instruction plus extra l bytes */ +#define instsize(l) (((l) + sizeof(Instruction) - 1)/sizeof(Instruction) + 1) + + +/* size (in elements) for a ISet instruction */ +#define CHARSETINSTSIZE instsize(CHARSETSIZE) + +/* size (in elements) for a IFunc instruction */ +#define funcinstsize(p) ((p)->i.aux + 2) + + +#define loopset(v,b) { int v; for (v = 0; v < CHARSETSIZE; v++) b; } + + +#define testchar(st,c) (((int)(st)[((c) >> 3)] & (1 << ((c) & 7)))) +#define setchar(st,c) ((st)[(c) >> 3] |= (1 << ((c) & 7))) + + +static int target (Instruction *p, int i); + + +static int sizei (const Instruction *i) { + switch((Opcode)i->i.code) { + case ISet: case ISpan: return CHARSETINSTSIZE; + case IFunc: return funcinstsize(i); + default: return 1; + } +} + + +static const char *val2str (lua_State *L, int idx) { + const char *k = lua_tostring(L, idx); + if (k != NULL) + return lua_pushfstring(L, "rule '%s'", k); + else + return lua_pushfstring(L, "rule ", luaL_typename(L, idx)); +} + + +static int getposition (lua_State *L, int t, int i) { + int res; + lua_getfenv(L, -1); + lua_rawgeti(L, -1, i); /* get key from pattern's environment */ + lua_gettable(L, t); /* get position from positions table */ + res = lua_tointeger(L, -1); + if (res == 0) { /* key has no registered position? */ + lua_rawgeti(L, -2, i); /* get key again */ + return luaL_error(L, "%s is not defined in given grammar", val2str(L, -1)); + } + lua_pop(L, 2); /* remove environment and position */ + return res; +} + + +/* +** {====================================================== +** Printing patterns (for debugging) +** ======================================================= +*/ + + +static void printcharset (const Charset st) { + int i; + printf("["); + for (i = 0; i <= UCHAR_MAX; i++) { + int first = i; + while (testchar(st, i) && i <= UCHAR_MAX) i++; + if (i - 1 == first) /* unary range? */ + printf("(%02x)", first); + else if (i - 1 > first) /* non-empty range? */ + printf("(%02x-%02x)", first, i - 1); + } + printf("]"); +} + + +static void printcapkind (int kind) { + const char *const modes[] = { + "close", "position", "constant", "backref", + "argument", "simple", "table", "function", + "query", "string", "substitution", "fold", + "runtime", "group"}; + printf("%s", modes[kind]); +} + + +static void printjmp (const Instruction *op, const Instruction *p) { + printf("-> "); + if (p->i.offset == 0) printf("FAIL"); + else printf("%d", (int)(dest(0, p) - op)); +} + + +static void printinst (const Instruction *op, const Instruction *p) { + const char *const names[] = { + "any", "char", "set", "span", "back", + "ret", "end", + "choice", "jmp", "call", "open_call", + "commit", "partial_commit", "back_commit", "failtwice", "fail", "giveup", + "func", + "fullcapture", "emptycapture", "emptycaptureidx", "opencapture", + "closecapture", "closeruntime" + }; + printf("%02ld: %s ", (long)(p - op), names[p->i.code]); + switch ((Opcode)p->i.code) { + case IChar: { + printf("'%c'", p->i.aux); + printjmp(op, p); + break; + } + case IAny: { + printf("* %d", p->i.aux); + printjmp(op, p); + break; + } + case IFullCapture: case IOpenCapture: + case IEmptyCapture: case IEmptyCaptureIdx: + case ICloseCapture: case ICloseRunTime: { + printcapkind(getkind(p)); + printf("(n = %d) (off = %d)", getoff(p), p->i.offset); + break; + } + case ISet: { + printcharset((p+1)->buff); + printjmp(op, p); + break; + } + case ISpan: { + printcharset((p+1)->buff); + break; + } + case IOpenCall: { + printf("-> %d", p->i.offset); + break; + } + case IChoice: { + printjmp(op, p); + printf(" (%d)", p->i.aux); + break; + } + case IJmp: case ICall: case ICommit: + case IPartialCommit: case IBackCommit: { + printjmp(op, p); + break; + } + default: break; + } + printf("\n"); +} + + +static void printpatt (Instruction *p) { + Instruction *op = p; + for (;;) { + printinst(op, p); + if ((Opcode)p->i.code == IEnd) break; + p += sizei(p); + } +} + + +#if 0 +static void printcap (Capture *cap) { + printcapkind(cap->kind); + printf(" (idx: %d - size: %d) -> %p\n", cap->idx, cap->siz, cap->s); +} + + +static void printcaplist (Capture *cap) { + for (; cap->s; cap++) printcap(cap); +} +#endif + +/* }====================================================== */ + + +/* +** {====================================================== +** Virtual Machine +** ======================================================= +*/ + + +typedef struct Stack { + const char *s; + const Instruction *p; + int caplevel; +} Stack; + + +#define getstackbase(L, ptop) ((Stack *)lua_touserdata(L, stackidx(ptop))) + + +static int runtimecap (lua_State *L, Capture *close, Capture *ocap, + const char *o, const char *s, int ptop); + + +static Capture *doublecap (lua_State *L, Capture *cap, int captop, int ptop) { + Capture *newc; + if (captop >= INT_MAX/((int)sizeof(Capture) * 2)) + luaL_error(L, "too many captures"); + newc = (Capture *)lua_newuserdata(L, captop * 2 * sizeof(Capture)); + memcpy(newc, cap, captop * sizeof(Capture)); + lua_replace(L, caplistidx(ptop)); + return newc; +} + + +static Stack *doublestack (lua_State *L, Stack **stacklimit, int ptop) { + Stack *stack = getstackbase(L, ptop); + Stack *newstack; + int n = *stacklimit - stack; + int max, newn; + lua_getfield(L, LUA_REGISTRYINDEX, MAXSTACKIDX); + max = lua_tointeger(L, -1); + lua_pop(L, 1); + if (n >= max) + luaL_error(L, "too many pending calls/choices"); + newn = 2*n; if (newn > max) newn = max; + newstack = (Stack *)lua_newuserdata(L, newn * sizeof(Stack)); + memcpy(newstack, stack, n * sizeof(Stack)); + lua_replace(L, stackidx(ptop)); + *stacklimit = newstack + newn; + return newstack + n; + +} + + +static void adddyncaptures (const char *s, Capture *base, int n, int fd) { + int i; + assert(base[0].kind == Cruntime && base[0].siz == 0); + base[0].idx = fd; /* first returned capture */ + for (i = 1; i < n; i++) { /* add extra captures */ + base[i].siz = 1; /* mark it as closed */ + base[i].s = s; + base[i].kind = Cruntime; + base[i].idx = fd + i; /* stack index */ + } + base[n].kind = Cclose; /* add closing entry */ + base[n].siz = 1; + base[n].s = s; +} + + +#define condfailed(p) { int f = p->i.offset; if (f) p+=f; else goto fail; } + +static const char *match (lua_State *L, + const char *o, const char *s, const char *e, + Instruction *op, Capture *capture, int ptop) { + Stack stackbase[INITBACK]; + Stack *stacklimit = stackbase + INITBACK; + Stack *stack = stackbase; /* point to first empty slot in stack */ + int capsize = INITCAPSIZE; + int captop = 0; /* point to first empty slot in captures */ + const Instruction *p = op; + stack->p = &giveup; stack->s = s; stack->caplevel = 0; stack++; + lua_pushlightuserdata(L, stackbase); + for (;;) { +#if defined(DEBUG) + printf("s: |%s| stck: %d c: %d ", + s, stack - getstackbase(L, ptop), captop); + printinst(op, p); +#endif + switch ((Opcode)p->i.code) { + case IEnd: { + assert(stack == getstackbase(L, ptop) + 1); + capture[captop].kind = Cclose; + capture[captop].s = NULL; + return s; + } + case IGiveup: { + assert(stack == getstackbase(L, ptop)); + return NULL; + } + case IRet: { + assert(stack > getstackbase(L, ptop) && (stack - 1)->s == NULL); + p = (--stack)->p; + continue; + } + case IAny: { + int n = p->i.aux; + if (n <= e - s) { p++; s += n; } + else condfailed(p); + continue; + } + case IChar: { + if ((byte)*s == p->i.aux && s < e) { p++; s++; } + else condfailed(p); + continue; + } + case ISet: { + int c = (byte)*s; + if (testchar((p+1)->buff, c) && s < e) + { p += CHARSETINSTSIZE; s++; } + else condfailed(p); + continue; + } + case IBack: { + int n = p->i.aux; + if (n > s - o) goto fail; + s -= n; p++; + continue; + } + case ISpan: { + for (; s < e; s++) { + int c = (byte)*s; + if (!testchar((p+1)->buff, c)) break; + } + p += CHARSETINSTSIZE; + continue; + } + case IFunc: { + const char *r = (p+1)->f(s, e, o, (p+2)->buff); + if (r != NULL) { s = r; p += funcinstsize(p); } + else condfailed(p); + continue; + } + case IJmp: { + p += p->i.offset; + continue; + } + case IChoice: { + if (stack == stacklimit) + stack = doublestack(L, &stacklimit, ptop); + stack->p = dest(0, p); + stack->s = s - p->i.aux; + stack->caplevel = captop; + stack++; + p++; + continue; + } + case ICall: { + if (stack == stacklimit) + stack = doublestack(L, &stacklimit, ptop); + stack->s = NULL; + stack->p = p + 1; /* save return address */ + stack++; + p += p->i.offset; + continue; + } + case ICommit: { + assert(stack > getstackbase(L, ptop) && (stack - 1)->s != NULL); + stack--; + p += p->i.offset; + continue; + } + case IPartialCommit: { + assert(stack > getstackbase(L, ptop) && (stack - 1)->s != NULL); + (stack - 1)->s = s; + (stack - 1)->caplevel = captop; + p += p->i.offset; + continue; + } + case IBackCommit: { + assert(stack > getstackbase(L, ptop) && (stack - 1)->s != NULL); + s = (--stack)->s; + captop = stack->caplevel; + p += p->i.offset; + continue; + } + case IFailTwice: + assert(stack > getstackbase(L, ptop)); + stack--; + /* go through */ + case IFail: + fail: { /* pattern failed: try to backtrack */ + do { /* remove pending calls */ + assert(stack > getstackbase(L, ptop)); + s = (--stack)->s; + } while (s == NULL); + captop = stack->caplevel; + p = stack->p; + continue; + } + case ICloseRunTime: { + int fr = lua_gettop(L) + 1; /* stack index of first result */ + int ncap = runtimecap(L, capture + captop, capture, o, s, ptop); + lua_Integer res = lua_tointeger(L, fr) - 1; /* offset */ + int n = lua_gettop(L) - fr; /* number of new captures */ + if (res == -1) { /* may not be a number */ + if (!lua_toboolean(L, fr)) { /* false value? */ + lua_settop(L, fr - 1); /* remove results */ + goto fail; /* and fail */ + } + else if (lua_isboolean(L, fr)) /* true? */ + res = s - o; /* keep current position */ + } + if (res < s - o || res > e - o) + luaL_error(L, "invalid position returned by match-time capture"); + s = o + res; /* update current position */ + captop -= ncap; /* remove nested captures */ + lua_remove(L, fr); /* remove first result (offset) */ + if (n > 0) { /* captures? */ + if ((captop += n + 1) >= capsize) { + capture = doublecap(L, capture, captop, ptop); + capsize = 2 * captop; + } + adddyncaptures(s, capture + captop - n - 1, n, fr); + } + p++; + continue; + } + case ICloseCapture: { + const char *s1 = s - getoff(p); + assert(captop > 0); + if (capture[captop - 1].siz == 0 && + s1 - capture[captop - 1].s < UCHAR_MAX) { + capture[captop - 1].siz = s1 - capture[captop - 1].s + 1; + p++; + continue; + } + else { + capture[captop].siz = 1; /* mark entry as closed */ + goto capture; + } + } + case IEmptyCapture: case IEmptyCaptureIdx: + capture[captop].siz = 1; /* mark entry as closed */ + goto capture; + case IOpenCapture: + capture[captop].siz = 0; /* mark entry as open */ + goto capture; + case IFullCapture: + capture[captop].siz = getoff(p) + 1; /* save capture size */ + capture: { + capture[captop].s = s - getoff(p); + capture[captop].idx = p->i.offset; + capture[captop].kind = getkind(p); + if (++captop >= capsize) { + capture = doublecap(L, capture, captop, ptop); + capsize = 2 * captop; + } + p++; + continue; + } + case IOpenCall: { + lua_rawgeti(L, penvidx(ptop), p->i.offset); + luaL_error(L, "reference to %s outside a grammar", val2str(L, -1)); + } + default: assert(0); return NULL; + } + } +} + +/* }====================================================== */ + + +/* +** {====================================================== +** Verifier +** ======================================================= +*/ + + +/* +** check whether pattern may go from 'p' to 'e' without consuming any +** input. Raises an error if it detects a left recursion. 'op' points +** the beginning of the pattern. If pattern belongs to a grammar, +** 'rule' is the stack index where is its corresponding key (only for +** error messages) and 'posttable' is the stack index with a table +** mapping rule keys to the position of their code in the pattern. +*/ +static int verify (lua_State *L, Instruction *op, const Instruction *p, + Instruction *e, int postable, int rule) { + static const char dummy[] = ""; + Stack back[MAXBACKVER]; + int backtop = 0; /* point to first empty slot in back */ + while (p != e) { + switch ((Opcode)p->i.code) { + case IRet: { + p = back[--backtop].p; + continue; + } + case IChoice: { + if (backtop >= MAXBACKVER) + return luaL_error(L, "too many pending calls/choices"); + back[backtop].p = dest(0, p); + back[backtop++].s = dummy; + p++; + continue; + } + case ICall: { + assert((p + 1)->i.code != IRet); /* no tail call */ + if (backtop >= MAXBACKVER) + return luaL_error(L, "too many pending calls/choices"); + back[backtop].s = NULL; + back[backtop++].p = p + 1; + goto dojmp; + } + case IOpenCall: { + int i; + if (postable == 0) /* grammar still not fixed? */ + goto fail; /* to be verified later */ + for (i = 0; i < backtop; i++) { + if (back[i].s == NULL && back[i].p == p + 1) + return luaL_error(L, "%s is left recursive", val2str(L, rule)); + } + if (backtop >= MAXBACKVER) + return luaL_error(L, "too many pending calls/choices"); + back[backtop].s = NULL; + back[backtop++].p = p + 1; + p = op + getposition(L, postable, p->i.offset); + continue; + } + case IBackCommit: + case ICommit: { + assert(backtop > 0 && p->i.offset > 0); + backtop--; + goto dojmp; + } + case IPartialCommit: { + assert(backtop > 0); + if (p->i.offset > 0) goto dojmp; /* forward jump */ + else { /* loop will be detected when checking corresponding rule */ + assert(postable != 0); + backtop--; + p++; /* just go on now */ + continue; + } + } + case IBack: { + if (p->i.aux == 1 && isfixcheck(p + 1)) { /* char test? */ + p++; /* skip back instruction */ + p += sizei(p); /* skip char test */ + } + else { /* standard lookbehind code */ + assert((Opcode)(p - 1)->i.code == IChoice); /* look behind */ + backtop--; + p += (p - 1)->i.offset; + assert((Opcode)(p - 1)->i.code == IFail); /* look behind */ + } + continue; + } + case IAny: + case IChar: + case ISet: { + const Instruction *next = p + sizei(p); + if ((Opcode)next->i.code == IBack) + p = next + 1; /* continue after the back instruction */ + else if (p->i.offset == 0) goto fail; + else /* jump */ + p += p->i.offset; + continue; + } + case IJmp: + dojmp: { + p += p->i.offset; + continue; + } + case IFailTwice: /* 'not' predicate */ + goto fail; /* body could have failed; try to backtrack it */ + case IFail: { + if (p > op && (p - 1)->i.code == IBackCommit) { /* 'and' predicate? */ + p++; /* pretend it succeeded and go ahead */ + continue; + } + /* else failed: go through */ + } + fail: { /* pattern failed: try to backtrack */ + do { + if (backtop-- == 0) + return 1; /* no more backtracking */ + } while (back[backtop].s == NULL); + p = back[backtop].p; + continue; + } + case ISpan: + case IOpenCapture: case ICloseCapture: + case IEmptyCapture: case IEmptyCaptureIdx: + case IFullCapture: { + p += sizei(p); + continue; + } + case ICloseRunTime: { + goto fail; /* be liberal in this case */ + } + case IFunc: { + const char *r = (p+1)->f(dummy, dummy, dummy, (p+2)->buff); + if (r != NULL) { p += funcinstsize(p); } + else condfailed(p); + continue; + } + case IEnd: /* cannot happen (should stop before it) */ + default: assert(0); return 0; + } + } + assert(backtop == 0); + return 0; +} + + +static void checkrule (lua_State *L, Instruction *op, int from, int to, + int postable, int rule) { + int i; + int lastopen = 0; /* more recent OpenCall seen in the code */ + for (i = from; i < to; i += sizei(op + i)) { + if (op[i].i.code == IPartialCommit && op[i].i.offset < 0) { /* loop? */ + int start = dest(op, i); + assert(op[start - 1].i.code == IChoice && + dest(op, start - 1) == target(op, i + 1)); + if (start <= lastopen) { /* loop does contain an open call? */ + if (!verify(L, op, op + start, op + i, postable, rule)) /* check body */ + luaL_error(L, "possible infinite loop in %s", val2str(L, rule)); + } + } + else if (op[i].i.code == IOpenCall) + lastopen = i; + } + assert(op[i - 1].i.code == IRet); + verify(L, op, op + from, op + to - 1, postable, rule); +} + + + + +/* }====================================================== */ + + +/* +** {====================================================== +** Building Patterns +** ======================================================= +*/ + +enum charsetanswer { NOINFO, ISCHARSET, VALIDSTARTS }; + +typedef struct CharsetTag { + enum charsetanswer tag; + Charset cs; +} CharsetTag; + + +static Instruction *getpatt (lua_State *L, int idx, int *size); + + +static void check2test (Instruction *p, int n) { + assert(ischeck(p) && n != 0); + p->i.offset = n; +} + + +/* +** invert array slice p[0]-p[e] (both inclusive) +*/ +static void invert (Instruction *p, int e) { + int i; + for (i = 0; i < e; i++, e--) { + Instruction temp = p[i]; + p[i] = p[e]; + p[e] = temp; + } +} + + +/* +** rotate array slice p[0]-p[e] (both inclusive) 'n' steps +** to the 'left' +*/ +static void rotate (Instruction *p, int e, int n) { + invert(p, n - 1); + invert(p + n, e - n); + invert(p, e); +} + + +#define op_step(p) ((p)->i.code == IAny ? (p)->i.aux : 1) + + +static int skipchecks (Instruction *p, int up, int *pn) { + int i, n = 0; + for (i = 0; isfixcheck(p + i); i += sizei(p + i)) { + int st = op_step(p + i); + if (n + st > MAXOFF - up) break; + n += st; + } + *pn = n; + return i; +} + + +#define ismovablecap(op) (ismovable(op) && getoff(op) < MAXOFF) + +static void optimizecaptures (Instruction *p) { + int i; + int limit = 0; + for (i = 0; p[i].i.code != IEnd; i += sizei(p + i)) { + if (isjmp(p + i) && dest(p, i) >= limit) + limit = dest(p, i) + 1; /* do not optimize jump targets */ + else if (i >= limit && ismovablecap(p + i) && isfixcheck(p + i + 1)) { + int end, n, j; /* found a border capture|check */ + int maxoff = getoff(p + i); + int start = i; + /* find first capture in the group */ + while (start > limit && ismovablecap(p + start - 1)) { + start--; + if (getoff(p + start) > maxoff) maxoff = getoff(p + start); + } + end = skipchecks(p + i + 1, maxoff, &n) + i; /* find last check */ + if (n == 0) continue; /* first check is too big to move across */ + assert(n <= MAXOFF && start <= i && i < end); + for (j = start; j <= i; j++) + p[j].i.aux += (n << 4); /* correct offset of captures to be moved */ + rotate(p + start, end - start, i - start + 1); /* move them up */ + i = end; + assert(isfixcheck(p + start) && iscapture(p + i)); + } + } +} + + +static int target (Instruction *p, int i) { + while (p[i].i.code == IJmp) i += p[i].i.offset; + return i; +} + + +static void optimizejumps (Instruction *p) { + int i; + for (i = 0; p[i].i.code != IEnd; i += sizei(p + i)) { + if (isjmp(p + i)) + p[i].i.offset = target(p, dest(p, i)) - i; + } +} + + +static void optimizechoice (Instruction *p) { + assert(p->i.code == IChoice); + if (isfixcheck(p + 1)) { + int lc = sizei(p + 1); + rotate(p, lc, 1); + assert(isfixcheck(p) && (p + lc)->i.code == IChoice); + (p + lc)->i.aux = op_step(p); + check2test(p, (p + lc)->i.offset); + (p + lc)->i.offset -= lc; + } +} + + +/* +** A 'headfail' pattern is a pattern that can only fails in its first +** instruction, which must be a check. +*/ +static int isheadfail (Instruction *p) { + if (!ischeck(p)) return 0; + /* check that other operations cannot fail */ + for (p += sizei(p); p->i.code != IEnd; p += sizei(p)) + if (!isnofail(p)) return 0; + return 1; +} + + +#define checkpattern(L, idx) ((Instruction *)luaL_checkudata(L, idx, PATTERN_T)) + + +/* +** Return the number of elements in the ktable of a pattern. +** in Lua 5.2, default "environment" for patterns is nil, not +** a table. Treat it as an empty table. +*/ +static int ktablelen (lua_State *L, int idx) { + if (!lua_istable(L, idx)) return 0; + else return lua_objlen(L, idx); +} + + +/* +** join the elements of the ktable from pattern 'p1' into the ktable of +** the pattern at the top of the stack ('p'). If 'p1' has no elements, +** 'p' keeps its original ktable. If 'p' has no elements, it shares +** 'p1' ktable. Otherwise, this function creates a new ktable for 'p'. +** Return the offset of original 'p' elements in the new ktable. +*/ +static int jointable (lua_State *L, int p1) { + int n, n1, i; + lua_getfenv(L, p1); + n1 = ktablelen(L, -1); /* number of elements in p1's env */ + lua_getfenv(L, -2); + if (n1 == 0 || lua_equal(L, -2, -1)) { + lua_pop(L, 2); + return 0; /* no need to change anything */ + } + n = ktablelen(L, -1); /* number of elements in p's env */ + if (n == 0) { + lua_pop(L, 1); /* removes p env */ + lua_setfenv(L, -2); /* p now shares p1's env */ + return 0; /* no need to correct anything */ + } + lua_createtable(L, n + n1, 0); + /* stack: p; p1 env; p env; new p env */ + for (i = 1; i <= n; i++) { + lua_rawgeti(L, -2, i); + lua_rawseti(L, -2, i); + } + for (i = 1; i <= n1; i++) { + lua_rawgeti(L, -3, i); + lua_rawseti(L, -2, n + i); + } + lua_setfenv(L, -4); /* new table becomes p env */ + lua_pop(L, 2); /* remove p1 env and old p env */ + return n; +} + + +#define copypatt(p1,p2,sz) memcpy(p1, p2, (sz) * sizeof(Instruction)); + +#define pattsize(L,idx) (lua_objlen(L, idx)/sizeof(Instruction) - 1) + + +static int addpatt (lua_State *L, Instruction *p, int p1idx) { + Instruction *p1 = (Instruction *)lua_touserdata(L, p1idx); + int sz = pattsize(L, p1idx); + int corr = jointable(L, p1idx); + copypatt(p, p1, sz + 1); + if (corr != 0) { + Instruction *px; + for (px = p; px < p + sz; px += sizei(px)) { + if (isfenvoff(px) && px->i.offset != 0) + px->i.offset += corr; + } + } + return sz; +} + + +static void setinstaux (Instruction *i, Opcode op, int offset, int aux) { + assert(aux <= MAXAUX); + i->i.code = op; + i->i.offset = offset; + i->i.aux = aux; +} + +#define setinst(i,op,off) setinstaux(i,op,off,0) + +#define setinstcap(i,op,idx,k,n) setinstaux(i,op,idx,((k) | ((n) << 4))) + + +/* +** create a new ktable for pattern at the stack top, mapping +** '1' to the value at stack position 'vidx'. +*/ +static int value2fenv (lua_State *L, int vidx) { + lua_createtable(L, 1, 0); + lua_pushvalue(L, vidx); + lua_rawseti(L, -2, 1); + lua_setfenv(L, -2); + return 1; +} + + +static Instruction *newpatt (lua_State *L, size_t n) { + Instruction *p; + if (n >= MAXPATTSIZE - 1) + luaL_error(L, "pattern too big"); + p = (Instruction *)lua_newuserdata(L, (n + 1) * sizeof(Instruction)); + luaL_getmetatable(L, PATTERN_T); + lua_setmetatable(L, -2); + setinst(p + n, IEnd, 0); + return p; +} + + +static void fillcharset (Instruction *p, Charset cs) { + switch (p[0].i.code) { + case ISet: { + loopset(i, cs[i] = p[1].buff[i]); + break; + } + case IChar: { + loopset(i, cs[i] = 0); + setchar(cs, p[0].i.aux); + break; + } + default: { /* any char may start unhandled instructions */ + loopset(i, cs[i] = 0xff); + break; + } + } +} + + +/* +** Function 'tocharset' gets information about which chars may be a +** valid start for a pattern. +*/ + +static enum charsetanswer tocharset (Instruction *p, CharsetTag *c) { + if (isfixcheck(p)) { + fillcharset(p, c->cs); + if ((p + sizei(p))->i.code == IEnd && op_step(p) == 1) + c->tag = ISCHARSET; + else + c->tag = VALIDSTARTS; + } + else + c->tag = NOINFO; + return c->tag; +} + + +static int exclusiveset (Charset c1, Charset c2) { + /* non-empty intersection? */ + loopset(i, {if ((c1[i] & c2[i]) != 0) return 0;}); + return 1; /* no intersection */ +} + + +static int exclusive (CharsetTag *c1, CharsetTag *c2) { + if (c1->tag == NOINFO || c2->tag == NOINFO) + return 0; /* one of them is not filled */ + else return exclusiveset(c1->cs, c2->cs); +} + + +static Instruction *newcharset (lua_State *L) { + Instruction *p = newpatt(L, CHARSETINSTSIZE); + p[0].i.code = ISet; + p[0].i.offset = 0; + loopset(i, p[1].buff[i] = 0); + return p; +} + + +static int set_l (lua_State *L) { + size_t l; + const char *s = luaL_checklstring(L, 1, &l); + if (l == 1) + getpatt(L, 1, NULL); /* a unit set is equivalent to a literal */ + else { + Instruction *p = newcharset(L); + while (l--) { + setchar(p[1].buff, (byte)(*s)); + s++; + } + } + return 1; +} + + +static int range_l (lua_State *L) { + int arg; + int top = lua_gettop(L); + Instruction *p = newcharset(L); + for (arg = 1; arg <= top; arg++) { + int c; + size_t l; + const char *r = luaL_checklstring(L, arg, &l); + luaL_argcheck(L, l == 2, arg, "range must have two characters"); + for (c = (byte)r[0]; c <= (byte)r[1]; c++) + setchar(p[1].buff, c); + } + return 1; +} + + +static int nter_l (lua_State *L) { + Instruction *p; + luaL_argcheck(L, !lua_isnoneornil(L, 1), 1, "non-nil value expected"); + p = newpatt(L, 1); + setinst(p, IOpenCall, value2fenv(L, 1)); + return 1; +} + + + +static int testpattern (lua_State *L, int idx) { + if (lua_touserdata(L, idx)) { /* value is a userdata? */ + if (lua_getmetatable(L, idx)) { /* does it have a metatable? */ + luaL_getmetatable(L, PATTERN_T); + if (lua_rawequal(L, -1, -2)) { /* does it have the correct mt? */ + lua_pop(L, 2); /* remove both metatables */ + return 1; + } + } + } + return 0; +} + + +static Instruction *fix_l (lua_State *L, int t) { + Instruction *p; + int i; + int totalsize = 2; /* include initial call and jump */ + int n = 0; /* number of rules */ + int base = lua_gettop(L); + lua_newtable(L); /* to store relative positions of each rule */ + lua_pushinteger(L, 1); /* default initial rule */ + /* collect patterns and compute sizes */ + lua_pushnil(L); + while (lua_next(L, t) != 0) { + int l; + if (lua_tonumber(L, -2) == 1 && lua_isstring(L, -1)) { + lua_replace(L, base + 2); /* use this value as initial rule */ + continue; + } + if (!testpattern(L, -1)) + luaL_error(L, "%s is not a pattern", val2str(L, -2)); + l = pattsize(L, -1) + 1; /* space for pattern + ret */ + if (totalsize >= MAXPATTSIZE - l) + luaL_error(L, "grammar too large"); + luaL_checkstack(L, LUA_MINSTACK, "grammar has too many rules"); + lua_insert(L, -2); /* put key on top */ + lua_pushvalue(L, -1); /* duplicate key (for lua_next) */ + lua_pushvalue(L, -1); /* duplicate key (to index positions table)) */ + lua_pushinteger(L, totalsize); /* position for this rule */ + lua_settable(L, base + 1); /* store key=>position in positions table */ + totalsize += l; + n++; + } + luaL_argcheck(L, n > 0, t, "empty grammar"); + p = newpatt(L, totalsize); /* create new pattern */ + p++; /* save space for call */ + setinst(p++, IJmp, totalsize - 1); /* after call, jumps to the end */ + for (i = 1; i <= n; i++) { /* copy all rules into new pattern */ + p += addpatt(L, p, base + 1 + i*2); + setinst(p++, IRet, 0); + } + p -= totalsize; /* back to first position */ + totalsize = 2; /* go through each rule's position */ + for (i = 1; i <= n; i++) { /* check all rules */ + int l = pattsize(L, base + 1 + i*2) + 1; + checkrule(L, p, totalsize, totalsize + l, base + 1, base + 2 + i*2); + totalsize += l; + } + lua_pushvalue(L, base + 2); /* get initial rule */ + lua_gettable(L, base + 1); /* get its position in postions table */ + i = lua_tonumber(L, -1); /* convert to number */ + lua_pop(L, 1); + if (i == 0) /* is it defined? */ + luaL_error(L, "initial rule not defined in given grammar"); + setinst(p, ICall, i); /* first instruction calls initial rule */ + /* correct calls */ + for (i = 0; i < totalsize; i += sizei(p + i)) { + if (p[i].i.code == IOpenCall) { + int pos = getposition(L, base + 1, p[i].i.offset); + p[i].i.code = (p[target(p, i + 1)].i.code == IRet) ? IJmp : ICall; + p[i].i.offset = pos - i; + } + } + optimizejumps(p); + lua_replace(L, t); /* put new pattern in old's position */ + lua_settop(L, base); /* remove rules and positions table */ + return p; +} + + +static Instruction *any (lua_State *L, int n, int extra, int *offsetp) { + int offset = offsetp ? *offsetp : 0; + Instruction *p = newpatt(L, (n - 1)/UCHAR_MAX + extra + 1); + Instruction *p1 = p + offset; + for (; n > UCHAR_MAX; n -= UCHAR_MAX) + setinstaux(p1++, IAny, 0, UCHAR_MAX); + setinstaux(p1++, IAny, 0, n); + if (offsetp) *offsetp = p1 - p; + return p; +} + + +static Instruction *getpatt (lua_State *L, int idx, int *size) { + Instruction *p; + switch (lua_type(L, idx)) { + case LUA_TSTRING: { + size_t i, len; + const char *s = lua_tolstring(L, idx, &len); + p = newpatt(L, len); + for (i = 0; i < len; i++) + setinstaux(p + i, IChar, 0, (byte)s[i]); + lua_replace(L, idx); + break; + } + case LUA_TNUMBER: { + int n = lua_tointeger(L, idx); + if (n == 0) /* empty pattern? */ + p = newpatt(L, 0); + else if (n > 0) + p = any(L, n, 0, NULL); + else if (-n <= UCHAR_MAX) { + p = newpatt(L, 2); + setinstaux(p, IAny, 2, -n); + setinst(p + 1, IFail, 0); + } + else { + int offset = 2; /* space for ITestAny & IChoice */ + p = any(L, -n - UCHAR_MAX, 3, &offset); + setinstaux(p, IAny, offset + 1, UCHAR_MAX); + setinstaux(p + 1, IChoice, offset, UCHAR_MAX); + setinst(p + offset, IFailTwice, 0); + } + lua_replace(L, idx); + break; + } + case LUA_TBOOLEAN: { + if (lua_toboolean(L, idx)) /* true? */ + p = newpatt(L, 0); /* empty pattern (always succeeds) */ + else { + p = newpatt(L, 1); + setinst(p, IFail, 0); + } + lua_replace(L, idx); + break; + } + case LUA_TTABLE: { + p = fix_l(L, idx); + break; + } + case LUA_TFUNCTION: { + p = newpatt(L, 2); + setinstcap(p, IOpenCapture, value2fenv(L, idx), Cruntime, 0); + setinstcap(p + 1, ICloseRunTime, 0, Cclose, 0); + lua_replace(L, idx); + break; + } + default: { + p = checkpattern(L, idx); + break; + } + } + if (size) *size = pattsize(L, idx); + return p; +} + + +static int getpattl (lua_State *L, int idx) { + int size; + getpatt(L, idx, &size); + return size; +} + + +static int pattern_l (lua_State *L) { + lua_settop(L, 1); + getpatt(L, 1, NULL); + return 1; +} + + +#define isany(p) ((p)->i.code == IAny && ((p) + 1)->i.code == IEnd) +#define isfail(p) ((p)->i.code == IFail) +#define issucc(p) ((p)->i.code == IEnd) + +static int concat_l (lua_State *L) { + /* p1; p2; */ + int l1, l2; + Instruction *p1 = getpatt(L, 1, &l1); + Instruction *p2 = getpatt(L, 2, &l2); + if (isfail(p1) || issucc(p2)) + lua_pushvalue(L, 1); /* fail * x == fail; x * true == x */ + else if (isfail(p2) || issucc(p1)) + lua_pushvalue(L, 2); /* x * fail == fail; true * x == x */ + else if (isany(p1) && isany(p2)) + any(L, p1->i.aux + p2->i.aux, 0, NULL); + else { + Instruction *op = newpatt(L, l1 + l2); + Instruction *p = op + addpatt(L, op, 1); + addpatt(L, p, 2); + optimizecaptures(op); + } + return 1; +} + + +static int diff_l (lua_State *L) { + int l1, l2; + Instruction *p1 = getpatt(L, 1, &l1); + Instruction *p2 = getpatt(L, 2, &l2); + CharsetTag st1, st2; + if (tocharset(p1, &st1) == ISCHARSET && tocharset(p2, &st2) == ISCHARSET) { + Instruction *p = newcharset(L); + loopset(i, p[1].buff[i] = st1.cs[i] & ~st2.cs[i]); + } + else if (isheadfail(p2)) { + Instruction *p = newpatt(L, l2 + 1 + l1); + p += addpatt(L, p, 2); + check2test(p - l2, l2 + 1); + setinst(p++, IFail, 0); + addpatt(L, p, 1); + } + else { /* !e2 . e1 */ + /* !e -> choice L1; e; failtwice; L1: ... */ + Instruction *p = newpatt(L, 1 + l2 + 1 + l1); + Instruction *pi = p; + setinst(p++, IChoice, 1 + l2 + 1); + p += addpatt(L, p, 2); + setinst(p++, IFailTwice, 0); + addpatt(L, p, 1); + optimizechoice(pi); + } + return 1; +} + + +static int unm_l (lua_State *L) { + Instruction *p = getpatt(L, 1, NULL); + if (isfail(p)) { /* -false? */ + newpatt(L, 0); /* true */ + return 1; + } + else if (issucc(p)) { /* -true? */ + Instruction *p1 = newpatt(L, 1); /* false */ + setinst(p1, IFail, 0); + return 1; + } + else { /* -A == '' - A */ + lua_pushliteral(L, ""); + lua_insert(L, 1); + return diff_l(L); + } +} + + +static int pattand_l (lua_State *L) { + int l1; + CharsetTag st1; + Instruction *p1 = getpatt(L, 1, &l1); + if (isfail(p1) || issucc(p1)) + lua_pushvalue(L, 1); /* &fail == fail; &true == true */ + else if (tocharset(p1, &st1) == ISCHARSET) { + Instruction *p = newpatt(L, l1 + 1); + copypatt(p, p1, l1); p += l1; + setinstaux(p, IBack, 0, 1); + } + else { /* Choice L1; p1; BackCommit L2; L1: Fail; L2: */ + Instruction *p = newpatt(L, 1 + l1 + 2); + setinst(p++, IChoice, 1 + l1 + 1); + p += addpatt(L, p, 1); + setinst(p++, IBackCommit, 2); + setinst(p, IFail, 0); + } + return 1; +} + + +static int nocalls (const Instruction *p) { + for (; (Opcode)p->i.code != IEnd; p += sizei(p)) + if ((Opcode)p->i.code == IOpenCall) return 0; + return 1; +} + + +static int pattbehind (lua_State *L) { + int l1; + CharsetTag st1; + Instruction *p1 = getpatt(L, 1, &l1); + int n = luaL_optint(L, 2, 1); + luaL_argcheck(L, n <= MAXAUX, 2, "lookbehind delta too large"); + if (!nocalls(p1)) + luaL_error(L, "lookbehind pattern cannot contain non terminals"); + if (isfail(p1) || issucc(p1)) + lua_pushvalue(L, 1); /* tag == NOINFO) return 1; + assert(p1->i.offset != 0); + switch (p1->i.code) { + case IChar: return testchar(st2->cs, p1->i.aux); + case ISet: return !exclusiveset(st2->cs, (p1 + 1)->buff); + default: assert(p1->i.code == IAny); return 1; + } +} + + +static Instruction *basicUnion (lua_State *L, Instruction *p1, int l1, + int l2, int *size, CharsetTag *st2) { + Instruction *op; + CharsetTag st1; + tocharset(p1, &st1); + if (st1.tag == ISCHARSET && st2->tag == ISCHARSET) { + Instruction *p = auxnew(L, &op, size, CHARSETINSTSIZE); + setinst(p, ISet, 0); + loopset(i, p[1].buff[i] = st1.cs[i] | st2->cs[i]); + } + else if (exclusive(&st1, st2) || isheadfail(p1)) { + Instruction *p = auxnew(L, &op, size, l1 + 1 + l2); + copypatt(p, p1, l1); + check2test(p, l1 + 1); + p += l1; + setinst(p++, IJmp, l2 + 1); + addpatt(L, p, 2); + } + else { + /* choice L1; e1; commit L2; L1: e2; L2: ... */ + Instruction *p = auxnew(L, &op, size, 1 + l1 + 1 + l2); + setinst(p++, IChoice, 1 + l1 + 1); + copypatt(p, p1, l1); p += l1; + setinst(p++, ICommit, 1 + l2); + addpatt(L, p, 2); + optimizechoice(p - (1 + l1 + 1)); + } + return op; +} + + +static Instruction *separateparts (lua_State *L, Instruction *p1, int l1, + int l2, int *size, CharsetTag *st2) { + int sp = firstpart(p1, l1); + if (sp == 0) /* first part is entire p1? */ + return basicUnion(L, p1, l1, l2, size, st2); + else if ((p1 + sp - 1)->i.code == ICommit || !interfere(p1, sp, st2)) { + Instruction *p; + int init = *size; + int end = init + sp; + *size = end; + p = separateparts(L, p1 + sp, l1 - sp, l2, size, st2); + copypatt(p + init, p1, sp); + (p + end - 1)->i.offset = *size - (end - 1); + return p; + } + else { /* must change back to non-optimized choice */ + Instruction *p; + int init = *size; + int end = init + sp + 1; /* needs one extra instruction (choice) */ + int sizefirst = sizei(p1); /* size of p1's first instruction (the test) */ + *size = end; + p = separateparts(L, p1 + sp, l1 - sp, l2, size, st2); + copypatt(p + init, p1, sizefirst); /* copy the test */ + (p + init)->i.offset++; /* correct jump (because of new instruction) */ + init += sizefirst; + setinstaux(p + init, IChoice, sp - sizefirst + 1, 1); init++; + copypatt(p + init, p1 + sizefirst, sp - sizefirst - 1); + init += sp - sizefirst - 1; + setinst(p + init, ICommit, *size - (end - 1)); + return p; + } +} + + +static int union_l (lua_State *L) { + int l1, l2; + int size = 0; + Instruction *p1 = getpatt(L, 1, &l1); + Instruction *p2 = getpatt(L, 2, &l2); + CharsetTag st2; + if (isfail(p1)) /* check for simple identities */ + lua_pushvalue(L, 2); /* fail / a == a */ + else if (isfail(p2) || issucc(p1)) + lua_pushvalue(L, 1); /* a / fail == a; true / a == true */ + else { + tocharset(p2, &st2); + separateparts(L, p1, l1, l2, &size, &st2); + } + return 1; +} + + +static int repeatcharset (lua_State *L, Charset cs, int l1, int n) { + /* e; ...; e; span; */ + int i; + Instruction *p = newpatt(L, n*l1 + CHARSETINSTSIZE); + for (i = 0; i < n; i++) { + p += addpatt(L, p, 1); + } + setinst(p, ISpan, 0); + loopset(k, p[1].buff[k] = cs[k]); + return 1; +} + + +static Instruction *repeatheadfail (lua_State *L, int l1, int n) { + /* e; ...; e; L2: e'(L1); jump L2; L1: ... */ + int i; + Instruction *p = newpatt(L, (n + 1)*l1 + 1); + Instruction *op = p; + for (i = 0; i < n; i++) { + p += addpatt(L, p, 1); + } + p += addpatt(L, p, 1); + check2test(p - l1, l1 + 1); + setinst(p, IJmp, -l1); + return op; +} + + +static Instruction *repeats (lua_State *L, Instruction *p1, int l1, int n) { + /* e; ...; e; choice L1; L2: e; partialcommit L2; L1: ... */ + int i; + Instruction *op = newpatt(L, (n + 1)*l1 + 2); + Instruction *p = op; + if (!verify(L, p1, p1, p1 + l1, 0, 0)) + luaL_error(L, "loop body may accept empty string"); + for (i = 0; i < n; i++) { + p += addpatt(L, p, 1); + } + setinst(p++, IChoice, 1 + l1 + 1); + p += addpatt(L, p, 1); + setinst(p, IPartialCommit, -l1); + return op; +} + + +static void optionalheadfail (lua_State *L, int l1, int n) { + Instruction *op = newpatt(L, n * l1); + Instruction *p = op; + int i; + for (i = 0; i < n; i++) { + p += addpatt(L, p, 1); + check2test(p - l1, (n - i)*l1); + } +} + + +static void optionals (lua_State *L, int l1, int n) { + /* choice L1; e; partialcommit L2; L2: ... e; L1: commit L3; L3: ... */ + int i; + Instruction *op = newpatt(L, n*(l1 + 1) + 1); + Instruction *p = op; + setinst(p++, IChoice, 1 + n*(l1 + 1)); + for (i = 0; i < n; i++) { + p += addpatt(L, p, 1); + setinst(p++, IPartialCommit, 1); + } + setinst(p - 1, ICommit, 1); /* correct last commit */ + optimizechoice(op); +} + + +static int star_l (lua_State *L) { + int l1; + int n = luaL_checkint(L, 2); + Instruction *p1 = getpatt(L, 1, &l1); + if (n >= 0) { + CharsetTag st; + Instruction *op; + if (tocharset(p1, &st) == ISCHARSET) + return repeatcharset(L, st.cs, l1, n); + if (isheadfail(p1)) + op = repeatheadfail(L, l1, n); + else + op = repeats(L, p1, l1, n); + optimizecaptures(op); + optimizejumps(op); + } + else { + if (isheadfail(p1)) + optionalheadfail(L, l1, -n); + else + optionals(L, l1, -n); + } + return 1; +} + + +static int getlabel (lua_State *L, int labelidx) { + if (labelidx == 0) return 0; + else return value2fenv(L, labelidx); +} + + +static int capture_aux (lua_State *L, int kind, int labelidx) { + int l1, n; + Instruction *p1 = getpatt(L, 1, &l1); + int lc = skipchecks(p1, 0, &n); + if (lc == l1) { /* got whole pattern? */ + /* may use a IFullCapture instruction at its end */ + Instruction *p = newpatt(L, l1 + 1); + int label = getlabel(L, labelidx); + p += addpatt(L, p, 1); + setinstcap(p, IFullCapture, label, kind, n); + } + else { /* must use open-close pair */ + Instruction *op = newpatt(L, 1 + l1 + 1); + Instruction *p = op; + setinstcap(p++, IOpenCapture, getlabel(L, labelidx), kind, 0); + p += addpatt(L, p, 1); + setinstcap(p, ICloseCapture, 0, Cclose, 0); + optimizecaptures(op); + } + return 1; +} + + +static int capture_l (lua_State *L) { return capture_aux(L, Csimple, 0); } +static int tcapture_l (lua_State *L) { return capture_aux(L, Ctable, 0); } +static int capsubst_l (lua_State *L) { return capture_aux(L, Csubst, 0); } + +static int rcapture_l (lua_State *L) { + switch (lua_type(L, 2)) { + case LUA_TFUNCTION: return capture_aux(L, Cfunction, 2); + case LUA_TTABLE: return capture_aux(L, Cquery, 2); + case LUA_TSTRING: return capture_aux(L, Cstring, 2); + default: return luaL_argerror(L, 2, "invalid replacement value"); + } +} + + +static int fold_l (lua_State *L) { + luaL_checktype(L, 2, LUA_TFUNCTION); + return capture_aux(L, Cfold, 2); +} + + +static int group_l (lua_State *L) { + if (lua_isnoneornil(L, 2)) + return capture_aux(L, Cgroup, 0); + else { + luaL_checkstring(L, 2); + return capture_aux(L, Cgroup, 2); + } +} + + +static int position_l (lua_State *L) { + Instruction *p = newpatt(L, 1); + setinstcap(p, IEmptyCapture, 0, Cposition, 0); + return 1; +} + + +static int backref_l (lua_State *L) { + Instruction *p = newpatt(L, 1); + int n = getlabel(L, 1); + setinstcap(p, IEmptyCaptureIdx, n, Cbackref, 0); + return 1; +} + + +static int argcap_l (lua_State *L) { + int n = luaL_checkint(L, 1); + Instruction *p = newpatt(L, 1); + luaL_argcheck(L, 0 < n && n <= SHRT_MAX, 1, "invalid argument index"); + setinstcap(p, IEmptyCapture, n, Carg, 0); + return 1; +} + + +static int matchtime_l (lua_State *L) { + int l1 = getpattl(L, 1); + Instruction *op = newpatt(L, 1 + l1 + 1); + Instruction *p = op; + luaL_checktype(L, 2, LUA_TFUNCTION); + setinstcap(p++, IOpenCapture, value2fenv(L, 2), Cruntime, 0); + p += addpatt(L, p, 1); + setinstcap(p, ICloseRunTime, 0, Cclose, 0); + optimizecaptures(op); + return 1; +} + + +static int capconst_l (lua_State *L) { + int i, j; + int n = lua_gettop(L); + Instruction *p = newpatt(L, n > 1 ? n + 2 : n); + lua_createtable(L, n, 0); /* new environment for the new pattern */ + if (n > 1) setinstcap(p++, IOpenCapture, 0, Cgroup, 0); + for (i = j = 1; i <= n; i++) { + if (lua_isnil(L, i)) + setinstcap(p++, IEmptyCaptureIdx, 0, Cconst, 0); + else { + setinstcap(p++, IEmptyCaptureIdx, j, Cconst, 0); + lua_pushvalue(L, i); + lua_rawseti(L, -2, j++); + } + } + if (n > 1) setinstcap(p++, ICloseCapture, 0, Cclose, 0); + lua_setfenv(L, -2); /* set environment */ + return 1; +} + + +/* }====================================================== */ + + +/* +** {====================================================== +** User-Defined Patterns +** ======================================================= +*/ + +static void l_newpf (lua_State *L, PattFunc f, const void *ud, size_t l) { + int n = instsize(l) + 1; + Instruction *p = newpatt(L, n); + if (n > MAXAUX) luaL_error(L, "pattern data too long"); + p[0].i.code = IFunc; + p[0].i.aux = n - 2; + p[0].i.offset = 0; + p[1].f = f; + memcpy(p[2].buff, ud, l); +} + +/* }====================================================== */ + + +/* +** {====================================================== +** Captures +** ======================================================= +*/ + + +typedef struct CapState { + Capture *cap; /* current capture */ + Capture *ocap; /* (original) capture list */ + lua_State *L; + int ptop; /* index of last argument to 'match' */ + const char *s; /* original string */ + int valuecached; /* value stored in cache slot */ +} CapState; + + +#define captype(cap) ((cap)->kind) + +#define isclosecap(cap) (captype(cap) == Cclose) + +#define closeaddr(c) ((c)->s + (c)->siz - 1) + +#define isfullcap(cap) ((cap)->siz != 0) + +#define getfromenv(cs,v) lua_rawgeti((cs)->L, penvidx((cs)->ptop), v) +#define pushluaval(cs) getfromenv(cs, (cs)->cap->idx) + +#define pushsubject(cs, c) lua_pushlstring((cs)->L, (c)->s, (c)->siz - 1) + + +#define updatecache(cs,v) { if ((v) != (cs)->valuecached) updatecache_(cs,v); } + + +static void updatecache_ (CapState *cs, int v) { + getfromenv(cs, v); + lua_replace(cs->L, subscache(cs)); + cs->valuecached = v; +} + + +static int pushcapture (CapState *cs); + + +static Capture *findopen (Capture *cap) { + int n = 0; + for (;;) { + cap--; + if (isclosecap(cap)) n++; + else if (!isfullcap(cap)) + if (n-- == 0) return cap; + } +} + + +static Capture *nextcap (Capture *cap) { + if (isfullcap(cap)) return cap + 1; + else { + int n = 0; + for (;;) { + cap++; + if (isclosecap(cap)) { + if (n-- == 0) return cap + 1; + } + else if (!isfullcap(cap)) n++; + } + } +} + + +static int pushallvalues (CapState *cs, int addextra) { + Capture *co = cs->cap; + int n = 0; + if (isfullcap(cs->cap++)) { + pushsubject(cs, co); /* push whole match */ + return 1; + } + while (!isclosecap(cs->cap)) + n += pushcapture(cs); + if (addextra || n == 0) { /* need extra? */ + lua_pushlstring(cs->L, co->s, cs->cap->s - co->s); /* push whole match */ + n++; + } + cs->cap++; /* skip close entry */ + return n; +} + + +static Capture *findback (CapState *cs, Capture *cap) { + lua_State *L = cs->L; + for (;;) { + if (cap == cs->ocap) { /* not found */ + const char *s = lua_tostring(L, -1); + if (s == NULL) s = lua_pushfstring(L, "(a %s)", luaL_typename(L, -1)); + luaL_error(L, "back reference '%s' not found", s); + } + cap--; + if (isclosecap(cap)) + cap = findopen(cap); + else if (!isfullcap(cap)) + continue; /* opening an enclosing capture: skip and get previous */ + if (captype(cap) == Cgroup) { + getfromenv(cs, cap->idx); /* get group name */ + if (lua_equal(L, -2, -1)) { /* right group? */ + lua_pop(L, 2); /* remove reference name and group name */ + return cap; + } + else lua_pop(L, 1); /* remove group name */ + } + } +} + + +static int backrefcap (CapState *cs) { + int n; + Capture *curr = cs->cap; + pushluaval(cs); /* reference name */ + cs->cap = findback(cs, curr); + n = pushallvalues(cs, 0); + cs->cap = curr + 1; + return n; +} + + +static int tablecap (CapState *cs) { + lua_State *L = cs->L; + int n = 0; + lua_newtable(L); + if (isfullcap(cs->cap++)) + return 1; /* table is empty */ + while (!isclosecap(cs->cap)) { + if (captype(cs->cap) == Cgroup && cs->cap->idx != 0) { /* named group? */ + int k; + pushluaval(cs); /* push group name */ + k = pushallvalues(cs, 0); + if (k == 0) { /* no value? */ + lua_pop(L, 1); /* remove group name */ + continue; /* and go on */ + } + else if (k > 1) + lua_pop(L, k - 1); /* keep just one value */ + lua_settable(L, -3); + } + else { + int i; + int k = pushcapture(cs); + for (i = k; i > 0; i--) + lua_rawseti(L, -(i + 1), n + i); + n += k; + } + } + cs->cap++; /* skip close entry */ + return 1; +} + + +static int querycap (CapState *cs) { + int idx = cs->cap->idx; + int n = pushallvalues(cs, 0); + if (n > 1) /* extra captures? */ + lua_pop(cs->L, n - 1); /* throw them away */ + updatecache(cs, idx); + lua_gettable(cs->L, subscache(cs)); + if (!lua_isnil(cs->L, -1)) + return 1; + else { + lua_pop(cs->L, 1); /* remove value */ + return 0; + } +} + + +static int foldcap (CapState *cs) { + int n; + lua_State *L = cs->L; + int idx = cs->cap->idx; + if (isfullcap(cs->cap++) || isclosecap(cs->cap) || (n = pushcapture(cs)) == 0) + return luaL_error(L, "no initial value for fold capture"); + if (n > 1) + lua_pop(L, n - 1); /* leave only one result */ + while (!isclosecap(cs->cap)) { + updatecache(cs, idx); + lua_pushvalue(L, subscache(cs)); /* get folding function */ + lua_insert(L, -2); /* put it before accumulator */ + n = pushcapture(cs); /* get other captures */ + lua_call(L, n + 1, 1); /* call folding function */ + } + cs->cap++; /* skip close entry */ + return 1; +} + + +static int functioncap (CapState *cs) { + int n; + int top = lua_gettop(cs->L); + pushluaval(cs); + n = pushallvalues(cs, 0); + lua_call(cs->L, n, LUA_MULTRET); + return lua_gettop(cs->L) - top; +} + + +static int runtimecap (lua_State *L, Capture *close, Capture *ocap, + const char *o, const char *s, int ptop) { + CapState cs; + int n; + Capture *open = findopen(close); + assert(captype(open) == Cruntime); + close->kind = Cclose; + close->s = s; + cs.ocap = ocap; cs.cap = open; cs.L = L; + cs.s = o; cs.valuecached = 0; cs.ptop = ptop; + luaL_checkstack(L, 4, "too many runtime captures"); + pushluaval(&cs); + lua_pushvalue(L, SUBJIDX); /* push original subject */ + lua_pushinteger(L, s - o + 1); /* current position */ + n = pushallvalues(&cs, 0); + lua_call(L, n + 2, LUA_MULTRET); + return close - open; +} + + + +typedef struct StrAux { + int isstring; + union { + Capture *cp; + struct { + const char *s; + const char *e; + } s; + } u; +} StrAux; + +#define MAXSTRCAPS 10 + +static int getstrcaps (CapState *cs, StrAux *cps, int n) { + int k = n++; + cps[k].isstring = 1; + cps[k].u.s.s = cs->cap->s; + if (!isfullcap(cs->cap++)) { + while (!isclosecap(cs->cap)) { + if (n >= MAXSTRCAPS) /* too many captures? */ + cs->cap = nextcap(cs->cap); /* skip it */ + else if (captype(cs->cap) == Csimple) + n = getstrcaps(cs, cps, n); + else { + cps[n].isstring = 0; + cps[n].u.cp = cs->cap; + cs->cap = nextcap(cs->cap); + n++; + } + } + cs->cap++; /* skip close */ + } + cps[k].u.s.e = closeaddr(cs->cap - 1); + return n; +} + + +/* +** add next capture (which should be a string) to buffer +*/ +static int addonestring (luaL_Buffer *b, CapState *cs, const char *what); + + +static void stringcap (luaL_Buffer *b, CapState *cs) { + StrAux cps[MAXSTRCAPS]; + int n; + size_t len, i; + const char *c; + updatecache(cs, cs->cap->idx); + c = lua_tolstring(cs->L, subscache(cs), &len); + n = getstrcaps(cs, cps, 0) - 1; + for (i = 0; i < len; i++) { + if (c[i] != '%' || c[++i] < '0' || c[i] > '9') + luaL_addchar(b, c[i]); + else { + int l = c[i] - '0'; + if (l > n) + luaL_error(cs->L, "invalid capture index (%d)", l); + else if (cps[l].isstring) + luaL_addlstring(b, cps[l].u.s.s, cps[l].u.s.e - cps[l].u.s.s); + else { + Capture *curr = cs->cap; + cs->cap = cps[l].u.cp; + if (addonestring(b, cs, "capture") == 0) + luaL_error(cs->L, "no values in capture index %d", l); + cs->cap = curr; + } + } + } +} + + +static void substcap (luaL_Buffer *b, CapState *cs) { + const char *curr = cs->cap->s; + if (isfullcap(cs->cap)) /* no nested captures? */ + luaL_addlstring(b, curr, cs->cap->siz - 1); /* keep original text */ + else { + cs->cap++; + while (!isclosecap(cs->cap)) { + const char *next = cs->cap->s; + luaL_addlstring(b, curr, next - curr); /* add text up to capture */ + if (addonestring(b, cs, "replacement") == 0) /* no capture value? */ + curr = next; /* keep original text in final result */ + else + curr = closeaddr(cs->cap - 1); /* continue after match */ + } + luaL_addlstring(b, curr, cs->cap->s - curr); /* add last piece of text */ + } + cs->cap++; /* go to next capture */ +} + + +static int addonestring (luaL_Buffer *b, CapState *cs, const char *what) { + switch (captype(cs->cap)) { + case Cstring: + stringcap(b, cs); /* add capture directly to buffer */ + return 1; + case Csubst: + substcap(b, cs); /* add capture directly to buffer */ + return 1; + default: { + lua_State *L = cs->L; + int n = pushcapture(cs); + if (n > 0) { + if (n > 1) lua_pop(L, n - 1); /* only one result */ + if (!lua_isstring(L, -1)) + luaL_error(L, "invalid %s value (a %s)", what, luaL_typename(L, -1)); + luaL_addvalue(b); + } + return n; + } + } +} + + +static int pushcapture (CapState *cs) { + luaL_checkstack(cs->L, 4, "too many captures"); + switch (captype(cs->cap)) { + case Cposition: { + lua_pushinteger(cs->L, cs->cap->s - cs->s + 1); + cs->cap++; + return 1; + } + case Cconst: { + pushluaval(cs); + cs->cap++; + return 1; + } + case Carg: { + int arg = (cs->cap++)->idx; + if (arg + FIXEDARGS > cs->ptop) + return luaL_error(cs->L, "reference to absent argument #%d", arg); + lua_pushvalue(cs->L, arg + FIXEDARGS); + return 1; + } + case Csimple: { + int k = pushallvalues(cs, 1); + if (k > 1) + lua_insert(cs->L, -k); /* whole match is first result */ + return k; + } + case Cruntime: { + int n = 0; + while (!isclosecap(cs->cap++)) { + luaL_checkstack(cs->L, 4, "too many captures"); + lua_pushvalue(cs->L, (cs->cap - 1)->idx); + n++; + } + return n; + } + case Cstring: { + luaL_Buffer b; + luaL_buffinit(cs->L, &b); + stringcap(&b, cs); + luaL_pushresult(&b); + return 1; + } + case Csubst: { + luaL_Buffer b; + luaL_buffinit(cs->L, &b); + substcap(&b, cs); + luaL_pushresult(&b); + return 1; + } + case Cgroup: { + if (cs->cap->idx == 0) /* anonymous group? */ + return pushallvalues(cs, 0); /* add all nested values */ + else { /* named group: add no values */ + cs->cap = nextcap(cs->cap); /* skip capture */ + return 0; + } + } + case Cbackref: return backrefcap(cs); + case Ctable: return tablecap(cs); + case Cfunction: return functioncap(cs); + case Cquery: return querycap(cs); + case Cfold: return foldcap(cs); + default: assert(0); return 0; + } +} + + +static int getcaptures (lua_State *L, const char *s, const char *r, int ptop) { + Capture *capture = (Capture *)lua_touserdata(L, caplistidx(ptop)); + int n = 0; + if (!isclosecap(capture)) { /* is there any capture? */ + CapState cs; + cs.ocap = cs.cap = capture; cs.L = L; + cs.s = s; cs.valuecached = 0; cs.ptop = ptop; + do { /* collect their values */ + n += pushcapture(&cs); + } while (!isclosecap(cs.cap)); + } + if (n == 0) { /* no capture values? */ + lua_pushinteger(L, r - s + 1); /* return only end position */ + n = 1; + } + return n; +} + +/* }====================================================== */ + + +static int version_l (lua_State *L) { + lua_pushstring(L, VERSION); + return 1; +} + + +static int type_l (lua_State *L) { + if (testpattern(L, 1)) + lua_pushliteral(L, "pattern"); + else + lua_pushnil(L); + return 1; +} + + +static void createcat (lua_State *L, const char *catname, int (catf) (int)) { + Instruction *p = newcharset(L); + int i; + for (i = 0; i < CHAR_MAX; i++) + if (catf(i)) setchar(p[1].buff, i); + lua_setfield(L, -2, catname); +} + + +static int locale_l (lua_State *L) { + if (lua_isnoneornil(L, 1)) { + lua_settop(L, 0); + lua_createtable(L, 0, 12); + } + else { + luaL_checktype(L, 1, LUA_TTABLE); + lua_settop(L, 1); + } + createcat(L, "alnum", isalnum); + createcat(L, "alpha", isalpha); + createcat(L, "cntrl", iscntrl); + createcat(L, "digit", isdigit); + createcat(L, "graph", isgraph); + createcat(L, "lower", islower); + createcat(L, "print", isprint); + createcat(L, "punct", ispunct); + createcat(L, "space", isspace); + createcat(L, "upper", isupper); + createcat(L, "xdigit", isxdigit); + return 1; +} + + +static int setmax (lua_State *L) { + luaL_optinteger(L, 1, -1); + lua_settop(L, 1); + lua_setfield(L, LUA_REGISTRYINDEX, MAXSTACKIDX); + return 0; +} + + +static int printpat_l (lua_State *L) { + Instruction *p = getpatt(L, 1, NULL); + int n, i; + lua_getfenv(L, 1); + n = ktablelen(L, -1); + printf("["); + for (i = 1; i <= n; i++) { + printf("%d = ", i); + lua_rawgeti(L, -1, i); + if (lua_isstring(L, -1)) + printf("%s ", lua_tostring(L, -1)); + else + printf("%s ", lua_typename(L, lua_type(L, -1))); + lua_pop(L, 1); + } + printf("]\n"); + printpatt(p); + return 0; +} + + +static int matchl (lua_State *L) { + Capture capture[INITCAPSIZE]; + const char *r; + size_t l; + Instruction *p = getpatt(L, 1, NULL); + const char *s = luaL_checklstring(L, SUBJIDX, &l); + int ptop = lua_gettop(L); + lua_Integer ii = luaL_optinteger(L, 3, 1); + size_t i = (ii > 0) ? + (((size_t)ii <= l) ? (size_t)ii - 1 : l) : + (((size_t)-ii <= l) ? l - ((size_t)-ii) : 0); + lua_pushnil(L); /* subscache */ + lua_pushlightuserdata(L, capture); /* caplistidx */ + lua_getfenv(L, 1); /* penvidx */ + r = match(L, s, s + i, s + l, p, capture, ptop); + if (r == NULL) { + lua_pushnil(L); + return 1; + } + return getcaptures(L, s, r, ptop); +} + + +static struct luaL_Reg pattreg[] = { + {"match", matchl}, + {"print", printpat_l}, + {"locale", locale_l}, + {"setmaxstack", setmax}, + {"B", pattbehind}, + {"C", capture_l}, + {"Cf", fold_l}, + {"Cc", capconst_l}, + {"Cg", group_l}, + {"Cp", position_l}, + {"Cb", backref_l}, + {"Carg", argcap_l}, + {"Cmt", matchtime_l}, + {"Cs", capsubst_l}, + {"Ct", tcapture_l}, + {"P", pattern_l}, + {"R", range_l}, + {"S", set_l}, + {"V", nter_l}, + {"type", type_l}, + {"version", version_l}, + {NULL, NULL} +}; + + +static struct luaL_Reg metapattreg[] = { + {"__add", union_l}, + {"__pow", star_l}, + {"__sub", diff_l}, + {"__mul", concat_l}, + {"__div", rcapture_l}, + {"__unm", unm_l}, + {"__len", pattand_l}, + {NULL, NULL} +}; + + +int luaopen_lpeg (lua_State *L); +int luaopen_lpeg (lua_State *L) { + lua_pushcfunction(L, (lua_CFunction)&l_newpf); /* new-pattern function */ + lua_setfield(L, LUA_REGISTRYINDEX, KEYNEWPATT); /* register it */ + luaL_newmetatable(L, PATTERN_T); + lua_pushnumber(L, MAXBACK); + lua_setfield(L, LUA_REGISTRYINDEX, MAXSTACKIDX); + luaL_register(L, NULL, metapattreg); + luaL_register(L, "lpeg", pattreg); + lua_pushliteral(L, "__index"); + lua_pushvalue(L, -2); + lua_settable(L, -4); + return 1; +} + diff --git a/lpeg/lpeg.h b/lpeg/lpeg.h new file mode 100644 index 00000000..13d7acef --- /dev/null +++ b/lpeg/lpeg.h @@ -0,0 +1,38 @@ +/* +** $Id: lpeg.h,v 1.1 2009/12/23 16:15:36 roberto Exp $ +** LPeg - PEG pattern matching for Lua +** Copyright 2009, Lua.org & PUC-Rio (see 'lpeg.html' for license) +** written by Roberto Ierusalimschy +*/ + +#ifndef lpeg_h +#define lpeg_h + +#include "lua.h" + + +#define KEYNEWPATT "lpeg.newpf" + + +/* +** type of extension functions that define new "patterns" for LPEG +** It should return the new current position or NULL if match fails +*/ +typedef const char *(*PattFunc) (const char *s, /* current position */ + const char *e, /* string end */ + const char *o, /* string start */ + const void *ud); /* user data */ + +/* +** function to create new patterns based on 'PattFunc' functions. +** This function is available at *registry[KEYNEWPATT]. (Notice +** the extra indirection; the userdata at the registry points to +** a variable that points to the function. In ANSI C a void* cannot +** point to a function.) +*/ +typedef void (*Newpf) (lua_State *L, + PattFunc f, /* pattern */ + const void *ud, /* (user) data to be passed to 'f' */ + size_t l); /* size of data to be passed to 'f' */ + +#endif diff --git a/lpeg/lpeg.html b/lpeg/lpeg.html new file mode 100644 index 00000000..bb92b279 --- /dev/null +++ b/lpeg/lpeg.html @@ -0,0 +1,1423 @@ + + + + LPeg - Parsing Expression Grammars For Lua + + + + + + + +
+ +
+ +
LPeg
+
+ Parsing Expression Grammars For Lua, version 0.10 +
+
+ +
+ + + +
+ + +

Introduction

+ +

+LPeg is a new pattern-matching library for Lua, +based on + +Parsing Expression Grammars (PEGs). +This text is a reference manual for the library. +For a more formal treatment of LPeg, +as well as some discussion about its implementation, +see + +A Text Pattern-Matching Tool based on Parsing Expression Grammars. +(You may also be interested in my +talk about LPeg +given at the III Lua Workshop.) +

+ +

+Following the Snobol tradition, +LPeg defines patterns as first-class objects. +That is, patterns are regular Lua values +(represented by userdata). +The library offers several functions to create +and compose patterns. +With the use of metamethods, +several of these functions are provided as infix or prefix +operators. +On the one hand, +the result is usually much more verbose than the typical +encoding of patterns using the so called +regular expressions +(which typically are not regular expressions in the formal sense). +On the other hand, +first-class patterns allow much better documentation +(as it is easy to comment the code, +to break complex definitions in smaller parts, etc.) +and are extensible, +as we can define new functions to create and compose patterns. +

+ +

+For a quick glance of the library, +the following table summarizes its basic operations +for creating patterns: +

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
OperatorDescription
lpeg.P(string)Matches string literally
lpeg.P(n)Matches exactly n characters
lpeg.S(string)Matches any character in string (Set)
lpeg.R("xy")Matches any character between x and y (Range)
patt^nMatches at least n repetitions of patt
patt^-nMatches at most n repetitions of patt
patt1 * patt2Matches patt1 followed by patt2
patt1 + patt2Matches patt1 or patt2 + (ordered choice)
patt1 - patt2Matches patt1 if patt2 does not match
-pattEquivalent to ("" - patt)
#pattMatches patt but consumes no input
lpeg.B(patt, n)Matches patt n characters behind + the current position, consuming no input
+ +

As a very simple example, +lpeg.R("09")^1 creates a pattern that +matches a non-empty sequence of digits. +As a not so simple example, +-lpeg.P(1) +(which can be written as lpeg.P(-1) +or simply -1 for operations expecting a pattern) +matches an empty string only if it cannot match a single character; +so, it succeeds only at the subject's end. +

+ +

+LPeg also offers the re module, +which implements patterns following a regular-expression style +(e.g., [09]+). +(This module is 250 lines of Lua code, +and of course uses LPeg to parse regular expressions and +translate them to regular LPeg patterns.) +

+ + +

Functions

+ + +

lpeg.match (pattern, subject [, init])

+

+The matching function. +It attempts to match the given pattern against the subject string. +If the match succeeds, +returns the index in the subject of the first character after the match, +or the captured values +(if the pattern captured any value). +

+ +

+An optional numeric argument init makes the match +starts at that position in the subject string. +As usual in Lua libraries, +a negative value counts from the end. +

+ +

+Unlike typical pattern-matching functions, +match works only in anchored mode; +that is, it tries to match the pattern with a prefix of +the given subject string (at position init), +not with an arbitrary substring of the subject. +So, if we want to find a pattern anywhere in a string, +we must either write a loop in Lua or write a pattern that +matches anywhere. +This second approach is easy and quite efficient; +see examples. +

+ +

lpeg.type (value)

+

+If the given value is a pattern, +returns the string "pattern". +Otherwise returns nil. +

+ +

lpeg.version ()

+

+Returns a string with the running version of LPeg. +

+ +

lpeg.setmaxstack (max)

+

+Sets the maximum size for the backtrack stack used by LPeg to +track calls and choices. +Most well-written patterns need little backtrack levels and +therefore you seldom need to change this maximum; +but a few useful patterns may need more space. +Before changing this maximum you should try to rewrite your +pattern to avoid the need for extra space. +

+ + +

Basic Constructions

+ +

+The following operations build patterns. +All operations that expect a pattern as an argument +may receive also strings, tables, numbers, booleans, or functions, +which are translated to patterns according to +the rules of function lpeg.P. +

+ + + +

lpeg.P (value)

+

+Converts the given value into a proper pattern, +according to the following rules: +

+
    + +

  • +If the argument is a pattern, +it is returned unmodified. +

    • + +

  • +If the argument is a string, +it is translated to a pattern that matches literally the string. +

    • + +

  • +If the argument is a non-negative number n, +the result is a pattern that matches exactly n characters. +

    • + +

  • +If the argument is a negative number -n, +the result is a pattern that +succeeds only if the input string does not have n characters: +lpeg.P(-n) +is equivalent to -lpeg.P(n) +(see the unary minus operation). +

    • + +

  • +If the argument is a boolean, +the result is a pattern that always succeeds or always fails +(according to the boolean value), +without consuming any input. +

    • + +

  • +If the argument is a table, +it is interpreted as a grammar +(see Grammars). +

    • + +

  • +If the argument is a function, +returns a pattern equivalent to a +match-time capture over the empty string. +

    • + +
+ + +

lpeg.B(patt [, n])

+

+Returns a pattern that +matches only if the input string matches patt +starting n positions behind the current position. +(The default value for n is 1.) +If the current position is less than or equal to n, +this pattern fails. +

+ +

+Like the and predicate, +this pattern never consumes any input, +independently of success or failure, +and it never produces any capture. +

+ +

+The pattern patt cannot contain any open reference +to grammar rules (see grammars). +

+ +

+(This is an experimental feature. +There is a good chance it will change in future versions.) +

+ + +

lpeg.R ({range})

+

+Returns a pattern that matches any single character +belonging to one of the given ranges. +Each range is a string xy of length 2, +representing all characters with code +between the codes of x and y +(both inclusive). +

+ +

+As an example, the pattern +lpeg.R("09") matches any digit, +and lpeg.R("az", "AZ") matches any ASCII letter. +

+ + +

lpeg.S (string)

+

+Returns a pattern that matches any single character that +appears in the given string. +(The S stands for Set.) +

+ +

+As an example, the pattern +lpeg.S("+-*/") matches any arithmetic operator. +

+ +

+Note that, if s is a character +(that is, a string of length 1), +then lpeg.P(s) is equivalent to lpeg.S(s) +which is equivalent to lpeg.R(s..s). +Note also that both lpeg.S("") and lpeg.R() +are patterns that always fail. +

+ + +

lpeg.V (v)

+

+This operation creates a non-terminal (a variable) +for a grammar. +The created non-terminal refers to the rule indexed by v +in the enclosing grammar. +(See Grammars for details.) +

+ + +

lpeg.locale ([table])

+

+Returns a table with patterns for matching some character classes +according to the current locale. +The table has fields named +alnum, +alpha, +cntrl, +digit, +graph, +lower, +print, +punct, +space, +upper, and +xdigit, +each one containing a correspondent pattern. +Each pattern matches any single character that belongs to its class. +

+ +

+If called with an argument table, +then it creates those fields inside the given table and +returns that table. +

+ + +

#patt

+

+Returns a pattern that +matches only if the input string matches patt, +but without consuming any input, +independently of success or failure. +(This pattern is called an and predicate +and it is equivalent to +&patt in the original PEG notation.) +

+ + +

+This pattern never produces any capture. +

. + + +

-patt

+

+Returns a pattern that +matches only if the input string does not match patt. +It does not consume any input, +independently of success or failure. +(This pattern is equivalent to +!patt in the original PEG notation.) +

+ +

+As an example, the pattern +-lpeg.P(1) matches only the end of string. +

+ +

+This pattern never produces any captures, +because either patt fails +or -patt fails. +(A failing pattern never produces captures.) +

+ + +

patt1 + patt2

+

+Returns a pattern equivalent to an ordered choice +of patt1 and patt2. +(This is denoted by patt1 / patt2 in the original PEG notation, +not to be confused with the / operation in LPeg.) +It matches either patt1 or patt2, +with no backtracking once one of them succeeds. +The identity element for this operation is the pattern +lpeg.P(false), +which always fails. +

+ +

+If both patt1 and patt2 are +character sets, +this operation is equivalent to set union. +

+
+lower = lpeg.R("az")
+upper = lpeg.R("AZ")
+letter = lower + upper
+
+ + +

patt1 - patt2

+

+Returns a pattern equivalent to !patt2 patt1. +This pattern asserts that the input does not match +patt2 and then matches patt1. +

+ +

+When succeeded, +this pattern produces all captures from patt1. +It never produces any capture from patt2 +(as either patt2 fails or +patt1 - patt2 fails). +

+ +

+If both patt1 and patt2 are +character sets, +this operation is equivalent to set difference. +Note that -patt is equivalent to "" - patt +(or 0 - patt). +If patt is a character set, +1 - patt is its complement. +

+ + +

patt1 * patt2

+

+Returns a pattern that matches patt1 +and then matches patt2, +starting where patt1 finished. +The identity element for this operation is the +pattern lpeg.P(true), +which always succeeds. +

+ +

+(LPeg uses the * operator +[instead of the more obvious ..] +both because it has +the right priority and because in formal languages it is +common to use a dot for denoting concatenation.) +

+ + +

patt^n

+

+If n is nonnegative, +this pattern is +equivalent to pattn patt*. +It matches at least n occurrences of patt. +

+ +

+Otherwise, when n is negative, +this pattern is equivalent to (patt?)-n. +That is, it matches at most |n| +occurrences of patt. +

+ +

+In particular, patt^0 is equivalent to patt*, +patt^1 is equivalent to patt+, +and patt^-1 is equivalent to patt? +in the original PEG notation. +

+ +

+In all cases, +the resulting pattern is greedy with no backtracking +(also called a possessive repetition). +That is, it matches only the longest possible sequence +of matches for patt. +

+ + + +

Grammars

+ +

+With the use of Lua variables, +it is possible to define patterns incrementally, +with each new pattern using previously defined ones. +However, this technique does not allow the definition of +recursive patterns. +For recursive patterns, +we need real grammars. +

+ +

+LPeg represents grammars with tables, +where each entry is a rule. +

+ +

+The call lpeg.V(v) +creates a pattern that represents the nonterminal +(or variable) with index v in a grammar. +Because the grammar still does not exist when +this function is evaluated, +the result is an open reference to the respective rule. +

+ +

+A table is fixed when it is converted to a pattern +(either by calling lpeg.P or by using it wherein a +pattern is expected). +Then every open reference created by lpeg.V(v) +is corrected to refer to the rule indexed by v in the table. +

+ +

+When a table is fixed, +the result is a pattern that matches its initial rule. +The entry with index 1 in the table defines its initial rule. +If that entry is a string, +it is assumed to be the name of the initial rule. +Otherwise, LPeg assumes that the entry 1 itself is the initial rule. +

+ +

+As an example, +the following grammar matches strings of a's and b's that +have the same number of a's and b's: +

+
+equalcount = lpeg.P{
+  "S";   -- initial rule name
+  S = "a" * lpeg.V"B" + "b" * lpeg.V"A" + "",
+  A = "a" * lpeg.V"S" + "b" * lpeg.V"A" * lpeg.V"A",
+  B = "b" * lpeg.V"S" + "a" * lpeg.V"B" * lpeg.V"B",
+} * -1
+
+

+It is equivalent to the following grammar in standard PEG notation: +

+
+  S <- 'a' B / 'b' A / ''
+  A <- 'a' S / 'b' A A
+  B <- 'b' S / 'a' B B
+
+ + +

Captures

+ +

+A capture is a pattern that creates values +(the so called semantic information) when it matches. +LPeg offers several kinds of captures, +which produces values based on matches and combine these values to +produce new values. +Each capture may produce zero or more values. +

+ +

+The following table summarizes the basic captures: +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OperationWhat it Produces
lpeg.C(patt)the match for patt plus all captures + made by patt
lpeg.Carg(n)the value of the nth extra argument to + lpeg.match (matches the empty string)
lpeg.Cb(name)the values produced by the previous + group capture named name + (matches the empty string)
lpeg.Cc(values)the given values (matches the empty string)
lpeg.Cf(patt, func)a folding of the captures from patt
lpeg.Cg(patt [, name])the values produced by patt, + optionally tagged with name
lpeg.Cp()the current position (matches the empty string)
lpeg.Cs(patt)the match for patt + with the values from nested captures replacing their matches
lpeg.Ct(patt)a table with all captures from patt
patt / stringstring, with some marks replaced by captures + of patt
patt / tabletable[c], where c is the (first) + capture of patt
patt / functionthe returns of function applied to the captures + of patt
lpeg.Cmt(patt, function)the returns of function applied to the captures + of patt; the application is done at match time
+ +

+A capture pattern produces its values every time it succeeds. +For instance, +a capture inside a loop produces as many values as matched by the loop. +A capture produces a value only when it succeeds. +For instance, +the pattern lpeg.C(lpeg.P"a"^-1) +produces the empty string when there is no "a" +(because the pattern "a"? succeeds), +while the pattern lpeg.C("a")^-1 +does not produce any value when there is no "a" +(because the pattern "a" fails). +

+ +

+Usually, +LPeg evaluates all captures only after (and if) the entire match succeeds. +During the match time it only gathers enough information +to produce the capture values later. +As a particularly important consequence, +most captures cannot affect the way a pattern matches a subject. +The only exception to this rule is the +so-called match-time capture. +When a match-time capture matches, +it forces the immediate evaluation of all its nested captures +and then calls its corresponding function, +which tells whether the match succeeds and also +what values are produced. +

+ +

lpeg.C (patt)

+

+Creates a simple capture, +which captures the substring of the subject that matches patt. +The captured value is a string. +If patt has other captures, +their values are returned after this one. +

+ + +

lpeg.Carg (n)

+

+Creates an argument capture. +This pattern matches the empty string and +produces the value given as the nth extra +argument given in the call to lpeg.match. +

+ + +

lpeg.Cb (name)

+

+Creates a back capture. +This pattern matches the empty string and +produces the values produced by the most recent +group capture named name. +

+ +

+Most recent means the last +complete +outermost +group capture with the given name. +A Complete capture means that the entire pattern +corresponding to the capture has matched. +An Outermost capture means that the capture is not inside +another complete capture. +

+ + +

lpeg.Cc ([value, ...])

+

+Creates a constant capture. +This pattern matches the empty string and +produces all given values as its captured values. +

+ + +

lpeg.Cf (patt, func)

+

+Creates a fold capture. +If patt produces a list of captures +C1 C2 ... Cn, +this capture will produce the value +func(...func(func(C1, C2), C3)..., + Cn), +that is, it will fold +(or accumulate, or reduce) +the captures from patt using function func. +

+ +

+This capture assumes that patt should produce +at least one capture with at least one value (of any type), +which becomes the initial value of an accumulator. +(If you need a specific initial value, +you may prefix a constant capture to patt.) +For each subsequent capture +LPeg calls func +with this accumulator as the first argument and all values produced +by the capture as extra arguments; +the value returned by this call +becomes the new value for the accumulator. +The final value of the accumulator becomes the captured value. +

+ +

+As an example, +the following pattern matches a list of numbers separated +by commas and returns their addition: +

+
+-- matches a numeral and captures its value
+number = lpeg.R"09"^1 / tonumber
+
+-- matches a list of numbers, captures their values
+list = number * ("," * number)^0
+
+-- auxiliary function to add two numbers
+function add (acc, newvalue) return acc + newvalue end
+
+-- folds the list of numbers adding them
+sum = lpeg.Cf(list, add)
+
+-- example of use
+print(sum:match("10,30,43"))   --> 83
+
+ + +

lpeg.Cg (patt [, name])

+

+Creates a group capture. +It groups all values returned by patt +into a single capture. +The group may be anonymous (if no name is given) +or named with the given name. +

+ +

+An anonymous group serves to join values from several captures into +a single capture. +A named group has a different behavior. +In most situations, a named group returns no values at all. +Its values are only relevant for a following +back capture or when used +inside a table capture. +

+ + +

lpeg.Cp ()

+

+Creates a position capture. +It matches the empty string and +captures the position in the subject where the match occurs. +The captured value is a number. +

+ + +

lpeg.Cs (patt)

+

+Creates a substitution capture, +which captures the substring of the subject that matches patt, +with substitutions. +For any capture inside patt with a value, +the substring that matched the capture is replaced by the capture value +(which should be a string). +The final captured value is the string resulting from +all replacements. +

+ + +

lpeg.Ct (patt)

+

+Creates a table capture. +This capture creates a table and puts all values from all anonymous captures +made by patt inside this table in successive integer keys, +starting at 1. +Moreover, +for each named capture group created by patt, +the first value of the group is put into the table +with the group name as its key. +The captured value is only the table. +

+ + +

patt / string

+

+Creates a string capture. +It creates a capture string based on string. +The captured value is a copy of string, +except that the character % works as an escape character: +any sequence in string of the form %n, +with n between 1 and 9, +stands for the match of the n-th capture in patt. +The sequence %0 stands for the whole match. +The sequence %% stands for a single %. + + +

patt / table

+

+Creates a query capture. +It indexes the given table using as key the first value captured by +patt, +or the whole match if patt produced no value. +The value at that index is the final value of the capture. +If the table does not have that key, +there is no captured value. +

+ + +

patt / function

+

+Creates a function capture. +It calls the given function passing all captures made by +patt as arguments, +or the whole match if patt made no capture. +The values returned by the function +are the final values of the capture. +In particular, +if function returns no value, +there is no captured value. +

+ + +

lpeg.Cmt(patt, function)

+

+Creates a match-time capture. +Unlike all other captures, +this one is evaluated immediately when a match occurs. +It forces the immediate evaluation of all its nested captures +and then calls function. +

+ +

+The given function gets as arguments the entire subject, +the current position (after the match of patt), +plus any capture values produced by patt. +

+ +

+The first value returned by function +defines how the match happens. +If the call returns a number, +the match succeeds +and the returned number becomes the new current position. +(Assuming a subject s and current position i, +the returned number must be in the range [i, len(s) + 1].) +If the call returns true, +the match succeeds without consuming any input. +(So, to return true is equivalent to return i.) +If the call returns false, nil, or no value, +the match fails. +

+ +

+Any extra values returned by the function become the +values produced by the capture. +

+ + + + +

Some Examples

+ +

Using a Pattern

+

+This example shows a very simple but complete program +that builds and uses a pattern: +

+
+local lpeg = require "lpeg"
+
+-- matches a word followed by end-of-string
+p = lpeg.R"az"^1 * -1
+
+print(p:match("hello"))        --> 6
+print(lpeg.match(p, "hello"))  --> 6
+print(p:match("1 hello"))      --> nil
+
+

+The pattern is simply a sequence of one or more lower-case letters +followed by the end of string (-1). +The program calls match both as a method +and as a function. +In both sucessful cases, +the match returns +the index of the first character after the match, +which is the string length plus one. +

+ + +

Name-value lists

+

+This example parses a list of name-value pairs and returns a table +with those pairs: +

+
+lpeg.locale(lpeg)   -- adds locale entries into 'lpeg' table
+
+local space = lpeg.space^0
+local name = lpeg.C(lpeg.alpha^1) * space
+local sep = lpeg.S(",;") * space
+local pair = lpeg.Cg(name * "=" * space * name) * sep^-1
+local list = lpeg.Cf(lpeg.Ct("") * pair^0, rawset)
+t = list:match("a=b, c = hi; next = pi")  --> { a = "b", c = "hi", next = "pi" }
+
+

+Each pair has the format name = name followed by +an optional separator (a comma or a semicolon). +The pair pattern encloses the pair in a group pattern, +so that the names become the values of a single capture. +The list pattern then folds these captures. +It starts with an empty table, +created by a table capture matching an empty string; +then for each capture (a pair of names) it applies rawset +over the accumulator (the table) and the capture values (the pair of names). +rawset returns the table itself, +so the accumulator is always the table. +

+ +

Splitting a string

+

+The following code builds a pattern that +splits a string using a given pattern +sep as a separator: +

+
+function split (s, sep)
+  sep = lpeg.P(sep)
+  local elem = lpeg.C((1 - sep)^0)
+  local p = elem * (sep * elem)^0
+  return lpeg.match(p, s)
+end
+
+

+First the function ensures that sep is a proper pattern. +The pattern elem is a repetition of zero of more +arbitrary characters as long as there is not a match against +the separator. It also captures its result. +The pattern p matches a list of elements separated +by sep. +

+ +

+If the split results in too many values, +it may overflow the maximum number of values +that can be returned by a Lua function. +In this case, +we should collect these values in a table: +

+
+function split (s, sep)
+  sep = lpeg.P(sep)
+  local elem = lpeg.C((1 - sep)^0)
+  local p = lpeg.Ct(elem * (sep * elem)^0)   -- make a table capture
+  return lpeg.match(p, s)
+end
+
+ + +

Searching for a pattern

+

+The primitive match works only in anchored mode. +If we want to find a pattern anywhere in a string, +we must write a pattern that matches anywhere. +

+ +

+Because patterns are composable, +we can write a function that, +given any arbitrary pattern p, +returns a new pattern that searches for p +anywhere in a string. +There are several ways to do the search. +One way is like this: +

+
+function anywhere (p)
+  return lpeg.P{ p + 1 * lpeg.V(1) }
+end
+
+

+This grammar has a straight reading: +it matches p or skips one character and tries again. +

+ +

+If we want to know where the pattern is in the string +(instead of knowing only that it is there somewhere), +we can add position captures to the pattern: +

+
+local I = lpeg.Cp()
+function anywhere (p)
+  return lpeg.P{ I * p * I + 1 * lpeg.V(1) }
+end
+
+ +

+Another option for the search is like this: +

+
+local I = lpeg.Cp()
+function anywhere (p)
+  return (1 - lpeg.P(p))^0 * I * p * I
+end
+
+

+Again the pattern has a straight reading: +it skips as many characters as possible while not matching p, +and then matches p (plus appropriate captures). +

+ +

+If we want to look for a pattern only at word boundaries, +we can use the following transformer: +

+ +
+local t = lpeg.locale()
+
+function atwordboundary (p)
+  return lpeg.P{
+    [1] = p + t.alpha^0 * (1 - t.alpha)^1 * lpeg.V(1)
+  }
+end
+
+ + +

Balanced parentheses

+

+The following pattern matches only strings with balanced parentheses: +

+
+b = lpeg.P{ "(" * ((1 - lpeg.S"()") + lpeg.V(1))^0 * ")" }
+
+

+Reading the first (and only) rule of the given grammar, +we have that a balanced string is +an open parenthesis, +followed by zero or more repetitions of either +a non-parenthesis character or +a balanced string (lpeg.V(1)), +followed by a closing parenthesis. +

+ + +

Global substitution

+

+The next example does a job somewhat similar to string.gsub. +It receives a pattern and a replacement value, +and substitutes the replacement value for all occurrences of the pattern +in a given string: +

+
+function gsub (s, patt, repl)
+  patt = lpeg.P(patt)
+  patt = lpeg.Cs((patt / repl + 1)^0)
+  return lpeg.match(patt, s)
+end
+
+

+As in string.gsub, +the replacement value can be a string, +a function, or a table. +

+ + +

Comma-Separated Values (CSV)

+

+This example breaks a string into comma-separated values, +returning all fields: +

+
+local field = '"' * lpeg.Cs(((lpeg.P(1) - '"') + lpeg.P'""' / '"')^0) * '"' +
+                    lpeg.C((1 - lpeg.S',\n"')^0)
+
+local record = field * (',' * field)^0 * (lpeg.P'\n' + -1)
+
+function csv (s)
+  return lpeg.match(record, s)
+end
+
+

+A field is either a quoted field +(which may contain any character except an individual quote, +which may be written as two quotes that are replaced by one) +or an unquoted field +(which cannot contain commas, newlines, or quotes). +A record is a list of fields separated by commas, +ending with a newline or the string end (-1). +

+ +

+As it is, +the previous pattern returns each field as a separated result. +If we add a table capture in the definition of record, +the pattern will return instead a single table +containing all fields: +

+
+local record = lpeg.Ct(field * (',' * field)^0) * (lpeg.P'\n' + -1)
+
+ + +

UTF-8 and Latin 1

+

+It is not difficult to use LPeg to convert a string from +UTF-8 encoding to Latin 1 (ISO 8859-1): +

+ +
+-- convert a two-byte UTF-8 sequence to a Latin 1 character
+local function f2 (s)
+  local c1, c2 = string.byte(s, 1, 2)
+  return string.char(c1 * 64 + c2 - 12416)
+end
+
+local utf8 = lpeg.R("\0\127")
+           + lpeg.R("\194\195") * lpeg.R("\128\191") / f2
+
+local decode_pattern = lpeg.Cs(utf8^0) * -1
+
+

+In this code, +the definition of UTF-8 is already restricted to the +Latin 1 range (from 0 to 255). +Any encoding outside this range (as well as any invalid encoding) +will not match that pattern. +

+ +

+As the definition of decode_pattern demands that +the pattern matches the whole input (because of the -1 at its end), +any invalid string will simply fail to match, +without any useful information about the problem. +We can improve this situation redefining decode_pattern +as follows: +

+
+local function er (_, i) error("invalid encoding at position " .. i) end
+
+local decode_pattern = lpeg.Cs(utf8^0) * (-1 + lpeg.P(er))
+
+

+Now, if the pattern utf8^0 stops +before the end of the string, +an appropriate error function is called. +

+ + +

UTF-8 and Unicode

+

+We can extend the previous patterns to handle all Unicode code points. +Of course, +we cannot translate them to Latin 1 or any other one-byte encoding. +Instead, our translation results in a array with the code points +represented as numbers. +The full code is here: +

+
+-- decode a two-byte UTF-8 sequence
+local function f2 (s)
+  local c1, c2 = string.byte(s, 1, 2)
+  return c1 * 64 + c2 - 12416
+end
+
+-- decode a three-byte UTF-8 sequence
+local function f3 (s)
+  local c1, c2, c3 = string.byte(s, 1, 3)
+  return (c1 * 64 + c2) * 64 + c3 - 925824
+end
+
+-- decode a four-byte UTF-8 sequence
+local function f4 (s)
+  local c1, c2, c3, c4 = string.byte(s, 1, 4)
+  return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168
+end
+
+local cont = lpeg.R("\128\191")   -- continuation byte
+
+local utf8 = lpeg.R("\0\127") / string.byte
+           + lpeg.R("\194\223") * cont / f2
+           + lpeg.R("\224\239") * cont * cont / f3
+           + lpeg.R("\240\244") * cont * cont * cont / f4
+
+local decode_pattern = lpeg.Ct(utf8^0) * -1
+
+ + +

Lua's long strings

+

+A long string in Lua starts with the pattern [=*[ +and ends at the first occurrence of ]=*] with +exactly the same number of equal signs. +If the opening brackets are followed by a newline, +this newline is discharged +(that is, it is not part of the string). +

+ +

+To match a long string in Lua, +the pattern must capture the first repetition of equal signs and then, +whenever it finds a candidate for closing the string, +check whether it has the same number of equal signs. +

+ +
+equals = lpeg.P"="^0
+open = "[" * lpeg.Cg(equals, "init") * "[" * lpeg.P"\n"^-1
+close = "]" * lpeg.C(equals) * "]"
+closeeq = lpeg.Cmt(close * lpeg.Cb("init"), function (s, i, a, b) return a == b end)
+string = open * lpeg.C((lpeg.P(1) - closeeq)^0) * close /
+  function (s, o) return s end
+
+ +

+The open pattern matches [=*[, +capturing the repetitions of equal signs in a group named init; +it also discharges an optional newline, if present. +The close pattern matches ]=*], +also capturing the repetitions of equal signs. +The closeeq pattern first matches close; +then it uses a back capture to recover the capture made +by the previous open, +which is named init; +finally it uses a match-time capture to check +whether both captures are equal. +The string pattern starts with an open, +then it goes as far as possible until matching closeeq, +and then matches the final close. +The final function capture simply discards +the capture made by close. +

+ + +

Arithmetic expressions

+

+This example is a complete parser and evaluator for simple +arithmetic expressions. +We write it in two styles. +The first approach first builds a syntax tree and then +traverses this tree to compute the expression value: +

+
+-- Lexical Elements
+local Space = lpeg.S(" \n\t")^0
+local Number = lpeg.C(lpeg.P"-"^-1 * lpeg.R("09")^1) * Space
+local FactorOp = lpeg.C(lpeg.S("+-")) * Space
+local TermOp = lpeg.C(lpeg.S("*/")) * Space
+local Open = "(" * Space
+local Close = ")" * Space
+
+-- Grammar
+local Exp, Term, Factor = lpeg.V"Exp", lpeg.V"Term", lpeg.V"Factor"
+G = lpeg.P{ Exp,
+  Exp = lpeg.Ct(Factor * (FactorOp * Factor)^0);
+  Factor = lpeg.Ct(Term * (TermOp * Term)^0);
+  Term = Number + Open * Exp * Close;
+}
+
+G = Space * G * -1
+
+-- Evaluator
+function eval (x)
+  if type(x) == "string" then
+    return tonumber(x)
+  else
+    local op1 = eval(x[1])
+    for i = 2, #x, 2 do
+      local op = x[i]
+      local op2 = eval(x[i + 1])
+      if (op == "+") then op1 = op1 + op2
+      elseif (op == "-") then op1 = op1 - op2
+      elseif (op == "*") then op1 = op1 * op2
+      elseif (op == "/") then op1 = op1 / op2
+      end
+    end
+    return op1
+  end
+end
+
+-- Parser/Evaluator
+function evalExp (s)
+  local t = lpeg.match(G, s)
+  if not t then error("syntax error", 2) end
+  return eval(t)
+end
+
+-- small example
+print(evalExp"3 + 5*9 / (1+1) - 12")   --> 13.5
+
+ +

+The second style computes the expression value on the fly, +without building the syntax tree. +The following grammar takes this approach. +(It assumes the same lexical elements as before.) +

+
+-- Auxiliary function
+function eval (v1, op, v2)
+  if (op == "+") then return v1 + v2
+  elseif (op == "-") then return v1 - v2
+  elseif (op == "*") then return v1 * v2
+  elseif (op == "/") then return v1 / v2
+  end
+end
+
+-- Grammar
+local V = lpeg.V
+G = lpeg.P{ "Exp",
+  Exp = lpeg.Cf(V"Factor" * lpeg.Cg(FactorOp * V"Factor")^0, eval);
+  Factor = lpeg.Cf(V"Term" * lpeg.Cg(TermOp * V"Term")^0, eval);
+  Term = Number / tonumber + Open * V"Exp" * Close;
+}
+
+-- small example
+print(lpeg.match(G, "3 + 5*9 / (1+1) - 12"))   --> 13.5
+
+

+Note the use of the fold (accumulator) capture. +To compute the value of an expression, +the accumulator starts with the value of the first factor, +and then applies eval over +the accumulator, the operator, +and the new factor for each repetition. +

+ + + +

Download

+ +

LPeg +source code.

+ + +

License

+ +

+Copyright © 2008 Lua.org, PUC-Rio. +

+

+Permission is hereby granted, free of charge, +to any person obtaining a copy of this software and +associated documentation files (the "Software"), +to deal in the Software without restriction, +including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, +and to permit persons to whom the Software is +furnished to do so, +subject to the following conditions: +

+ +

+The above copyright notice and this permission notice +shall be included in all copies or substantial portions of the Software. +

+ +

+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +

+ +
+ +
+ +
+

+$Id: lpeg.html,v 1.64 2011/02/16 15:01:58 roberto Exp $ +

+
+ +
+ + + diff --git a/lpeg/lpeg.o b/lpeg/lpeg.o new file mode 100644 index 0000000000000000000000000000000000000000..f566774f7a0f7769007dcbf23f29e0c825361d1d GIT binary patch literal 48688 zcmeIbeRx#W)i-=*5@1X)Cur1YZ5?gUL=+Q6n^f96Lnd$rCmIEmRxJsW#6&`pCNqeN zFmw{+7zWeUwzRd+O>bLktM^^3En4-KgpYt)E21cB<-__hV??4>i1?BBx7OZ!X6L{> z*ZW@Yf6sHD>mqZ`Z?C=f+H0@9_TFpnb68OwoauJC40XATZyV_+g&Ia(bNarLZ!3+H z4Aba2ynKok8nogC+spdRvka@(4EDNX{jQ)r!LoDg4@(BD*mjQ0rJ z)jX@4xdiQoq7k3{l4Z9R4cnji;^CrE%f6u~Va47rwCv9kzZ@MMwLkEeyi*f;Yj%A4 zKNEK`VA=bu-qs?I)q7@9u(zQo-->N5Otv9yEPr#OSzXp&*6%krK0V`bcAN^$HoN=q z(-d3bHq4$L-1;6j@wd9{XrA&=gzl zHX<{s?Sby6eUA}!WE@Clx~g3}sw;M`m>m7o-T7Gb(=6X#$02daBhgPMMIHdfl`daw zz&&)Id6pk#+dKTWdrP11THie1+W>|JKaq^hr`#Hn-077`YYa9X8Geyu9W;%Xp0q}yUj2+x~IF%STz`Nja<)Kp=Xiu z9H*d&IY==#_V;F!K8y!kk>C0&{<*CG0dM$8b7QBgvj2nZ*gIzIOEdhC6<_w82a+8Y zwAtZbuqq=zMh>6-#_}dB^r98_?;bHXdZfy|;j7J!m9G9oc5J^H8!^K(h~UE^ueq^y z^lHJF*SZ7fnoj z<;&60-eA$ho~O-j6V30h7^(?vTQjfC`~LW}LVwB7?D(>6MtqLTZy)yCANxx_NG$kbbabo&f3OL~nP*iR)3dlW z%~g*8m_Etuxt|g?&UT?mlmE^{QK}ToGXMG;eK+`S^nGs&`UmVW`sw&^TfiPEJIv1m zQjMuk)H}WCbtGnyFkCVQm(@z^8Zr?uMjr+&w z57mGBc-$Ce{kV^cJD4i$pZGvoX+BrEHn{9(Vh1MlznAsmeskk4kN?!{p?{yA^>{Jz zE%}e>lOjJiH(uw0j&|n*8R}y`-TjgA(G@ONDgdT8pt`!@| zw`{X}m)Y|^lA8Je=HRLO6+Fd8PFQ|b^wwNBnZMvMU_S#7W5sXHA38nNKdZ@ZwzlaLv%4+qLKu2llGq`n^m38$n>4VLDOCUbmJM?!=->jfq zG;Hz!dPN_ne~?bMH(T}xaOQ~^Qmx+HgCc2a-WzS-)puqzz z#DfVB94o*47?nl)Et$x<%lbJ#JorH}O)8N1;rDU+eQM8NhereURs}unfn z7mXNAzU}^^qc9pgeLA}0HLK^KZ!o(^;1JZe`i3G`*raTMlC6P(&$Ik?qd|ViIsUSK z$sr{EL=6zPJcEW?)V6FF>}D`~61wGPbnm0y$VaSt8emiQ;k(Yai8_Vv+JOBs4PB@{ z^_w|p4&u$@cfj7J{Dt3s*EjkyTwI~w-tV^$`0b+uBU$L?*18wp^m*ovgSX!bYyW&j zQgsUR2*D(HP!dsfNYqa{=uq|g7EGpzuk4z{(uOPii||g<@n+=F}wJPtq4iB zAM+PTX!t4)`g_Ag6Yuz=FZP9NdA@IO!l~%uv#2Kz&LW@lZrbvWr-pQYv_iw6PP~0| zbTm~>dkdFR-E+{q`z#p3&>uDWbbml&+n@B_LbGnGcSXMaUTiRnGxl~BTD`aD#|B;e zXfG;unogQu&<+<32jkzzTbp_#i6?AQ7~eVa^{LgdX27M`?l3 zVawhfu%Ay%h1kB8U$V=yw=Fl>cf;_d`}V~?a0lXdKtI*Sr0UfX7nDS;7PODn*p2WN zAEX5%-&`0(-}DBd+TvjEvZ7L`u3Y$zl>Z_z_mG(TAqYtd>B8h3J|R(uAz|^#qwtJb zSflvtzb_v%oK(f96;Wq~`~&Y~vvL9`x%5<+s1&;%a&u4dFLvSR(;pX=)&-Zh7m$ zKm6ejObE{hb07+SWE-=$sYhSM`(}I@%(E(VbCH7PQ}l4g#(Z7=>VpX^kB*X2<=LF^jsP2xi6Y(?IRhT2x9}D5sXH zgeolde+~;sc0gX;|IHC@4+}-=NIm`L+K=!n!F`YO$VMp&5mKsQpHdPPpRx*s#uz#8 z803^=kgQ{n4@aDQ_8o&fcMS5xG05+aL4I}&(gQ^JLYjJD{#=GGBwF$DiKs*RF-eP} zbpIqXO7LQvH6mtA_7^EnGdioqkX{xDT&i!bgk8vWLnm`^P^NhnDSM~7IKgIOoT z;e}GeO7>XmCT|6sPW87r*7nW9=04-1RC|nhNEemUBlm|`FWyN|8krn8$HF_oCORn|9p`DwSD z4`{S#8?)Rv$CV~k({Ot)*;hXKo|RjPb($olO<~vO#wwS!y}PKNnBfv40oWm)4NBZ8 z?^*8z?9meUYxk@@JQK~Z#1W97#}SQ8MZgI6kg?;U`+V)bP`meP_r>bI_nxi&$S138 zLJ#JqE8OXdmi3E(2FsNE7seQ}AEINL7E8xy|l-kt#!hKg4bHsJqjQ9?g1;vM$Mq9-S0k@fZnhxwf;%H8-9X zJvu(z3?`-hl8q-)G;}lTN0CaZa&zNm8Q<}QFCpH0Q@mdV?+5~lvk=3`-bOJD6e03O zc5qQ!n)=H65xdbyk`Z(oL{Zgx>iF&9v;S3gFaeH+g{78>3AP#?dio=0I%gp4qoqGg zyIz&!akTyDQ^IG95e6)udjL&sS^{AXh{d-l!{UUt!o|Dl270-lE3dWZe6<5o6W7hQ3 zmYfa_E4Na*ww7Z1MK2yL+D$Da+A~n`Mpth1_K_3K?wc{w?yK%OYsY3%&tH*CotdR*DN9Qw?)Kc;l7J23)U@iVs!trPN+ zSVS}Qj(D!_c?e)9mgAsTC_X zA_uJs%M*FmZ~r}Lzd}MMnqm7Sq%yysxLYUep zMGUBM;n-)<#EkCkJ!yxUUQ5HV7W29+kp-Sl&|GI|H??P?3b*1JvA#+USrBXH1^&|Y z2Fc$INTYaH8pTj=TAw1G!_Ewz44Ysp-!G(|*gM|b)Zy`69w`pPa~w+e4aEVVF0b~OvHt*y z?T|dHcb>6&B4QH=M%hTj%D}3)m$NTvsfkZJ(--~rhrADv#Y;t}(y zet*1UXLZF9bM1p*r#c?2vv+QGe$TFO9Tv#hSoY4%_ELC`c$a*{I#Olaa<7gX*|YkhOLL5Hnblhi z8$cje)t09jn1hn#m9S7&Z&u_SqDJ^6^1RiXPZ^YMVp(E$f%aEA5LZSY%jy^&`VV^E zMSBpTT|$W|Le;-~*NGWTHnpzUTjk*F%J(fVnbnH?2CFxFaxs!EySu0yptQKwTU3iMVR)|4tQMbWHyhnO*0 zkd=G?(1QrD`OHo85K5*fn+Uq4lh>%^pnXuM2=<2Z`>cvZ931Y z+4D6#Tk$*gBMcBrpdB`2>4qioH_xZU1L(QvDnoB^nSt0gsCr6#hWqlWLUYXuD2fUh zV+a#q2-5KOC0+vs+=#jML%2sGwFfD@B>f2)R`L~2S%f}Vc9^z9q)V}SPv#0^n-m*o zn<3Z9B+u_bZEl>A(+xMd5Q0xXF><2$=nM}wLd~8WstF<+(?HbIkM6s?1d-iY@oDGz zj==F4uvr9iQ38dDX%u5SKp1s2@>aF$a}+sr9|uEF*~Om3SB3D)PZWYnp)X%l9L^Cl z#rNOCcJkPGC#wS*5WdXb%ysC&j^8j0mlGkINJJpubqTT)5=M#H7wWn<;id31Juu8{ zk`GC8h7fz}&61^?n`%AE7DP0so>T#S(|LxcA18sNQzek|mn4!mWG?<{8iE|fUkGxt z)Z9RPUVgQ?2|?7}Mcgp`kzAPn85jol!4I)HK+9?v{{+0qa>@?EcBuSxFGD+sYWE^6 zaC<)7_K13hf5?%J0X|}fok;#CjfKA09Ug2ehhUdwhf!#*K|$W2oeU!ueK~Y`C#Y!N zIRZ(j@7n`b+(%{|EO{@w?3>snJ%FFjT(Ospp*Mw}#Pbx${)Nm*2Mmxic@WR+U!!+? z2D{p3c0U0m`ss;g&z~p>oiNqYo z*lMnb1f|0(K;8!#l6RmfdA)!{QmV*lPZ7<2Gjc=8$sfRaQns;ktZB>y4V9Ef3L0@O z#}JRkQbRo27)v!VtS8??snq${{LllF=kun>zQiuX-igN~STphXp3Y;aR?ctymxqeO zxpD=x>0Kd`eMk+xtQA!j_1k6O3Xdmj7ScfxLJ(LwdHcwJNrMpG;a4ChtBH3C?F;ly zLgTl3lLt{1DCsFOe%2Bm9E&1xU>9{=uF%1rgW=P$*Agip7fzu< zJ0ZpeOq?*zV#}a6pLT^%O*Oy`y_6cSL}8%fcnJ+qW5%~J-28?yhjpRW#AUl;Bjduk z_D+4Boz@mM+mM3g^_cpmD=Q6eU*dkUNN8^~y&Hg973zI~Doz%T>BQGnFrZom9Z+e% zR%^&fy)ML|9K{GKTYhnBKhbZ$DVt6dqdn6GF&B+fKzF=?T$id` zD2M2Ty^DNR@q!B!qf|yeGGfuW?JVF2_%*@Onhh%fvJ5s*TdKr!TpWFFwud>->x z`<2uou1{`^X4DYN4RpAfmdMa<&3oSxUi zHwQvnt$1)7R`t(yo-}kl>`AWCb_40Uf_7EYVYte$X+Am+f#O}^@vhgJ?0HzZbw00_ z8P8&wu^Ab#B2^17V$%79tS{?ezc!%}xSF4T!F?q5{(S6~!As9ADrO;LBjcCM#V$@E zj|VhSr+Q!-uIwAZfudp~d-#a1;S9tnI1GiLA)SnHO{Bk`)A+PR4h#-YCm4Fk$;D_p9iS;k zDduxL+ab31zJYR@6FSoIJzH6}3LyRHZVC4x$vQCOayXlo_^@RBWTQP|f!GctX2|IM!T?%?Sc3FvL zFUUgmx6+oVe=AD#t)9CTD^Vsn$)No+Y5c4y`Byc?p|EjE?G9lJEEMA9YWUGk1=N$OvnWI%okVwppWFF6p4(>C{|Ak<16kM+$Tl z_bJm!&HhA&kIW19D2rqcJ+GVHKZPa`j3i%m0({jC(pMSFXP3QB=2ALj!cKVN~=3#dtCmf`0gnhJ)z^(ThZ6oX?PmYesIaVN2`EKGzyT`gV$XTAMNdz+=o(E&b zi#XYrl<*+}CGjgbR_FyZ>`Ugxiz}mFdcyDF?>O@(o104Z#5Q~A$kOV0 z2K!KUBZ>As>@1T%=UE)A!f(GUY~fN0!jPS)d9Z8`G6}Yj1Ip z6*xUbaR>~1MZw3>_ly!EKyL_AuSl#Bgs{UPpMYb(kAVPwe**jaQMf=cBv>I3@of%u zxIrT?-P4lqU=HWB725TJ)(WlAkQ&a==Y8zYxj7N4fGpO)+gXY5D=-b|0mvJW7YLNR zpWKCN>GQ2I=fev5)RbUF=cTZ+*%ppqPLOCswMbtGP+vD{(UB|kqVOyqo5fIG+TQ5| zCRYA9KR#f^4q@=b#1hAA5Cwk4ewI$p4rTdXJYesQy>ph|6D&oLeOKgJ+8$v}e8?;c zJ;M6;!Fb8RBu*be*&4-|`C@v1$C2xwN75$}#>_7z+w`G!PvTL~%6I~Y=me`niHkmw zjet3#zG+D(m)8CWR3gB5Ir@j}X{ptO+zFDo;8t0y-1!uZui}kmwf1Y=$~t#IFtF4L zY0LvwdOtya#$=Npk_(QZUg{^ua0}Z196Bl8J}QPTV_m<(ECrk6)5nL;g$I69O~&?O zGKR=4wz^f!_ixf4)A|ADANT}0tsi{XNqq+F(NsKZ<@+PKXt+8$uMxkeXg?n0qyx1Q zN05RxZvAqWCJr`IX95hgR8$Qk@7f{{z}`PI4=Y|f8mt%&nlnG5?D2bxHk^RectcUM z$PoXtKx{cJCi4L$CkElAi-TD>dPR|t+F+KDi{-etC;<@>urHcVO^@F?`lXXswYzW zIC3V5u5BS>5Svj5HWU;E;fp=sD~>$UW)$v&*pbSo&7L6bESf#J&<`>O)74Z-VqX&n zp8}|FV(0`pk4Ee*J_9Fi27)CAaK0(8um*2yB>lZNk667okjq@!dlq_`gF%Q@xJ&nG>DAt8E&LI~@ z>QNOs!L8`m>kde_!&3lEsM(B?zeC=>;N`zzV>P?~g3;EBS9=0@4S*yGz*Iae4vtw% zA==2>v;C#>JxKXTYP1Ko%(jV)G+vX!si_eL7zo%Ci-=Ehil|3T=(zyiQ>k&i3VZPl z`f`lGcIsYeY7-Qxs<>HJ3OtxSq|)FN^f=UvyF(zpMEJpU+$#XB4YN`=WARJz^LZ{` z`KfsSC!3U1&6`!RLx`&E36|`LF7r^d@j=uPHl_A&YbrjvrC?}E zs(%C!X{xDsIea>zGUM&dIHiT-NC^KUY;uanyl->oUWitld>gZCF~4ECKeDFK5|e~3 z8KmvqwOc`uyo#ldl3FvcGh+$Z9)vJg?ZXZ6CJB-~ia(_O3Z%XiCX0|T$2Bk_CnvKB zLoc+cuz{Q%ejmec+3U3Tf);wRs2`6wRI~>_bP>f)+#|UZw>xkPy<71oIe+33#M)dq z-X+*^r0JiSPWGHk6&VlNgx$ei!N&LcX`R6>k~jnNz$yy01Q1R@W5B^oOXjO~V`*ya zGM?bDojnCZ5Y2fnOU(XJ7cSM#mW>esVyenWskJ(>B`BFOs!*k@UQu;Dz^z6rVL8%- zvACiU0cSzw!DFS1vcrja?^9UOoeqVgwwxIq1NNn!g7FS0L8JumhKbMa5859&e0ulC znEM4UM@imCvm?@iUXd&yzSYz?|;XuiyqWX*L*EmtZ84BSyf5Ij$Nti--D&m_(1z$G|#}mhJz*6 zkyoh(6)#|}Fr0iI(>vwAx$FD}Hp-oky3TA&FVpj*{v`ByICj9aO8#!aLH1s3+0R=g zN34~jU!nobyXh=0&s8gnhg}1!XXhs80wFVJ8^;dfXyO~Y$i(NSOq}IK^`<3%3~?cM zsTKb-yKzvb8ZNOO+1O1{2njtS`Bi8tBXy^f`aGf*bBIi8M03L4jysX)B_E5zD)HV- z(#C`6zkC#(&!OYMxj2H3X~`Z0rNy~)oR}9gXL{cP8{~;a=n1rpm`HK$?TLxoSSU}$ zaCjM=u*dQQLB*G!0g=6>te=j<^gKXD9uFb=p~0+E=y;uf^&%JB87;Oeu-Jz8$H_ci zC7qgHB_+NG?%Wu3j1*%#pImRCBlfLygd5{4=Bwp2Pvzb7Lu4AE;ly<)9wkG9$iHzh z^7P^J*leS5reF)4-o!!oNlkvc(~W-MKOwPthHL1arashwGEelzsDh zNMOy3brUtMJ80FNoO@jU6n(Ej3^ZIC8_5ozOuQ)*%Wyp(u}u|+nKX(!k=gJ#A;q9W zan~n5fkB9QtP;<|IL&9quE!py>Z5*K2KU`m5$K*^^#{t5+OHq8{&g zHG$rlqZIoCgnOFoS~nX&;w1_Qpy@Fm8Z=HM6La(Hbz=jf)IrKqTkULlk?+F9EG%3_ zf04_`eO&XQ=lkGcq#SQxdXx+8}Lq2yzHc;q(n~}JeFf_nR1Ng;!F};iz_KB=w6_2to zqt^+d1a&_hSruLC86Sz78zWdIRJthWf?Te_L?uK}=QxQ^x{qjlSUqX|_~vqPYQL<5 zH8jSenqg6cBsFPFsf()jFoQU%gZjV&CsuA@n=-o#X&#FRR_#j^4Q3nbxd*btpFv@8 zTdXkmAZcP7c@EZ3Fy2T>cr|DOy+em!7@Qf!s1nWZfRY><>lfqe*1_?J{mexD2C;LB z`GqIpCmQJW%op(st7{%>b!N}sAprE*LF-BFePY;~CQ}=_1(idZv#_v1Er;Hgg>(Cn zv>4_F$D3SoXeaqem?f5HEL@ZA!`qi68B6tF$jtGz)c8$3u#%KB2d`0}2bw**;6hMl z;$qSVQvFtZDZ)m}z zha&~g_~KW&d`I>**|Xgkl$W~2tIvlLH~crL^-yBkCS*R=JDWW>vh9+G79TzEEr1DB zq8@0&R6P#Mru;UHm}(IkM5HGWpSTqhdPGp9fL`=P+6@|kwt5cM*zbpNh*{2)#?*7W zE7AQX)$$+{Avw8Aw2cbnJw` zWWNs^*ARlauDJwf8nx zVj0TtR~%G&pjeb}2Be5A-U&qmag-bf-N--SI|v2id83fN;BY8;E+k0%cRyx!BnU;4 z{N^7I(r+O2NunD@OZ!#OShE{3968cMGBGY|?UP6#=xMFvoV?M6*9K{$Ya;bK7mxZY zc9KLRx%Bq%FP?+X`?Z??u>Yan1I-)B!+`lNW$%vhr;$@?$k|ESj_3KO_WEP1$pEJ!F(I2aEGbE6S%{>?l_747Vdq*E^K>@e^|V8V^RPnzMH+h7(TB%xE-& z^FadX0{u=+;*u0|5ys#cPxkX6X`DypqtW89G=>yfT7D6r*La_E{hc)24_}7y0moFi zB8lOD(3#@wB7BA-efHV!O4Ni7`fR}K#5w4a1n&XTkbpVlfa5%kJpW<;onFbnDTx}q zl7TZI^e!J>8M}cspzs@k8vzvKKxS78cw0dHDk$l>MNcGmqB z@oCfHQ1Nn>n1o@t9ZybSi90p5NX}m+YkRYWLA3?PWFfq4^=73-?JnsORf-akJLK zsiPSYr|l;HjSh7BHV{}V9nUmP!8?g6nb#uW4G!<-SBOdEzkn$UGg644^{YS zLxrlHcn9G|RFZaJc}D;zhW6IbdPaRN0%t(+88VtAAlL+6xGAhgV69{~w2A3K98qda zX9Bcd#Mhw6QsO`wD}yAz8P=cJYx+Uz&bZrRG2)0j14be z(rd?9zndE;p62TgPRcfWC=`*JYQLML_hHiGDeVO48JIXHgJ>`=mcMc(oIzqe0$!=F z!{8h{vuD@B*2dq9L52L2vU6y?acW8l?OxIB0}bgk?lIzfIA2RC`(fVcY|YH*xcYXn;Z zBBszcc1U}%PfZwD_@OQQqpx4Eexl$D$cjP%v9-PJ zCWAOEu5Y{58>;UJM>-q31h2*QOS{7Lp+!c0Yg=S7{&qCi8=ZlJj8j+3;;7bK*OB*C`CzoElv_$+|DDH1GF8sFFnCS}yjmZrLu-1ledM9^zjq@gl&NI&U0t~gbcQqOv z?OiS5mi9Iy)ZPY3QT{@R+u7J;sA7z+mc<>djYhbBA^tXzbVUFCNMq-%Mpqaz-(++} z7IuYO!VyksYHw{YIwNi2mc@;zUwfp(nA~L)w|T#XVoSW<;`XK{aZf@*5^phZBgcUH zj8JoZr_t5O3TR_lj7~f?wn5>m=EWUGsJ^w;XzyrjTfldr_Qi{#%Z~caa7%sb z0{KW4QTI*tEv?~OC_CL5H?=HjgjiG={Aq2Kjb=11?g-y{yoZ*CrCgp$W95x`P!(3z z(-~=P^qx1l>pZ9echl>k{%=DO-j*(JYhzQ`+u0b3bap`yLp3ZCg{>76aY+i7<%sqc z%2GwQwTmoZ+onuU(m}DEjUBBpBQh34@+768=%&Wb#5ka2ajz-k~-Q%hS*xY66%-rhl?!XPF$ zzzC>BuW=(@g;V`U>QPmu%TU_UDkJJ_pxd-BTG-f(e&B6rY*pQnWu%@wwXN}%jwWNl z0?|}{@J(&!D}((WZ=uZL6V>@v~{fK@l3PeB2#tz8!hqeptWX&M@#%*F;%BiS+j ziuMq$JKeV7c5nT{E^-vAD|^2VqmUOYhnfWaMH*dGdnc@*wLxiK(kUXgVzP|?eetR0 zQjP`227gHMF1!^U5{kW$97C#*@W`$q53uw93C!Q52eAcd!h5iEp7F!Q0)I)|D%Eb(ZK(48ldfB_0Qv| z9oFy$Z8OtvrTnFP`lnxr!5DN9vmF1HYX53+&)=xPuegT)P5geR(wJ3M^-XW_yoHzm zMJ@`$$#k}Rr%k$YnQ*V25~S0+yI(0MiDe*1q%_^2NKyju6#yz-gH)WZVXHImO|Bv+abs;lcG#pmsu<1xA-m*w3fc~z6Vuuvs$ z5gz{~)9$CVz}l>9){I*9wMrZwAFoFen1F zU#9m{I+t+;_haG`$i1EQN4DqT@Q-175aU zdXasus%i98QJ%t^O55gI4gG= zvTv2VeujJEwZ`oyUj#1-UldBYysF$#+Aom|pUZsb(orRrA(*N&`=4~bot+o;xoeLr zgOBTD5f5As(jV}OyP>}|B*QB5@v(7mU2cC)UU5~P*Qb2DKNlmx7=+vO9?6USCAax? z+~(JywSr(zXX@#~7kFh^@Hft~f+!2tpDJr+u6u(6qJHNGi)^H@&RxuwXyPV#yL_oDwv-(2b@ z=RA}9vz0qX-I9O!o6JLYFZ#+twr}zm37J-t(i!@*)R~{!2ZCgkx*V;6Iw3E;dqUT1 zG->(Y+sUlF4TV_+aBAgM$Eb*g67G{>$*+`Tsacb~YFtfhd^{_VSGm^h{xzD~pUXbu zUP;GMXpoX0+2Fv)2W*v?g?mx&HtyY5p!SsKEk~YWU?URa6&TwXd5!AaEAe->`iuOe zSG=6e*SYkUX=#KNeiyHk@kJXmW7?_C{a4ZR;CEqY(~;aNX(u^pOWi+8wP_&t>eRiJ zd#h?+vWbUe{(jkpYiPWd{d^V~)PWo}c!-FJ(-rKS_scwCl9FY=jq3>q#!i;>mz_h2 z;7@<%!&ULf4uF3-w~y0T8D_CMA;a{E&~!jfM$jV-XJ zvGu6S@*%$$9apc%12PY{SIw&I*f{vqxO+{On`Uf6&c8_-FZs1}Kj!&V+J{Ry*SoUL zIKG_Z7I_fj)z18n6NL2SaT}bBFF}iosu928if6X#~o%9`wkCIzG--(;! z>CbpfP?XA!MS7sq?kd01-y~p8+oPO2Zlb@2L0{I^FQ@$U?3A78_qU2%^Yg?nl|yaq zET4vzL?M{RQx0QS}7{WJb+PvhVW8J zui_Xr65lCtRq^W-E{%la8NleuI>zf7#zNr4zfS61*$Fi#;Zcc`-O)w!0mAbq@T2i- z@xyT@!uuu8b8w`^B)u{Oj#&VUOFSwKWHO%VT8DqcN6pt%x4KW_`Ep*;K*YHIfPXBf z#Ah=q{$z0^O#^WIXZ$1joswR)%aan1O8>*L1Yp}G-Y@$l`A@oD!aw4ZKan4e`^As( z4~h3n{D%^MN8)wVDRJ?-AHN^rAImTM4X^QmeJ1e@WMH^>ZI9n<*ay*_IIqi*Rwwa(iT6wTR*6SX69BKjaoYu) z`qu-pU*M6i<&wUGCJ?w@7C%O>#MggK0KBHf?aw4`NSxQmz}HK>_Dn&~>u}uuLE<}$ z1pc7-;qQKM`$_{>{o;8^-!E}qlOoTn63;(RFyOT%ZVyO&SmL}^1fG<5=|zJ65%FUj zQuGp+Vi})G+$#q_Ue_TH&8bPw4(Zr<%?3OdIO)?X{fV-elO%oZ<)T1dli~Jj5|2(7 zxa|oYz1|>y>zPrXb|C z3~qla@jj^lRj*%3JU<}ld5wbG2PIy5rNFm}AL9|=F7#}5Mdi&ViL2*2SuaY?!Xbv^ zU0H@|y|JlTJhK>!iT|?hSU%68Tm$@--RQZp498>#i7$a(x-gy8_-6+EgADkW8Sq@F z=6L)mW_vumBm-WN0jE!?ACJ$R40uBZd~pW+_6#_EG4^=nuFHV`CIh}91O8|Rd@uvP zI|IHy1AZt2ek21vngO2xKXp7km>KZXGT>)rz|YHoUz7p=Mh5)fGT>GQ{JR4V z_`(c$M+Q8a0lzB){?iQjeHrjSXTUdQz@N{6zmWldCjg{ydAbp_e6sH1rpC4#(#3B`vy>FgrkwBWF1y7J;+V@p?aV|@dk zPYgG9h1K~xI!OkekpE(6fE1x>MVe@7y47fiEM9!;)Qi7y={F3VaBFWvWV!Dh)e7+N>-w?utaMVUq%Pui)ZpWEaKKTjOtZWj1l#WV5 zEGZ@3(dkt@;+BtjrA}E&5f;=(mKtI8z{=)hqoC(QqWDcGQcIKE0O?E`eyE`OHoAj+ zXj|Mgv@|v02j}+im&>G+Ww=MXPyJc>@(TJ}R$6YfM(P)kwxnM17+lcU+1cL7DPSDwYV73TF;PN6O)ZVB4V*}( z0QN$KlnNl7Dl9dfNQH2FaNf@PTR`2S_)sC~L_T21lF)g_lrShf+}=T1f(y9AwWft$ z0ES2x4CQf3RDN-&<5n&s+&-orVQTtD@LsAU7|WR@Xz3Cso9UO%Hn((%)6Fei^3ZY@ zj`87yb4!;vbZy| zMBT>f8ue}bex$yYQ%Sct&Yil~6%%gPpNO(aQyW#ZjiUC{ekf#YLy*R~gHS4}Bl;eT zk1msHuppg2)k+CM`o@lhk*20D_&;JT8%#PCFe5osWtn*!EFdgMDI*=t6i-BnGv|?3 ze&$2bLM7OafUOJ3VM{_#9Db!U>7=&>>~4rTbVWyB({e;o1|4o+*xugSSkI|&h1f5l z9N~5xXNqeNwVvA`$1_^_d>#M#F^{E-;yrXZ@fYPf&54g@!09u($Kyl)_w4cTnX-H*|5r2M zjTk@aqLrQuW%2XP5~pJ3;vc=HLf3r`dQ1oS`h!M4Pow9r0N|PUT&v-m9eiY1$#`Dl zL$Oa)?mskqzJ@<57ox=fyBfY*!>`xycQyP54gZ&h->Bhd$+#l%rx=!!r(WVpZ|5M5 zt~QN+0iG58n;KrH;RSMuP5kRMyh!5oBST`iGmMKgdY%7|G`tRUs@wq$*Yz`j3y6IuD0*FQZ5poY?ROfkm%CrX7lNUZ^N@yzG<<;!Sx_~|J}dB#t`>=t{JOn; zU4PZ+bvxgo;Z2}Z z<-VffH)(jTj58l2zlQ7bU!dW-{PhwiImzC<_(#_wjb7LDT^g?Ui{EH?Gx96>AJ%YP z{=Z6`%59+n<@J;$+Xd zo%~9}b${|V4W}5cvbSLe|369oUug8Y|MbZq1eHs+qxf8@;krE6N}SrAKC8GbK*L2Jz_^WX5EPEB$;f z@v;njEC>A$T_QuP#-HSIN&Nc``o$8z!$Chm;W zcb_8Co%H8s!0(kf=|`VeJfz{bAWiv=7bLFqCLfKx8ofT>7}oHm8lUV_g`8Bb?)T2s z@LM%{pTt$Uv`V3JB_tTG9EsE8Jp5Djy-VVXzv6TD8I((`2T24kJ>Z%maVMW= zB(C_V_3x8k5&4K-*Td5q9z~jxXSas0)bO*@1N>8R-m2j-4gbBwRk{3K3Z(CJ;P**+UeWmI@+2JegOdJW2Kr0B zhC*;T!*kBY11{2o&gU!*zY}Rn&dW4>wT1^YyiUXGHC)$cRKsnJ-qvuP&;QbJU7o*a zxGqmZ;;LQxPzGHeY4m!#oN$gvSN5sw>Qsr7essG!SHrJIn$pi38op4&Kh*HMG~7*2 z6PFwRbp3xt!*xG!fy9-Z^j|g8b(u!5*K3}J>-?8#xGv|vYq(DTkcRi7JXPO+YPe3H zTf|~$H)YPfFymuk3f57%qB&gXUw*X_AS!|%}Kyi4LrZ^Ke=Khx-!Y4pF>aJ^oe zHGGXmKVx#bzPD-k%^C2!G+dYe_ZlA8`21bNmuvXj8SpPOT(`GL=ceV<>orrub^p_# z;kunH)o|S}-K*g`{lglr({Iu68?}1v)NozS4>RCjYPinl{PWWFZPfUeYq+lGCJk@U z=zk`0HIArxMQ(ALzuu0gOI+F8DT2&6TccmA@vqcyU7p1fSLJSycKf(SAJX_dtKoNR zxKSbmpzKW=zCgouKl8YT*JgyV=>*ua1B(2b!UawzYAn+XgP`&6}8C?%bob;@Z3xf{&QzZS18R!!luG`y} z8m{Xh=Rz)?+gG25oUY+|`<}1iHzBLi|NR=Sm-{CT*XgHC73C^D(D^sIE|<7d56@}z zG=F=HsSV?m4D<<&{zTATEBSw_(Juh5 z?pF_MxIVtS%8(hC(vNy){X~hA{&jhNqv3kHJgnh$$fMeEyM{+Jd_==_|K_%FT=xa54ozLAGu9y3$gHM74z_nYW*T;p=HC%7sN2iH=lzk!o zDfyq4IO&=C3!VR`>t&5z_cK4ZRHPFhef<3o4cGnQDVL?`>u{SY_ZtFN?R&Shx3kLA z^t#`kq2apxS82HJubMSnmuHQJ+nPMT&~SY`y(t`AoP<@FzV}9n^bV6D3aAPw-FK&m@hW<`w%?yJ+;^(dfVK zpg#p^bWPFde~M?t{}K(qM#C@H@SkaTm4kmh(&(zu=yiFn)99t>On-w$zYd6!XOV{M z>xhVhPn5af{{q+Obv}1$^rW}lS8F)Y ztM}jfG@R&ZPCysgHeIA=qF3+Ht;aoGO8@6b{tpN^MIX%*zy=K`{w0$BF%2jF>OIeX z4JZ0bCH)Q!Cwlc>*-i~7dU_9qu07%>C69W)Zoh^TeTAe?XgKqi_ByQLL_b~9k7zj4 zlY^y;6eE9#p5A|<%cJ2$uigVx``${QGbFv5&l1l3J;cQ@yc!?kUoGj2HJs^BU~)sf zuc-LbdpC5IJLuK>x0M=B{A(n=rQyV1y$4vU;Y5G6q^I{>>7pN^SMNjCX*ki(mGo*J zq3G3nm1=*RaH79X(l6Ec5dYze@CR2^!-@Vol76*@6TNybx=+K2{(4EjPQ!^_y?44^ z!-@VzN&kR`6TNzWdxM4({Q^n%f02>5CmWy%$VZsrX6p|1XJ`JMceAywZXHRpOQd|8I%c zI`GXBpYOo`Ch0u_{9?6;K08v@y8ta6%y~)aDu#Yyx-x#YvnnQoesQC;(Hu;RO0&` zc%|f@$bb)Lz(*W-pXAR9LmH`_72Br4C%xw>$%0 znE|&l;I$6CPU>@h2D~l<-kbsN$bc`+fJYtp@adwVR%gKbGT`eR_#Vk;y#s$t;tx3R z^%CFUz#oP!D8A6*JBmF4C;aHZ$M47fJ~uD*Aw_$xl88R*L$xT;sB1Apsl zLT<}}Pd-cFwGO;m;_6(LlBYr9bq@L`CEo17lM?T6;HP^E}D}I*Hdg z@O;Uq*@3URSjgYu!1E=ar4D>p@`*a|T1me;1K#Jr_eehL9C(Ms*E?{JY)7vUm)l&L z1|zcg6kaLm<~r!t%jaeXUP}#!>ml)z;-lKN-+}Lx^1bB5<-DD~V?-Cp$o47c?Pogh zsKlo`aP|GEW(TgmKNWT0>H{Qd{8HscJuHTy#v_HR?@zICnw)C=asd%)cvP+{EeD=j zPdV_YTo?btfvfLKseVn#^iRbny;83VSKpcPJMcPM^x#r-I$PnxCkr{%cMBD+z9$ux_M-6UDT4kt4qSatYO4cR z-;;XFfvfLHO&llqrTEMF!+Zy>z8kg7fvfL8?Qr1gyHSR;U&UX2XDaBx)pw)TIq>LY zA>U&TTzxmnGhwX!>bp_34qSaV>Rtz~*6%wVxQZLR@4!{Ot626MRWB7cTk62o_o3E1 za23b+vjbQ0kR1+O#f1_MT*a?Gci<|nSSJ$F_mBfu--p`iz*FCcqKzF~s$3Nx4mxo4eU1MI_9oJa literal 0 HcmV?d00001 diff --git a/lpeg/makefile b/lpeg/makefile new file mode 100644 index 00000000..87169a22 --- /dev/null +++ b/lpeg/makefile @@ -0,0 +1,42 @@ +LIBNAME = lpeg +LUADIR = /usr/include/lua5.1/ + +COPT = -O2 -DNDEBUG + +CWARNS = -Wall -Wextra -pedantic \ + -Waggregate-return \ + -Wbad-function-cast \ + -Wcast-align \ + -Wcast-qual \ + -Wdeclaration-after-statement \ + -Wdisabled-optimization \ + -Wmissing-prototypes \ + -Wnested-externs \ + -Wpointer-arith \ + -Wshadow \ + -Wsign-compare \ + -Wstrict-prototypes \ + -Wundef \ + -Wwrite-strings \ + # -Wunreachable-code \ + + +CFLAGS = $(CWARNS) $(COPT) -ansi -I$(LUADIR) +CC = gcc + +# For Linux +DLLFLAGS = -shared -fpic +ENV = + +# For Mac OS +# ENV = MACOSX_DEPLOYMENT_TARGET=10.4 +# DLLFLAGS = -bundle -undefined dynamic_lookup + +lpeg.so: lpeg.o + env $(ENV) $(CC) $(DLLFLAGS) lpeg.o -o lpeg.so + +lpeg.o: makefile lpeg.c lpeg.h + +test: test.lua re.lua lpeg.so + test.lua + diff --git a/lpeg/re.html b/lpeg/re.html new file mode 100644 index 00000000..d67e23a3 --- /dev/null +++ b/lpeg/re.html @@ -0,0 +1,486 @@ + + + + LPeg.re - Regex syntax for LPEG + + + + + + + +
+ +
+ +
LPeg.re
+
+ Regex syntax for LPEG +
+
+ +
+ + + +
+ +

The re Module

+ +

+The re module +(provided by file re.lua in the distribution) +supports a somewhat conventional regex syntax +for pattern usage within LPeg. +

+ +

+The next table summarizes re's syntax. +A p represents an arbitrary pattern; +num represents a number ([0-9]+); +name represents an identifier +([a-zA-Z][a-zA-Z0-9_]*). +Constructions are listed in order of decreasing precedence. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
SyntaxDescription
( p ) grouping
'string' literal string
"string" literal string
[class] character class
. any character
%namepattern defs[name] or a pre-defined pattern
namenon terminal
<name>non terminal
{} position capture
{ p } simple capture
{: p :} anonymous group capture
{:name: p :} named group capture
{~ p ~} substitution capture
=name back reference +
p ? optional match
p * zero or more repetitions
p + one or more repetitions
p^num exactly n repetitions
p^+numat least n repetitions
p^-numat most n repetitions
p -> 'string' string capture
p -> "string" string capture
p -> {} table capture
p -> name function/query/string capture +equivalent to p / defs[name]
p => name match-time capture +equivalent to lpeg.Cmt(p, defs[name])
& p and predicate
! p not predicate
p1 p2 concatenation
p1 / p2 ordered choice
(name <- p)+ grammar
+

+Any space appearing in a syntax description can be +replaced by zero or more space characters and Lua-style comments +(-- until end of line). +

+ +

+Character classes define sets of characters. +An initial ^ complements the resulting set. +A range x-y includes in the set +all characters with codes between the codes of x and y. +A pre-defined class %name includes all +characters of that class. +A simple character includes itself in the set. +The only special characters inside a class are ^ +(special only if it is the first character); +] +(can be included in the set as the first character, +after the optional ^); +% (special only if followed by a letter); +and - +(can be included in the set as the first or the last character). +

+ +

+Currently the pre-defined classes are similar to those from the +Lua's string library +(%a for letters, +%A for non letters, etc.). +There is also a class %nl +containing only the newline character, +which is particularly handy for grammars written inside long strings, +as long strings do not interpret escape sequences like \n. +

+ + +

Functions

+ +

re.compile (string, [, defs])

+

+Compiles the given string and +returns an equivalent LPeg pattern. +The given string may define either an expression or a grammar. +The optional defs table provides extra Lua values +to be used by the pattern. +

+ +

re.find (subject, pattern [, init])

+

+Searches the given pattern in the given subject. +If it finds a match, +returns the index where this occurrence starts, +plus the captures made by the pattern (if any). +Otherwise, returns nil. +

+ +

+An optional numeric argument init makes the search +starts at that position in the subject string. +As usual in Lua libraries, +a negative value counts from the end. +

+ +

re.match (subject, pattern)

+

+Matches the given pattern against the given subject. +

+ +

re.updatelocale ()

+

+Updates the pre-defined character classes to the current locale. +

+ + +

Some Examples

+ +

A complete simple program

+

+The next code shows a simple complete Lua program using +the re module: +

+
+local re = require"re"
+
+-- find the position of the first number in a string
+print(re.find("the number 423 is odd", "[0-9]+"))  --> 12
+
+-- similar, but also captures (and returns) the number
+print(re.find("the number 423 is odd", "{[0-9]+}"))  --> 12    423
+
+-- returns all words in a string
+print(re.match("the number 423 is odd", "({%a+} / .)*"))
+--> the    number    is    odd
+
+ + +

Balanced parentheses

+

+The following call will produce the same pattern produced by the +Lua expression in the +balanced parentheses example: +

+
+b = re.compile[[  balanced <- "(" ([^()] / balanced)* ")"  ]]
+
+ +

String reversal

+

+The next example reverses a string: +

+
+rev = re.compile[[ R <- (!.) -> '' / ({.} R) -> '%2%1']]
+print(rev:match"0123456789")   --> 9876543210
+
+ +

CSV decoder

+

+The next example replicates the CSV decoder: +

+
+record = re.compile[[
+  record <- ( field (',' field)* ) -> {} (%nl / !.)
+  field <- escaped / nonescaped
+  nonescaped <- { [^,"%nl]* }
+  escaped <- '"' {~ ([^"] / '""' -> '"')* ~} '"'
+]]
+
+ +

Lua's long strings

+

+The next example matches Lua long strings: +

+
+c = re.compile([[
+  longstring <- ('[' {:eq: '='* :} '[' close) -> void
+  close <- ']' =eq ']' / . close
+]], {void = function () end})
+
+print(c:match'[==[]]===]]]]==]===[]')   --> 17
+
+ +

Abstract Syntax Trees

+

+This example shows a simple way to build an +abstract syntax tree (AST) for a given grammar. +To keep our example simple, +let us consider the following grammar +for lists of names: +

+
+p = re.compile[[
+      listname <- (name s)*
+      name <- [a-z][a-z]*
+      s <- %s*
+]]
+
+

+Now, we will add captures to build a corresponding AST. +As a first step, the pattern will build a table to +represent each non terminal; +terminals will be represented by their corresponding strings: +

+
+c = re.compile[[
+      listname <- (name s)* -> {}
+      name <- {[a-z][a-z]*} -> {}
+      s <- %s*
+]]
+
+

+Now, a match against "hi hello bye" +results in the table +{{"hi"}, {"hello"}, {"bye"}}. +

+

+For such a simple grammar, +this AST is more than enough; +actually, the tables around each single name +are already overkilling. +More complex grammars, +however, may need some more structure. +Specifically, +it would be useful if each table had +a tag field telling what non terminal +that table represents. +We can add such a tag using +named group captures: +

+
+x = re.compile[[
+      listname <- ({:tag: '' -> 'list':} (name s)*) -> {}
+      name <- ({:tag: '' -> 'id':} {[a-z][a-z]*}) -> {}
+      s <- ' '*
+]]
+
+

+With these group captures, +a match against "hi hello bye" +results in the following table: +

+
+{tag="list",
+  {tag="id", "hi"},
+  {tag="id", "hello"},
+  {tag="id", "bye"}
+}
+
+ + +

Indented blocks

+

+This example breaks indented blocks into tables, +respecting the indentation: +

+
+p = re.compile[[
+  block <- ({:ident:' '*:} line
+           ((=ident !' ' line) / &(=ident ' ') block)*) -> {}
+  line <- {[^%nl]*} %nl
+]]
+
+

+As an example, +consider the following text: +

+
+t = p:match[[
+first line
+  subline 1
+  subline 2
+second line
+third line
+  subline 3.1
+    subline 3.1.1
+  subline 3.2
+]]
+
+

+The resulting table t will be like this: +

+
+   {'first line'; {'subline 1'; 'subline 2'; ident = '  '};
+    'second line';
+    'third line'; { 'subline 3.1'; {'subline 3.1.1'; ident = '    '};
+                    'subline 3.2'; ident = '  '};
+    ident = ''}
+
+ +

Macro expander

+

+This example implements a simple macro expander. +Macros must be defined as part of the pattern, +following some simple rules: +

+
+p = re.compile[[
+      text <- {~ item* ~}
+      item <- macro / [^()] / '(' item* ')'
+      arg <- ' '* {~ (!',' item)* ~}
+      args <- '(' arg (',' arg)* ')'
+      -- now we define some macros
+      macro <- ('apply' args) -> '%1(%2)'
+             / ('add' args) -> '%1 + %2'
+             / ('mul' args) -> '%1 * %2'
+]]
+
+print(p:match"add(mul(a,b), apply(f,x))")   --> a * b + f(x)
+
+

+A text is a sequence of items, +wherein we apply a substitution capture to expand any macros. +An item is either a macro, +any character different from parentheses, +or a parenthesized expression. +A macro argument (arg) is a sequence +of items different from a comma. +(Note that a comma may appear inside an item, +e.g., inside a parenthesized expression.) +Again we do a substitution capture to expand any macro +in the argument before expanding the outer macro. +args is a list of arguments separated by commas. +Finally we define the macros. +Each macro is a string substitution; +it replaces the macro name and its arguments by its corresponding string, +with each %n replaced by the n-th argument. +

+ +

Patterns

+

+This example shows the complete syntax +of patterns accepted by re. +

+
+p = [=[
+
+pattern         <- exp !.
+exp             <- S (alternative / grammar)
+
+alternative     <- seq ('/' S seq)*
+seq             <- prefix*
+prefix          <- '&' S prefix / '!' S prefix / suffix
+suffix          <- primary S (([+*?]
+                            / '^' [+-]? num
+                            / '->' S (string / '{}' / name)
+                            / '=>' S name) S)*
+
+primary         <- '(' exp ')' / string / class / defined
+                 / '{:' (name ':')? exp ':}'
+                 / '=' name
+                 / '{}'
+                 / '{~' exp '~}'
+                 / '{' exp '}'
+                 / '.'
+                 / name S !arrow
+                 / '<' name '>'          -- old-style non terminals
+
+grammar         <- definition+
+definition      <- name S arrow exp
+
+class           <- '[' '^'? item (!']' item)* ']'
+item            <- defined / range / .
+range           <- . '-' [^]]
+
+S               <- (%s / '--' [^%nl]*)*   -- spaces and comments
+name            <- [A-Za-z][A-Za-z0-9_]*
+arrow           <- '<-'
+num             <- [0-9]+
+string          <- '"' [^"]* '"' / "'" [^']* "'"
+defined         <- '%' name
+
+]=]
+
+print(re.match(p, p))   -- a self description must match itself
+
+ + + +

License

+ +

+Copyright © 2008-2010 Lua.org, PUC-Rio. +

+

+Permission is hereby granted, free of charge, +to any person obtaining a copy of this software and +associated documentation files (the "Software"), +to deal in the Software without restriction, +including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, +and to permit persons to whom the Software is +furnished to do so, +subject to the following conditions: +

+ +

+The above copyright notice and this permission notice +shall be included in all copies or substantial portions of the Software. +

+ +

+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +

+ +
+ +
+ +
+

+$Id: re.html,v 1.17 2011/01/10 15:08:06 roberto Exp $ +

+
+ +
+ + + diff --git a/lpeg/re.lua b/lpeg/re.lua new file mode 100644 index 00000000..2320c206 --- /dev/null +++ b/lpeg/re.lua @@ -0,0 +1,248 @@ +-- $Id: re.lua,v 1.39 2010/11/04 19:44:18 roberto Exp $ + +-- imported functions and modules +local tonumber, type, print, error = tonumber, type, print, error +local setmetatable = setmetatable +local m = require"lpeg" + +-- 'm' will be used to parse expressions, and 'mm' will be used to +-- create expressions; that is, 're' runs on 'm', creating patterns +-- on 'mm' +local mm = m + +-- pattern's metatable +local mt = getmetatable(mm.P(0)) + + + +-- No more global accesses after this point +local version = _VERSION +if version == "Lua 5.2" then _ENV = nil end + + +local any = m.P(1) + + +-- Pre-defined names +local Predef = { nl = m.P"\n" } + + +local mem +local fmem +local gmem + + +local function updatelocale () + mm.locale(Predef) + Predef.a = Predef.alpha + Predef.c = Predef.cntrl + Predef.d = Predef.digit + Predef.g = Predef.graph + Predef.l = Predef.lower + Predef.p = Predef.punct + Predef.s = Predef.space + Predef.u = Predef.upper + Predef.w = Predef.alnum + Predef.x = Predef.xdigit + Predef.A = any - Predef.a + Predef.C = any - Predef.c + Predef.D = any - Predef.d + Predef.G = any - Predef.g + Predef.L = any - Predef.l + Predef.P = any - Predef.p + Predef.S = any - Predef.s + Predef.U = any - Predef.u + Predef.W = any - Predef.w + Predef.X = any - Predef.x + mem = {} -- restart memoization + fmem = {} + gmem = {} + local mt = {__mode = "v"} + setmetatable(mem, mt) + setmetatable(fmem, mt) + setmetatable(gmem, mt) +end + + +updatelocale() + + + +local I = m.P(function (s,i) print(i, s:sub(1, i-1)); return i end) + + +local function getdef (id, Defs) + local c = Defs and Defs[id] + if not c then error("undefined name: " .. id) end + return c +end + + +local function patt_error (s, i) + local msg = (#s < i + 20) and s:sub(i) + or s:sub(i,i+20) .. "..." + msg = ("pattern error near '%s'"):format(msg) + error(msg, 2) +end + +local function mult (p, n) + local np = mm.P(true) + while n >= 1 do + if n%2 >= 1 then np = np * p end + p = p * p + n = n/2 + end + return np +end + +local function equalcap (s, i, c) + if type(c) ~= "string" then return nil end + local e = #c + i + if s:sub(i, e - 1) == c then return e else return nil end +end + + +local S = (m.S(" \f\n\r\t\v") + "--" * (any - Predef.nl)^0)^0 + +local name = m.R("AZ", "az") * m.R("AZ", "az", "__", "09")^0 + +local arrow = S * "<-" + +local exp_follow = m.P"/" + ")" + "}" + ":}" + "~}" + (name * arrow) + -1 + +name = m.C(name) + + +-- identifiers only have meaning in a given environment +local Identifier = name * m.Carg(1) + +local num = m.C(m.R"09"^1) * S / tonumber + +local String = "'" * m.C((any - "'")^0) * "'" + + '"' * m.C((any - '"')^0) * '"' + + +local defined = "%" * Identifier / function (c,Defs) + local cat = Defs and Defs[c] or Predef[c] + if not cat then error ("name '" .. c .. "' undefined") end + return cat +end + +local Range = m.Cs(any * (m.P"-"/"") * (any - "]")) / mm.R + +local item = defined + Range + m.C(any) + +local Class = + "[" + * (m.C(m.P"^"^-1)) -- optional complement symbol + * m.Cf(item * (item - "]")^0, mt.__add) / + function (c, p) return c == "^" and any - p or p end + * "]" + +local function adddef (t, k, Defs, exp) + if t[k] then + error("'"..k.."' already defined as a rule") + else + t[k] = exp + end + return t +end + +local function firstdef (n, Defs, r) return adddef({n}, n, Defs, r) end + + + +local exp = m.P{ "Exp", + Exp = S * ( m.V"Grammar" + + m.Cf(m.V"Seq" * ("/" * S * m.V"Seq")^0, mt.__add) ); + Seq = m.Cf(m.Cc(m.P"") * m.V"Prefix"^0 , mt.__mul) + * (#exp_follow + patt_error); + Prefix = "&" * S * m.V"Prefix" / mt.__len + + "!" * S * m.V"Prefix" / mt.__unm + + m.V"Suffix"; + Suffix = m.Cf(m.V"Primary" * S * + ( ( m.P"+" * m.Cc(1, mt.__pow) + + m.P"*" * m.Cc(0, mt.__pow) + + m.P"?" * m.Cc(-1, mt.__pow) + + "^" * ( m.Cg(num * m.Cc(mult)) + + m.Cg(m.C(m.S"+-" * m.R"09"^1) * m.Cc(mt.__pow)) + ) + + "->" * S * ( m.Cg(String * m.Cc(mt.__div)) + + m.P"{}" * m.Cc(nil, m.Ct) + + m.Cg(Identifier / getdef * m.Cc(mt.__div)) + ) + + "=>" * S * m.Cg(Identifier / getdef * m.Cc(m.Cmt)) + ) * S + )^0, function (a,b,f) return f(a,b) end ); + Primary = "(" * m.V"Exp" * ")" + + String / mm.P + + Class + + defined + + "{:" * (name * ":" + m.Cc(nil)) * m.V"Exp" * ":}" / + function (n, p) return mm.Cg(p, n) end + + "=" * name / function (n) return mm.Cmt(mm.Cb(n), equalcap) end + + m.P"{}" / mm.Cp + + "{~" * m.V"Exp" * "~}" / mm.Cs + + "{" * m.V"Exp" * "}" / mm.C + + m.P"." * m.Cc(any) + + name * -arrow / mm.V + + "<" * name * ">" / mm.V; + Definition = Identifier * arrow * m.V"Exp"; + Grammar = m.Cf(m.V"Definition" / firstdef * m.Cg(m.V"Definition")^0, adddef) / + mm.P +} + +local pattern = S * exp / mm.P * (-any + patt_error) + + +local function compile (p, defs) + if mm.type(p) == "pattern" then return p end -- already compiled + local cp = pattern:match(p, 1, defs) + if not cp then error("incorrect pattern", 3) end + return cp +end + +local function match (s, p, i) + local cp = mem[p] + if not cp then + cp = compile(p) + mem[p] = cp + end + return cp:match(s, i or 1) +end + +local function find (s, p, i) + local cp = fmem[p] + if not cp then + cp = compile(p) + cp = mm.P{ mm.Cp() * cp + 1 * mm.V(1) } + fmem[p] = cp + end + return cp:match(s, i or 1) +end + +local function gsub (s, p, rep) + local g = gmem[p] or {} -- ensure gmem[p] is not collected while here + gmem[p] = g + local cp = g[rep] + if not cp then + cp = compile(p) + cp = mm.Cs((cp / rep + 1)^0) + g[rep] = cp + end + return cp:match(s) +end + + +-- exported names +local re = { + compile = compile, + match = match, + find = find, + gsub = gsub, + updatelocale = updatelocale, +} + +if version == "Lua 5.1" then _G.re = re end + +return re diff --git a/lpeg/test.lua b/lpeg/test.lua new file mode 100755 index 00000000..533a6d2d --- /dev/null +++ b/lpeg/test.lua @@ -0,0 +1,1188 @@ +#!/usr/bin/env lua5.1 + +-- $Id: test.lua,v 1.82 2010/12/03 14:49:54 roberto Exp $ + +require"strict" -- just to be pedantic + +local m = require"lpeg" + +local debug = require"debug" + + +-- compatibility with Lua 5.2 +local unpack = table.unpack or unpack + + +-- most tests here do not need much stack space +m.setmaxstack(5) + +any = m.P(1) +space = m.S" \t\n"^0 + +local function checkeq (x, y, p) +if p then print(x,y) end + if type(x) ~= "table" then assert(x == y) + else + for k,v in pairs(x) do checkeq(v, y[k], p) end + for k,v in pairs(y) do checkeq(v, x[k], p) end + end +end + + +mt = getmetatable(m.P(1)) + + +local allchar = {} +for i=0,255 do allchar[i + 1] = i end +allchar = string.char(unpack(allchar)) +assert(#allchar == 256) + +local function cs2str (c) + return m.match(m.Cs((c + m.P(1)/"")^0), allchar) +end + +local function eqcharset (c1, c2) + assert(cs2str(c1) == cs2str(c2)) +end + + +print"General tests for LPeg library" + +assert(type(m.version()) == "string") +print("version " .. m.version()) +assert(m.type("alo") ~= "pattern") +assert(m.type(io.input) ~= "pattern") +assert(m.type(m.P"alo") == "pattern") + +-- tests for some basic optimizations +assert(m.match(m.P(false) + "a", "a") == 2) +assert(m.match(m.P(true) + "a", "a") == 1) +assert(m.match("a" + m.P(false), "b") == nil) +assert(m.match("a" + m.P(true), "b") == 1) + +assert(m.match(m.P(false) * "a", "a") == nil) +assert(m.match(m.P(true) * "a", "a") == 2) +assert(m.match("a" * m.P(false), "a") == nil) +assert(m.match("a" * m.P(true), "a") == 2) + +assert(m.match(#m.P(false) * "a", "a") == nil) +assert(m.match(#m.P(true) * "a", "a") == 2) +assert(m.match("a" * #m.P(false), "a") == nil) +assert(m.match("a" * #m.P(true), "a") == 2) + + +-- tests for locale +do + assert(m.locale(m) == m) + local t = {} + assert(m.locale(t, m) == t) + local x = m.locale() + for n,v in pairs(x) do + assert(type(n) == "string") + eqcharset(v, m[n]) + end +end + + + +assert(m.match(3, "aaaa")) +assert(m.match(4, "aaaa")) +assert(not m.match(5, "aaaa")) +assert(m.match(-3, "aa")) +assert(not m.match(-3, "aaa")) +assert(not m.match(-3, "aaaa")) +assert(not m.match(-4, "aaaa")) +assert(m.P(-5):match"aaaa") + +assert(m.match("a", "alo") == 2) +assert(m.match("al", "alo") == 3) +assert(not m.match("alu", "alo")) +assert(m.match(true, "") == 1) + +digit = m.S"0123456789" +upper = m.S"ABCDEFGHIJKLMNOPQRSTUVWXYZ" +lower = m.S"abcdefghijklmnopqrstuvwxyz" +letter = m.S"" + upper + lower +alpha = letter + digit + m.R() + +eqcharset(m.S"", m.P(false)) +eqcharset(upper, m.R("AZ")) +eqcharset(lower, m.R("az")) +eqcharset(upper + lower, m.R("AZ", "az")) +eqcharset(upper + lower, m.R("AZ", "cz", "aa", "bb", "90")) +eqcharset(digit, m.S"01234567" + "8" + "9") +eqcharset(upper, letter - lower) +eqcharset(m.S(""), m.R()) +assert(cs2str(m.S("")) == "") + +eqcharset(m.S"\0", "\0") +eqcharset(m.S"\1\0\2", m.R"\0\2") +eqcharset(m.S"\1\0\2", m.R"\1\2" + "\0") +eqcharset(m.S"\1\0\2" - "\0", m.R"\1\2") + +word = alpha^1 * (1 - alpha)^0 + +assert((word^0 * -1):match"alo alo") +assert(m.match(word^1 * -1, "alo alo")) +assert(m.match(word^2 * -1, "alo alo")) +assert(not m.match(word^3 * -1, "alo alo")) + +assert(not m.match(word^-1 * -1, "alo alo")) +assert(m.match(word^-2 * -1, "alo alo")) +assert(m.match(word^-3 * -1, "alo alo")) + +eos = m.P(-1) + +assert(m.match(digit^0 * letter * digit * eos, "1298a1")) +assert(not m.match(digit^0 * letter * eos, "1257a1")) + +b = { + [1] = "(" * (((1 - m.S"()") + #m.P"(" * m.V(1))^0) * ")" +} + +assert(m.match(b, "(al())()")) +assert(not m.match(b * eos, "(al())()")) +assert(m.match(b * eos, "((al())()(é))")) +assert(not m.match(b, "(al()()")) + +assert(not m.match(letter^1 - "for", "foreach")) +assert(m.match(letter^1 - ("for" * eos), "foreach")) +assert(not m.match(letter^1 - ("for" * eos), "for")) + +function basiclookfor (p) + return m.P { + [1] = p + (1 * m.V(1)) + } +end + +function caplookfor (p) + return basiclookfor(p:C()) +end + +assert(m.match(caplookfor(letter^1), " 4achou123...") == "achou") +a = {m.match(caplookfor(letter^1)^0, " two words, one more ")} +checkeq(a, {"two", "words", "one", "more"}) + +assert(m.match( basiclookfor((#m.P(b) * 1) * m.Cp()), " ( (a)") == 7) + +a = {m.match(m.C(digit^1 * m.Cc"d") + m.C(letter^1 * m.Cc"l"), "123")} +checkeq(a, {"123", "d"}) + +a = {m.match(m.C(digit^1) * "d" * -1 + m.C(letter^1 * m.Cc"l"), "123d")} +checkeq(a, {"123"}) + +a = {m.match(m.C(digit^1 * m.Cc"d") + m.C(letter^1 * m.Cc"l"), "abcd")} +checkeq(a, {"abcd", "l"}) + +a = {m.match(m.Cc(10,20,30) * 'a' * m.Cp(), 'aaa')} +checkeq(a, {10,20,30,2}) +a = {m.match(m.Cp() * m.Cc(10,20,30) * 'a' * m.Cp(), 'aaa')} +checkeq(a, {1,10,20,30,2}) +a = m.match(m.Ct(m.Cp() * m.Cc(10,20,30) * 'a' * m.Cp()), 'aaa') +checkeq(a, {1,10,20,30,2}) +a = m.match(m.Ct(m.Cp() * m.Cc(7,8) * m.Cc(10,20,30) * 'a' * m.Cp()), 'aaa') +checkeq(a, {1,7,8,10,20,30,2}) +a = {m.match(m.Cc() * m.Cc() * m.Cc(1) * m.Cc(2,3,4) * m.Cc() * 'a', 'aaa')} +checkeq(a, {1,2,3,4}) + +a = {m.match(m.Cp() * letter^1 * m.Cp(), "abcd")} +checkeq(a, {1, 5}) + + +t = {m.match({[1] = m.C(m.C(1) * m.V(1) + -1)}, "abc")} +checkeq(t, {"abc", "a", "bc", "b", "c", "c", ""}) + + +-- test for small capture boundary +for i = 250,260 do + assert(#m.match(m.C(i), string.rep('a', i)) == i) + assert(#m.match(m.C(m.C(i)), string.rep('a', i)) == i) +end + + +-- tests for any*n +for n = 1, 550 do + local x_1 = string.rep('x', n - 1) + local x = x_1 .. 'a' + assert(not m.P(n):match(x_1)) + assert(m.P(n):match(x) == n + 1) + assert(n < 4 or m.match(m.P(n) + "xxx", x_1) == 4) + assert(m.C(n):match(x) == x) + assert(m.C(m.C(n)):match(x) == x) + assert(m.P(-n):match(x_1) == 1) + assert(not m.P(-n):match(x)) + assert(n < 13 or m.match(m.Cc(20) * ((n - 13) * m.P(10)) * 3, x) == 20) + local n3 = math.floor(n/3) + assert(m.match(n3 * m.Cp() * n3 * n3, x) == n3 + 1) +end + +assert(m.P(0):match("x") == 1) +assert(m.P(0):match("") == 1) +assert(m.C(0):match("x") == "") +assert(m.match(m.Cc(0) * m.P(10) + m.Cc(1) * "xuxu", "xuxu") == 1) +assert(m.match(m.Cc(0) * m.P(10) + m.Cc(1) * "xuxu", "xuxuxuxuxu") == 0) +assert(m.match(m.C(m.P(2)^1), "abcde") == "abcd") +p = m.Cc(0) * 1 + m.Cc(1) * 2 + m.Cc(2) * 3 + m.Cc(3) * 4 + + +-- test for alternation optimization +assert(m.match(m.P"a"^1 + "ab" + m.P"x"^0, "ab") == 2) +assert(m.match((m.P"a"^1 + "ab" + m.P"x"^0 * 1)^0, "ab") == 3) +assert(m.match(m.P"ab" + "cd" + "" + "cy" + "ak", "98") == 1) +assert(m.match(m.P"ab" + "cd" + "ax" + "cy", "ax") == 3) +assert(m.match("a" * m.P"b"^0 * "c" + "cd" + "ax" + "cy", "ax") == 3) +assert(m.match((m.P"ab" + "cd" + "ax" + "cy")^0, "ax") == 3) +assert(m.match(m.P(1) * "x" + m.S"" * "xu" + "ay", "ay") == 3) +assert(m.match(m.P"abc" + "cde" + "aka", "aka") == 4) +assert(m.match(m.S"abc" * "x" + "cde" + "aka", "ax") == 3) +assert(m.match(m.S"abc" * "x" + "cde" + "aka", "aka") == 4) +assert(m.match(m.S"abc" * "x" + "cde" + "aka", "cde") == 4) +assert(m.match(m.S"abc" * "x" + "ide" + m.S"ab" * "ka", "aka") == 4) +assert(m.match("ab" + m.S"abc" * m.P"y"^0 * "x" + "cde" + "aka", "ax") == 3) +assert(m.match("ab" + m.S"abc" * m.P"y"^0 * "x" + "cde" + "aka", "aka") == 4) +assert(m.match("ab" + m.S"abc" * m.P"y"^0 * "x" + "cde" + "aka", "cde") == 4) +assert(m.match("ab" + m.S"abc" * m.P"y"^0 * "x" + "ide" + m.S"ab" * "ka", "aka") == 4) +assert(m.match("ab" + m.S"abc" * m.P"y"^0 * "x" + "ide" + m.S"ab" * "ka", "ax") == 3) +assert(m.match(m.P(1) * "x" + "cde" + m.S"ab" * "ka", "aka") == 4) +assert(m.match(m.P(1) * "x" + "cde" + m.P(1) * "ka", "aka") == 4) +assert(m.match(m.P(1) * "x" + "cde" + m.P(1) * "ka", "cde") == 4) +assert(m.match(m.P"eb" + "cd" + m.P"e"^0 + "x", "ee") == 3) +assert(m.match(m.P"ab" + "cd" + m.P"e"^0 + "x", "abcd") == 3) +assert(m.match(m.P"ab" + "cd" + m.P"e"^0 + "x", "eeex") == 4) +assert(m.match(m.P"ab" + "cd" + m.P"e"^0 + "x", "cd") == 3) +assert(m.match(m.P"ab" + "cd" + m.P"e"^0 + "x", "x") == 1) +assert(m.match(m.P"ab" + "cd" + m.P"e"^0 + "x" + "", "zee") == 1) +assert(m.match(m.P"ab" + "cd" + m.P"e"^1 + "x", "abcd") == 3) +assert(m.match(m.P"ab" + "cd" + m.P"e"^1 + "x", "eeex") == 4) +assert(m.match(m.P"ab" + "cd" + m.P"e"^1 + "x", "cd") == 3) +assert(m.match(m.P"ab" + "cd" + m.P"e"^1 + "x", "x") == 2) +assert(m.match(m.P"ab" + "cd" + m.P"e"^1 + "x" + "", "zee") == 1) + +pi = "3.14159 26535 89793 23846 26433 83279 50288 41971 69399 37510" +assert(m.match(m.Cs((m.P"1" / "a" + m.P"5" / "b" + m.P"9" / "c" + 1)^0), pi) == + m.match(m.Cs((m.P(1) / {["1"] = "a", ["5"] = "b", ["9"] = "c"})^0), pi)) +print"+" + + +-- tests for capture optimizations +assert(m.match((m.P(3) + 4 * m.Cp()) * "a", "abca") == 5) +t = {m.match(((m.P"a" + m.Cp()) * m.P"x")^0, "axxaxx")} +checkeq(t, {3, 6}) + +-- test for table captures +t = m.match(m.Ct(letter^1), "alo") +checkeq(t, {}) + +t, n = m.match(m.Ct(m.C(letter)^1) * m.Cc"t", "alo") +assert(n == "t" and table.concat(t) == "alo") + +t = m.match(m.Ct(m.C(m.C(letter)^1)), "alo") +assert(table.concat(t, ";") == "alo;a;l;o") + +t = m.match(m.Ct(m.C(m.C(letter)^1)), "alo") +assert(table.concat(t, ";") == "alo;a;l;o") + +t = m.match(m.Ct(m.Ct((m.Cp() * letter * m.Cp())^1)), "alo") +assert(table.concat(t[1], ";") == "1;2;2;3;3;4") + +t = m.match(m.Ct(m.C(m.C(1) * 1 * m.C(1))), "alo") +checkeq(t, {"alo", "a", "o"}) + + +-- tests for groups +p = m.Cg(1) -- no capture +assert(p:match('x') == 'x') +p = m.Cg(m.P(true)/function () end * 1) -- no value +assert(p:match('x') == 'x') +p = m.Cg(m.Cg(m.Cg(m.C(1)))) +assert(p:match('x') == 'x') +p = m.Cg(m.Cg(m.Cg(m.C(1))^0) * m.Cg(m.Cc(1) * m.Cc(2))) +t = {p:match'abc'} +checkeq(t, {'a', 'b', 'c', 1, 2}) + +-- test for non-pattern as arguments to pattern functions + +p = { ('a' * m.V(1))^-1 } * m.P'b' * { 'a' * m.V(2); m.V(1)^-1 } +assert(m.match(p, "aaabaac") == 7) + +-- a large table capture +t = m.match(m.Ct(m.C('a')^0), string.rep("a", 10000)) +assert(#t == 10000 and t[1] == 'a' and t[#t] == 'a') + + +-- test for errors +local function checkerr (msg, ...) + assert(m.match({ m.P(msg) + 1 * m.V(1) }, select(2, pcall(...)))) +end + +-- checkerr("rule '1' is left recursive", m.match, { m.V(1) * 'a' }, "a") +checkerr("rule '1' outside a grammar", m.match, m.V(1), "") +checkerr("rule 'hiii' outside a grammar", m.match, m.V('hiii'), "") +checkerr("rule 'hiii' is not defined", m.match, { m.V('hiii') }, "") +checkerr("rule is not defined", m.match, { m.V{} }, "") + +checkerr("rule 'A' is not a pattern", m.P, { A = {} }) +checkerr("rule is not a pattern", m.P, { [print] = {} }) + +-- test for non-pattern as arguments to pattern functions + +p = { ('a' * m.V(1))^-1 } * m.P'b' * { 'a' * m.V(2); m.V(1)^-1 } +assert(m.match(p, "aaabaac") == 7) + +-- a large table capture +t = m.match(m.Ct(m.C('a')^0), string.rep("a", 10000)) +assert(#t == 10000 and t[1] == 'a' and t[#t] == 'a') + + +-- test for errors +local function checkerr (msg, ...) + assert(m.match({ m.P(msg) + 1 * m.V(1) }, select(2, pcall(...)))) +end + +checkerr("rule '1' is left recursive", m.match, { m.V(1) * 'a' }, "a") +checkerr("rule '1' outside a grammar", m.match, m.V(1), "") +checkerr("rule 'hiii' outside a grammar", m.match, m.V('hiii'), "") +checkerr("rule 'hiii' is not defined", m.match, { m.V('hiii') }, "") +checkerr("rule is not defined", m.match, { m.V({}) }, "") + +print('+') + + +-- bug in 0.10 (rechecking a grammar, after tail-call optimization) +m.P{ m.P { (m.P(3) + "xuxu")^0 * m.V"xuxu", xuxu = m.P(1) } } + +local V = m.V + +local Space = m.S(" \n\t")^0 +local Number = m.C(m.R("09")^1) * Space +local FactorOp = m.C(m.S("+-")) * Space +local TermOp = m.C(m.S("*/")) * Space +local Open = "(" * Space +local Close = ")" * Space + + +local function f_factor (v1, op, v2, d) + assert(d == nil) + if op == "+" then return v1 + v2 + else return v1 - v2 + end +end + + +local function f_term (v1, op, v2, d) + assert(d == nil) + if op == "*" then return v1 * v2 + else return v1 / v2 + end +end + +G = m.P{ "Exp", + Exp = m.Cf(V"Factor" * m.Cg(FactorOp * V"Factor")^0, f_factor); + Factor = m.Cf(V"Term" * m.Cg(TermOp * V"Term")^0, f_term); + Term = Number / tonumber + Open * V"Exp" * Close; +} + +G = Space * G * -1 + +for _, s in ipairs{" 3 + 5*9 / (1+1) ", "3+4/2", "3+3-3- 9*2+3*9/1- 8"} do + assert(m.match(G, s) == loadstring("return "..s)()) +end + + +-- test for grammars (errors deep in calling non-terminals) +g = m.P{ + [1] = m.V(2) + "a", + [2] = "a" * m.V(3) * "x", + [3] = "b" * m.V(3) + "c" +} + +assert(m.match(g, "abbbcx") == 7) +assert(m.match(g, "abbbbx") == 2) + + +-- tests for \0 +assert(m.match(m.R("\0\1")^1, "\0\1\0") == 4) +assert(m.match(m.S("\0\1ab")^1, "\0\1\0a") == 5) +assert(m.match(m.P(1)^3, "\0\1\0a") == 5) +assert(not m.match(-4, "\0\1\0a")) +assert(m.match("\0\1\0a", "\0\1\0a") == 5) +assert(m.match("\0\0\0", "\0\0\0") == 4) +assert(not m.match("\0\0\0", "\0\0")) + + +-- tests for predicates +assert(not m.match(-m.P("a") * 2, "alo")) +assert(m.match(- -m.P("a") * 2, "alo") == 3) +assert(m.match(#m.P("a") * 2, "alo") == 3) +assert(m.match(##m.P("a") * 2, "alo") == 3) +assert(not m.match(##m.P("c") * 2, "alo")) +assert(m.match(m.Cs((##m.P("a") * 1 + m.P(1)/".")^0), "aloal") == "a..a.") +assert(m.match(m.Cs((#((#m.P"a")/"") * 1 + m.P(1)/".")^0), "aloal") == "a..a.") +assert(m.match(m.Cs((- -m.P("a") * 1 + m.P(1)/".")^0), "aloal") == "a..a.") +assert(m.match(m.Cs((-((-m.P"a")/"") * 1 + m.P(1)/".")^0), "aloal") == "a..a.") + + +-- look-behind predicate +assert(not m.match(m.B'a', 'a')) +assert(m.match(1 * m.B'a', 'a') == 2) +assert(not m.match(m.B(1), 'a')) +assert(m.match(1 * m.B(1), 'a') == 2) +assert(m.match(-m.B(1), 'a') == 1) + +B = #letter * -m.B(letter) + -letter * m.B(letter) +x = m.Ct({ (B * m.Cp())^-1 * (1 * m.V(1) + m.P(true)) }) +checkeq(m.match(x, 'ar cal c'), {1,3,4,7,9,10}) +checkeq(m.match(x, ' ar cal '), {2,4,5,8}) +checkeq(m.match(x, ' '), {}) +checkeq(m.match(x, 'aloalo'), {1,7}) + +assert(m.match(B, "a") == 1) +assert(m.match(1 * B, "a") == 2) +assert(not m.B(-letter):match("")) +assert((-m.B(letter)):match("") == 1) + +assert((4 * m.B(letter, 4)):match("aaaaaaaa") == 5) +assert(not (4 * m.B(letter, 5)):match("aaaaaaaa")) +assert((4 * -m.B(letter, 5)):match("aaaaaaaa") == 5) + +assert((3 * m.B(m.C(1))):match("12345") == 4) + + +-- bug in 0.9 +assert(m.match(('a' * #m.P'b'), "ab") == 2) +assert(not m.match(('a' * #m.P'b'), "a")) + +assert(not m.match(#m.S'567', "")) +assert(m.match(#m.S'567' * 1, "6") == 2) + + +-- tests for Tail Calls + +-- create a grammar for a simple DFA for even number of 0s and 1s +-- finished in '$': +-- +-- ->1 <---0---> 2 +-- ^ ^ +-- | | +-- 1 1 +-- | | +-- V V +-- 3 <---0---> 4 +-- +-- this grammar should keep no backtracking information + +p = m.P{ + [1] = '0' * m.V(2) + '1' * m.V(3) + '$', + [2] = '0' * m.V(1) + '1' * m.V(4), + [3] = '0' * m.V(4) + '1' * m.V(1), + [4] = '0' * m.V(3) + '1' * m.V(2), +} + +assert(p:match(string.rep("00", 10000) .. "$")) +assert(p:match(string.rep("01", 10000) .. "$")) +assert(p:match(string.rep("011", 10000) .. "$")) +assert(not p:match(string.rep("011", 10001) .. "$")) + + +-- this grammar does need backtracking info. +local lim = 10000 +p = m.P{ '0' * m.V(1) + '0' } +assert(not pcall(m.match, p, string.rep("0", lim))) +m.setmaxstack(2*lim) +assert(not pcall(m.match, p, string.rep("0", lim))) +m.setmaxstack(2*lim + 2) +assert(pcall(m.match, p, string.rep("0", lim))) + +-- tests for optional start position +assert(m.match("a", "abc", 1)) +assert(m.match("b", "abc", 2)) +assert(m.match("c", "abc", 3)) +assert(not m.match(1, "abc", 4)) +assert(m.match("a", "abc", -3)) +assert(m.match("b", "abc", -2)) +assert(m.match("c", "abc", -1)) +assert(m.match("abc", "abc", -4)) -- truncate to position 1 + +assert(m.match("", "abc", 10)) -- empty string is everywhere! +assert(m.match("", "", 10)) +assert(not m.match(1, "", 1)) +assert(not m.match(1, "", -1)) +assert(not m.match(1, "", 0)) + +print("+") + + +-- tests for argument captures +assert(not pcall(m.Carg, 0)) +assert(not pcall(m.Carg, -1)) +assert(not pcall(m.Carg, 2^18)) +assert(not pcall(m.match, m.Carg(1), 'a', 1)) +assert(m.match(m.Carg(1), 'a', 1, print) == print) +x = {m.match(m.Carg(1) * m.Carg(2), '', 1, 10, 20)} +checkeq(x, {10, 20}) + +assert(m.match(m.Cmt(m.Cg(m.Carg(3), "a") * + m.Cmt(m.Cb("a"), function (s,i,x) + assert(s == "a" and i == 1); + return i, x+1 + end) * + m.Carg(2), function (s,i,a,b,c) + assert(s == "a" and i == 1 and c == nil); + return i, 2*a + 3*b + end) * "a", + "a", 1, false, 100, 1000) == 2*1001 + 3*100) + + +-- tests for Lua functions + +t = {} +s = "" +p = function (s1, i) assert(s == s1); t[#t + 1] = i; return nil end +s = "hi, this is a test" +assert(m.match(((p - m.P(-1)) + 2)^0, s) == string.len(s) + 1) +assert(#t == string.len(s)/2 and t[1] == 1 and t[2] == 3) + +assert(not m.match(p, s)) + +p = mt.__add(function (s, i) return i end, function (s, i) return nil end) +assert(m.match(p, "alo")) + +p = mt.__mul(function (s, i) return i end, function (s, i) return nil end) +assert(not m.match(p, "alo")) + + +t = {} +p = function (s1, i) assert(s == s1); t[#t + 1] = i; return i end +s = "hi, this is a test" +assert(m.match((m.P(1) * p)^0, s) == string.len(s) + 1) +assert(#t == string.len(s) and t[1] == 2 and t[2] == 3) + +t = {} +p = m.P(function (s1, i) assert(s == s1); t[#t + 1] = i; + return i <= s1:len() and i + 1 end) +s = "hi, this is a test" +assert(m.match(p^0, s) == string.len(s) + 1) +assert(#t == string.len(s) + 1 and t[1] == 1 and t[2] == 2) + +p = function (s1, i) return m.match(m.P"a"^1, s1, i) end +assert(m.match(p, "aaaa") == 5) +assert(m.match(p, "abaa") == 2) +assert(not m.match(p, "baaa")) + +assert(not pcall(m.match, function () return 2^20 end, s)) +assert(not pcall(m.match, function () return 0 end, s)) +assert(not pcall(m.match, function (s, i) return i - 1 end, s)) +assert(not pcall(m.match, m.P(1)^0 * function (_, i) return i - 1 end, s)) +assert(m.match(m.P(1)^0 * function (_, i) return i end * -1, s)) +assert(not pcall(m.match, m.P(1)^0 * function (_, i) return i + 1 end, s)) +assert(m.match(m.P(function (s, i) return s:len() + 1 end) * -1, s)) +assert(not pcall(m.match, m.P(function (s, i) return s:len() + 2 end) * -1, s)) +assert(not m.match(m.P(function (s, i) return s:len() end) * -1, s)) +assert(m.match(m.P(1)^0 * function (_, i) return true end, s) == + string.len(s) + 1) +for i = 1, string.len(s) + 1 do + assert(m.match(function (_, _) return i end, s) == i) +end + +p = (m.P(function (s, i) return i%2 == 0 and i + 1 end) + + m.P(function (s, i) return i%2 ~= 0 and i + 2 <= s:len() and i + 3 end))^0 + * -1 +assert(p:match(string.rep('a', 14000))) + +-- tests for Function Replacements +f = function (a, ...) if a ~= "x" then return {a, ...} end end + +t = m.match(m.C(1)^0/f, "abc") +checkeq(t, {"a", "b", "c"}) + +t = m.match(m.C(1)^0/f/f, "abc") +checkeq(t, {{"a", "b", "c"}}) + +t = m.match(m.P(1)^0/f/f, "abc") -- no capture +checkeq(t, {{"abc"}}) + +t = m.match((m.P(1)^0/f * m.Cp())/f, "abc") +checkeq(t, {{"abc"}, 4}) + +t = m.match((m.C(1)^0/f * m.Cp())/f, "abc") +checkeq(t, {{"a", "b", "c"}, 4}) + +t = m.match((m.C(1)^0/f * m.Cp())/f, "xbc") +checkeq(t, {4}) + +t = m.match(m.C(m.C(1)^0)/f, "abc") +checkeq(t, {"abc", "a", "b", "c"}) + +g = function (...) return 1, ... end +t = {m.match(m.C(1)^0/g/g, "abc")} +checkeq(t, {1, 1, "a", "b", "c"}) + +t = {m.match(m.Cc(nil,nil,4) * m.Cc(nil,3) * m.Cc(nil, nil) / g / g, "")} +t1 = {1,1,nil,nil,4,nil,3,nil,nil} +for i=1,10 do assert(t[i] == t1[i]) end + +t = {m.match((m.C(1) / function (x) return x, x.."x" end)^0, "abc")} +checkeq(t, {"a", "ax", "b", "bx", "c", "cx"}) + +t = m.match(m.Ct((m.C(1) / function (x,y) return y, x end * m.Cc(1))^0), "abc") +checkeq(t, {nil, "a", 1, nil, "b", 1, nil, "c", 1}) + +-- tests for Query Replacements + +assert(m.match(m.C(m.C(1)^0)/{abc = 10}, "abc") == 10) +assert(m.match(m.C(1)^0/{a = 10}, "abc") == 10) +assert(m.match(m.S("ba")^0/{ab = 40}, "abc") == 40) +t = m.match(m.Ct((m.S("ba")/{a = 40})^0), "abc") +checkeq(t, {40}) + +assert(m.match(m.Cs((m.C(1)/{a=".", d=".."})^0), "abcdde") == ".bc....e") +assert(m.match(m.Cs((m.C(1)/{f="."})^0), "abcdde") == "abcdde") +assert(m.match(m.Cs((m.C(1)/{d="."})^0), "abcdde") == "abc..e") +assert(m.match(m.Cs((m.C(1)/{e="."})^0), "abcdde") == "abcdd.") +assert(m.match(m.Cs((m.C(1)/{e=".", f="+"})^0), "eefef") == "..+.+") +assert(m.match(m.Cs((m.C(1))^0), "abcdde") == "abcdde") +assert(m.match(m.Cs(m.C(m.C(1)^0)), "abcdde") == "abcdde") +assert(m.match(1 * m.Cs(m.P(1)^0), "abcdde") == "bcdde") +assert(m.match(m.Cs((m.C('0')/'x' + 1)^0), "abcdde") == "abcdde") +assert(m.match(m.Cs((m.C('0')/'x' + 1)^0), "0ab0b0") == "xabxbx") +assert(m.match(m.Cs((m.C('0')/'x' + m.P(1)/{b=3})^0), "b0a0b") == "3xax3") +assert(m.match(m.P(1)/'%0%0'/{aa = -3} * 'x', 'ax') == -3) +assert(m.match(m.C(1)/'%0%1'/{aa = 'z'}/{z = -3} * 'x', 'ax') == -3) + +assert(m.match(m.Cs(m.Cc(0) * (m.P(1)/"")), "4321") == "0") + +assert(m.match(m.Cs((m.P(1) / "%0")^0), "abcd") == "abcd") +assert(m.match(m.Cs((m.P(1) / "%0.%0")^0), "abcd") == "a.ab.bc.cd.d") +assert(m.match(m.Cs((m.P("a") / "%0.%0" + 1)^0), "abcad") == "a.abca.ad") +assert(m.match(m.C("a") / "%1%%%0", "a") == "a%a") +assert(m.match(m.Cs((m.P(1) / ".xx")^0), "abcd") == ".xx.xx.xx.xx") +assert(m.match(m.Cp() * m.P(3) * m.Cp()/"%2%1%1 - %0 ", "abcde") == + "411 - abc ") + +assert(pcall(m.match, m.P(1)/"%0", "abc")) +assert(not pcall(m.match, m.P(1)/"%1", "abc")) -- out of range +assert(not pcall(m.match, m.P(1)/"%9", "abc")) -- out of range + +p = m.C(1) +p = p * p; p = p * p; p = p * p * m.C(1) / "%9 - %1" +assert(p:match("1234567890") == "9 - 1") + +assert(m.match(m.Cc(print), "") == print) + +-- too many captures (just ignore extra ones) +p = m.C(1)^0 / "%2-%9-%0-%9" +assert(p:match"01234567890123456789" == "1-8-01234567890123456789-8") +s = string.rep("12345678901234567890", 20) +assert(m.match(m.C(1)^0 / "%9-%1-%0-%3", s) == "9-1-" .. s .. "-3") + +-- string captures with non-string subcaptures +p = m.Cc('alo') * m.C(1) / "%1 - %2 - %1" +assert(p:match'x' == 'alo - x - alo') + +assert(not pcall(m.match, m.Cc(true) / "%1", "a")) + +-- long strings for string capture +l = 10000 +s = string.rep('a', l) .. string.rep('b', l) .. string.rep('c', l) + +p = (m.C(m.P'a'^1) * m.C(m.P'b'^1) * m.C(m.P'c'^1)) / '%3%2%1' + +assert(p:match(s) == string.rep('c', l) .. + string.rep('b', l) .. + string.rep('a', l)) + +print"+" + +-- accumulator capture +function f (x) return x + 1 end +assert(m.match(m.Cf(m.Cc(0) * m.C(1)^0, f), "alo alo") == 7) + +t = {m.match(m.Cf(m.Cc(1,2,3), error), "")} +checkeq(t, {1}) +p = m.Cf(m.Ct(true) * m.Cg(m.C(m.R"az"^1) * "=" * m.C(m.R"az"^1) * ";")^0, + rawset) +t = p:match("a=b;c=du;xux=yuy;") +checkeq(t, {a="b", c="du", xux="yuy"}) + + +-- tests for loop checker + +local function haveloop (p) + assert(not pcall(function (p) return p^0 end, m.P(p))) +end + +haveloop(m.P("x")^-4) +assert(m.match(((m.P(0) + 1) * m.S"al")^0, "alo") == 3) +assert(m.match((("x" + #m.P(1))^-4 * m.S"al")^0, "alo") == 3) +haveloop("") +haveloop(m.P("x")^0) +haveloop(m.P("x")^-1) +haveloop(m.P("x") + 1 + 2 + m.P("a")^-1) +haveloop(-m.P("ab")) +haveloop(- -m.P("ab")) +haveloop(# #(m.P("ab") + "xy")) +haveloop(- #m.P("ab")^0) +haveloop(# -m.P("ab")^1) +haveloop(#m.V(3)) +haveloop(m.V(3) + m.V(1) + m.P('a')^-1) +haveloop({[1] = m.V(2) * m.V(3), [2] = m.V(3), [3] = m.P(0)}) +assert(m.match(m.P{[1] = m.V(2) * m.V(3), [2] = m.V(3), [3] = m.P(1)}^0, "abc") + == 3) +assert(m.match(m.P""^-3, "a") == 1) + +local function find (p, s) + return m.match(basiclookfor(p), s) +end + + +local function badgrammar (g, exp) + local err, msg = pcall(m.P, g) + assert(not err) + if exp then assert(find(exp, msg)) end +end + +badgrammar({[1] = m.V(1)}, "rule '1'") +badgrammar({[1] = m.V(2)}, "rule '2'") -- invalid non-terminal +badgrammar({[1] = m.V"x"}, "rule 'x'") -- invalid non-terminal +badgrammar({[1] = m.V{}}, "rule ") -- invalid non-terminal +badgrammar({[1] = #m.P("a") * m.V(1)}, "rule '1'") +badgrammar({[1] = -m.P("a") * m.V(1)}, "rule '1'") +badgrammar({[1] = -1 * m.V(1)}, "rule '1'") +badgrammar({[1] = 1 * m.V(2), [2] = m.V(2)}, "rule '2'") +badgrammar({[1] = m.P(0), [2] = 1 * m.V(1)^0}, "loop in rule '2'") +badgrammar({ m.V(2), m.V(3)^0, m.P"" }, "rule '2'") +badgrammar({ m.V(2) * m.V(3)^0, m.V(3)^0, m.P"" }, "rule '1'") +badgrammar({ #(m.V(1) * 'a') }, "rule '1'") +badgrammar({ -(m.V(1) * 'a') }, "rule '1'") + +assert(m.match({'a' * -m.V(1)}, "aaa") == 2) +assert(m.match({'a' * -m.V(1)}, "aaaa") == nil) + + +-- simple tests for maximum sizes: +local p = m.P"a" +for i=1,14 do p = p * p end + +p = {} +for i=1,100 do p[i] = m.P"a" end +p = m.P(p) + + +-- strange values for rule labels + +p = m.P{ "print", + print = m.V(print), + [print] = m.V(_G), + [_G] = m.P"a", + } + +assert(p:match("a")) + +-- initial rule +g = {} +for i = 1, 10 do g["i"..i] = "a" * m.V("i"..i+1) end +g.i11 = m.P"" +for i = 1, 10 do + g[1] = "i"..i + local p = m.P(g) + assert(p:match("aaaaaaaaaaa") == 11 - i + 1) +end + +print"+" + + +-- tests for back references +assert(not pcall(m.match, m.Cb('x'), '')) +assert(not pcall(m.match, m.Cg(1, 'a') * m.Cb('b'), 'a')) + +p = m.Cg(m.C(1) * m.C(1), "k") * m.Ct(m.Cb("k")) +t = p:match("ab") +checkeq(t, {"a", "b"}) + + +t = {} +function foo (p) t[#t + 1] = p; return p .. "x" end + +p = m.Cg(m.C(2) / foo, "x") * m.Cb"x" * + m.Cg(m.Cb('x') / foo, "x") * m.Cb"x" * + m.Cg(m.Cb('x') / foo, "x") * m.Cb"x" * + m.Cg(m.Cb('x') / foo, "x") * m.Cb"x" +x = {p:match'ab'} +checkeq(x, {'abx', 'abxx', 'abxxx', 'abxxxx'}) +checkeq(t, {'ab', + 'ab', 'abx', + 'ab', 'abx', 'abxx', + 'ab', 'abx', 'abxx', 'abxxx'}) + + + +-- tests for match-time captures + +local function id (s, i, ...) + return true, ... +end + +assert(m.Cmt(m.Cs((m.Cmt(m.S'abc' / { a = 'x', c = 'y' }, id) + + m.R'09'^1 / string.char + + m.P(1))^0), id):match"acb98+68c" == "xyb\98+\68y") + +p = m.P{'S', + S = m.V'atom' * space + + m.Cmt(m.Ct("(" * space * (m.Cmt(m.V'S'^1, id) + m.P(true)) * ")" * space), id), + atom = m.Cmt(m.C(m.R("AZ", "az", "09")^1), id) +} +x = p:match"(a g () ((b) c) (d (e)))" +checkeq(x, {'a', 'g', {}, {{'b'}, 'c'}, {'d', {'e'}}}); + +x = {(m.Cmt(1, id)^0):match(string.rep('a', 500))} +assert(#x == 500) + +local function id(s, i, x) + if x == 'a' then return i + 1, 1, 3, 7 + else return nil, 2, 4, 6, 8 + end +end + +p = ((m.P(id) + m.Cmt(2, id) + m.Cmt(1, id)))^0 +assert(table.concat{p:match('abababab')} == string.rep('137', 4)) + +local function ref (s, i, x) + return m.match(x, s, i - x:len()) +end + +assert(m.Cmt(m.P(1)^0, ref):match('alo') == 4) +assert((m.P(1) * m.Cmt(m.P(1)^0, ref)):match('alo') == 4) +assert(not (m.P(1) * m.Cmt(m.C(1)^0, ref)):match('alo')) + +ref = function (s,i,x) return i == tonumber(x) and i, 'xuxu' end + +assert(m.Cmt(1, ref):match'2') +assert(not m.Cmt(1, ref):match'1') +assert(m.Cmt(m.P(1)^0, ref):match'03') + +function ref (s, i, a, b) + if a == b then return i, a:upper() end +end + +p = m.Cmt(m.C(m.R"az"^1) * "-" * m.C(m.R"az"^1), ref) +p = (any - p)^0 * p * any^0 * -1 + +assert(p:match'abbbc-bc ddaa' == 'BC') + + +c = '[' * m.Cg(m.P'='^0, "init") * '[' * + { m.Cmt(']' * m.C(m.P'='^0) * ']' * m.Cb("init"), function (_, _, s1, s2) + return s1 == s2 end) + + 1 * m.V(1) } / function () end + +assert(c:match'[==[]]====]]]]==]===[]' == 18) +assert(c:match'[[]=]====]=]]]==]===[]' == 14) +assert(not c:match'[[]=]====]=]=]==]===[]') + + + +------------------------------------------------------------------- +-- Tests for 're' module +------------------------------------------------------------------- + +local re = require "re" + +local match, compile = re.match, re.compile + + + +assert(match("a", ".") == 2) +assert(match("a", "''") == 1) +assert(match("", " ! . ") == 1) +assert(not match("a", " ! . ")) +assert(match("abcde", " ( . . ) * ") == 5) +assert(match("abbcde", " [a-c] +") == 5) +assert(match("0abbc1de", "'0' [a-c]+ '1'") == 7) +assert(match("0zz1dda", "'0' [^a-c]+ 'a'") == 8) +assert(match("abbc--", " [a-c] + +") == 5) +assert(match("abbc--", " [ac-] +") == 2) +assert(match("abbc--", " [-acb] + ") == 7) +assert(not match("abbcde", " [b-z] + ")) +assert(match("abb\"de", '"abb"["]"de"') == 7) +assert(match("abceeef", "'ac' ? 'ab' * 'c' { 'e' * } / 'abceeef' ") == "eee") +assert(match("abceeef", "'ac'? 'ab'* 'c' { 'f'+ } / 'abceeef' ") == 8) +local t = {match("abceefe", "( ( & 'e' {} ) ? . ) * ")} +checkeq(t, {4, 5, 7}) +local t = {match("abceefe", "((&&'e' {})? .)*")} +checkeq(t, {4, 5, 7}) +local t = {match("abceefe", "( ( ! ! 'e' {} ) ? . ) *")} +checkeq(t, {4, 5, 7}) +local t = {match("abceefe", "(( & ! & ! 'e' {})? .)*")} +checkeq(t, {4, 5, 7}) + +assert(match("cccx" , "'ab'? ('ccc' / ('cde' / 'cd'*)? / 'ccc') 'x'+") == 5) +assert(match("cdx" , "'ab'? ('ccc' / ('cde' / 'cd'*)? / 'ccc') 'x'+") == 4) +assert(match("abcdcdx" , "'ab'? ('ccc' / ('cde' / 'cd'*)? / 'ccc') 'x'+") == 8) + +assert(match("abc", "a <- (. a)?") == 4) +b = "balanced <- '(' ([^()] / balanced)* ')'" +assert(match("(abc)", b)) +assert(match("(a(b)((c) (d)))", b)) +assert(not match("(a(b ((c) (d)))", b)) + +b = compile[[ balanced <- "(" ([^()] / balanced)* ")" ]] +assert(b == m.P(b)) +assert(b:match"((((a))(b)))") + +local g = [[ + S <- "0" B / "1" A / "" -- balanced strings + A <- "0" S / "1" A A -- one more 0 + B <- "1" S / "0" B B -- one more 1 +]] +assert(match("00011011", g) == 9) + +local g = [[ + S <- ("0" B / "1" A)* + A <- "0" / "1" A A + B <- "1" / "0" B B +]] +assert(match("00011011", g) == 9) +assert(match("000110110", g) == 9) +assert(match("011110110", g) == 3) +assert(match("000110010", g) == 1) + +s = "aaaaaaaaaaaaaaaaaaaaaaaa" +assert(match(s, "'a'^3") == 4) +assert(match(s, "'a'^0") == 1) +assert(match(s, "'a'^+3") == s:len() + 1) +assert(not match(s, "'a'^+30")) +assert(match(s, "'a'^-30") == s:len() + 1) +assert(match(s, "'a'^-5") == 6) +for i = 1, s:len() do + assert(match(s, string.format("'a'^+%d", i)) >= i + 1) + assert(match(s, string.format("'a'^-%d", i)) <= i + 1) + assert(match(s, string.format("'a'^%d", i)) == i + 1) +end +assert(match("01234567890123456789", "[0-9]^3+") == 19) + + +assert(match("01234567890123456789", "({....}{...}) -> '%2%1'") == "4560123") +t = match("0123456789", "{.}*->{}") +checkeq(t, {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9"}) +assert(match("012345", "( (..) -> '%0%0' ) -> {}")[1] == "0101") + +eqcharset(compile"[]]", "]") +eqcharset(compile"[][]", m.S"[]") +eqcharset(compile"[]-]", m.S"-]") +eqcharset(compile"[-]", m.S"-") +eqcharset(compile"[az-]", m.S"a-z") +eqcharset(compile"[-az]", m.S"a-z") +eqcharset(compile"[a-z]", m.R"az") +eqcharset(compile"[]['\"]", m.S[[]['"]]) + +eqcharset(compile"[^]]", any - "]") +eqcharset(compile"[^][]", any - m.S"[]") +eqcharset(compile"[^]-]", any - m.S"-]") +eqcharset(compile"[^]-]", any - m.S"-]") +eqcharset(compile"[^-]", any - m.S"-") +eqcharset(compile"[^az-]", any - m.S"a-z") +eqcharset(compile"[^-az]", any - m.S"a-z") +eqcharset(compile"[^a-z]", any - m.R"az") +eqcharset(compile"[^]['\"]", any - m.S[[]['"]]) + +-- tests for comments in 're' +e = compile[[ +A <- B -- \t \n %nl .<> <- -> -- +B <- 'x' --]] +assert(e:match'xy' == 2) + +-- tests for 're' with pre-definitions +defs = {digits = m.R"09", letters = m.R"az"} +e = compile("%letters (%letters / %digits)*", defs) +assert(e:match"x123" == 5) + +e = compile([[ + S <- A+ + A <- %letters+ B + B <- %digits+ +]], defs) + +e = compile("{[0-9]+'.'?[0-9]*} -> sin", math) +assert(e:match("2.34") == math.sin(2.34)) + + +function eq (_, _, a, b) return a == b end + +c = re.compile([[ + longstring <- '[' {:init: '='* :} '[' close + close <- ']' =init ']' / . close +]]) + +assert(c:match'[==[]]===]]]]==]===[]' == 17) +assert(c:match'[[]=]====]=]]]==]===[]' == 14) +assert(not c:match'[[]=]====]=]=]==]===[]') + +c = re.compile" '[' {:init: '='* :} '[' (!(']' =init ']') .)* ']' =init ']' !. " + +assert(c:match'[==[]]===]]]]==]') +assert(c:match'[[]=]====]=][]==]===[]]') +assert(not c:match'[[]=]====]=]=]==]===[]') + +assert(re.find("hi alalo", "{:x:..:} =x") == 4) +assert(re.find("hi alalo", "{:x:..:} =x", 4) == 4) +assert(not re.find("hi alalo", "{:x:..:} =x", 5)) +assert(re.find("hi alalo", "'al'", 5) == 6) +assert(re.find("hi aloalolo", "{:x:..:} =x") == 8) +assert(re.find("alo alohi x x", "{:word:%w+:}%W*(=word)!%w") == 11) + +assert(re.gsub("alo alo", "[abc]", "x") == "xlo xlo") +assert(re.gsub("alo alo", "%w+", ".") == ". .") +assert(re.gsub("hi, how are you", "[aeiou]", string.upper) == + "hI, hOw ArE yOU") + +s = 'hi [[a comment[=]=] ending here]] and [=[another]]=]]' +c = re.compile" '[' {:i: '='* :} '[' (!(']' =i ']') .)* ']' { =i } ']' " +assert(re.gsub(s, c, "%2") == 'hi and =]') +assert(re.gsub(s, c, "%0") == s) +assert(re.gsub('[=[hi]=]', c, "%2") == '=') + +assert(re.find("", "!.") == 1) +assert(re.find("alo", "!.") == 4) + +function addtag (s, i, t, tag) t.tag = tag; return i, t end + +c = re.compile([[ + doc <- block !. + block <- (start (block / { [^<]+ })* -> {} end?) => addtag + start <- '<' {:tag: [a-z]+ :} '>' + end <- '' +]], {addtag = addtag}) + +x = c:match[[ +hihellobuttotheend]] +checkeq(x, {tag='x', 'hi', {tag = 'b', 'hello'}, 'but', + {'totheend'}}) + +assert(not pcall(compile, "x <- 'a' x <- 'b'")) +assert(not pcall(compile, "'x' -> x", {x = 3})) + + +-- tests for look-ahead captures +x = {re.match("alo", "&(&{.}) !{'b'} {&(...)} &{..} {...} {!.}")} +checkeq(x, {"", "alo", ""}) + +assert(re.match("aloalo", + "{~ (((&'al' {.}) -> 'A%1' / (&%l {.}) -> '%1%1') / .)* ~}") + == "AallooAalloo") + +-- bug in 0.9 (and older versions), due to captures in look-aheads +x = re.compile[[ {~ (&(. ([a-z]* -> '*')) ([a-z]+ -> '+') ' '*)* ~} ]] +assert(x:match"alo alo" == "+ +") + +-- valid capture in look-ahead (used inside the look-ahead itself) +x = re.compile[[ + S <- &({:two: .. :} . =two) {[a-z]+} / . S +]] +assert(x:match("hello aloaLo aloalo xuxu") == "aloalo") + + +p = re.compile[[ + block <- ({:ident:' '*:} line + ((=ident !' ' line) / &(=ident ' ') block)*) -> {} + line <- {[^%nl]*} %nl +]] + +t= p:match[[ +1 + 1.1 + 1.2 + 1.2.1 + +2 + 2.1 +]] +checkeq(t, {"1", {"1.1", "1.2", {"1.2.1", "", ident = " "}, ident = " "}, + "2", {"2.1", ident = " "}, ident = ""}) + + +-- nested grammars +p = re.compile[[ + s <- a b !. + b <- ( x <- ('b' x)? ) + a <- ( x <- 'a' x? ) +]] + +assert(p:match'aaabbb') +assert(p:match'aaa') +assert(not p:match'bbb') +assert(not p:match'aaabbba') + +-- testing groups +t = {re.match("abc", "{:S <- {:.:} {S} / '':}")} +checkeq(t, {"a", "bc", "b", "c", "c", ""}) + +t = re.match("1234", "({:a:.:} {:b:.:} {:c:.{.}:}) -> {}") +checkeq(t, {a="1", b="2", c="4"}) +t = re.match("1234", "({:a:.:} {:b:{.}{.}:} {:c:{.}:}) -> {}") +checkeq(t, {a="1", b="2", c="4"}) +t = re.match("12345", "({:.:} {:b:{.}{.}:} {:{.}{.}:}) -> {}") +checkeq(t, {"1", b="2", "4", "5"}) +t = re.match("12345", "({:.:} {:{:b:{.}{.}:}:} {:{.}{.}:}) -> {}") +checkeq(t, {"1", "23", "4", "5"}) +t = re.match("12345", "({:.:} {{:b:{.}{.}:}} {:{.}{.}:}) -> {}") +checkeq(t, {"1", "23", "4", "5"}) + + +-- testing pre-defined names +assert(os.setlocale("C") == "C") + +function eqlpeggsub (p1, p2) + local s1 = cs2str(re.compile(p1)) + local s2 = string.gsub(allchar, "[^" .. p2 .. "]", "") +if s1 ~= s2 then print(s1,s2) end + assert(s1 == s2) +end + + +eqlpeggsub("%w", "%w") +eqlpeggsub("%a", "%a") +eqlpeggsub("%l", "%l") +eqlpeggsub("%u", "%u") +eqlpeggsub("%p", "%p") +eqlpeggsub("%d", "%d") +eqlpeggsub("%x", "%x") +eqlpeggsub("%s", "%s") + +eqlpeggsub("%W", "%W") +eqlpeggsub("%A", "%A") +eqlpeggsub("%L", "%L") +eqlpeggsub("%U", "%U") +eqlpeggsub("%P", "%P") +eqlpeggsub("%D", "%D") +eqlpeggsub("%X", "%X") +eqlpeggsub("%S", "%S") + +eqlpeggsub("[%w]", "%w") +eqlpeggsub("[_%w]", "_%w") +eqlpeggsub("[^%w]", "%W") +eqlpeggsub("[%W%S]", "%W%S") + +re.updatelocale() + + +-- testing nested substitutions x string captures + +p = re.compile[[ + text <- {~ item* ~} + item <- macro / [^()] / '(' item* ')' + arg <- ' '* {~ (!',' item)* ~} + args <- '(' arg (',' arg)* ')' + macro <- ('apply' args) -> '%1(%2)' + / ('add' args) -> '%1 + %2' + / ('mul' args) -> '%1 * %2' +]] + +assert(p:match"add(mul(a,b), apply(f,x))" == "a * b + f(x)") + +rev = re.compile[[ R <- (!.) -> '' / ({.} R) -> '%2%1']] + +assert(rev:match"0123456789" == "9876543210") + +print"OK" + + diff --git a/lua-int64/Makefile b/lua-int64/Makefile new file mode 100644 index 00000000..82eacaa6 --- /dev/null +++ b/lua-int64/Makefile @@ -0,0 +1,6 @@ +install : + gcc -g -Wall -fPIC --shared -o int64.so int64.c + +clean : + rm int64.so + diff --git a/lua-int64/README b/lua-int64/README new file mode 100644 index 00000000..b08921aa --- /dev/null +++ b/lua-int64/README @@ -0,0 +1 @@ +See https://github.com/cloudwu/lua-int64 diff --git a/lua-int64/int64.c b/lua-int64/int64.c new file mode 100644 index 00000000..abdbb6de --- /dev/null +++ b/lua-int64/int64.c @@ -0,0 +1,317 @@ +#include +#include +#include +#include +#include +#include + +static int64_t +_int64(lua_State *L, int index) { + int type = lua_type(L,index); + int64_t n = 0; + switch(type) { + case LUA_TNUMBER: { + lua_Number d = lua_tonumber(L,index); + n = (int64_t)d; + break; + } + case LUA_TSTRING: { + size_t len = 0; + const uint8_t * str = (const uint8_t *)lua_tolstring(L, index, &len); + if (len>8) { + return luaL_error(L, "The string (length = %d) is not an int64 string", len); + } + int i = 0; + uint64_t n64 = 0; + for (i=0;i<(int)len;i++) { + n64 |= (uint64_t)str[i] << (i*8); + } + n = (int64_t)n64; + break; + } + case LUA_TLIGHTUSERDATA: { + void * p = lua_touserdata(L,index); + n = (intptr_t)p; + break; + } + default: + return luaL_error(L, "argument 1 error type %s", lua_typename(L,1)); + } + return n; +} + +static inline void +_pushint64(lua_State *L, int64_t n) { + void * p = (void *)(intptr_t)n; + lua_pushlightuserdata(L,p); +} + +static int +int64_add(lua_State *L) { + int64_t a = _int64(L,1); + int64_t b = _int64(L,2); + _pushint64(L, a+b); + + return 1; +} + +static int +int64_sub(lua_State *L) { + int64_t a = _int64(L,1); + int64_t b = _int64(L,2); + _pushint64(L, a-b); + + return 1; +} + +static int +int64_mul(lua_State *L) { + int64_t a = _int64(L,1); + int64_t b = _int64(L,2); + _pushint64(L, a * b); + + return 1; +} + +static int +int64_div(lua_State *L) { + int64_t a = _int64(L,1); + int64_t b = _int64(L,2); + if (b == 0) { + return luaL_error(L, "div by zero"); + } + _pushint64(L, a / b); + + return 1; +} + +static int +int64_mod(lua_State *L) { + int64_t a = _int64(L,1); + int64_t b = _int64(L,2); + if (b == 0) { + return luaL_error(L, "mod by zero"); + } + _pushint64(L, a % b); + + return 1; +} + +static int64_t +_pow64(int64_t a, int64_t b) { + if (b == 1) { + return a; + } + int64_t a2 = a * a; + if (b % 2 == 1) { + return _pow64(a2, b/2) * a; + } else { + return _pow64(a2, b/2); + } +} + +static int +int64_pow(lua_State *L) { + int64_t a = _int64(L,1); + int64_t b = _int64(L,2); + int64_t p; + if (b > 0) { + p = _pow64(a,b); + } else if (b == 0) { + p = 1; + } else { + return luaL_error(L, "pow by nagtive number %d",(int)b); + } + _pushint64(L, p); + + return 1; +} + +static int +int64_unm(lua_State *L) { + int64_t a = _int64(L,1); + _pushint64(L, -a); + return 1; +} + +static int +int64_new(lua_State *L) { + int top = lua_gettop(L); + int64_t n; + switch(top) { + case 0 : + lua_pushlightuserdata(L,NULL); + break; + case 1 : + n = _int64(L,1); + _pushint64(L,n); + break; + default: { + int base = luaL_checkinteger(L,2); + if (base < 2) { + luaL_error(L, "base must be >= 2"); + } + const char * str = lua_tostring(L, 1); + n = strtoll(str, NULL, base); + _pushint64(L,n); + break; + } + } + return 1; +} + +static int +int64_eq(lua_State *L) { + // __eq metamethod can't be invoke by lightuserdata + int64_t a = _int64(L,1); + int64_t b = _int64(L,2); + lua_pushboolean(L,a == b); + return 1; +} + +static int +int64_lt(lua_State *L) { + int64_t a = _int64(L,1); + int64_t b = _int64(L,2); + lua_pushboolean(L,a < b); + return 1; +} + +static int +int64_le(lua_State *L) { + int64_t a = _int64(L,1); + int64_t b = _int64(L,2); + lua_pushboolean(L,a <= b); + return 1; +} + +static int +int64_len(lua_State *L) { + int64_t a = _int64(L,1); + lua_pushnumber(L,(lua_Number)a); + return 1; +} + +static int +tostring(lua_State *L) { + static char hex[16] = "0123456789ABCDEF"; + uintptr_t n = (uintptr_t)lua_touserdata(L,1); + if (lua_gettop(L) == 1) { + luaL_Buffer b; + luaL_buffinitsize(L , &b , 28); + luaL_addstring(&b, "int64: 0x"); + int i; + bool strip = true; + for (i=15;i>=0;i--) { + int c = (n >> (i*4)) & 0xf; + if (strip && c ==0) { + continue; + } + strip = false; + luaL_addchar(&b, hex[c]); + } + if (strip) { + luaL_addchar(&b , '0'); + } + luaL_pushresult(&b); + } else { + int base = luaL_checkinteger(L,2); + int shift =0, mask =0; + switch(base) { + case 0: { + unsigned char buffer[8]; + int i; + for (i=0;i<8;i++) { + buffer[i] = (n >> (i*8)) & 0xff; + } + lua_pushlstring(L,(const char *)buffer, 8); + return 1; + } + case 10: { + int64_t dec = (int64_t)n; + luaL_Buffer b; + luaL_buffinitsize(L , &b , 28); + if (dec<0) { + luaL_addchar(&b, '-'); + dec = -dec; + } + int buffer[32]; + int i; + for (i=0;i<32;i++) { + buffer[i] = dec%10; + dec /= 10; + if (dec == 0) + break; + } + while (i>=0) { + luaL_addchar(&b, hex[buffer[i]]); + --i; + } + luaL_pushresult(&b); + return 1; + } + case 2: + shift = 1; + mask = 1; + break; + case 8: + shift = 3; + mask = 7; + break; + case 16: + shift = 4; + mask = 0xf; + break; + default: + luaL_error(L, "Unsupport base %d",base); + break; + } + int i; + char buffer[64]; + for (i=0;i<64;i+=shift) { + buffer[i/shift] = hex[(n>>(64-shift-i)) & mask]; + } + lua_pushlstring(L, buffer, 64 / shift); + } + return 1; +} + +static void +make_mt(lua_State *L) { + luaL_Reg lib[] = { + { "__add", int64_add }, + { "__sub", int64_sub }, + { "__mul", int64_mul }, + { "__div", int64_div }, + { "__mod", int64_mod }, + { "__unm", int64_unm }, + { "__pow", int64_pow }, + { "__eq", int64_eq }, + { "__lt", int64_lt }, + { "__le", int64_le }, + { "__len", int64_len }, + { "__tostring", tostring }, + { NULL, NULL }, + }; + luaL_newlib(L,lib); +} + +int +luaopen_int64(lua_State *L) { + if (sizeof(intptr_t)!=sizeof(int64_t)) { + return luaL_error(L, "Only support 64bit architecture"); + } + lua_pushlightuserdata(L,NULL); + make_mt(L); + lua_setmetatable(L,-2); + lua_pop(L,1); + + lua_newtable(L); + lua_pushcfunction(L, int64_new); + lua_setfield(L, -2, "new"); + lua_pushcfunction(L, tostring); + lua_setfield(L, -2, "tostring"); + + return 1; +} + diff --git a/lua-int64/int64.so b/lua-int64/int64.so new file mode 100755 index 0000000000000000000000000000000000000000..b48db918f3fb7f917e93f5605db3d9f43eade328 GIT binary patch literal 24305 zcmeHvdw5jUx%b*LlQ2NY3?y7d8EzIVf>)zPdr~jk&yXQ(GPha|XQ=g_-NPZFi$d2OM)g3Uz ztg?}W1@Jc$e=Ey>efEX7@4fx(cfNA)%&nWI{&U9%`TeiGuUiBZvK($PQh3K?3LgdQ z&x9n`fdGcXqgnK~WYOQ71-~JSpLJREmjfR+Oq|oQ=pTpfaQZ*YqCX}JZf1$|NEZM5 zv*_QR#Xn#Xz@PK@cL2lH|I1nMvsv)=EO>F2cs|dfzatA?ody3g@Bsdt$8rF}m1`<+ z-4$h+bjvtD80;wxWZ}BMHkR7cEE?S9MFUs`i%P)=*tUDpbY7p+q#k`U-+0L5bQm@07`A`Z zpS#3C*#?z%a!TDu@NqcmFB|l zz^7>aNe;YI>jxZoKgjG6z0S>zfYzh}K`{z!zx!O%6Op zgDZ|L4t$B$uXW(B7AXBj2Y!XtZ+76{AFuRxI`CCmf0qNli8>G*oetd8`gsvzNoi#;kpS*e4h>1O-JHS+3-mcQKthoe6kHcWW%S}@NOGUV?Q4!ZMeO?nW?kh-k+j# z?7b1qt@oc$M`r40-WLZK{bEKy49yz=)I|ICN&Ye7scH6gO8$GqQ5A~DEYgIrzY69Me_fP zcxrlm>m+|0@zmt{$|N5no|;-;iR8aYJT52Cm;8;yQh^MC0w@>mjh^Hpg*D3ia z#8Xr0+bQ{R#8VULYn1#b;;Cu$ZIQf-cxn=T>%h}>G~09j5C{mfwL9S&JS{I{Cm$Pk zF6)JXFx%c)as7s5rI}iG2D&1VXSOf<)%1YqeHw){J1(3#Eg&j7W)~uXzr(PpnESSQ z^v`ZH)o+-Worwp0(^gX>={C%^oQEJ^FiQlFA4~;5dkEoKz)ObH2-F^W7$E7279QT-TY!_C+TcJfi{_KcRbaWe4>5h6J~q) zQ*FO7+fMX;gu=F!pEXaEcaz`t^0TP{ccLIQ;7NQnHQ@5y`8v?P-;-#qG<%-Z+Fl{s zPyRAE_@QrX`RNaRew<9-yl%6tyu0rlG~nry#I)e}nnW+J9(@e8{ z;~@&j>{#?N8W227%U+YN!`{9p;pIemuXJ{z{3NBC?VrxRn5IW#0QU#J5a(B)&C@A)rLX$g0G5-~}@2l$1bo-_P({AV5-Ol?0 zZubu7HkEdJ&TKno`ku?EfY(u&zZGuTaDKncep7c0h$LD+YM^7$o9LXZwv&eo+J7-5 zGUKD4E;m29V}N2v*Ve`_ko{5BE;4ItzXAu$jO=Ud*?0UI*`r}+W}nXcUr~bLicYO1 zbw`hp%%wp4n`OAZ=jaO41*gV&%%svBeQ&vGoKE!^=3Mkl14i-^ODZMitGcvETV%@+2UZE`{t6FJZ#V!;bowJuuZcJ8?%&zjSzMh_qQkHNuev)wa$j%j#Xnt<}o!(=HEBU6Wo6^X6N7uvy|P5_8< z7bTY+i%rydTZOrw3e$=*?$H5DgN}u+_C4gwtonu7F+MaIkI)AT$L^Y-9-3N7so&^K zJwve?sl${AU)yzq>0qw<0+~-9Hgn_HaaV~gBd#vg)1i;i72M`VrRi(Jqx2vIZLQ4+ zBXyY0e&64J53+U%nH`0#^6u@(`7R#U+dVF`>NIYZyTPDKAM82NT1y4M(SK>Xr0JPv zNQJi@Q$lm^uH89QT`8%H@{{?99Y1qicHD`4*?)@o*`6k#EHbuC*>7&@$MjinA& zX6i60iS6sYK!u+CXt=9M6*}Kup`ScNE>E;Z$T5z-yD8H44jIk!HCxqTww*LTc@JZs zi>RXm%gwpR%(l}eHAdsKX>^-yzW~%O-I#mZcDHupHu(Z>+Fmv19wnb`s{Fcyi3us` z3iU~WP4j)HZV=7v*3SNhFHrQCKOC;;vLTZMybP;$?_c!OLw*mb6Sw1M?7(311Nw1{y6)_x-_5V{hXeJ=cp^|44qUM; zut3a?!_g=)J5n9M-zXt7zC2LUEH*a8lZ}nh7<9=B*5ASd-S6?NNYDC|I{%JotY7On z+66gA41ZNb3=qTaIhI*V)0Gz|O;)IC^oG>GUQv=S4sGh)R2yYV> zBa92&OZTB^(94jY{MX>%|3g}WD*qkQeQ0fYxU?@LoriQ5Zr%!{-$EKe`Wn)EkgiAH zydUW+NMAx4M(2JOX)XrAJly?PBAth{7|#L~NZ&`Apk4tjZ#U9Mk^ThfTS#9=T7(}o zpCR3YbSgR>dND(Xaoc)fH2aMU@UEw+*LR^c_1F*jTab6# zJLI=QelkP;qUD3&^T+ubI9~(jYv6nhoUeiN zHE_NL&ey>C8aQ7A|MN9~_a5@#oSZ!4_$eU|T8l-PPEGLIoD%#tpo7=a{9fzR!Uj4fsl-}4*IOKrh&B_An&98JytaoH7SOR*J7#=9 z6Z+?>*r)AzjSuIC7eX@rlUg73B6+R{{5+NX4lTzE9m(6*u&4df z^ah=7)oGJX@6zf0I(<^72X%T}r@ztZyE^@&PIGBR362Rmy;!G61D=sMF&*Rk@_kJ$k)Q8p))1edA@cf&~Y!U$oXMPE$CqN96T~yxx%47xH>S zUO&j|1(#@hUJuCY|9HJ0ukYjae7t^-*X!~6JYJ8->+g8Ior*4fE-77b>B2>ezp~`A z6)RV*E?;xys#TW<=4`A?HYAe86<9hPiv|`JEiNjZe_2w3m(EqZm6MVnHKIgvxvxkrXEu@%UNRj`uKb>fe?gM>&p3G|eFtaPOp*6(;wINf z&MQ2m=$Z05&SU-&&H@3?|E5aFuN0n6=@uqC0C>9SZJ2AOH@`u6?p3ruze0HKqu^aL zC*@s2TD!^EH50FBDU2F0eyWtgX~IM9J$t-#rV7oOy9{T~_rd3s{+LL5wUV=h)IDp7 zY?cZYM1G4<86%At{O|?K)As-*;wfC(aTU;@Fx_2yTME`l z1FCy>*Y1|Wuffn5MXamq8{->b(!PqgfU5FwrP9=)@mD*cggGywTOxAXWFA81oD+DBtv301rTk3Or9y0@gZScsvK<@Unz`*iiV@-plvLbj@SWrGXdmr zw|2DD=IE<7N6Rcni^4+V0|Fjd1Uek_N+Z>hK9Ide&(I9B7m5$f}cWgayG}WVu zw$|q8TAQQmEJvS_qs`J0JGfjgBP~Nkhqa^4Hb*zw9NlC&nn{%%md%tMa92u0@^j<} z6&zLuo5rDnHN9?pXn`>5EKflSDJDZ=58O%9Q-cQ&9CX=YDkPIGTig~>LOY_ikkq+} zu|FWEDe07*{U7QFacqIp720XaR?a)Eh`Ok3cS~b7;$fj1#?NcxyE!8I4#8+wcR?-P za`)&sCQj8$Zg=ankX{WN_bb(ix|O~sd$O)AxG(Dvp7Dgtm^0+@Cx8ldD z=~&g(e@m(K!s8?4@!u4G8az!bF#r0ImdVIJUzXa_)pMj}N-c?f#+``3-PPGL;hM>? z$MJ$rFNB~_D*bl1j8d1ON9Aw^K}}@c+x24q@zr63-Ah&N={kfmqXzn{8c^Qee?odY zfO-h-N?8*aa($c{?WKyB=XoF-YR>+P6da=)u5)2OsG_Ph4csbg?nt zkhRCo_`zojoRY)&(xI!X7OG>7+4%*d-T4>fk2S^_erUL6iP5940E9ezAd$fY;m*H& z^wQB+q07wCVuX-e0Q{@?MKyx-Na!7Jd@1j$0t4j896`vai)Avpz;Ne}9!*Y2m^a4q zxCDLt7%9zRX+ET0l^;ochM`>gr^8n<8vfWKcq0YMjN&fKC9mUhEISwM3c%{Hi7i~1 z!Hx&3Ga>1O(Wr`X2DzE2NDR;EY|@!b)^<5p6_DK&?NzfE$PiIyQ?+f*m-5oK7wQ1D z@{779x zvUz?@L$X*tNEg>NR7H~2VWD)RRiQ{G#Bw!nT~r0v_)xC8I=r>0mNLOdZy*puB$RAM zUW$PRo`*qWB*4{m@k82_kG#uL-hJLj-nc2{O+Dh8S(r@W5zo~{C)cd8NQ~U8bMH%) zdap^9x=ISK+4fP&TU`2r_Zs}Wce9h&r;()hUhl^?`WKr^KT6h~@}6>CQix2XSw+&D zO78Q%>n%}=DJ1Wn@fO?&&&4I)Wi{R=ZaLL&D*@2N-6cuze-sAU1hlk0t>`5o7l1+E1J-v@8xCUReyH=}r{ zo@<#5DYpBaM<02lME&=EG>a5fUTale&*$Ffnp?Qp`%CY1x6ixZ^_7CsBi@1+yanDu z?`Bj9{cGCna-ng_wxAJwHO3y& zZ%o7>m!D4Jb5N~Fs}qzjtWek*iG~vCg$VTRqliQsY69{^naFP&r9N`5+sfI|N(beI z)dl|${8)AD6Wmxhl}wwWwN?Be#`cT zaH1GRn6F}>#T9tSr|&{5byZ}zfo5>k8WbZ@7i|#ogI9d?9$2M+$_7{BgHu5tjpCcs z`p_-mV0}VVN0amcsQipng#oTU6u(6WMQb&vI+s|Ljb$CQtVC7qO65&i()tjJVFSJt z4b%N859&1NaEWd7b4?z2c##J%Khrb_(cuzJex~JG35jh(PoH_P>{iQeo0sWC8uK&# zpoKO0nSRK^hWck2J+sh3@iYBs8uK&#k%cw+nWi-aY^$jBnPq>l=`cNz#{5iQjlPf$ zsvoAWv9O`>vy9feD7$T5rmst5ex@I@uqHp#k6YMK|15jLro;3f;%9n> zg$?!3vY9p=rY}xoex@I?uqHp#4_nw!|15jNro;4OY0S^`LfufQ{+V86VMG11Y_Uy; z>C4iXpXpy)Sd*XW-&ojC|15jMro;4`Y0S^`1U#~+cF^Q!dZL95_0O_NHXWufNMnAc zKew9>}Ob(^2*BNo==XZoiWHq<}Mx@|g4znI4SOi$GB zqUriFJ;}m``e)f>n-0@c)0m&>Mhk25GkvRt4fW5mm`#W2WE%4`{j`NO`I-Kag$?!3 zvi&w4rk_h=ey0C|M{IRXn*2mG$+}4O z{JLr(x!O>?RuollZ-Avr@wQG%n!>R-4a#;V2w5x~36X)O8Zo#Q$-OO#65(dha)*ne zSX6GGP!z5WZjE6RiePOuY?%{9Rf%XUjto`u%_<1tGgMz!g$$!`gHw_8t^8mY1{g)_ zX=y4{!)9sKH(tnyqWt8f7|ESs4pAF^32JFw^ulQX52~Rq3kx|E1_>F*Q9s zd<@x_XqLmV?&d7C2o#ofM%kX{&p7?6*3{j*a6VKlfgC@Zto(U?jZ=DiMF-_ed9b~H z2O-LUkqYO}^L3o^{2Wx|p@(^9g$au0%GjRg$2cW99UMRFak>#QntNlO=ifLjk=81H z%A4cYu&4&XWh0E|`#9aF9drI{&*g7$*z^1$r##mLVdw-4xkfYSqx3fHg3t-Wz}IP7`9FHV>ERK9#>d*<(S*zczJO8AP1&k>-uR{JPm?E zHA_t4}c4y$8U?2x=J)b`GD z#+NIDR~;EL%jq0#uiab8>J^IQ=U*1E9rI&B@U>@*Qje*)OC1IdR+h)nMM%R#UsLv- zy8fqwU5!6(w|rlac)0w{o0RUwy_xpCKTZ~0NKGeXC9g(=7&eSi!ewM@O7YgoIGxf&{?k_X( z3D&+|nfOHXOT+o+)@awJ`q2|TnK1Av4n(nTp(lA9Qn&p2!~ueJiwlQ>64ws_HYqh@ z7oJr4xr9Y@|9Zp+gUoYEQ((q0*M<6HW0>o*MB+oQOF3}L_nhNFhHg(or0bJ!vu|p= zSJyM&@6FmzqYs4b$QJ*d(4qKC9POb?;y&@z44V^>k4SuI{67LN>&;QsM}ZGlZ>O|A zPct;?f}PelFPWGRo{qQhNB(&Q#MdQ??}RkY3mO=IU*o(!VUyPXlg4?uL5s#|d?Nq6 zdf;}A(>|Jn^J0MSYP?Y6JhlI*#xK%1&(5#VIE^3VhbP`?Jf>rT#(7@6TcScN)i_Uc zZ`Sx~jq|KE<7+j}lhE?pDcF2n<2=7CznKEQ1-Kz52%bdtYyWjxpC>G9G*0h-sXdoD z+Q|;>ho6VEx@ed92ZPRjX1B(j{mIk7IR^GY=|5$`yMYh4FXk&*^#45z-k$~kEDJsg zgRmd{fU|!aD{*_|ik=R999E_B?-bVflEj%yr>nB~*^ouQCX0R|3w~D?e0LWAKg^>4 zLKgfb;DK~;Eb6T+`tN1IKh1*Y;W=fvc0%v8>AIXN&CLBvS@ge>1z)56IIr*eEc(GL z_&0%PMxv@>M;86R2R_59tu*mC@Zt7z6`1yo*tTvJRk1`ofjJ{g>IH+VuD@=5u;Qu> z*VB%!!PObu)vIix>S(Yg60HnHg7Ott5HrF8udf;-;Y7H)Xz{{DO9Tmmb=A$_7MBS1 zmaIBiU%#F6DZh5L%x2B{6>H0b%uzN>r*o0-(_}7kVvB;6gndFVJ>{h+nhI1DeHyJO zYYtO=BF)ox*i}`2C#{syvxzjH?Xzs?`oUY zq`AFB)V}9!1u0sa#tz*I7Tey^yexLt)jPnF3;V{iyoGP({o2{-#csSqJXBzNT-&C-+Gl#(TgMi}B&V%!)4TRc z*0!l{MbNp6uhhugI9P6j%f7g>z*o@M_FQ+iZF5PLW_EejnYS$dtJ@xDZjqc>a", a > b) +print("#", #a) diff --git a/lua-protobuf/Makefile b/lua-protobuf/Makefile new file mode 100644 index 00000000..94f837e7 --- /dev/null +++ b/lua-protobuf/Makefile @@ -0,0 +1,14 @@ +.PHONY : all clean + +all : protobuf.so addressbook.pb + +LIBSRCS = context.c varint.c array.c pattern.c register.c proto.c map.c alloc.c rmessage.c wmessage.c bootstrap.c stringpool.c decode.c + +protobuf.so : pbc-lua.c $(LIBSRCS) + gcc -O2 -Wall --shared -fPIC -o $@ $^ + +addressbook.pb : addressbook.proto + protoc -o$@ $< + +clean : + rm protobuf.so addressbook.pb \ No newline at end of file diff --git a/lua-protobuf/README b/lua-protobuf/README new file mode 100644 index 00000000..94a03da5 --- /dev/null +++ b/lua-protobuf/README @@ -0,0 +1,2 @@ +See https://github.com/cloudwu/pbc + diff --git a/lua-protobuf/addressbook.proto b/lua-protobuf/addressbook.proto new file mode 100644 index 00000000..883a7881 --- /dev/null +++ b/lua-protobuf/addressbook.proto @@ -0,0 +1,39 @@ +// See README.txt for information and build instructions. + +package tutorial; + +option java_package = "com.example.tutorial"; +option java_outer_classname = "AddressBookProtos"; + +message Person { + required string name = 1; + required int32 id = 2; // Unique ID number for this person. + optional string email = 3; + + enum PhoneType { + MOBILE = 0; + HOME = 1; + WORK = 2; + } + + message PhoneNumber { + required string number = 1; + optional PhoneType type = 2 [default = HOME]; + } + + repeated PhoneNumber phone = 4; + repeated int32 test = 5 [packed=true]; + + extensions 10 to max; +} + +message Ext { + extend Person { + optional int32 test = 10; + } +} + +// Our address book file is just one of these. +message AddressBook { + repeated Person person = 1; +} diff --git a/lua-protobuf/alloc.c b/lua-protobuf/alloc.c new file mode 100644 index 00000000..e8d4320e --- /dev/null +++ b/lua-protobuf/alloc.c @@ -0,0 +1,84 @@ +#include +#include + +static int _g = 0; + +void * _pbcM_malloc(size_t sz) { + ++ _g; + return malloc(sz); +} + +void _pbcM_free(void *p) { + if (p) { + -- _g; + free(p); + } +} + +void* _pbcM_realloc(void *p, size_t sz) { + return realloc(p,sz); +} + +void _pbcM_memory() { + printf("%d\n",_g); +} + +struct heap_page { + struct heap_page * next; +}; + +struct heap { + struct heap_page *current; + int size; + int used; +}; + +struct heap * +_pbcH_new(int pagesize) { + int cap = 1024; + while(cap < pagesize) { + cap *= 2; + } + struct heap * h = _pbcM_malloc(sizeof(struct heap)); + h->current = _pbcM_malloc(sizeof(struct heap_page) + cap); + h->size = cap; + h->used = 0; + h->current->next = NULL; + return h; +} + +void +_pbcH_delete(struct heap *h) { + struct heap_page * p = h->current; + struct heap_page * next = p->next; + for(;;) { + _pbcM_free(p); + if (next == NULL) + break; + p = next; + next = p->next; + } + _pbcM_free(h); +} + +void* +_pbcH_alloc(struct heap *h, int size) { + size = (size + 3) & ~3; + if (h->size - h->used < size) { + struct heap_page * p; + if (size < h->size) { + p = _pbcM_malloc(sizeof(struct heap_page) + h->size); + } else { + p = _pbcM_malloc(sizeof(struct heap_page) + size); + } + p->next = h->current; + h->current = p; + h->used = size; + return (p+1); + } else { + char * buffer = (char *)(h->current + 1); + buffer += h->used; + h->used += size; + return buffer; + } +} diff --git a/lua-protobuf/alloc.h b/lua-protobuf/alloc.h new file mode 100644 index 00000000..036acbbe --- /dev/null +++ b/lua-protobuf/alloc.h @@ -0,0 +1,25 @@ +#ifndef PROTOBUF_C_ALLOC_H +#define PROTOBUF_C_ALLOC_H + +#include +#include + +void * _pbcM_malloc(size_t sz); +void _pbcM_free(void *p); +void * _pbcM_realloc(void *p, size_t sz); +void _pbcM_memory(); + +struct heap; + +struct heap * _pbcH_new(int pagesize); +void _pbcH_delete(struct heap *); +void* _pbcH_alloc(struct heap *, int size); + +#define HMALLOC(size) ((h) ? _pbcH_alloc(h, size) : _pbcM_malloc(size)) + +#define malloc _pbcM_malloc +#define free _pbcM_free +#define realloc _pbcM_realloc +#define memory _pbcM_memory + +#endif diff --git a/lua-protobuf/array.c b/lua-protobuf/array.c new file mode 100644 index 00000000..b6004d0c --- /dev/null +++ b/lua-protobuf/array.c @@ -0,0 +1,139 @@ +#include "pbc.h" +#include "array.h" +#include "alloc.h" + +#include +#include + +struct array { + int number; + struct heap *heap; + union _pbc_var * a; +}; + +#define INNER_FIELD ((PBC_ARRAY_CAP - sizeof(struct array)) / sizeof(pbc_var)) + +void +_pbcA_open(pbc_array _array) { + struct array * a = (struct array *)_array; + a->number = 0; + a->heap = NULL; + a->a = (union _pbc_var *)(a+1); +} + +void +_pbcA_open_heap(pbc_array _array, struct heap *h) { + struct array * a = (struct array *)_array; + a->number = 0; + a->heap = h; + a->a = (union _pbc_var *)(a+1); +} + +void +_pbcA_close(pbc_array _array) { + struct array * a = (struct array *)_array; + if (a->heap == NULL && a->a != NULL && (union _pbc_var *)(a+1) != a->a) { + _pbcM_free(a->a); + a->a = NULL; + } +} + +void +_pbcA_push(pbc_array _array, pbc_var var) { + struct array * a = (struct array *)_array; + if (a->number == 0) { + a->a = (union _pbc_var *)(a+1); + } else if (a->number >= INNER_FIELD) { + if (a->number == INNER_FIELD) { + int cap = 1; + while (cap <= a->number + 1) + cap *= 2; + struct heap * h = a->heap; + union _pbc_var * outer = HMALLOC(cap * sizeof(union _pbc_var)); + memcpy(outer , a->a , INNER_FIELD * sizeof(pbc_var)); + a->a = outer; + } else { + int size=a->number; + if (((size + 1) ^ size) > size) { + struct heap * h = a->heap; + if (h) { + void * old = a->a; + a->a = _pbcH_alloc(h, sizeof(union _pbc_var) * (size+1) * 2); + memcpy(a->a, old, sizeof(union _pbc_var) * size); + } else { + a->a=_pbcM_realloc(a->a,sizeof(union _pbc_var) * (size+1) * 2); + } + } + } + } + a->a[a->number] = *var; + ++ a->number; +} + +void +_pbcA_index(pbc_array _array, int idx, pbc_var var) +{ + struct array * a = (struct array *)_array; + var[0] = a->a[idx]; +} + +void * +_pbcA_index_p(pbc_array _array, int idx) +{ + struct array * a = (struct array *)_array; + return &(a->a[idx]); +} + +int +pbc_array_size(pbc_array _array) { + struct array * a = (struct array *)_array; + return a->number; +} + +uint32_t +pbc_array_integer(pbc_array array, int index, uint32_t *hi) { + pbc_var var; + _pbcA_index(array , index , var); + if (hi) { + *hi = var->integer.hi; + } + return var->integer.low; +} + +double +pbc_array_real(pbc_array array, int index) { + pbc_var var; + _pbcA_index(array , index , var); + return var->real; +} + +struct pbc_slice * +pbc_array_slice(pbc_array _array, int index) { + struct array * a = (struct array *)_array; + if (index <0 || index > a->number) { + return NULL; + } + return (struct pbc_slice *) &(a->a[index]); +} + +void +pbc_array_push_integer(pbc_array array, uint32_t low, uint32_t hi) { + pbc_var var; + var->integer.low = low; + var->integer.hi = hi; + _pbcA_push(array,var); +} + +void +pbc_array_push_slice(pbc_array array, struct pbc_slice *s) { + pbc_var var; + var->m = *s; + _pbcA_push(array,var); +} + +void +pbc_array_push_real(pbc_array array, double v) { + pbc_var var; + var->real = v; + _pbcA_push(array,var); +} diff --git a/lua-protobuf/array.h b/lua-protobuf/array.h new file mode 100644 index 00000000..cc7f0a9a --- /dev/null +++ b/lua-protobuf/array.h @@ -0,0 +1,31 @@ +#ifndef PROTOBUF_C_ARRAY_H +#define PROTOBUF_C_ARRAY_H + +#include "varint.h" +#include "pbc.h" +#include "alloc.h" + +typedef union _pbc_var { + struct longlong integer; + double real; + struct { + const char * str; + int len; + } s; + struct { + int id; + const char * name; + } e; + struct pbc_slice m; + void * p[2]; +} pbc_var[1]; + +void _pbcA_open(pbc_array); +void _pbcA_open_heap(pbc_array, struct heap *h); +void _pbcA_close(pbc_array); + +void _pbcA_push(pbc_array, pbc_var var); +void _pbcA_index(pbc_array , int idx, pbc_var var); +void * _pbcA_index_p(pbc_array _array, int idx); + +#endif diff --git a/lua-protobuf/bootstrap.c b/lua-protobuf/bootstrap.c new file mode 100644 index 00000000..a2b790ec --- /dev/null +++ b/lua-protobuf/bootstrap.c @@ -0,0 +1,303 @@ +#include "pbc.h" +#include "map.h" +#include "context.h" +#include "pattern.h" +#include "proto.h" +#include "alloc.h" +#include "bootstrap.h" +#include "stringpool.h" +#include "array.h" +#include "descriptor.pbc.h" + +#include +#include +#include +#include + +/* + +// Descriptor + +// google.protobuf.Descriptor.proto encoded in descriptor.pbc.h with proto pbc.file . + +package pbc; + +message field { + optional string name = 1; + optional int32 id = 2; + optional int32 label = 3; // 0 optional 1 required 2 repeated + optional int32 type = 4; // type_id + optional string type_name = 5; + optional int32 default_int = 6; + optional string default_string = 7; + optional double default_real = 8; +} + +message file { + optional string name = 1; + repeated string dependency = 2; + + repeated string message_name = 3; + repeated int32 message_size = 4; + repeated field message_field = 5; + + repeated string enum_name = 6; + repeated int32 enum_size = 7; + repeated string enum_string = 8; + repeated int32 enum_id = 9; +} + +*/ + +struct field_t { + struct pbc_slice name; + int32_t id; + int32_t label; + int32_t type; + struct pbc_slice type_name; + int32_t default_integer; + struct pbc_slice default_string; + double default_real; +}; + +struct file_t { + struct pbc_slice name; // string + pbc_array dependency; // string + pbc_array message_name; // string + pbc_array message_size; // int32 + pbc_array message_field; // field_t + pbc_array enum_name; // string + pbc_array enum_size; // int32 + pbc_array enum_string; // string + pbc_array enum_id; // int32 +}; + +static void +set_enum_one(struct pbc_env *p, struct file_t *file, const char *name, int start, int sz) { + struct map_kv *table = malloc(sz * sizeof(struct map_kv)); + int i; + for (i=0;ienum_id, start+i, id); + _pbcA_index(file->enum_string, start+i, string); + table[i].id = (int)id->integer.low; + table[i].pointer = (void *)string->s.str; + } + _pbcP_push_enum(p,name,table,sz); + + free(table); +} + +static void +set_enums(struct pbc_env *p, struct file_t *file) { + int n = pbc_array_size(file->enum_size); + int i; + int start = 0; + for (i=0;ienum_name,i,name); + pbc_var var; + _pbcA_index(file->enum_size,i,var); + set_enum_one(p, file, name->s.str, start , (int)var->integer.low); + start += var->integer.low; + } +} + +static void +set_default(struct _field *f, struct field_t *input) { + switch (f->type) { + case PTYPE_DOUBLE: + case PTYPE_FLOAT: + f->default_v->real = input->default_real; + break; + case PTYPE_STRING: + case PTYPE_ENUM: + f->default_v->m = input->default_string; + break; + default: + f->default_v->integer.low = input->default_integer; + break; + } +} + +static void +set_msg_one(struct pbc_pattern * FIELD_T, struct pbc_env *p, struct file_t *file, const char *name, int start, int sz , pbc_array queue) { + int i; + for (i=0;imessage_field, start+i, _field); + struct field_t field; + + int ret = pbc_pattern_unpack(FIELD_T, &_field->m, &field); + if (ret != 0) { + continue; + } + struct _field f; + f.id = field.id; + f.name = field.name.buffer; + f.type = field.type; + f.label = field.label; + f.type_name.n = field.type_name.buffer; + set_default(&f, &field); + + _pbcP_push_message(p,name, &f , queue); + + // don't need to close pattern since no array + } + _pbcP_init_message(p, name); +} + +static void +set_msgs(struct pbc_pattern * FIELD_T, struct pbc_env *p, struct file_t *file , pbc_array queue) { + int n = pbc_array_size(file->message_size); + int i; + int start = 0; + for (i=0;imessage_name,i,name); + pbc_var sz; + _pbcA_index(file->message_size,i,sz); + set_msg_one(FIELD_T, p, file, name->s.str, start , (int)sz->integer.low , queue); + start += sz->integer.low; + } +} + +static void +set_field_one(struct pbc_env *p, struct _field *f) { + const char * type_name = f->type_name.n; + if (f->type == PTYPE_MESSAGE) { + f->type_name.m = _pbcM_sp_query(p->msgs, type_name); +// printf("MESSAGE: %s %p\n",type_name, f->type_name.m); + } else if (f->type == PTYPE_ENUM) { + f->type_name.e = _pbcM_sp_query(p->enums, type_name); +// printf("ENUM: %s %p ",type_name, f->type_name.e); + const char * str = f->default_v->s.str; + if (str && str[0]) { + int err = _pbcM_si_query(f->type_name.e->name, str , &(f->default_v->e.id)); + if (err < 0) + goto _default; + f->default_v->e.name = _pbcM_ip_query(f->type_name.e->id, f->default_v->e.id); +// printf("[%s %d]\n",str,f->default_v->e.id); + } else { +_default: + memcpy(f->default_v, f->type_name.e->default_v, sizeof(pbc_var)); +// printf("(%s %d)\n",f->default_v->e.name,f->default_v->e.id); + } + } +} + +void +_pbcB_register_fields(struct pbc_env *p, pbc_array queue) { + int sz = pbc_array_size(queue); + int i; + for (i=0;im.buffer; + set_field_one(p, f); + } +} + +static void +_set_string(struct _pattern_field * f) { + f->ptype = PTYPE_STRING; + f->ctype = CTYPE_VAR; + f->defv->s.str = ""; + f->defv->s.len = 0; +} + +static void +_set_int32(struct _pattern_field * f) { + f->ptype = PTYPE_INT32; + f->ctype = CTYPE_INT32; +} + +static void +_set_double(struct _pattern_field * f) { + f->ptype = PTYPE_DOUBLE; + f->ctype = CTYPE_DOUBLE; +} + +static void +_set_message_array(struct _pattern_field *f) { + f->ptype = PTYPE_MESSAGE; + f->ctype = CTYPE_ARRAY; +} + +static void +_set_string_array(struct _pattern_field * f) { + f->ptype = PTYPE_STRING; + f->ctype = CTYPE_ARRAY; +} + +static void +_set_int32_array(struct _pattern_field * f) { + f->ptype = PTYPE_INT32; + f->ctype = CTYPE_ARRAY; +} + +#define SET_PATTERN(pat , idx , pat_type, field_name , type) \ + pat->f[idx].id = idx+1 ; \ + pat->f[idx].offset = offsetof(struct pat_type, field_name); \ + _set_##type(&pat->f[idx]); + +#define F(idx,field_name,type) SET_PATTERN(FIELD_T, idx, field_t ,field_name, type) +#define D(idx,field_name,type) SET_PATTERN(FILE_T, idx, file_t ,field_name, type) + +static int +register_internal(struct pbc_env * p, struct pbc_slice *slice) { + struct pbc_pattern * FIELD_T = _pbcP_new(p,8); + F(0,name,string); + F(1,id,int32); + F(2,label,int32); + F(3,type,int32); + F(4,type_name,string); + F(5,default_integer,int32); + F(6,default_string,string); + F(7,default_real,double); + + struct pbc_pattern * FILE_T = _pbcP_new(p,10); + + D(0,name,string); + D(1,dependency,string_array); + D(2,message_name,string_array); + D(3,message_size,int32_array); + D(4,message_field,message_array); + D(5,enum_name,string_array); + D(6,enum_size,int32_array); + D(7,enum_string,string_array); + D(8,enum_id,int32_array); + + int ret = 0; + + struct file_t file; + int r = pbc_pattern_unpack(FILE_T, slice, &file); + if (r != 0) { + ret = 1; + goto _return; + } + + _pbcM_sp_insert(p->files , file.name.buffer, NULL); + + pbc_array queue; + _pbcA_open(queue); + + set_enums(p, &file); + set_msgs(FIELD_T, p, &file, queue); + _pbcB_register_fields(p, queue); + + _pbcA_close(queue); + pbc_pattern_close_arrays(FILE_T, &file); + +_return: + free(FIELD_T); + free(FILE_T); + return ret; +} + +void +_pbcB_init(struct pbc_env * p) { + struct pbc_slice slice = { pbc_descriptor,sizeof(pbc_descriptor) }; + register_internal(p,&slice); +} diff --git a/lua-protobuf/bootstrap.h b/lua-protobuf/bootstrap.h new file mode 100644 index 00000000..48566f58 --- /dev/null +++ b/lua-protobuf/bootstrap.h @@ -0,0 +1,10 @@ +#ifndef PROTOBUF_C_BOOTSTRAP_H +#define PROTOBUF_C_BOOTSTRAP_H + +#include "proto.h" +#include "pbc.h" + +void _pbcB_init(struct pbc_env *); +void _pbcB_register_fields(struct pbc_env *, pbc_array queue); + +#endif diff --git a/lua-protobuf/context.c b/lua-protobuf/context.c new file mode 100644 index 00000000..ac40cdec --- /dev/null +++ b/lua-protobuf/context.c @@ -0,0 +1,278 @@ +#include "pbc.h" +#include "alloc.h" +#include "varint.h" +#include "context.h" + +#include +#include +#include +#include + +#define INNER_ATOM ((PBC_CONTEXT_CAP - sizeof(struct context)) / sizeof(struct atom)) + +static char * +wiretype_decode(uint8_t *buffer, int cap , struct atom *a , int start) +{ + uint8_t temp[10]; + struct longlong r; + int len; + if (cap >= 10) { + len = _pbcV_decode(buffer, &r); + if (r.hi !=0) + return NULL; + } else { + memcpy(temp, buffer , cap); + len = _pbcV_decode(temp, &r); + if (len > cap || r.hi !=0) + return NULL; + } + + int wiretype = r.low & 7; + a->wire_id = r.low; + buffer += len; + start += len; + cap -=len; + + switch (wiretype) { + case WT_VARINT : + if (cap >=10) { + len = _pbcV_decode(buffer, &a->v.i); + } else { + memcpy(temp, buffer , cap); + len = _pbcV_decode(temp, &a->v.i); + if (cap < len) + return NULL; + } + return (char *)buffer+len; + case WT_BIT64 : + if (cap < 8) + return NULL; + a->v.i.low = buffer[0] | + buffer[1] << 8 | + buffer[2] << 16 | + buffer[3] << 24; + a->v.i.hi = buffer[4] | + buffer[5] << 8 | + buffer[6] << 16 | + buffer[7] << 24; + return (char *)buffer + 8; + case WT_LEND : + if (cap >=10) { + len = _pbcV_decode(buffer, &r); + } else { + memcpy(temp, buffer , cap); + len = _pbcV_decode(temp, &r); + } + if (cap < len + r.low || r.hi !=0) + return NULL; + a->v.s.start = start + len; + a->v.s.end = start + len + r.low; + return (char *)buffer + len + r.low; + case WT_BIT32 : + if (cap < 4) + return NULL; + a->v.i.low = buffer[0] | + buffer[1] << 8 | + buffer[2] << 16 | + buffer[3] << 24; + a->v.i.hi = 0; + return (char *)buffer + 4; + default: + return NULL; + } +} + +static inline int +_decode_varint(uint8_t * buffer, int size , struct atom * a) { + a->wire_id = WT_VARINT; + if (size < 10) { + uint8_t temp[10]; + memcpy(temp,buffer,size); + return _pbcV_decode(temp , &(a->v.i)); + } else { + return _pbcV_decode(buffer , &(a->v.i)); + } +} + +static int +_open_packed_varint(struct context * ctx , uint8_t * buffer, int size) { + struct atom * a = (struct atom *)(ctx + 1); + + int i; + + for (i=0;ia = a; + } else { + int cap = 64; + ctx->a = malloc(cap * sizeof(struct atom)); + while (size > 0) { + if (i >= cap) { + cap = cap + 64; + ctx->a = realloc(ctx->a, cap * sizeof(struct atom)); + continue; + } + int len = _decode_varint(buffer, size, &a[i]); + buffer += len; + size -= len; + + ++i; + } + memcpy(ctx->a, a , sizeof(struct atom) * INNER_ATOM); + } + ctx->number = i; + + return i; +} + +int +_pbcC_open_packed(pbc_ctx _ctx, int ptype, void *buffer, int size) { + struct context * ctx = (struct context *)_ctx; + ctx->buffer = buffer; + ctx->size = size; + ctx->number = 0; + ctx->a = NULL; + + if (buffer == NULL || size == 0) { + return 0; + } + + int bits = 0; + + switch (ptype) { + case PTYPE_INT64: + case PTYPE_UINT64: + case PTYPE_INT32: + case PTYPE_BOOL: + case PTYPE_UINT32: + case PTYPE_ENUM: + case PTYPE_SINT32: + case PTYPE_SINT64: + return _open_packed_varint(ctx , buffer, size); + case PTYPE_DOUBLE: + case PTYPE_FIXED64: + case PTYPE_SFIXED64: + ctx->number = size / 8; + bits = 64; + break; + case PTYPE_FLOAT: + case PTYPE_FIXED32: + case PTYPE_SFIXED32: + ctx->number = size / 4; + bits = 32; + break; + default: + return 0; + } + + struct atom * a = (struct atom *)(ctx + 1); + + if (ctx->number > INNER_ATOM) { + ctx->a = malloc(ctx->number * sizeof(struct atom)); + a = ctx->a; + } else { + ctx->a = a; + } + + int i; + if (bits == 64) { + uint8_t * data = buffer; + for (i=0;inumber;i++) { + a[i].wire_id = WT_BIT64; + a[i].v.i.low = data[0] | + data[1] << 8 | + data[2] << 16 | + data[3] << 24; + a[i].v.i.hi = data[4] | + data[5] << 8 | + data[6] << 16 | + data[7] << 24; + data += 8; + } + } else { + uint8_t * data = buffer; + for (i=0;inumber;i++) { + a[i].wire_id = WT_BIT32; + a[i].v.i.low = data[0] | + data[1] << 8 | + data[2] << 16 | + data[3] << 24; + a[i].v.i.hi = 0; + data += 4; + } + } + + return ctx->number; +} + +int +_pbcC_open(pbc_ctx _ctx , void *buffer, int size) { + struct context * ctx = (struct context *)_ctx; + ctx->buffer = buffer; + ctx->size = size; + + if (buffer == NULL || size == 0) { + ctx->number = 0; + ctx->a = NULL; + return 0; + } + + struct atom * a = (struct atom *)(ctx + 1); + + int i; + int start = 0; + + ctx->a = a; + + for (i=0;i 0) { + int cap = 64; + ctx->a = malloc(cap * sizeof(struct atom)); + while (size > 0) { + if (i >= cap) { + cap = cap + 64; + ctx->a = realloc(ctx->a, cap * sizeof(struct atom)); + continue; + } + char * next = wiretype_decode(buffer, size , &ctx->a[i] , start); + if (next == NULL) { + return -i; + } + start += next - (char *)buffer; + size -= next - (char *)buffer; + buffer = next; + ++i; + } + memcpy(ctx->a, a , sizeof(struct atom) * INNER_ATOM); + } + ctx->number = i; + + return i; +} + + +void +_pbcC_close(pbc_ctx _ctx) { + struct context * ctx = (struct context *)_ctx; + if (ctx->a != NULL && (struct atom *)(ctx+1) != ctx->a) { + free(ctx->a); + ctx->a = NULL; + } +} diff --git a/lua-protobuf/context.h b/lua-protobuf/context.h new file mode 100644 index 00000000..e19ed91c --- /dev/null +++ b/lua-protobuf/context.h @@ -0,0 +1,140 @@ +#ifndef PROTOBUF_C_CONTEXT_H +#define PROTOBUF_C_CONTEXT_H + +#include + +#include "array.h" + +#define PBC_CONTEXT_CAP 256 + +// wiretype + +#define WT_VARINT 0 +#define WT_BIT64 1 +#define WT_LEND 2 +#define WT_BIT32 5 + +#define CTYPE_INT32 1 +#define CTYPE_INT64 2 +#define CTYPE_DOUBLE 3 +#define CTYPE_FLOAT 4 +#define CTYPE_POINTER 5 +#define CTYPE_BOOL 6 +#define CTYPE_INT8 7 +#define CTYPE_INT16 8 +#define CTYPE_ARRAY 9 +#define CTYPE_VAR 10 +#define CTYPE_PACKED 11 + +#define PTYPE_DOUBLE 1 +#define PTYPE_FLOAT 2 +#define PTYPE_INT64 3 // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT64 if negative values are likely. +#define PTYPE_UINT64 4 +#define PTYPE_INT32 5 // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT32 if negative values are likely. +#define PTYPE_FIXED64 6 +#define PTYPE_FIXED32 7 +#define PTYPE_BOOL 8 +#define PTYPE_STRING 9 +#define PTYPE_GROUP 10 // Tag-delimited aggregate. +#define PTYPE_MESSAGE 11 // Length-delimited aggregate. +#define PTYPE_BYTES 12 +#define PTYPE_UINT32 13 +#define PTYPE_ENUM 14 +#define PTYPE_SFIXED32 15 +#define PTYPE_SFIXED64 16 +#define PTYPE_SINT32 17 // Uses ZigZag encoding. +#define PTYPE_SINT64 18 // Uses ZigZag encoding. + +struct slice { + int start; + int end; +}; + +struct atom { + int wire_id; + union { + struct slice s; + struct longlong i; + } v; +}; + +struct context { + char * buffer; + int size; + int number; + struct atom * a; +}; + +typedef struct _pbc_ctx { char _data[PBC_CONTEXT_CAP]; } pbc_ctx[1]; + +int _pbcC_open(pbc_ctx , void *buffer, int size); // <=0 failed +int _pbcC_open_packed(pbc_ctx _ctx, int ptype, void *buffer, int size); +void _pbcC_close(pbc_ctx); + +static inline double +read_double(struct atom * a) { + union { + uint64_t i; + double d; + } u; + u.i = (uint64_t) a->v.i.low | (uint64_t) a->v.i.hi << 32; + return u.d; +} + +static inline float +read_float(struct atom * a) { + union { + uint32_t i; + float f; + } u; + u.i = a->v.i.low; + return u.f; +} + +static inline void +double_encode(double v , uint8_t * buffer) { + union { + double v; + uint64_t e; + } u; + u.v = v; + buffer[0] = (uint8_t) (u.e & 0xff); + buffer[1] = (uint8_t) (u.e >> 8 & 0xff); + buffer[2] = (uint8_t) (u.e >> 16 & 0xff); + buffer[3] = (uint8_t) (u.e >> 24 & 0xff); + buffer[4] = (uint8_t) (u.e >> 32 & 0xff); + buffer[5] = (uint8_t) (u.e >> 40 & 0xff); + buffer[6] = (uint8_t) (u.e >> 48 & 0xff); + buffer[7] = (uint8_t) (u.e >> 56 & 0xff); +} + +static inline void +float_encode(float v , uint8_t * buffer) { + union { + float v; + uint32_t e; + } u; + u.v = v; + buffer[0] = (uint8_t) (u.e & 0xff); + buffer[1] = (uint8_t) (u.e >> 8 & 0xff); + buffer[2] = (uint8_t) (u.e >> 16 & 0xff); + buffer[3] = (uint8_t) (u.e >> 24 & 0xff); +} + +#define CHECK_LEND(a,err) if ((a->wire_id & 7) != WT_LEND) return err; + +#if 0 +/* maybe we don't need check these wire type */ +#define CHECK_VARINT(a,err) if ((a->wire_id & 7) != WT_VARINT) return err; +#define CHECK_BIT32(a,err) if ((a->wire_id & 7) != WT_BIT32) return err; +#define CHECK_BIT64(a,err) if ((a->wire_id & 7) != WT_BIT64) return err; + +#else + +#define CHECK_VARINT(a,err) +#define CHECK_BIT32(a,err) +#define CHECK_BIT64(a,err) + +#endif + +#endif diff --git a/lua-protobuf/decode.c b/lua-protobuf/decode.c new file mode 100644 index 00000000..440cfbb3 --- /dev/null +++ b/lua-protobuf/decode.c @@ -0,0 +1,349 @@ +#include "pbc.h" +#include "alloc.h" +#include "context.h" +#include "proto.h" +#include "varint.h" + +#include + +static const char * TYPENAME[] = { + "invalid", // 0 + "integer", // 1 + "real", // 2 + "boolean", // 3 + "enum", // 4 + "string", // 5 + "message", // 6 + "fixed64", // 7 + "fixed32", // 8 + "bytes", // 9 + "int64", // 10 + "uint", // 11 +}; + +static int +call_unknown(pbc_decoder f, void * ud, int id, struct atom *a, uint8_t * start) { + union pbc_value v; + switch (a->wire_id) { + case WT_VARINT: + v.i.low = a->v.i.low; + v.i.hi = a->v.i.hi; + f(ud, PBC_INT, TYPENAME[PBC_INT], &v, id , NULL); + break; + case WT_BIT64: + v.i.low = a->v.i.low; + v.i.hi = a->v.i.hi; + f(ud, PBC_FIXED64, TYPENAME[PBC_FIXED64], &v, id , NULL); + break; + case WT_LEND: + v.s.buffer = (char*)start + a->v.s.start; + v.s.len = a->v.s.end - a->v.s.start; + f(ud, PBC_BYTES, TYPENAME[PBC_BYTES], &v, id , NULL); + break; + case WT_BIT32: + v.i.low = a->v.i.low; + v.i.hi = 0; + f(ud, PBC_FIXED32, TYPENAME[PBC_FIXED32], &v, id , NULL); + break; + default: + return 1; + } + return 0; +} + +static int +call_type(pbc_decoder pd, void * ud, struct _field *f, struct atom *a, uint8_t * start) { + union pbc_value v; + const char * typename = NULL; + int type = _pbcP_type(f, &typename); + assert(type != 0); + if (typename == NULL) { + typename = TYPENAME[type & ~PBC_REPEATED]; + } + switch (f->type) { + case PTYPE_DOUBLE: + CHECK_BIT64(a, -1); + v.f = read_double(a); + break; + case PTYPE_FLOAT: + CHECK_BIT32(a, -1); + v.f = (double) read_float(a); + break; + case PTYPE_ENUM: + CHECK_VARINT(a, -1); + v.e.id = a->v.i.low; + v.e.name = _pbcM_ip_query(f->type_name.e->id , v.e.id); + break; + case PTYPE_INT64: + case PTYPE_UINT64: + CHECK_VARINT(a, -1); + v.i.low = a->v.i.low; + v.i.hi = a->v.i.hi; + break; + case PTYPE_FIXED64: + case PTYPE_SFIXED64: + CHECK_BIT64(a, -1); + v.i.low = a->v.i.low; + v.i.hi = a->v.i.hi; + break; + case PTYPE_INT32: + case PTYPE_UINT32: + case PTYPE_BOOL: + CHECK_VARINT(a, -1); + v.i.low = a->v.i.low; + v.i.hi = 0; + break; + case PTYPE_FIXED32: + case PTYPE_SFIXED32: + CHECK_BIT32(a, -1); + v.i.low = a->v.i.low; + v.i.hi = 0; + break; + case PTYPE_SINT32: + CHECK_VARINT(a, -1); + v.i.low = a->v.i.low; + v.i.hi = a->v.i.hi; + _pbcV_dezigzag32((struct longlong *)&(v.i)); + break; + case PTYPE_SINT64: + CHECK_VARINT(a, -1); + v.i.low = a->v.i.low; + v.i.hi = a->v.i.hi; + _pbcV_dezigzag64((struct longlong *)&(v.i)); + break; + case PTYPE_STRING: + case PTYPE_BYTES: + case PTYPE_MESSAGE: + CHECK_LEND(a, -1); + v.s.buffer = start + a->v.s.start; + v.s.len = a->v.s.end - a->v.s.start; + break; + default: + assert(0); + break; + } + pd(ud, type, typename, &v, f->id, f->name); + return 0; +} + +static int +call_array(pbc_decoder pd, void * ud, struct _field *f, uint8_t * buffer , int size) { + union pbc_value v; + const char * typename = NULL; + int type = _pbcP_type(f, &typename); + assert(type != 0); + if (typename == NULL) { + typename = TYPENAME[type & ~PBC_REPEATED]; + } + v.i.hi = 0; + int i; + switch(f->type) { + case PTYPE_DOUBLE: + if (size % 8 != 0) { + return -1; + } + for (i=0;iid, f->name); + } + return size/8; + case PTYPE_FLOAT: + if (size % 4 != 0) + return -1; + for (i=0;iid, f->name); + } + return size/4; + case PTYPE_FIXED32: + case PTYPE_SFIXED32: + if (size % 4 != 0) + return -1; + for (i=0;iid, f->name); + } + return size/4; + case PTYPE_FIXED64: + case PTYPE_SFIXED64: + if (size % 8 != 0) + return -1; + for (i=0;iid, f->name); + } + return size/8; + case PTYPE_INT64: + case PTYPE_UINT64: + case PTYPE_INT32: + case PTYPE_UINT32: + case PTYPE_BOOL: { + int n = 0; + while (size > 0) { + int len; + if (size >= 10) { + len = _pbcV_decode(buffer, (struct longlong *)&(v.i)); + } else { + uint8_t temp[10]; + memcpy(temp, buffer, size); + len = _pbcV_decode(buffer, (struct longlong *)&(v.i)); + if (len > size) + return -1; + } + pd(ud, type , typename, &v, f->id, f->name); + buffer += len; + size -= len; + ++n; + } + return n; + } + case PTYPE_ENUM: { + int n = 0; + while (size > 0) { + int len; + if (size >= 10) { + len = _pbcV_decode(buffer, (struct longlong *)&(v.i)); + } else { + uint8_t temp[10]; + memcpy(temp, buffer, size); + len = _pbcV_decode(buffer, (struct longlong *)&(v.i)); + if (len > size) + return -1; + } + v.e.id = v.i.low; + v.e.name = _pbcM_ip_query(f->type_name.e->id , v.i.low); + pd(ud, type , typename, &v, f->id, f->name); + buffer += len; + size -= len; + ++n; + } + return n; + } + case PTYPE_SINT32: { + int n = 0; + while (size > 0) { + int len; + if (size >= 10) { + len = _pbcV_decode(buffer, (struct longlong *)&(v.i)); + _pbcV_dezigzag32((struct longlong *)&(v.i)); + } else { + uint8_t temp[10]; + memcpy(temp, buffer, size); + len = _pbcV_decode(buffer, (struct longlong *)&(v.i)); + if (len > size) + return -1; + _pbcV_dezigzag32((struct longlong *)&(v.i)); + } + pd(ud, type , typename, &v, f->id, f->name); + buffer += len; + size -= len; + ++n; + } + return n; + } + case PTYPE_SINT64: { + int n = 0; + while (size > 0) { + int len; + if (size >= 10) { + len = _pbcV_decode(buffer, (struct longlong *)&(v.i)); + _pbcV_dezigzag64((struct longlong *)&(v.i)); + } else { + uint8_t temp[10]; + memcpy(temp, buffer, size); + len = _pbcV_decode(buffer, (struct longlong *)&(v.i)); + if (len > size) + return -1; + _pbcV_dezigzag64((struct longlong *)&(v.i)); + } + pd(ud, type , typename, &v, f->id, f->name); + buffer += len; + size -= len; + ++n; + } + return n; + } + default: + return -1; + } +} + +int +pbc_decode(struct pbc_env * env, const char * typename , struct pbc_slice * slice, pbc_decoder pd, void *ud) { + struct _message * msg = _pbcP_get_message(env, typename); + if (msg == NULL) { + env->lasterror = "Proto not found"; + return -1; + } + if (slice->len == 0) { + return 0; + } + pbc_ctx _ctx; + int count = _pbcC_open(_ctx,slice->buffer,slice->len); + if (count <= 0) { + env->lasterror = "decode context error"; + _pbcC_close(_ctx); + return count - 1; + } + struct context * ctx = (struct context *)_ctx; + uint8_t * start = slice->buffer; + + int i; + for (i=0;inumber;i++) { + int id = ctx->a[i].wire_id >> 3; + struct _field * f = _pbcM_ip_query(msg->id , id); + if (f==NULL) { + int err = call_unknown(pd,ud,id,&ctx->a[i],start); + if (err) { + _pbcC_close(_ctx); + return -i-1; + } + } else if (f->label == LABEL_PACKED) { + struct atom * a = &ctx->a[i]; + int n = call_array(pd, ud, f , start + a->v.s.start , a->v.s.end - a->v.s.start); + if (n < 0) { + _pbcC_close(_ctx); + return -i-1; + } + } else { + if (call_type(pd,ud,f,&ctx->a[i],start) != 0) { + _pbcC_close(_ctx); + return -i-1; + } + } + } + + _pbcC_close(_ctx); + return ctx->number; +} + diff --git a/lua-protobuf/descriptor.pbc.h b/lua-protobuf/descriptor.pbc.h new file mode 100644 index 00000000..88dca861 --- /dev/null +++ b/lua-protobuf/descriptor.pbc.h @@ -0,0 +1,271 @@ +static unsigned char pbc_descriptor[] = { +72,1,72,2,72,3,72,4,72,5,72,6,72,7,72,8, +72,9,72,10,72,11,72,12,72,13,72,14,72,15,72,16, +72,17,72,18,72,1,72,2,72,3,72,1,72,2,72,3, +72,0,72,1,72,2,50,42,103,111,111,103,108,101,46,112, +114,111,116,111,98,117,102,46,70,105,101,108,100,68,101,115, +99,114,105,112,116,111,114,80,114,111,116,111,46,84,121,112, +101,0,50,43,103,111,111,103,108,101,46,112,114,111,116,111, +98,117,102,46,70,105,101,108,100,68,101,115,99,114,105,112, +116,111,114,80,114,111,116,111,46,76,97,98,101,108,0,50, +41,103,111,111,103,108,101,46,112,114,111,116,111,98,117,102, +46,70,105,108,101,79,112,116,105,111,110,115,46,79,112,116, +105,109,105,122,101,77,111,100,101,0,50,35,103,111,111,103, +108,101,46,112,114,111,116,111,98,117,102,46,70,105,101,108, +100,79,112,116,105,111,110,115,46,67,84,121,112,101,0,66, +12,84,89,80,69,95,68,79,85,66,76,69,0,66,11,84, +89,80,69,95,70,76,79,65,84,0,66,11,84,89,80,69, +95,73,78,84,54,52,0,66,12,84,89,80,69,95,85,73, +78,84,54,52,0,66,11,84,89,80,69,95,73,78,84,51, +50,0,66,13,84,89,80,69,95,70,73,88,69,68,54,52, +0,66,13,84,89,80,69,95,70,73,88,69,68,51,50,0, +66,10,84,89,80,69,95,66,79,79,76,0,66,12,84,89, +80,69,95,83,84,82,73,78,71,0,66,11,84,89,80,69, +95,71,82,79,85,80,0,66,13,84,89,80,69,95,77,69, +83,83,65,71,69,0,66,11,84,89,80,69,95,66,89,84, +69,83,0,66,12,84,89,80,69,95,85,73,78,84,51,50, +0,66,10,84,89,80,69,95,69,78,85,77,0,66,14,84, +89,80,69,95,83,70,73,88,69,68,51,50,0,66,14,84, +89,80,69,95,83,70,73,88,69,68,54,52,0,66,12,84, +89,80,69,95,83,73,78,84,51,50,0,66,12,84,89,80, +69,95,83,73,78,84,54,52,0,66,15,76,65,66,69,76, +95,79,80,84,73,79,78,65,76,0,66,15,76,65,66,69, +76,95,82,69,81,85,73,82,69,68,0,66,15,76,65,66, +69,76,95,82,69,80,69,65,84,69,68,0,66,6,83,80, +69,69,68,0,66,10,67,79,68,69,95,83,73,90,69,0, +66,13,76,73,84,69,95,82,85,78,84,73,77,69,0,66, +7,83,84,82,73,78,71,0,66,5,67,79,82,68,0,66, +13,83,84,82,73,78,71,95,80,73,69,67,69,0,56,18, +56,3,56,3,56,3,10,11,100,101,115,99,114,105,112,116, +111,114,0,32,1,32,9,32,7,32,2,32,8,32,3,32, +3,32,3,32,4,32,9,32,3,32,5,32,1,32,1,32, +1,32,1,32,7,32,2,32,1,32,2,42,51,24,2,16, +1,32,11,42,36,103,111,111,103,108,101,46,112,114,111,116, +111,98,117,102,46,70,105,108,101,68,101,115,99,114,105,112, +116,111,114,80,114,111,116,111,0,10,5,102,105,108,101,0, +42,13,32,9,24,0,10,5,110,97,109,101,0,16,1,42, +16,32,9,24,0,10,8,112,97,99,107,97,103,101,0,16, +2,42,19,32,9,24,2,10,11,100,101,112,101,110,100,101, +110,99,121,0,16,3,42,55,24,2,16,4,32,11,42,32, +103,111,111,103,108,101,46,112,114,111,116,111,98,117,102,46, +68,101,115,99,114,105,112,116,111,114,80,114,111,116,111,0, +10,13,109,101,115,115,97,103,101,95,116,121,112,101,0,42, +56,24,2,16,5,32,11,42,36,103,111,111,103,108,101,46, +112,114,111,116,111,98,117,102,46,69,110,117,109,68,101,115, +99,114,105,112,116,111,114,80,114,111,116,111,0,10,10,101, +110,117,109,95,116,121,112,101,0,42,57,24,2,16,6,32, +11,42,39,103,111,111,103,108,101,46,112,114,111,116,111,98, +117,102,46,83,101,114,118,105,99,101,68,101,115,99,114,105, +112,116,111,114,80,114,111,116,111,0,10,8,115,101,114,118, +105,99,101,0,42,57,24,2,16,7,32,11,42,37,103,111, +111,103,108,101,46,112,114,111,116,111,98,117,102,46,70,105, +101,108,100,68,101,115,99,114,105,112,116,111,114,80,114,111, +116,111,0,10,10,101,120,116,101,110,115,105,111,110,0,42, +46,24,0,16,8,32,11,42,28,103,111,111,103,108,101,46, +112,114,111,116,111,98,117,102,46,70,105,108,101,79,112,116, +105,111,110,115,0,10,8,111,112,116,105,111,110,115,0,42, +58,24,0,16,9,32,11,42,31,103,111,111,103,108,101,46, +112,114,111,116,111,98,117,102,46,83,111,117,114,99,101,67, +111,100,101,73,110,102,111,0,10,17,115,111,117,114,99,101, +95,99,111,100,101,95,105,110,102,111,0,42,13,32,9,24, +0,10,5,110,97,109,101,0,16,1,42,53,24,2,16,2, +32,11,42,37,103,111,111,103,108,101,46,112,114,111,116,111, +98,117,102,46,70,105,101,108,100,68,101,115,99,114,105,112, +116,111,114,80,114,111,116,111,0,10,6,102,105,101,108,100, +0,42,57,24,2,16,6,32,11,42,37,103,111,111,103,108, +101,46,112,114,111,116,111,98,117,102,46,70,105,101,108,100, +68,101,115,99,114,105,112,116,111,114,80,114,111,116,111,0, +10,10,101,120,116,101,110,115,105,111,110,0,42,54,24,2, +16,3,32,11,42,32,103,111,111,103,108,101,46,112,114,111, +116,111,98,117,102,46,68,101,115,99,114,105,112,116,111,114, +80,114,111,116,111,0,10,12,110,101,115,116,101,100,95,116, +121,112,101,0,42,56,24,2,16,4,32,11,42,36,103,111, +111,103,108,101,46,112,114,111,116,111,98,117,102,46,69,110, +117,109,68,101,115,99,114,105,112,116,111,114,80,114,111,116, +111,0,10,10,101,110,117,109,95,116,121,112,101,0,42,73, +24,2,16,5,32,11,42,47,103,111,111,103,108,101,46,112, +114,111,116,111,98,117,102,46,68,101,115,99,114,105,112,116, +111,114,80,114,111,116,111,46,69,120,116,101,110,115,105,111, +110,82,97,110,103,101,0,10,16,101,120,116,101,110,115,105, +111,110,95,114,97,110,103,101,0,42,49,24,0,16,7,32, +11,42,31,103,111,111,103,108,101,46,112,114,111,116,111,98, +117,102,46,77,101,115,115,97,103,101,79,112,116,105,111,110, +115,0,10,8,111,112,116,105,111,110,115,0,42,14,32,5, +24,0,10,6,115,116,97,114,116,0,16,1,42,12,32,5, +24,0,10,4,101,110,100,0,16,2,42,13,32,9,24,0, +10,5,110,97,109,101,0,16,1,42,15,32,5,24,0,10, +7,110,117,109,98,101,114,0,16,3,42,59,24,0,16,4, +32,14,42,43,103,111,111,103,108,101,46,112,114,111,116,111, +98,117,102,46,70,105,101,108,100,68,101,115,99,114,105,112, +116,111,114,80,114,111,116,111,46,76,97,98,101,108,0,10, +6,108,97,98,101,108,0,42,57,24,0,16,5,32,14,42, +42,103,111,111,103,108,101,46,112,114,111,116,111,98,117,102, +46,70,105,101,108,100,68,101,115,99,114,105,112,116,111,114, +80,114,111,116,111,46,84,121,112,101,0,10,5,116,121,112, +101,0,42,18,32,9,24,0,10,10,116,121,112,101,95,110, +97,109,101,0,16,6,42,17,32,9,24,0,10,9,101,120, +116,101,110,100,101,101,0,16,2,42,22,32,9,24,0,10, +14,100,101,102,97,117,108,116,95,118,97,108,117,101,0,16, +7,42,47,24,0,16,8,32,11,42,29,103,111,111,103,108, +101,46,112,114,111,116,111,98,117,102,46,70,105,101,108,100, +79,112,116,105,111,110,115,0,10,8,111,112,116,105,111,110, +115,0,42,13,32,9,24,0,10,5,110,97,109,101,0,16, +1,42,57,24,2,16,2,32,11,42,41,103,111,111,103,108, +101,46,112,114,111,116,111,98,117,102,46,69,110,117,109,86, +97,108,117,101,68,101,115,99,114,105,112,116,111,114,80,114, +111,116,111,0,10,6,118,97,108,117,101,0,42,46,24,0, +16,3,32,11,42,28,103,111,111,103,108,101,46,112,114,111, +116,111,98,117,102,46,69,110,117,109,79,112,116,105,111,110, +115,0,10,8,111,112,116,105,111,110,115,0,42,13,32,9, +24,0,10,5,110,97,109,101,0,16,1,42,15,32,5,24, +0,10,7,110,117,109,98,101,114,0,16,2,42,51,24,0, +16,3,32,11,42,33,103,111,111,103,108,101,46,112,114,111, +116,111,98,117,102,46,69,110,117,109,86,97,108,117,101,79, +112,116,105,111,110,115,0,10,8,111,112,116,105,111,110,115, +0,42,13,32,9,24,0,10,5,110,97,109,101,0,16,1, +42,55,24,2,16,2,32,11,42,38,103,111,111,103,108,101, +46,112,114,111,116,111,98,117,102,46,77,101,116,104,111,100, +68,101,115,99,114,105,112,116,111,114,80,114,111,116,111,0, +10,7,109,101,116,104,111,100,0,42,49,24,0,16,3,32, +11,42,31,103,111,111,103,108,101,46,112,114,111,116,111,98, +117,102,46,83,101,114,118,105,99,101,79,112,116,105,111,110, +115,0,10,8,111,112,116,105,111,110,115,0,42,13,32,9, +24,0,10,5,110,97,109,101,0,16,1,42,19,32,9,24, +0,10,11,105,110,112,117,116,95,116,121,112,101,0,16,2, +42,20,32,9,24,0,10,12,111,117,116,112,117,116,95,116, +121,112,101,0,16,3,42,48,24,0,16,4,32,11,42,30, +103,111,111,103,108,101,46,112,114,111,116,111,98,117,102,46, +77,101,116,104,111,100,79,112,116,105,111,110,115,0,10,8, +111,112,116,105,111,110,115,0,42,21,32,9,24,0,10,13, +106,97,118,97,95,112,97,99,107,97,103,101,0,16,1,42, +29,32,9,24,0,10,21,106,97,118,97,95,111,117,116,101, +114,95,99,108,97,115,115,110,97,109,101,0,16,8,42,30, +24,0,16,10,32,8,10,20,106,97,118,97,95,109,117,108, +116,105,112,108,101,95,102,105,108,101,115,0,48,0,42,40, +24,0,16,20,32,8,10,30,106,97,118,97,95,103,101,110, +101,114,97,116,101,95,101,113,117,97,108,115,95,97,110,100, +95,104,97,115,104,0,48,0,42,72,24,0,16,9,32,14, +42,41,103,111,111,103,108,101,46,112,114,111,116,111,98,117, +102,46,70,105,108,101,79,112,116,105,111,110,115,46,79,112, +116,105,109,105,122,101,77,111,100,101,0,10,13,111,112,116, +105,109,105,122,101,95,102,111,114,0,58,6,83,80,69,69, +68,0,42,30,24,0,16,16,32,8,10,20,99,99,95,103, +101,110,101,114,105,99,95,115,101,114,118,105,99,101,115,0, +48,0,42,32,24,0,16,17,32,8,10,22,106,97,118,97, +95,103,101,110,101,114,105,99,95,115,101,114,118,105,99,101, +115,0,48,0,42,30,24,0,16,18,32,8,10,20,112,121, +95,103,101,110,101,114,105,99,95,115,101,114,118,105,99,101, +115,0,48,0,42,68,24,2,16,231,7,32,11,42,36,103, +111,111,103,108,101,46,112,114,111,116,111,98,117,102,46,85, +110,105,110,116,101,114,112,114,101,116,101,100,79,112,116,105, +111,110,0,10,21,117,110,105,110,116,101,114,112,114,101,116, +101,100,95,111,112,116,105,111,110,0,42,34,24,0,16,1, +32,8,10,24,109,101,115,115,97,103,101,95,115,101,116,95, +119,105,114,101,95,102,111,114,109,97,116,0,48,0,42,42, +24,0,16,2,32,8,10,32,110,111,95,115,116,97,110,100, +97,114,100,95,100,101,115,99,114,105,112,116,111,114,95,97, +99,99,101,115,115,111,114,0,48,0,42,68,24,2,16,231, +7,32,11,42,36,103,111,111,103,108,101,46,112,114,111,116, +111,98,117,102,46,85,110,105,110,116,101,114,112,114,101,116, +101,100,79,112,116,105,111,110,0,10,21,117,110,105,110,116, +101,114,112,114,101,116,101,100,95,111,112,116,105,111,110,0, +42,60,24,0,16,1,32,14,42,35,103,111,111,103,108,101, +46,112,114,111,116,111,98,117,102,46,70,105,101,108,100,79, +112,116,105,111,110,115,46,67,84,121,112,101,0,10,6,99, +116,121,112,101,0,58,7,83,84,82,73,78,71,0,42,15, +32,8,24,0,10,7,112,97,99,107,101,100,0,16,2,42, +21,24,0,16,3,32,8,10,11,100,101,112,114,101,99,97, +116,101,100,0,48,0,42,29,32,9,24,0,10,21,101,120, +112,101,114,105,109,101,110,116,97,108,95,109,97,112,95,107, +101,121,0,16,9,42,68,24,2,16,231,7,32,11,42,36, +103,111,111,103,108,101,46,112,114,111,116,111,98,117,102,46, +85,110,105,110,116,101,114,112,114,101,116,101,100,79,112,116, +105,111,110,0,10,21,117,110,105,110,116,101,114,112,114,101, +116,101,100,95,111,112,116,105,111,110,0,42,68,24,2,16, +231,7,32,11,42,36,103,111,111,103,108,101,46,112,114,111, +116,111,98,117,102,46,85,110,105,110,116,101,114,112,114,101, +116,101,100,79,112,116,105,111,110,0,10,21,117,110,105,110, +116,101,114,112,114,101,116,101,100,95,111,112,116,105,111,110, +0,42,68,24,2,16,231,7,32,11,42,36,103,111,111,103, +108,101,46,112,114,111,116,111,98,117,102,46,85,110,105,110, +116,101,114,112,114,101,116,101,100,79,112,116,105,111,110,0, +10,21,117,110,105,110,116,101,114,112,114,101,116,101,100,95, +111,112,116,105,111,110,0,42,68,24,2,16,231,7,32,11, +42,36,103,111,111,103,108,101,46,112,114,111,116,111,98,117, +102,46,85,110,105,110,116,101,114,112,114,101,116,101,100,79, +112,116,105,111,110,0,10,21,117,110,105,110,116,101,114,112, +114,101,116,101,100,95,111,112,116,105,111,110,0,42,68,24, +2,16,231,7,32,11,42,36,103,111,111,103,108,101,46,112, +114,111,116,111,98,117,102,46,85,110,105,110,116,101,114,112, +114,101,116,101,100,79,112,116,105,111,110,0,10,21,117,110, +105,110,116,101,114,112,114,101,116,101,100,95,111,112,116,105, +111,110,0,42,60,24,2,16,2,32,11,42,45,103,111,111, +103,108,101,46,112,114,111,116,111,98,117,102,46,85,110,105, +110,116,101,114,112,114,101,116,101,100,79,112,116,105,111,110, +46,78,97,109,101,80,97,114,116,0,10,5,110,97,109,101, +0,42,25,32,9,24,0,10,17,105,100,101,110,116,105,102, +105,101,114,95,118,97,108,117,101,0,16,3,42,27,32,4, +24,0,10,19,112,111,115,105,116,105,118,101,95,105,110,116, +95,118,97,108,117,101,0,16,4,42,27,32,3,24,0,10, +19,110,101,103,97,116,105,118,101,95,105,110,116,95,118,97, +108,117,101,0,16,5,42,21,32,1,24,0,10,13,100,111, +117,98,108,101,95,118,97,108,117,101,0,16,6,42,21,32, +12,24,0,10,13,115,116,114,105,110,103,95,118,97,108,117, +101,0,16,7,42,24,32,9,24,0,10,16,97,103,103,114, +101,103,97,116,101,95,118,97,108,117,101,0,16,8,42,18, +32,9,24,1,10,10,110,97,109,101,95,112,97,114,116,0, +16,1,42,21,32,8,24,1,10,13,105,115,95,101,120,116, +101,110,115,105,111,110,0,16,2,42,59,24,2,16,1,32, +11,42,40,103,111,111,103,108,101,46,112,114,111,116,111,98, +117,102,46,83,111,117,114,99,101,67,111,100,101,73,110,102, +111,46,76,111,99,97,116,105,111,110,0,10,9,108,111,99, +97,116,105,111,110,0,42,13,32,5,24,2,10,5,112,97, +116,104,0,16,1,42,13,32,5,24,2,10,5,115,112,97, +110,0,16,2,26,34,103,111,111,103,108,101,46,112,114,111, +116,111,98,117,102,46,70,105,108,101,68,101,115,99,114,105, +112,116,111,114,83,101,116,0,26,36,103,111,111,103,108,101, +46,112,114,111,116,111,98,117,102,46,70,105,108,101,68,101, +115,99,114,105,112,116,111,114,80,114,111,116,111,0,26,32, +103,111,111,103,108,101,46,112,114,111,116,111,98,117,102,46, +68,101,115,99,114,105,112,116,111,114,80,114,111,116,111,0, +26,47,103,111,111,103,108,101,46,112,114,111,116,111,98,117, +102,46,68,101,115,99,114,105,112,116,111,114,80,114,111,116, +111,46,69,120,116,101,110,115,105,111,110,82,97,110,103,101, +0,26,37,103,111,111,103,108,101,46,112,114,111,116,111,98, +117,102,46,70,105,101,108,100,68,101,115,99,114,105,112,116, +111,114,80,114,111,116,111,0,26,36,103,111,111,103,108,101, +46,112,114,111,116,111,98,117,102,46,69,110,117,109,68,101, +115,99,114,105,112,116,111,114,80,114,111,116,111,0,26,41, +103,111,111,103,108,101,46,112,114,111,116,111,98,117,102,46, +69,110,117,109,86,97,108,117,101,68,101,115,99,114,105,112, +116,111,114,80,114,111,116,111,0,26,39,103,111,111,103,108, +101,46,112,114,111,116,111,98,117,102,46,83,101,114,118,105, +99,101,68,101,115,99,114,105,112,116,111,114,80,114,111,116, +111,0,26,38,103,111,111,103,108,101,46,112,114,111,116,111, +98,117,102,46,77,101,116,104,111,100,68,101,115,99,114,105, +112,116,111,114,80,114,111,116,111,0,26,28,103,111,111,103, +108,101,46,112,114,111,116,111,98,117,102,46,70,105,108,101, +79,112,116,105,111,110,115,0,26,31,103,111,111,103,108,101, +46,112,114,111,116,111,98,117,102,46,77,101,115,115,97,103, +101,79,112,116,105,111,110,115,0,26,29,103,111,111,103,108, +101,46,112,114,111,116,111,98,117,102,46,70,105,101,108,100, +79,112,116,105,111,110,115,0,26,28,103,111,111,103,108,101, +46,112,114,111,116,111,98,117,102,46,69,110,117,109,79,112, +116,105,111,110,115,0,26,33,103,111,111,103,108,101,46,112, +114,111,116,111,98,117,102,46,69,110,117,109,86,97,108,117, +101,79,112,116,105,111,110,115,0,26,31,103,111,111,103,108, +101,46,112,114,111,116,111,98,117,102,46,83,101,114,118,105, +99,101,79,112,116,105,111,110,115,0,26,30,103,111,111,103, +108,101,46,112,114,111,116,111,98,117,102,46,77,101,116,104, +111,100,79,112,116,105,111,110,115,0,26,36,103,111,111,103, +108,101,46,112,114,111,116,111,98,117,102,46,85,110,105,110, +116,101,114,112,114,101,116,101,100,79,112,116,105,111,110,0, +26,45,103,111,111,103,108,101,46,112,114,111,116,111,98,117, +102,46,85,110,105,110,116,101,114,112,114,101,116,101,100,79, +112,116,105,111,110,46,78,97,109,101,80,97,114,116,0,26, +31,103,111,111,103,108,101,46,112,114,111,116,111,98,117,102, +46,83,111,117,114,99,101,67,111,100,101,73,110,102,111,0, +26,40,103,111,111,103,108,101,46,112,114,111,116,111,98,117, +102,46,83,111,117,114,99,101,67,111,100,101,73,110,102,111, +46,76,111,99,97,116,105,111,110,0, +}; \ No newline at end of file diff --git a/lua-protobuf/map.c b/lua-protobuf/map.c new file mode 100644 index 00000000..8c75b9ee --- /dev/null +++ b/lua-protobuf/map.c @@ -0,0 +1,477 @@ +#include "map.h" +#include "alloc.h" + +#include +#include + +struct _pbcM_ip_slot { + int id; + void * pointer; + int next; +}; + +struct map_ip { + size_t array_size; + void ** array; + size_t hash_size; + struct _pbcM_ip_slot * slot; +}; + +struct _pbcM_si_slot { + const char *key; + size_t hash; + int id; + int next; +}; + +struct map_si { + size_t size; + struct _pbcM_si_slot slot[1]; +}; + +static size_t +calc_hash(const char *name) +{ + size_t len = strlen(name); + size_t h = len; + size_t step = (len>>5)+1; + size_t i; + for (i=len; i>=step; i-=step) + h = h ^ ((h<<5)+(h>>2)+(size_t)name[i-1]); + return h; +} + +struct map_si * +_pbcM_si_new(struct map_kv * table, int size) +{ + size_t sz = sizeof(struct map_si) + (size-1) * sizeof(struct _pbcM_si_slot); + struct map_si * ret = (struct map_si *)malloc(sz); + memset(ret,0,sz); + + ret->size = (size_t)size; + + int empty = 0; + int i; + + for (i=0;islot[hash]; + if (slot->key == NULL) { + slot->key = table[i].pointer; + slot->id = table[i].id; + slot->hash = hash_full; + } else { + while(ret->slot[empty].key != NULL) { + ++empty; + } + struct _pbcM_si_slot * empty_slot = &ret->slot[empty]; + empty_slot->next = slot->next; + slot->next = empty + 1; + empty_slot->id = table[i].id; + empty_slot->key = table[i].pointer; + empty_slot->hash = hash_full; + } + } + + return ret; +} + +void +_pbcM_si_delete(struct map_si *map) +{ + free(map); +} + +int +_pbcM_si_query(struct map_si *map, const char *key, int *result) +{ + size_t hash_full = calc_hash(key); + size_t hash = hash_full % map->size; + + struct _pbcM_si_slot * slot = &map->slot[hash]; + for (;;) { + if (slot->hash == hash_full && strcmp(slot->key, key) == 0) { + *result = slot->id; + return 0; + } + if (slot->next == 0) { + return 1; + } + slot = &map->slot[slot->next-1]; + } +} + +static struct map_ip * +_pbcM_ip_new_hash(struct map_kv * table, int size) +{ + struct map_ip * ret = (struct map_ip *)malloc(sizeof(struct map_ip)); + ret->array = NULL; + ret->array_size = 0; + ret->hash_size = (size_t)size; + ret->slot = malloc(sizeof(struct _pbcM_ip_slot) * size); + memset(ret->slot,0,sizeof(struct _pbcM_ip_slot) * size); + int empty = 0; + int i; + for (i=0;islot[hash]; + if (slot->pointer == NULL) { + slot->pointer = table[i].pointer; + slot->id = table[i].id; + } else { + while(ret->slot[empty].pointer != NULL) { + ++empty; + } + struct _pbcM_ip_slot * empty_slot = &ret->slot[empty]; + empty_slot->next = slot->next; + slot->next = empty + 1; + empty_slot->id = table[i].id; + empty_slot->pointer = table[i].pointer; + } + } + return ret; +} + +struct map_ip * +_pbcM_ip_new(struct map_kv * table, int size) +{ + int i; + int max = table[0].id; + if (max > size * 2 || max < 0) + return _pbcM_ip_new_hash(table,size); + for (i=1;i max) { + max = table[i].id; + if (max > size * 2) + return _pbcM_ip_new_hash(table,size); + } + } + struct map_ip * ret = (struct map_ip *)malloc(sizeof(struct map_ip)); + ret->hash_size = size; + ret->slot = NULL; + ret->array_size = max + 1; + ret->array = (void **)malloc((max+1) * sizeof(void *)); + memset(ret->array,0,(max+1) * sizeof(void *)); + for (i=0;iarray[table[i].id] = table[i].pointer; + } + return ret; +} + +void +_pbcM_ip_delete(struct map_ip * map) +{ + if (map) { + free(map->array); + free(map->slot); + free(map); + } +} + +static void +_inject(struct map_kv * table, struct map_ip *map) +{ + if (map->array) { + int n = 0; + int i; + for (i=0;i<(int)map->array_size;i++) { + if (map->array[i]) { + table[n].id = i; + table[n].pointer = map->array[i]; + ++ n; + } + } + } else { + int i; + for (i=0;i<(int)map->hash_size;i++) { + table[i].id = map->slot[i].id; + table[i].pointer = map->slot[i].pointer; + } + } +} + +struct map_ip * +_pbcM_ip_combine(struct map_ip *a, struct map_ip *b) +{ + int sz = (int)(a->hash_size + b->hash_size); + struct map_kv * table = malloc(sz * sizeof(struct map_kv)); + memset(table , 0 , sz * sizeof(struct map_kv)); + _inject(table, a); + _inject(table + a->hash_size, b); + struct map_ip * r = _pbcM_ip_new(table, sz); + free(table); + return r; +} + +void * +_pbcM_ip_query(struct map_ip * map, int id) +{ + if (map == NULL) + return NULL; + if (map->array) { + if (id>=0 && id<(int)map->array_size) + return map->array[id]; + return NULL; + } + int hash = (unsigned)id % map->hash_size; + struct _pbcM_ip_slot * slot = &map->slot[hash]; + for (;;) { + if (slot->id == id) { + return slot->pointer; + } + if (slot->next == 0) { + return NULL; + } + slot = &map->slot[slot->next-1]; + } +} + +struct _pbcM_sp_slot { + const char *key; + size_t hash; + void *pointer; + int next; +}; + +struct map_sp { + size_t cap; + size_t size; + struct heap *heap; + struct _pbcM_sp_slot * slot; +}; + +struct map_sp * +_pbcM_sp_new(int max , struct heap *h) +{ + struct map_sp * ret = (struct map_sp *)HMALLOC(sizeof(struct map_sp)); + int cap = 1; + while (cap < max) { + cap *=2; + } + ret->cap = cap; + ret->size = 0; + ret->slot = (struct _pbcM_sp_slot *)HMALLOC(ret->cap * sizeof(struct _pbcM_sp_slot)); + memset(ret->slot,0,sizeof(struct _pbcM_sp_slot) * ret->cap); + ret->heap = h; + return ret; +} + +void +_pbcM_sp_delete(struct map_sp *map) +{ + if (map && map->heap == NULL) { + _pbcM_free(map->slot); + _pbcM_free(map); + } +} + +static void _pbcM_sp_rehash(struct map_sp *map); + +static void +_pbcM_sp_insert_hash(struct map_sp *map, const char *key, size_t hash_full, void * value) +{ + if (map->cap > map->size) { + size_t hash = hash_full & (map->cap-1); + struct _pbcM_sp_slot * slot = &map->slot[hash]; + if (slot->key == NULL) { + slot->key = key; + slot->pointer = value; + slot->hash = hash_full; + } else { + int empty = (hash + 1) & (map->cap-1); + while(map->slot[empty].key != NULL) { + empty = (empty + 1) & (map->cap-1); + } + struct _pbcM_sp_slot * empty_slot = &map->slot[empty]; + empty_slot->next = slot->next; + slot->next = empty + 1; + empty_slot->pointer = value; + empty_slot->key = key; + empty_slot->hash = hash_full; + } + map->size++; + return; + } + _pbcM_sp_rehash(map); + _pbcM_sp_insert_hash(map, key, hash_full, value); +} + +static void +_pbcM_sp_rehash(struct map_sp *map) { + struct heap * h = map->heap; + struct _pbcM_sp_slot * old_slot = map->slot; + size_t size = map->size; + map->size = 0; + map->cap *= 2; + map->slot = (struct _pbcM_sp_slot *)HMALLOC(sizeof(struct _pbcM_sp_slot)*map->cap); + memset(map->slot,0,sizeof(struct _pbcM_sp_slot)*map->cap); + size_t i; + for (i=0;icap-1); + struct _pbcM_sp_slot * slot = &map->slot[hash]; + if (slot->key == NULL) { + if (map->cap <= map->size) + goto _rehash; + slot->key = key; + slot->hash = hash_full; + map->size++; + return &(slot->pointer); + } else { + for (;;) { + if (slot->hash == hash_full && strcmp(slot->key, key) == 0) + return &(slot->pointer); + if (slot->next == 0) { + break; + } + slot = &map->slot[slot->next-1]; + } + if (map->cap <= map->size) + goto _rehash; + + int empty = (hash + 1) & (map->cap-1); + while(map->slot[empty].key != NULL) { + empty = (empty + 1) & (map->cap-1); + } + struct _pbcM_sp_slot * empty_slot = &map->slot[empty]; + empty_slot->next = slot->next; + slot->next = empty + 1; + empty_slot->key = key; + empty_slot->hash = hash_full; + + map->size++; + + return &(empty_slot->pointer); + } +_rehash: + _pbcM_sp_rehash(map); + return _pbcM_sp_query_insert_hash(map, key, hash_full); +} + +void +_pbcM_sp_insert(struct map_sp *map, const char *key, void * value) +{ + _pbcM_sp_insert_hash(map,key,calc_hash(key),value); +} + +void ** +_pbcM_sp_query_insert(struct map_sp *map, const char *key) +{ + return _pbcM_sp_query_insert_hash(map,key,calc_hash(key)); +} + +void * +_pbcM_sp_query(struct map_sp *map, const char *key) +{ + if (map == NULL) + return NULL; + size_t hash_full = calc_hash(key); + size_t hash = hash_full & (map->cap -1); + + struct _pbcM_sp_slot * slot = &map->slot[hash]; + for (;;) { + if (slot->hash == hash_full && strcmp(slot->key, key) == 0) { + return slot->pointer; + } + if (slot->next == 0) { + return NULL; + } + slot = &map->slot[slot->next-1]; + } +} + +void +_pbcM_sp_foreach(struct map_sp *map, void (*func)(void *p)) +{ + size_t i; + for (i=0;icap;i++) { + if (map->slot[i].pointer) { + func(map->slot[i].pointer); + } + } +} + +void +_pbcM_sp_foreach_ud(struct map_sp *map, void (*func)(void *p, void *ud), void *ud) +{ + size_t i; + for (i=0;icap;i++) { + if (map->slot[i].pointer) { + func(map->slot[i].pointer,ud); + } + } +} + +static int +_find_first(struct map_sp *map) +{ + size_t i; + for (i=0;icap;i++) { + if (map->slot[i].pointer) { + return i; + } + } + return -1; +} + +static int +_find_next(struct map_sp *map, const char *key) +{ + size_t hash_full = calc_hash(key); + size_t hash = hash_full & (map->cap -1); + + struct _pbcM_sp_slot * slot = &map->slot[hash]; + for (;;) { + if (slot->hash == hash_full && strcmp(slot->key, key) == 0) { + int i = slot - map->slot + 1; + while(icap) { + if (map->slot[i].pointer) { + return i; + } + ++i; + } + return -1; + } + if (slot->next == 0) { + return -1; + } + slot = &map->slot[slot->next-1]; + } +} + +void * +_pbcM_sp_next(struct map_sp *map, const char ** key) +{ + if (map == NULL) { + *key = NULL; + return NULL; + } + int idx; + if (*key == NULL) { + idx = _find_first(map); + } else { + idx = _find_next(map, *key); + } + if (idx < 0) { + *key = NULL; + return NULL; + } + *key = map->slot[idx].key; + return map->slot[idx].pointer; +} + + + diff --git a/lua-protobuf/map.h b/lua-protobuf/map.h new file mode 100644 index 00000000..d0fc6a94 --- /dev/null +++ b/lua-protobuf/map.h @@ -0,0 +1,33 @@ +#ifndef PROTOBUF_C_MAP_H +#define PROTOBUF_C_MAP_H + +#include "alloc.h" + +struct map_ip; +struct map_si; +struct map_sp; + +struct map_kv { + int id; + void *pointer; +}; + +struct map_si * _pbcM_si_new(struct map_kv * table, int size); +int _pbcM_si_query(struct map_si *map, const char *key, int *result); +void _pbcM_si_delete(struct map_si *map); + +struct map_ip * _pbcM_ip_new(struct map_kv * table, int size); +struct map_ip * _pbcM_ip_combine(struct map_ip * a, struct map_ip * b); +void * _pbcM_ip_query(struct map_ip * map, int id); +void _pbcM_ip_delete(struct map_ip *map); + +struct map_sp * _pbcM_sp_new(int max, struct heap *h); +void _pbcM_sp_insert(struct map_sp *map, const char *key, void * value); +void * _pbcM_sp_query(struct map_sp *map, const char *key); +void ** _pbcM_sp_query_insert(struct map_sp *map, const char *key); +void _pbcM_sp_delete(struct map_sp *map); +void _pbcM_sp_foreach(struct map_sp *map, void (*func)(void *p)); +void _pbcM_sp_foreach_ud(struct map_sp *map, void (*func)(void *p, void *ud), void *ud); +void * _pbcM_sp_next(struct map_sp *map, const char ** key); + +#endif diff --git a/lua-protobuf/pattern.c b/lua-protobuf/pattern.c new file mode 100644 index 00000000..00a17ded --- /dev/null +++ b/lua-protobuf/pattern.c @@ -0,0 +1,1131 @@ +#include "pbc.h" +#include "alloc.h" +#include "context.h" +#include "varint.h" +#include "pattern.h" +#include "array.h" +#include "proto.h" +#include "map.h" + +#include +#include +#include +#include +#include +#include + +static void +set_default_v(void * output, int ctype, pbc_var defv) { + switch (ctype) { + case CTYPE_INT32: + *(uint32_t *)output = defv->integer.low; + break; + case CTYPE_INT64: + *(uint64_t *)output = (uint64_t)defv->integer.low | (uint64_t)defv->integer.hi << 32; + break; + case CTYPE_DOUBLE: + *(double *)output = defv->real; + break; + case CTYPE_FLOAT: + *(float *)output = (float)defv->real; + break; + case CTYPE_BOOL: + *(bool *)output = (defv->integer.low != 0); + break; + case CTYPE_INT8: + *(uint8_t *)output = (uint8_t)defv->integer.low; + break; + case CTYPE_INT16: + *(uint16_t *)output = (uint16_t)defv->integer.low; + break; + case CTYPE_VAR: + *(union _pbc_var *)output = *defv; + break; + } +} + +static void +_pattern_set_default(struct _pattern_field *field, char *output) { + if (field->ctype == CTYPE_ARRAY || field->ctype == CTYPE_PACKED) { + struct _pbc_array *array = (struct _pbc_array *)(output + field->offset); + _pbcA_open(array); + } else if (field->ptype == PTYPE_ENUM) { + pbc_var defv; + defv->integer.low = field->defv->e.id; + defv->integer.hi = 0; + set_default_v(output + field->offset, field->ctype, defv); + } + set_default_v(output + field->offset, field->ctype, field->defv); +} + +void +pbc_pattern_set_default(struct pbc_pattern *pat, void *output) { + int i; + for (i=0;icount;i++) { + _pattern_set_default(&pat->f[i], output); + } +} + +// pattern unpack + +static struct _pattern_field * +bsearch_pattern(struct pbc_pattern *pat, int id) +{ + int begin = 0; + int end = pat->count; + while (begin < end) { + int mid = (begin + end)/2; + struct _pattern_field * f = &pat->f[mid]; + if (id == f->id) { + return f; + } + if (id < f->id) { + end = mid; + } else { + begin = mid + 1; + } + } + return NULL; +} + +static inline int +write_real(int ctype, double v, void *out) { + switch(ctype) { + case CTYPE_DOUBLE: + *(double *)out = v; + return 0; + case CTYPE_FLOAT: + *(float *)out = (float)v; + return 0; + case CTYPE_VAR: + ((union _pbc_var *)out)->real = v; + return 0; + } + return -1; +} + +static inline int +write_longlong(int ctype, struct longlong *i, void *out) { + switch(ctype) { + case CTYPE_INT32: + *(uint32_t *)out = i->low; + return 0; + case CTYPE_INT64: + *(uint64_t *)out = (uint64_t)i->low | (uint64_t)i->hi << 32; + return 0; + case CTYPE_BOOL: + *(bool *)out = (i->low !=0) ; + return 0; + case CTYPE_INT8: + *(uint8_t *)out = (uint8_t)i->low; + return 0; + case CTYPE_INT16: + *(uint8_t *)out = (uint16_t)i->low; + return 0; + case CTYPE_VAR: + ((union _pbc_var *)out)->integer = *i; + return 0; + } + return -1; +} + +static inline int +write_integer(int ctype, struct atom *a, void *out) { + return write_longlong(ctype, &(a->v.i), out); +} + +static int unpack_array(int ptype, char *buffer, struct atom *, pbc_array _array); + +int +_pbcP_unpack_packed(uint8_t *buffer, int size, int ptype, pbc_array array) { + pbc_var var; + var->integer.hi = 0; + int i; + switch(ptype) { + case PTYPE_DOUBLE: + if (size % 8 != 0) + return -1; + for (i=0;ireal = u.d; + _pbcA_push(array, var); + } + return size/8; + case PTYPE_FLOAT: + if (size % 4 != 0) + return -1; + for (i=0;ireal = (double)u.f; + _pbcA_push(array, var); + } + return size/4; + case PTYPE_FIXED32: + case PTYPE_SFIXED32: + if (size % 4 != 0) + return -1; + for (i=0;iinteger.low = (uint32_t)buffer[i] | + (uint32_t)buffer[i+1] << 8 | + (uint32_t)buffer[i+2] << 16 | + (uint32_t)buffer[i+3] << 24; + _pbcA_push(array, var); + } + return size/4; + case PTYPE_FIXED64: + case PTYPE_SFIXED64: + if (size % 8 != 0) + return -1; + for (i=0;iinteger.low = (uint32_t)buffer[i] | + (uint32_t)buffer[i+1] << 8 | + (uint32_t)buffer[i+2] << 16 | + (uint32_t)buffer[i+3] << 24; + var->integer.hi = (uint32_t)buffer[i+4] | + (uint32_t)buffer[i+5] << 8 | + (uint32_t)buffer[i+6] << 16 | + (uint32_t)buffer[i+7] << 24; + _pbcA_push(array, var); + } + return size/8; + case PTYPE_INT64: + case PTYPE_UINT64: + case PTYPE_INT32: + case PTYPE_UINT32: + case PTYPE_ENUM: // enum must be integer type in pattern mode + case PTYPE_BOOL: { + int n = 0; + while (size > 0) { + int len; + if (size >= 10) { + len = _pbcV_decode(buffer, &(var->integer)); + } else { + uint8_t temp[10]; + memcpy(temp, buffer, size); + len = _pbcV_decode(buffer, &(var->integer)); + if (len > size) + return -1; + } + _pbcA_push(array, var); + buffer += len; + size -= len; + ++n; + } + return n; + } + case PTYPE_SINT32: { + int n = 0; + while (size > 0) { + int len; + if (size >= 10) { + len = _pbcV_decode(buffer, &(var->integer)); + _pbcV_dezigzag32(&(var->integer)); + } else { + uint8_t temp[10]; + memcpy(temp, buffer, size); + len = _pbcV_decode(buffer, &(var->integer)); + if (len > size) + return -1; + _pbcV_dezigzag32(&(var->integer)); + } + _pbcA_push(array, var); + buffer += len; + size -= len; + ++n; + } + return n; + } + case PTYPE_SINT64: { + int n = 0; + while (size > 0) { + int len; + if (size >= 10) { + len = _pbcV_decode(buffer, &(var->integer)); + _pbcV_dezigzag64(&(var->integer)); + } else { + uint8_t temp[10]; + memcpy(temp, buffer, size); + len = _pbcV_decode(buffer, &(var->integer)); + if (len > size) + return -1; + _pbcV_dezigzag64(&(var->integer)); + } + _pbcA_push(array, var); + buffer += len; + size -= len; + ++n; + } + return n; + } + } + return -1; +} + +static int +unpack_field(int ctype, int ptype, char * buffer, struct atom * a, void *out) { + if (ctype == CTYPE_ARRAY) { + return unpack_array(ptype, buffer, a , out); + } + if (ctype == CTYPE_PACKED) { + return _pbcP_unpack_packed((uint8_t *)buffer + a->v.s.start, a->v.s.end - a->v.s.start, ptype, out); + } + switch(ptype) { + case PTYPE_DOUBLE: + CHECK_BIT64(a, -1); + return write_real(ctype, read_double(a), out); + case PTYPE_FLOAT: + CHECK_BIT32(a, -1); + return write_real(ctype, read_float(a), out); + case PTYPE_INT64: + case PTYPE_UINT64: + case PTYPE_INT32: + case PTYPE_UINT32: + case PTYPE_ENUM: // enum must be integer type in pattern mode + case PTYPE_BOOL: + CHECK_VARINT(a, -1); + return write_integer(ctype, a , out); + case PTYPE_FIXED32: + case PTYPE_SFIXED32: + CHECK_BIT32(a, -1); + return write_integer(ctype, a , out); + case PTYPE_FIXED64: + case PTYPE_SFIXED64: + CHECK_BIT64(a, -1); + return write_integer(ctype, a , out); + case PTYPE_SINT32: { + CHECK_VARINT(a, -1); + struct longlong temp = a->v.i; + _pbcV_dezigzag32(&temp); + return write_longlong(ctype, &temp , out); + } + case PTYPE_SINT64: { + CHECK_LEND(a, -1); + struct longlong temp = a->v.i; + _pbcV_dezigzag64(&temp); + return write_longlong(ctype, &temp , out); + } + case PTYPE_MESSAGE: + CHECK_LEND(a, -1); + ((union _pbc_var *)out)->m.buffer = buffer + a->v.s.start; + ((union _pbc_var *)out)->m.len = a->v.s.end - a->v.s.start; + return 0; + case PTYPE_STRING: + case PTYPE_BYTES: + CHECK_LEND(a, -1); + ((struct pbc_slice *)out)->buffer = buffer + a->v.s.start; + ((struct pbc_slice *)out)->len = a->v.s.end - a->v.s.start; + return 0; + } + return -1; +} + +static int +unpack_array(int ptype, char *buffer, struct atom * a, pbc_array _array) { + pbc_var var; + int r = unpack_field(CTYPE_VAR, ptype, buffer, a , var); + if (r !=0 ) + return r; + _pbcA_push(_array , var); + + return 0; +} + +void +pbc_pattern_close_arrays(struct pbc_pattern *pat, void * data) { + int i; + for (i=0;icount;i++) { + if (pat->f[i].ctype == CTYPE_ARRAY || pat->f[i].ctype == CTYPE_PACKED) { + void *array = (char *)data + pat->f[i].offset; + _pbcA_close(array); + } + } +} + +static inline int +_pack_wiretype(uint32_t wt, struct pbc_slice *s) { + int len; + if (s->len < 10) { + uint8_t temp[10]; + len = _pbcV_encode32(wt, temp); + if (len > s->len) + return -1; + memcpy(s->buffer, temp, len); + } else { + len = _pbcV_encode32(wt, s->buffer); + } + s->buffer = (char *)s->buffer + len; + s->len -= len; + return len; +} + +static inline int +_pack_varint64(uint64_t i64, struct pbc_slice *s) { + int len; + if (s->len < 10) { + uint8_t temp[10]; + len = _pbcV_encode(i64, temp); + if (len > s->len) + return -1; + memcpy(s->buffer, temp, len); + } else { + len = _pbcV_encode(i64, s->buffer); + } + s->buffer = (char *)s->buffer + len; + s->len -= len; + return len; +} + +static inline int +_pack_sint32(uint32_t v, struct pbc_slice *s) { + int len; + if (s->len < 10) { + uint8_t temp[10]; + len = _pbcV_zigzag32(v, temp); + if (len > s->len) + return -1; + memcpy(s->buffer, temp, len); + } else { + len = _pbcV_zigzag32(v, s->buffer); + } + s->buffer = (char *)s->buffer + len; + s->len -= len; + return len; +} + +static inline int +_pack_sint64(uint64_t v, struct pbc_slice *s) { + int len; + if (s->len < 10) { + uint8_t temp[10]; + len = _pbcV_zigzag(v, temp); + if (len > s->len) + return -1; + memcpy(s->buffer, temp, len); + } else { + len = _pbcV_zigzag(v, s->buffer); + } + s->buffer = (char *)s->buffer + len; + s->len -= len; + return len; +} + +static inline void +_fix32_encode(uint32_t v , uint8_t *buffer) { + buffer[0] = (uint8_t) v; + buffer[1] = (uint8_t) (v >> 8); + buffer[2] = (uint8_t) (v >> 16); + buffer[3] = (uint8_t) (v >> 24); +} + +static inline void +_fix64_encode(struct longlong *v , uint8_t *buffer) { + _fix32_encode(v->low , buffer); + _fix32_encode(v->hi, buffer + 4); +} + +static int +_pack_number(int ptype , int ctype , struct pbc_slice *s, void *input) { + pbc_var var; + if (ctype == CTYPE_VAR) { + memcpy(var, input, sizeof(var)); + } else { + switch (ctype) { + case CTYPE_INT32: + var->integer.low = *(uint32_t *)input; + var->integer.hi = 0; + break; + case CTYPE_INT64: { + uint64_t v = *(uint64_t *)input; + var->integer.low = (uint32_t) (v & 0xffffffff); + var->integer.hi = (uint32_t) (v >> 32); + break; + } + case CTYPE_INT16: + var->integer.low = *(uint16_t *)input; + var->integer.hi = 0; + break; + case CTYPE_INT8: + var->integer.low = *(uint8_t *)input; + var->integer.hi = 0; + break; + case CTYPE_BOOL: + var->integer.low = *(bool *)input; + var->integer.hi = 0; + break; + case CTYPE_DOUBLE: + var->real = *(double *)input; + break; + case CTYPE_FLOAT: + var->real = *(float *)input; + break; + } + } + + switch(ptype) { + case PTYPE_FIXED64: + case PTYPE_SFIXED64: + if (s->len < 8) + return -1; + _fix64_encode(&(var->integer), s->buffer); + s->buffer = (char *)s->buffer + 8; + s->len -= 8; + return 8; + case PTYPE_DOUBLE: + if (s->len < 8) + return -1; + double_encode(var->real , s->buffer); + s->buffer = (char *)s->buffer + 8; + s->len -= 8; + return 8; + case PTYPE_FLOAT: + if (s->len < 4) + return -1; + float_encode((float)var->real , s->buffer); + s->buffer = (char *)s->buffer + 4; + s->len -= 4; + return 4; + case PTYPE_FIXED32: + case PTYPE_SFIXED32: + if (s->len < 4) + return -1; + _fix32_encode(var->integer.low, s->buffer); + s->buffer = (char *)s->buffer + 4; + s->len -= 4; + return 4; + case PTYPE_UINT64: + case PTYPE_INT64: + case PTYPE_INT32: + return _pack_varint64((uint64_t)var->integer.low | (uint64_t)var->integer.hi << 32, s); + case PTYPE_UINT32: + case PTYPE_BOOL: + case PTYPE_ENUM: + return _pack_wiretype(var->integer.low , s); + case PTYPE_SINT32: + return _pack_sint32(var->integer.low , s); + case PTYPE_SINT64: + return _pack_sint64((uint64_t)var->integer.low | (uint64_t)var->integer.hi << 32 , s); + default: + return -1; + } +} + +static int +_pack_field(struct _pattern_field *pf , int ctype, struct pbc_slice *s, void *input) { + int wiretype; + int ret = 0; + int len; + struct pbc_slice * input_slice; + struct pbc_slice string_slice; + + switch(pf->ptype) { + case PTYPE_FIXED64: + case PTYPE_SFIXED64: + case PTYPE_DOUBLE: + wiretype = WT_BIT64; + goto _number; + case PTYPE_FIXED32: + case PTYPE_SFIXED32: + case PTYPE_FLOAT: + wiretype = WT_BIT32; + goto _number; + case PTYPE_UINT64: + case PTYPE_INT64: + case PTYPE_INT32: + case PTYPE_BOOL: + case PTYPE_UINT32: + case PTYPE_ENUM: + case PTYPE_SINT32: + case PTYPE_SINT64: + wiretype = WT_VARINT; + goto _number; + case PTYPE_STRING: + wiretype = WT_LEND; + input_slice = input; + if (input_slice->len > 0) + goto _string; + string_slice.buffer = input_slice->buffer; + string_slice.len = strlen((const char *)string_slice.buffer) - input_slice->len; + input_slice = &string_slice; + + goto _string; + case PTYPE_MESSAGE: + case PTYPE_BYTES: + wiretype = WT_LEND; + goto _bytes; + default: + break; + } + + return 0; +_bytes: + input_slice = input; +_string: + len = _pack_wiretype(pf->id << 3 | WT_LEND , s); + if (len < 0) { + return len; + } + ret += len; + len = _pack_wiretype(input_slice->len , s); + if (len < 0) { + return len; + } + ret += len; + if (input_slice->len > s->len) + return -1; + memcpy(s->buffer , input_slice->buffer, input_slice->len); + ret += input_slice->len; + s->buffer = (char *)s->buffer + input_slice->len; + s->len -= input_slice->len; + + return ret; +_number: + len = _pack_wiretype(pf->id << 3 | wiretype , s); + if (len < 0) { + return len; + } + ret += len; + len = _pack_number(pf->ptype, ctype , s, input); + if (len < 0) { + return len; + } + ret += len; + + return ret; +} + +static int +_pack_repeated(struct _pattern_field *pf , struct pbc_slice *s, pbc_array array) { + int n = pbc_array_size(array); + int ret = 0; + if (n>0) { + int i; + for (i=0;ilen - len < n * width) + return -1; + int i; + for (i=0;iptype, CTYPE_VAR , s, _pbcA_index_p(array, i)); + } + + return len + n * width; +} + +static int +_pack_packed_varint(struct _pattern_field *pf , struct pbc_slice *slice, pbc_array array) { + struct pbc_slice s = * slice; + int n = pbc_array_size(array); + int estimate = n; + int estimate_len = _pack_wiretype(estimate , &s); + if (estimate_len < 0) { + return -1; + } + int i; + int packed_len = 0; + for (i=0;iptype, CTYPE_VAR , &s, _pbcA_index_p(array, i)); + if (len < 0) + return -1; + packed_len += len; + } + if (packed_len == estimate) { + *slice = s; + return packed_len + estimate_len; + } + uint8_t temp[10]; + struct pbc_slice header_slice = { temp , 10 }; + int header_len = _pack_wiretype(packed_len , &header_slice); + if (header_len == estimate_len) { + memcpy(slice->buffer , temp , header_len); + *slice = s; + return packed_len + estimate_len; + } + if (header_len + packed_len > slice->len) + return -1; + memmove((char *)slice->buffer + header_len , (char *)slice->buffer + estimate_len, packed_len); + memcpy(slice->buffer , temp , header_len); + slice->buffer = (char *)slice->buffer + packed_len + header_len; + slice->len -= packed_len + header_len; + return packed_len + header_len; +} + +static int +_pack_packed(struct _pattern_field *pf , struct pbc_slice *s, pbc_array array) { + int n = pbc_array_size(array); + if (n == 0) + return 0; + + int ret = 0; + int len; + len = _pack_wiretype(pf->id << 3 | WT_LEND , s); + if (len < 0) { + return len; + } + ret += len; + + switch (pf->ptype) { + case PTYPE_FIXED64: + case PTYPE_SFIXED64: + case PTYPE_DOUBLE: + len = _pack_packed_fixed(pf, 8, s , array); + if (len < 0) + return len; + break; + case PTYPE_FIXED32: + case PTYPE_SFIXED32: + case PTYPE_FLOAT: + len = _pack_packed_fixed(pf, 4, s , array); + if (len < 0) + return len; + break; + case PTYPE_UINT64: + case PTYPE_INT64: + case PTYPE_INT32: + case PTYPE_BOOL: + case PTYPE_UINT32: + case PTYPE_ENUM: + case PTYPE_SINT32: + case PTYPE_SINT64: + len = _pack_packed_varint(pf, s, array); + if (len < 0) + return len; + break; + } + ret += len; + + return ret; +} + +static bool +_is_default(struct _pattern_field * pf, void * in) { + switch (pf->ctype) { + case CTYPE_INT64: { + struct longlong * d64 = &pf->defv->integer; + return ((uint64_t)d64->low | (uint64_t)d64->hi << 32) == *(uint64_t *)in; + } + case CTYPE_DOUBLE: + return pf->defv->real == *(double *)in; + case CTYPE_FLOAT: + return (float)(pf->defv->real) == *(float *)in; + case CTYPE_INT32: + return pf->defv->integer.low == *(uint32_t *)in; + case CTYPE_INT16: + return (uint16_t)(pf->defv->integer.low) == *(uint16_t *)in; + case CTYPE_INT8: + return (uint8_t)(pf->defv->integer.low) == *(uint8_t *)in; + case CTYPE_BOOL: + if (pf->defv->integer.low) + return *(bool *)in == true; + else + return *(bool *)in == false; + } + if (pf->ptype == PTYPE_STRING) { + struct pbc_slice *slice = in; + if (slice->buffer == NULL) { + return pf->defv->s.str[0] == '\0'; + } + int len = slice->len; + if (len <= 0) { + return strcmp(pf->defv->s.str, slice->buffer) == 0; + } + return len == pf->defv->s.len && memcmp(pf->defv->s.str, slice->buffer, len)==0; + } + + return false; +} + +int +pbc_pattern_pack(struct pbc_pattern *pat, void *input, struct pbc_slice * s) +{ + struct pbc_slice slice = *s; + int i; + for (i=0;icount;i++) { + struct _pattern_field * pf = &pat->f[i]; + void * in = (char *)input + pf->offset; + int len = 0; + switch(pf->label) { + case LABEL_OPTIONAL: + if (_is_default(pf , in)) { + break; + } + case LABEL_REQUIRED: + len = _pack_field(pf, pf->ctype, &slice, in); + break; + case LABEL_REPEATED: + len = _pack_repeated(pf, &slice , in); + break; + case LABEL_PACKED: + len = _pack_packed(pf, &slice , in); + break; + } + if (len < 0) { + return len; + } + } + int len = (char *)slice.buffer - (char *)s->buffer; + int ret = s->len - len; + s->len = len; + return ret; +} + +int +pbc_pattern_unpack(struct pbc_pattern *pat, struct pbc_slice *s, void * output) { + if (s->len == 0) { + pbc_pattern_set_default(pat, output); + return 0; + } + pbc_ctx _ctx; + int r = _pbcC_open(_ctx, s->buffer, s->len); + if (r <= 0) { + pat->env->lasterror = "Pattern unpack open context error"; + _pbcC_close(_ctx); + return r-1; + } + + struct context * ctx = (struct context *)_ctx; + bool field[pat->count]; + memset(field, 0, sizeof(field)); + + int i; + int fc = 0; + + for (i=0;inumber;i++) { + struct _pattern_field * f = bsearch_pattern(pat, ctx->a[i].wire_id >> 3); + if (f) { + int index = f - pat->f; + if (field[index] == false) { + field[index] = true; + ++fc; + if ((f->ctype == CTYPE_ARRAY || f->ctype == CTYPE_PACKED)) { + struct _pbc_array *array = (struct _pbc_array *)(output + f->offset); + _pbcA_open(array); + } + } + char * out = (char *)output + f->offset; + if (unpack_field(f->ctype , f->ptype , ctx->buffer , &ctx->a[i], out) < 0) { + int j; + for (j=0;jcount;j++) { + if (field[j] == true && (pat->f[j].ctype == CTYPE_ARRAY || pat->f[j].ctype == CTYPE_PACKED)) { + void *array = (char *)output + pat->f[j].offset; + _pbcA_close(array); + } + } + _pbcC_close(_ctx); + pat->env->lasterror = "Pattern unpack field error"; + return -i-1; + } + } + } + _pbcC_close(_ctx); + if (fc != pat->count) { + for (i=0;icount;i++) { + if (field[i] == false) { + _pattern_set_default(&pat->f[i], output); + } + } + } + return 0; +} + +/* + format : key %type + %f float + %F double + %d int32 + %D int64 + %b bool + %h int16 + %c int8 + %s slice + %a array +*/ + +static int +_ctype(const char * ctype) { + if (ctype[0]!='%') + return -1; + switch (ctype[1]) { + case 'f': + return CTYPE_FLOAT; + case 'F': + return CTYPE_DOUBLE; + case 'd': + return CTYPE_INT32; + case 'D': + return CTYPE_INT64; + case 'b': + return CTYPE_BOOL; + case 'h': + return CTYPE_INT16; + case 'c': + return CTYPE_INT8; + case 's': + return CTYPE_VAR; + case 'a': + return CTYPE_ARRAY; + default: + return -1; + } +} + +static int +_ctype_size(const char *ctype) { + switch (ctype[1]) { + case 'f': + return sizeof(float); + case 'F': + return sizeof(double); + case 'd': + return sizeof(int32_t); + case 'D': + return sizeof(int64_t); + case 'b': + return sizeof(bool); + case 'h': + return sizeof(int16_t); + case 'c': + return sizeof(int8_t); + case 's': + return sizeof(struct pbc_slice); + case 'a': + return sizeof(pbc_array); + default: + return 0; + } +} + +static const char * +_copy_string(const char *format , char ** temp) { + char * output = *temp; + while (*format == ' ' || *format == '\t' || *format == '\n' || *format == '\r') { + ++format; + } + while (*format != '\0' && + *format != ' ' && + *format != '\t' && + *format != '\n' && + *format != '\r') { + *output = *format; + ++output; + ++format; + } + *output = '\0'; + ++output; + *temp = output; + + return format; +} + +static int +_scan_pattern(const char * format , char * temp) { + int n = 0; + for(;;) { + format = _copy_string(format , &temp); + if (format[0] == '\0') + return 0; + ++n; + format = _copy_string(format , &temp); + if (format[0] == '\0') + return n; + } +} + +static int +_comp_field(const void * a, const void * b) { + const struct _pattern_field * fa = a; + const struct _pattern_field * fb = b; + + return fa->id - fb->id; +} + +struct pbc_pattern * +_pbcP_new(struct pbc_env * env, int n) { + size_t sz = sizeof(struct pbc_pattern) + (sizeof(struct _pattern_field)) * (n-1); + struct pbc_pattern * ret = malloc(sz); + memset(ret, 0 , sz); + ret->count = n; + ret->env = env; + return ret; +} + +static int +_check_ctype(struct _field * field, struct _pattern_field *f) { + if (field->label == LABEL_REPEATED) { + return f->ctype != CTYPE_ARRAY; + } + if (field->label == LABEL_PACKED) { + return f->ctype != CTYPE_PACKED; + } + if (field->type == PTYPE_STRING || field->type == PTYPE_MESSAGE || field->type == PTYPE_BYTES) { + return f->ctype != CTYPE_VAR; + } + if (field->type == PTYPE_FLOAT || field->type == PTYPE_DOUBLE) { + return !(f->ctype == CTYPE_DOUBLE || f->ctype == CTYPE_FLOAT); + } + if (field->type == PTYPE_ENUM) { + return !(f->ctype == CTYPE_INT8 || + f->ctype == CTYPE_INT8 || + f->ctype == CTYPE_INT16 || + f->ctype == CTYPE_INT32 || + f->ctype == CTYPE_INT64); + } + + return f->ctype == CTYPE_VAR || f->ctype == CTYPE_ARRAY || f->ctype == CTYPE_PACKED || + f->ctype == CTYPE_DOUBLE || f->ctype == CTYPE_FLOAT; +} + +struct pbc_pattern * +_pattern_new(struct _message *m, const char *format) { + int len = strlen(format); + char temp[len+1]; + int n = _scan_pattern(format, temp); + struct pbc_pattern * pat = _pbcP_new(m->env, n); + int i; + + const char *ptr = temp; + + int offset = 0; + + for (i=0;if[i]); + struct _field * field = _pbcM_sp_query(m->name, ptr); + if (field == NULL) { + m->env->lasterror = "Pattern @new query none exist field"; + goto _error; + } + f->id = field->id; + f->ptype = field->type; + *f->defv = *field->default_v; + f->offset = offset; + f->label = field->label; + ptr += strlen(ptr) + 1; + f->ctype = _ctype(ptr); + if (f->ctype < 0) { + m->env->lasterror = "Pattern @new use an invalid ctype"; + goto _error; + } + + if (f->ctype == CTYPE_ARRAY && field->label == LABEL_PACKED) { + f->ctype = CTYPE_PACKED; + } + if (_check_ctype(field, f)) { + m->env->lasterror = "Pattern @new ctype check error"; + goto _error; + } + + offset += _ctype_size(ptr); + ptr += strlen(ptr) + 1; + } + + pat->count = n; + + qsort(pat->f , n , sizeof(struct _pattern_field), _comp_field); + + return pat; +_error: + free(pat); + return NULL; +} + +struct pbc_pattern * +pbc_pattern_new(struct pbc_env * env , const char * message, const char * format, ... ) { + struct _message *m = _pbcP_get_message(env, message); + if (m==NULL) { + env->lasterror = "Pattern new can't find proto"; + return NULL; + } + if (format[0]=='@') { + return _pattern_new(m , format+1); + } + + int len = strlen(format); + char temp[len+1]; + int n = _scan_pattern(format, temp); + struct pbc_pattern * pat = _pbcP_new(env, n); + int i; + va_list ap; + va_start(ap , format); + + const char *ptr = temp; + + for (i=0;if[i]); + struct _field * field = _pbcM_sp_query(m->name, ptr); + if (field == NULL) { + env->lasterror = "Pattern new query none exist field"; + goto _error; + } + f->id = field->id; + f->ptype = field->type; + *f->defv = *field->default_v; + f->offset = va_arg(ap, int); + f->label = field->label; + + ptr += strlen(ptr) + 1; + + f->ctype = _ctype(ptr); + if (f->ctype < 0) { + env->lasterror = "Pattern new use an invalid ctype"; + goto _error; + } + if (f->ctype == CTYPE_ARRAY && field->label == LABEL_PACKED) { + f->ctype = CTYPE_PACKED; + } + if (_check_ctype(field, f)) { + env->lasterror = "Pattern new ctype check error"; + goto _error; + } + + ptr += strlen(ptr) + 1; + } + + va_end(ap); + + pat->count = n; + + qsort(pat->f , n , sizeof(struct _pattern_field), _comp_field); + + return pat; +_error: + free(pat); + return NULL; +} + +void +pbc_pattern_delete(struct pbc_pattern * pat) { + free(pat); +} diff --git a/lua-protobuf/pattern.h b/lua-protobuf/pattern.h new file mode 100644 index 00000000..880d3d5d --- /dev/null +++ b/lua-protobuf/pattern.h @@ -0,0 +1,26 @@ +#ifndef PROTOBUF_C_PATTERN_H +#define PROTOBUF_C_PATTERN_H + +#include "pbc.h" +#include "context.h" +#include "array.h" + +struct _pattern_field { + int id; + int offset; + int ptype; + int ctype; + int label; + pbc_var defv; +}; + +struct pbc_pattern { + struct pbc_env * env; + int count; + struct _pattern_field f[1]; +}; + +struct pbc_pattern * _pbcP_new(struct pbc_env * env, int n); +int _pbcP_unpack_packed(uint8_t *buffer, int size, int ptype, pbc_array array); + +#endif diff --git a/lua-protobuf/pbc-lua.c b/lua-protobuf/pbc-lua.c new file mode 100644 index 00000000..e5930267 --- /dev/null +++ b/lua-protobuf/pbc-lua.c @@ -0,0 +1,1004 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "pbc.h" + +#if LUA_VERSION_NUM == 501 + +#define lua_rawlen lua_objlen +#define luaL_newlib(L ,reg) luaL_register(L,"protobuf.c",reg) +#define luaL_buffinit(L , _ ) +#define luaL_prepbuffsize( b , cap ) malloc(cap) +#define _Free(p) free(p) +#undef luaL_addsize +#define luaL_addsize(b , len) lua_pushlstring(L, temp , len) ; free(temp) +#define luaL_pushresult(b) +#define luaL_checkversion(L) + +#else + +#define _Free(p) + +#endif + +static inline void * +checkuserdata(lua_State *L, int index) { + void * ud = lua_touserdata(L,index); + if (ud == NULL) { + luaL_error(L, "userdata %d is nil",index); + } + return ud; +} + +static int +_env_new(lua_State *L) { + struct pbc_env * env = pbc_new(); + lua_pushlightuserdata(L, env); + return 1; +} + +static int +_env_delete(lua_State *L) { + struct pbc_env * env = lua_touserdata(L,1); + pbc_delete(env); + + return 0; +} + +static int +_env_register(lua_State *L) { + struct pbc_env * env = checkuserdata(L,1); + size_t sz = 0; + const char * buffer = luaL_checklstring(L, 2 , &sz); + struct pbc_slice slice; + slice.buffer = (void *)buffer; + slice.len = (int)sz; + int ret = pbc_register(env, &slice); + + if (ret) { + return luaL_error(L, "register fail"); + } + return 0; +} + +static int +_rmessage_new(lua_State *L) { + struct pbc_env * env = checkuserdata(L,1); + const char * typename = luaL_checkstring(L,2); + struct pbc_slice slice; + if (lua_isstring(L,3)) { + size_t sz = 0; + slice.buffer = (void *)lua_tolstring(L,3,&sz); + slice.len = (int)sz; + } else { + slice.buffer = lua_touserdata(L,3); + slice.len = luaL_checkinteger(L,4); + } + struct pbc_rmessage * m = pbc_rmessage_new(env, typename, &slice); + if (m==NULL) + return 0; + lua_pushlightuserdata(L,m); + return 1; +} + +static int +_rmessage_delete(lua_State *L) { + struct pbc_rmessage * m = checkuserdata(L,1); + pbc_rmessage_delete(m); + + return 0; +} + +static int +_rmessage_integer(lua_State *L) { + struct pbc_rmessage * m = checkuserdata(L,1); + const char * key = luaL_checkstring(L,2); + int index = luaL_checkinteger(L,3); + int32_t v = (int32_t)pbc_rmessage_integer(m, key, index, NULL); + + lua_pushinteger(L,v); + + return 1; +} + +static int +_rmessage_int32(lua_State *L) { + struct pbc_rmessage * m = checkuserdata(L,1); + const char * key = luaL_checkstring(L,2); + int index = luaL_checkinteger(L,3); + uint32_t v = pbc_rmessage_integer(m, key, index, NULL); + lua_pushlightuserdata(L,(void *)(intptr_t)v); + + return 1; +} + +static int +_rmessage_int64_ia64(lua_State *L) { + struct pbc_rmessage * m = checkuserdata(L,1); + const char * key = luaL_checkstring(L,2); + int index = luaL_checkinteger(L,3); + uint32_t v[2]; + v[0] = pbc_rmessage_integer(m, key, index, &v[1]); + uint64_t u64 = (uint64_t)v[0] | (uint64_t)v[1] << 32; + + lua_pushlightuserdata(L,(void *)u64); + + return 1; +} + +static int +_rmessage_int52(lua_State *L) { + struct pbc_rmessage * m = checkuserdata(L,1); + const char * key = luaL_checkstring(L,2); + int index = luaL_checkinteger(L,3); + uint32_t hi,low; + low = pbc_rmessage_integer(m, key, index, &hi); + int64_t v = (int64_t)((uint64_t)hi << 32 | (uint64_t)low); + lua_pushnumber(L,(lua_Number)v); + + return 1; +} + +static int +_rmessage_uint52(lua_State *L) { + struct pbc_rmessage * m = checkuserdata(L,1); + const char * key = luaL_checkstring(L,2); + int index = luaL_checkinteger(L,3); + uint32_t hi,low; + low = pbc_rmessage_integer(m, key, index, &hi); + uint64_t v = (uint64_t)hi << 32 | (uint64_t)low; + lua_pushnumber(L,(lua_Number)v); + + return 1; +} + +static int +_rmessage_real(lua_State *L) { + struct pbc_rmessage * m = checkuserdata(L,1); + const char * key = luaL_checkstring(L,2); + int index = luaL_checkinteger(L,3); + double v = pbc_rmessage_real(m, key, index); + + lua_pushnumber(L,v); + + return 1; +} + +static int +_rmessage_string(lua_State *L) { + struct pbc_rmessage * m = checkuserdata(L,1); + const char * key = luaL_checkstring(L,2); + int index = lua_tointeger(L,3); + int sz = 0; + const char * v = pbc_rmessage_string(m,key,index,&sz); + lua_pushlstring(L,v,sz); + return 1; +} + +static int +_rmessage_message(lua_State *L) { + struct pbc_rmessage * m = checkuserdata(L,1); + const char * key = luaL_checkstring(L,2); + int index = lua_tointeger(L,3); + struct pbc_rmessage * v = pbc_rmessage_message(m,key,index); + lua_pushlightuserdata(L,v); + return 1; +} + +static int +_rmessage_size(lua_State *L) { + struct pbc_rmessage * m = checkuserdata(L,1); + const char * key = luaL_checkstring(L,2); + + int sz = pbc_rmessage_size(m, key); + + lua_pushinteger(L, sz); + + return 1; +} + +static int +_env_type(lua_State *L) { + lua_settop(L,3); + struct pbc_env * env = checkuserdata(L,1); + const char * typename = luaL_checkstring(L,2); + if (lua_isnil(L,3)) { + int ret = pbc_type(env, typename, NULL, NULL); + lua_pushboolean(L,ret); + return 1; + } + const char * key = luaL_checkstring(L,3); + const char * type = NULL; + int ret = pbc_type(env, typename, key, &type); + lua_pushinteger(L,ret); + if (type == NULL) { + return 1; + } { + lua_pushstring(L, type); + return 2; + } +} + +static int +_wmessage_new(lua_State *L) { + struct pbc_env * env = checkuserdata(L,1); + const char * typename = luaL_checkstring(L,2); + struct pbc_wmessage * ret = pbc_wmessage_new(env, typename); + lua_pushlightuserdata(L,ret); + return 1; +} + +static int +_wmessage_delete(lua_State *L) { + struct pbc_wmessage * m = lua_touserdata(L,1); + pbc_wmessage_delete(m); + + return 0; +} + + +static int +_wmessage_integer(lua_State *L) { + struct pbc_wmessage * m = checkuserdata(L,1); + const char * key = luaL_checkstring(L,2); + int number = luaL_checkinteger(L,3); + uint32_t hi = 0; + if (number < 0) + hi = ~0; + pbc_wmessage_integer(m, key, number, hi); + + return 0; +} + +static int +_wmessage_real(lua_State *L) { + struct pbc_wmessage * m = checkuserdata(L,1); + const char * key = luaL_checkstring(L,2); + double number = luaL_checknumber(L,3); + pbc_wmessage_real(m, key, number); + + return 0; +} + +static int +_wmessage_string(lua_State *L) { + struct pbc_wmessage * m = checkuserdata(L,1); + const char * key = luaL_checkstring(L,2); + size_t len = 0; + const char * v = luaL_checklstring(L,3,&len); + int err = pbc_wmessage_string(m, key, v, (int)len); + if (err) { + return luaL_error(L, "Invalid enum %s", v); + } + + return 0; +} + +static int +_wmessage_message(lua_State *L) { + struct pbc_wmessage * m = checkuserdata(L,1); + const char * key = luaL_checkstring(L,2); + struct pbc_wmessage * ret = pbc_wmessage_message(m, key); + lua_pushlightuserdata(L, ret); + + return 1; +} + +static int +_wmessage_int32(lua_State *L) { + struct pbc_wmessage * m = checkuserdata(L,1); + const char * key = luaL_checkstring(L,2); + if (!lua_islightuserdata(L,3)) { + return luaL_error(L,"Need a lightuserdata for int32"); + } + void *number = lua_touserdata(L,3); + pbc_wmessage_integer(m, key, (uint32_t)(intptr_t)number , 0); + return 0; +} + +static int +_wmessage_int64_ia64(lua_State *L) { + struct pbc_wmessage * m = checkuserdata(L,1); + const char * key = luaL_checkstring(L,2); + luaL_checktype(L, 3, LUA_TLIGHTUSERDATA); + void * v = lua_touserdata(L,3); + uint64_t v64 = (uintptr_t)v; + pbc_wmessage_integer(m, key, (uint32_t)v64 , (uint32_t)(v64>>32)); + return 0; +} + +static int +_wmessage_int52(lua_State *L) { + struct pbc_wmessage * m = checkuserdata(L,1); + const char * key = luaL_checkstring(L,2); + int64_t number = (int64_t)(luaL_checknumber(L,3)); + uint32_t hi = (uint32_t)(number >> 32); + pbc_wmessage_integer(m, key, (uint32_t)number, hi); + + return 0; +} + +static int +_wmessage_uint52(lua_State *L) { + struct pbc_wmessage * m = checkuserdata(L,1); + const char * key = luaL_checkstring(L,2); + lua_Number v = (luaL_checknumber(L,3)); + if (v < 0) { + return luaL_error(L, "negative number : %f passed to unsigned field",v); + } + uint64_t number = (uint64_t)v; + uint32_t hi = (uint32_t)(number >> 32); + pbc_wmessage_integer(m, key, (uint32_t)number, hi); + + return 0; +} + +static int +_wmessage_buffer(lua_State *L) { + struct pbc_slice slice; + struct pbc_wmessage * m = checkuserdata(L,1); + pbc_wmessage_buffer(m , &slice); + lua_pushlightuserdata(L, slice.buffer); + lua_pushinteger(L, slice.len); + return 2; +} + +static int +_wmessage_buffer_string(lua_State *L) { + struct pbc_slice slice; + struct pbc_wmessage * m = checkuserdata(L,1); + pbc_wmessage_buffer(m , &slice); + lua_pushlstring(L, (const char *)slice.buffer, slice.len); + return 1; +} + +/* + lightuserdata env + */ +static int +_last_error(lua_State *L) { + struct pbc_env * env = checkuserdata(L, 1); + const char * err = pbc_error(env); + lua_pushstring(L,err); + return 1; +} + +/* + lightuserdata env + string message + string format + */ +static int +_pattern_new(lua_State *L) { + struct pbc_env * env = checkuserdata(L, 1); + const char * message = luaL_checkstring(L,2); + const char * format = luaL_checkstring(L,3); + struct pbc_pattern * pat = pbc_pattern_new(env, message, format); + if (pat == NULL) { + return luaL_error(L, "create patten %s (%s) failed", message , format); + } + lua_pushlightuserdata(L,pat); + + return 1; +} + +static int +_pattern_delete(lua_State *L) { + struct pbc_pattern * pat = lua_touserdata(L,1); + pbc_pattern_delete(pat); + + return 0; +} + +static void * +_push_value(lua_State *L, char * ptr, char type) { + switch(type) { + case 'u': { + uint64_t v = *(uint64_t*)ptr; + ptr += 8; + lua_pushnumber(L,(lua_Number)v); + break; + } + case 'i': { + int32_t v = *(int32_t*)ptr; + ptr += 4; + lua_pushinteger(L,v); + break; + } + case 'b': { + int32_t v = *(int32_t*)ptr; + ptr += 4; + lua_pushboolean(L,v); + break; + } + case 'p': { + uint32_t v = *(uint32_t*)ptr; + ptr += 4; + lua_pushlightuserdata(L,(void *)(intptr_t)v); + break; + } + case 'x': { + void * v; + memcpy(&v, ptr, 8); + lua_pushlightuserdata(L,v); + ptr += 8; + break; + } + case 'd': { + int64_t v = *(int64_t*)ptr; + ptr += 8; + lua_pushnumber(L,(lua_Number)v); + break; + } + case 'r': { + double v = *(double *)ptr; + ptr += 8; + lua_pushnumber(L,v); + break; + } + case 's': { + struct pbc_slice * slice = (struct pbc_slice *)ptr; + lua_pushlstring(L,slice->buffer, slice->len); + ptr += sizeof(struct pbc_slice); + break; + } + case 'm': { + struct pbc_slice * slice = (struct pbc_slice *)ptr; + lua_createtable(L,2,0); + lua_pushlightuserdata(L, slice->buffer); + lua_rawseti(L,-2,1); + lua_pushinteger(L,slice->len); + lua_rawseti(L,-2,2); + ptr += sizeof(struct pbc_slice); + break; + } + } + return ptr; +} + +static void +_push_array(lua_State *L, pbc_array array, char type, int index) { + switch (type) { + case 'I': { + int v = pbc_array_integer(array, index, NULL); + lua_pushinteger(L, v); + break; + } + case 'U': { + uint32_t hi = 0; + uint32_t low = pbc_array_integer(array, index, &hi); + uint64_t v = (uint64_t)hi << 32 | (uint64_t)low; + lua_pushnumber(L, (lua_Number)v); + break; + } + case 'D': { + uint32_t hi = 0; + uint32_t low = pbc_array_integer(array, index, &hi); + uint64_t v = (uint64_t)hi << 32 | (uint64_t)low; + lua_pushnumber(L, (lua_Number)((int64_t)v)); + break; + } + case 'B': { + int v = pbc_array_integer(array, index, NULL); + lua_pushboolean(L, v); + break; + } + case 'P': { + uint32_t v = pbc_array_integer(array, index, NULL); + lua_pushlightuserdata(L,(void *)(intptr_t)v); + break; + } + case 'X': { + uint32_t hi = 0; + uint32_t low = pbc_array_integer(array, index, &hi); + uint64_t v = (uint64_t)low | (uint64_t)hi << 32; + lua_pushlightuserdata(L, (void *)v); + break; + } + case 'R': { + double v = pbc_array_real(array, index); + lua_pushnumber(L, v); + break; + } + case 'S': { + struct pbc_slice * slice = pbc_array_slice(array, index); + lua_pushlstring(L, (const char *)slice->buffer,slice->len); + break; + } + case 'M': { + struct pbc_slice * slice = pbc_array_slice(array, index); + lua_createtable(L,2,0); + lua_pushlightuserdata(L,slice->buffer); + lua_rawseti(L,-2,1); + lua_pushinteger(L,slice->len); + lua_rawseti(L,-2,2); + break; + } + } + lua_rawseti(L,-2,index+1); +} + +/* + lightuserdata pattern + string format "ixrsmb" + integer size + lightuserdata buffer + integer buffer_len + */ +static int +_pattern_unpack(lua_State *L) { + struct pbc_pattern * pat = checkuserdata(L, 1); + if (pat == NULL) { + return luaL_error(L, "unpack pattern is NULL"); + } + size_t format_sz = 0; + const char * format = lua_tolstring(L,2,&format_sz); + int size = lua_tointeger(L,3); + struct pbc_slice slice; + if (lua_isstring(L,4)) { + size_t buffer_len = 0; + const char *buffer = luaL_checklstring(L,4,&buffer_len); + slice.buffer = (void *)buffer; + slice.len = buffer_len; + } else { + if (!lua_isuserdata(L,4)) { + return luaL_error(L, "Need a userdata"); + } + slice.buffer = lua_touserdata(L,4); + slice.len = luaL_checkinteger(L,5); + } + + char temp[size]; + int ret = pbc_pattern_unpack(pat, &slice, temp); + if (ret < 0) + return 0; + lua_checkstack(L, format_sz + 3); + int i; + char * ptr = temp; + bool array = false; + for (i=0;i= 'a' && type <='z') { + ptr = _push_value(L,ptr,type); + } else { + array = true; + int n = pbc_array_size((void *)ptr); + lua_createtable(L,n,0); + int j; + for (j=0;jbuffer = (void*)str; + slice->len = sz; + return ptr + sizeof(struct pbc_slice); + } + case 'm': { + struct pbc_slice * slice = (struct pbc_slice *)ptr; + if (lua_istable(L,index)) { + lua_rawgeti(L,index,1); + slice->buffer = lua_touserdata(L,-1); + lua_rawgeti(L,index,2); + slice->len = lua_tointeger(L,-1); + lua_pop(L,2); + } else { + size_t sz = 0; + const char * buffer = luaL_checklstring(L, index, &sz); + slice->buffer = (void *)buffer; + slice->len = sz; + } + return ptr + sizeof(struct pbc_slice); + } + default: + luaL_error(L,"unknown format %c", type); + return ptr; + } +} + +static void +_get_array_value(lua_State *L, pbc_array array, char type) { + switch(type) { + case 'I': { + int32_t v = luaL_checkinteger(L, -1); + uint32_t hi = 0; + if (v<0) { + hi = ~0; + } + pbc_array_push_integer(array, v, hi); + break; + } + case 'U' : { + uint64_t v = (uint64_t)luaL_checknumber(L, -1); + pbc_array_push_integer(array, (uint32_t)v, (uint32_t)(v >> 32)); + break; + } + case 'D' : { + int64_t v = (int64_t)luaL_checknumber(L, -1); + pbc_array_push_integer(array, (uint32_t)v, (uint32_t)(v >> 32)); + break; + } + case 'B': { + int32_t v = lua_toboolean(L, -1); + pbc_array_push_integer(array, v ? 1: 0, 0); + break; + } + case 'P': { + void *p = lua_touserdata(L, -1); + uint32_t v = (uint32_t)(intptr_t)p; + pbc_array_push_integer(array, v, 0); + break; + } + case 'X': { + luaL_checktype(L,-1,LUA_TLIGHTUSERDATA); + void * i64 = lua_touserdata(L,-1); + uint64_t v = (uintptr_t)i64; + pbc_array_push_integer(array, (uint32_t)v, (uint32_t)(v >> 32)); + break; + } + case 'R': { + double v = luaL_checknumber(L, -1); + pbc_array_push_real(array, v); + break; + } + case 'S': { + size_t sz = 0; + const char * str = luaL_checklstring(L, -1, &sz); + struct pbc_slice slice; + slice.buffer = (void*)str; + slice.len = sz; + pbc_array_push_slice(array, &slice); + break; + } + case 'M': { + struct pbc_slice slice; + if (lua_istable(L,-1)) { + lua_rawgeti(L,-1,1); + slice.buffer = lua_touserdata(L,-1); + lua_rawgeti(L,-2,2); + slice.len = lua_tointeger(L,-1); + lua_pop(L,2); + } else { + size_t sz = 0; + const char * buffer = luaL_checklstring(L, -1, &sz); + slice.buffer = (void *)buffer; + slice.len = sz; + } + pbc_array_push_slice(array, &slice); + break; + } + } +} + +/* + lightuserdata pattern + string format "ixrsmbp" + integer size + */ +static int +_pattern_pack(lua_State *L) { + struct pbc_pattern * pat = checkuserdata(L,1); + if (pat == NULL) { + return luaL_error(L, "pack pattern is NULL"); + } + size_t format_sz = 0; + const char * format = lua_tolstring(L,2,&format_sz); + int size = lua_tointeger(L,3); + + char data[size]; +// A trick , we don't need default value. zero buffer for array and message field. +// pbc_pattern_set_default(pat, data); + memset(data, 0 , size); + + char * ptr = data; + + int i; + + for (i=0;i= 'a' && format[i] <='z') { + ptr = _get_value(L, 4+i, ptr, format[i]); + } else { + if (!lua_istable(L,4+i)) { + luaL_error(L,"need table for array type"); + } + int j; + int n = lua_rawlen(L,4+i); + for (j=0;ji.low); + break; + case PBC_REAL: + lua_pushnumber(L, v->f); + break; + case PBC_BOOL: + lua_pushboolean(L, v->i.low); + break; + case PBC_ENUM: + lua_pushstring(L, v->e.name); + break; + case PBC_BYTES: + case PBC_STRING: + lua_pushlstring(L, (const char *)v->s.buffer , v->s.len); + break; + case PBC_MESSAGE: + lua_pushvalue(L, -3); + lua_pushstring(L, typename); + lua_pushlstring(L, (const char *)v->s.buffer , v->s.len); + lua_call(L, 2 , 1); + break; + case PBC_FIXED64: + lua_pushlstring(L, (const char *)&(v->i), 8); + break; + case PBC_FIXED32: + lua_pushlightuserdata(L,(void *)(intptr_t)v->i.low); + break; + case PBC_INT64: { + uint64_t v64 = (uint64_t)(v->i.hi) << 32 | (uint64_t)(v->i.low); + lua_pushnumber(L,(lua_Number)(int64_t)v64); + break; + } + case PBC_UINT: { + uint64_t v64 = (uint64_t)(v->i.hi) << 32 | (uint64_t)(v->i.low); + lua_pushnumber(L,(lua_Number)v64); + break; + } + default: + luaL_error(L, "Unknown type %s", typename); + break; + } +} + +/* + -3: function decode + -2: table key + -1: table id + */ +static void +decode_cb(void *ud, int type, const char * typename, union pbc_value *v, int id, const char *key) { + lua_State *L = ud; + if (type & PBC_REPEATED) { + push_value(L, type & ~PBC_REPEATED, typename, v); + new_array(L, id , key); // func.decode table.key table.id value array + int n = lua_rawlen(L,-1); + lua_insert(L, -2); // func.decode table.key table.id array value + lua_rawseti(L, -2 , n+1); // func.decode table.key table.id array + lua_pop(L,1); + } else { + push_value(L, type, typename, v); + lua_setfield(L, -3 , key); + } +} + +/* + :1 lightuserdata env + :2 function decode_message + :3 table target + :4 string type + :5 string data + :5 lightuserdata pointer + :6 integer len + + table + */ +static int +_decode(lua_State *L) { + struct pbc_env * env = checkuserdata(L,1); + luaL_checktype(L, 2 , LUA_TFUNCTION); + luaL_checktype(L, 3 , LUA_TTABLE); + const char * type = luaL_checkstring(L,4); + struct pbc_slice slice; + if (lua_type(L,5) == LUA_TSTRING) { + size_t len; + slice.buffer = (void *)luaL_checklstring(L,5,&len); + slice.len = (int)len; + } else { + slice.buffer = checkuserdata(L,5); + slice.len = luaL_checkinteger(L,6); + } + lua_pushvalue(L, 2); + lua_pushvalue(L, 3); + lua_newtable(L); + + int n = pbc_decode(env, type, &slice, decode_cb, L); + if (n<0) { + lua_pushboolean(L,0); + } else { + lua_pushboolean(L,1); + } + return 1; +} + +int +luaopen_protobuf_c(lua_State *L) { + luaL_Reg reg[] = { + {"_env_new" , _env_new }, + {"_env_delete" , _env_delete }, + {"_env_register" , _env_register }, + {"_env_type", _env_type }, + {"_rmessage_new" , _rmessage_new }, + {"_rmessage_delete" , _rmessage_delete }, + {"_rmessage_integer" , _rmessage_integer }, + {"_rmessage_int32", _rmessage_int32 }, + {"_rmessage_int64", _rmessage_int64_ia64 }, + {"_rmessage_int52", _rmessage_int52 }, + {"_rmessage_uint52", _rmessage_uint52 }, + {"_rmessage_real" , _rmessage_real }, + {"_rmessage_string" , _rmessage_string }, + {"_rmessage_message" , _rmessage_message }, + {"_rmessage_size" , _rmessage_size }, + {"_wmessage_new", _wmessage_new }, + {"_wmessage_delete", _wmessage_delete }, + {"_wmessage_integer", _wmessage_integer }, + {"_wmessage_real", _wmessage_real }, + {"_wmessage_string", _wmessage_string }, + {"_wmessage_message", _wmessage_message }, + {"_wmessage_int32", _wmessage_int32 }, + {"_wmessage_int64", _wmessage_int64_ia64 }, + {"_wmessage_int52", _wmessage_int52 }, + {"_wmessage_uint52", _wmessage_uint52 }, + {"_wmessage_buffer", _wmessage_buffer }, + {"_wmessage_buffer_string", _wmessage_buffer_string }, + {"_pattern_new", _pattern_new }, + {"_pattern_delete", _pattern_delete }, + {"_pattern_size", _pattern_size }, + {"_pattern_unpack", _pattern_unpack }, + {"_pattern_pack", _pattern_pack }, + {"_last_error", _last_error }, + {"_decode", _decode }, + {NULL,NULL}, + }; + + if (sizeof(void *) != sizeof(uint64_t)) { + luaL_error(L, "Skynet need ia64"); + } + + luaL_checkversion(L); + luaL_newlib(L, reg); + + return 1; +} diff --git a/lua-protobuf/pbc.h b/lua-protobuf/pbc.h new file mode 100644 index 00000000..dbc956c0 --- /dev/null +++ b/lua-protobuf/pbc.h @@ -0,0 +1,104 @@ +#ifndef PROTOBUF_C_H +#define PROTOBUF_C_H + +#include +#include + +#define PBC_ARRAY_CAP 64 + +#define PBC_NOEXIST -1 +#define PBC_INT 1 +#define PBC_REAL 2 +#define PBC_BOOL 3 +#define PBC_ENUM 4 +#define PBC_STRING 5 +#define PBC_MESSAGE 6 +#define PBC_FIXED64 7 +#define PBC_FIXED32 8 +#define PBC_BYTES 9 +#define PBC_INT64 10 +#define PBC_UINT 11 +#define PBC_UNKNOWN 12 +#define PBC_REPEATED 128 + +typedef struct _pbc_array { char _data[PBC_ARRAY_CAP]; } pbc_array[1]; + +struct pbc_slice { + void *buffer; + int len; +}; + +struct pbc_pattern; +struct pbc_env; +struct pbc_rmessage; +struct pbc_wmessage; + +struct pbc_env * pbc_new(void); +void pbc_delete(struct pbc_env *); +int pbc_register(struct pbc_env *, struct pbc_slice * slice); +int pbc_type(struct pbc_env *, const char * typename , const char * key , const char ** type); +const char * pbc_error(struct pbc_env *); + +// callback api +union pbc_value { + struct { + uint32_t low; + uint32_t hi; + } i; + double f; + struct pbc_slice s; + struct { + int id; + const char * name; + } e; +}; + +typedef void (*pbc_decoder)(void *ud, int type, const char * typename, union pbc_value *v, int id, const char *key); +int pbc_decode(struct pbc_env * env, const char * typename , struct pbc_slice * slice, pbc_decoder f, void *ud); + +// message api + +struct pbc_rmessage * pbc_rmessage_new(struct pbc_env * env, const char * typename , struct pbc_slice * slice); +void pbc_rmessage_delete(struct pbc_rmessage *); + +uint32_t pbc_rmessage_integer(struct pbc_rmessage * , const char *key , int index, uint32_t *hi); +double pbc_rmessage_real(struct pbc_rmessage * , const char *key , int index); +const char * pbc_rmessage_string(struct pbc_rmessage * , const char *key , int index, int *sz); +struct pbc_rmessage * pbc_rmessage_message(struct pbc_rmessage *, const char *key, int index); +int pbc_rmessage_size(struct pbc_rmessage *, const char *key); +int pbc_rmessage_next(struct pbc_rmessage *, const char **key); + +struct pbc_wmessage * pbc_wmessage_new(struct pbc_env * env, const char *typename); +void pbc_wmessage_delete(struct pbc_wmessage *); + +// for negative integer, pass -1 to hi +int pbc_wmessage_integer(struct pbc_wmessage *, const char *key, uint32_t low, uint32_t hi); +int pbc_wmessage_real(struct pbc_wmessage *, const char *key, double v); +int pbc_wmessage_string(struct pbc_wmessage *, const char *key, const char * v, int len); +struct pbc_wmessage * pbc_wmessage_message(struct pbc_wmessage *, const char *key); +void * pbc_wmessage_buffer(struct pbc_wmessage *, struct pbc_slice * slice); + +// array api + +int pbc_array_size(pbc_array); +uint32_t pbc_array_integer(pbc_array array, int index, uint32_t *hi); +double pbc_array_real(pbc_array array, int index); +struct pbc_slice * pbc_array_slice(pbc_array array, int index); + +void pbc_array_push_integer(pbc_array array, uint32_t low, uint32_t hi); +void pbc_array_push_slice(pbc_array array, struct pbc_slice *); +void pbc_array_push_real(pbc_array array, double v); + +struct pbc_pattern * pbc_pattern_new(struct pbc_env * , const char * message, const char *format, ...); +void pbc_pattern_delete(struct pbc_pattern *); + +// return unused bytes , -1 for error +int pbc_pattern_pack(struct pbc_pattern *, void *input, struct pbc_slice * s); + +// <0 for error +int pbc_pattern_unpack(struct pbc_pattern *, struct pbc_slice * s , void * output); + +void pbc_pattern_set_default(struct pbc_pattern * , void *data); +void pbc_pattern_close_arrays(struct pbc_pattern *, void *data); + +#endif diff --git a/lua-protobuf/proto.c b/lua-protobuf/proto.c new file mode 100644 index 00000000..fc8c59bd --- /dev/null +++ b/lua-protobuf/proto.c @@ -0,0 +1,252 @@ +#include "pbc.h" +#include "proto.h" +#include "pattern.h" +#include "map.h" +#include "alloc.h" +#include "stringpool.h" +#include "bootstrap.h" + +#include +#include + +const char * +pbc_error(struct pbc_env * p) { + const char *err = p->lasterror; + p->lasterror = ""; + return err; +} + +struct _message * +_pbcP_get_message(struct pbc_env * p , const char *name) { + return _pbcM_sp_query(p->msgs, name); +} + +struct pbc_env * +pbc_new(void) { + struct pbc_env * p = malloc(sizeof(*p)); + p->files = _pbcM_sp_new(0 , NULL); + p->enums = _pbcM_sp_new(0 , NULL); + p->msgs = _pbcM_sp_new(0 , NULL); + p->lasterror = ""; + + _pbcB_init(p); + + return p; +} + +static void +free_enum(void *p) { + struct _enum * e = p; + _pbcM_ip_delete(e->id); + _pbcM_si_delete(e->name); + + free(p); +} + +static void +free_stringpool(void *p) { + _pbcS_delete(p); +} + +static void +free_msg(void *p) { + struct _message * m = p; + if (m->id) + _pbcM_ip_delete(m->id); + free(m->def); + _pbcM_sp_foreach(m->name, free); + _pbcM_sp_delete(m->name); + free(p); +} + +void +pbc_delete(struct pbc_env *p) { + _pbcM_sp_foreach(p->enums, free_enum); + _pbcM_sp_delete(p->enums); + + _pbcM_sp_foreach(p->msgs, free_msg); + _pbcM_sp_delete(p->msgs); + + _pbcM_sp_foreach(p->files, free_stringpool); + _pbcM_sp_delete(p->files); + + free(p); +} + +struct _enum * +_pbcP_push_enum(struct pbc_env * p, const char *name, struct map_kv *table, int sz) { + void * check = _pbcM_sp_query(p->enums, name); + if (check) + return NULL; + struct _enum * v = malloc(sizeof(*v)); + v->key = name; + v->id = _pbcM_ip_new(table,sz); + v->name = _pbcM_si_new(table,sz); + v->default_v->e.id = table[0].id; + v->default_v->e.name = table[0].pointer; + + _pbcM_sp_insert(p->enums, name , v); + return v; +} + +void +_pbcP_push_message(struct pbc_env * p, const char *name, struct _field *f , pbc_array queue) { + struct _message * m = _pbcM_sp_query(p->msgs, name); + if (m==NULL) { + m = malloc(sizeof(*m)); + m->def = NULL; + m->key = name; + m->id = NULL; + m->name = _pbcM_sp_new(0 , NULL); + m->env = p; + _pbcM_sp_insert(p->msgs, name, m); + } + struct _field * field = malloc(sizeof(*field)); + memcpy(field,f,sizeof(*f)); + _pbcM_sp_insert(m->name, field->name, field); + pbc_var atom; + atom->m.buffer = field; + if (f->type == PTYPE_MESSAGE || f->type == PTYPE_ENUM) { + _pbcA_push(queue, atom); + } +} + +struct _iter { + int count; + struct map_kv * table; +}; + +static void +_count(void *p, void *ud) { + struct _iter *iter = ud; + iter->count ++; +} + +static void +_set_table(void *p, void *ud) { + struct _field * field = p; + struct _iter *iter = ud; + iter->table[iter->count].id = field->id; + iter->table[iter->count].pointer = field; + ++iter->count; +} + +struct _message * +_pbcP_init_message(struct pbc_env * p, const char *name) { + struct _message * m = _pbcM_sp_query(p->msgs, name); + if (m == NULL) { + m = malloc(sizeof(*m)); + m->def = NULL; + m->key = name; + m->id = NULL; + m->name = _pbcM_sp_new(0 , NULL); + m->env = p; + _pbcM_sp_insert(p->msgs, name, m); + + return m; + } + if (m->id) { + // extend message, delete old id map. + _pbcM_ip_delete(m->id); + } + struct _iter iter = { 0, NULL }; + _pbcM_sp_foreach_ud(m->name, _count, &iter); + iter.table = malloc(iter.count * sizeof(struct map_kv)); + iter.count = 0; + _pbcM_sp_foreach_ud(m->name, _set_table, &iter); + + m->id = _pbcM_ip_new(iter.table , iter.count); + + free(iter.table); + + return m; +} + +int +_pbcP_message_default(struct _message * m, const char * name, pbc_var defv) { + struct _field * f= _pbcM_sp_query(m->name, name); + if (f==NULL) { + // invalid key + defv->p[0] = NULL; + defv->p[1] = NULL; + return -1; + } + *defv = *(f->default_v); + return f->type; +} + +int +_pbcP_type(struct _field * field, const char ** type) { + if (field == NULL) { + return 0; + } + int ret = 0; + switch (field->type) { + case PTYPE_DOUBLE: + case PTYPE_FLOAT: + ret = PBC_REAL; + break; + case PTYPE_INT64: + case PTYPE_SINT64: + ret = PBC_INT64; + break; + case PTYPE_INT32: + case PTYPE_SINT32: + ret = PBC_INT; + break; + case PTYPE_UINT32: + case PTYPE_UINT64: + ret = PBC_UINT; + break; + case PTYPE_FIXED32: + case PTYPE_SFIXED32: + ret = PBC_FIXED32; + break; + case PTYPE_SFIXED64: + case PTYPE_FIXED64: + ret = PBC_FIXED64; + break; + case PTYPE_BOOL: + ret = PBC_BOOL; + break; + case PTYPE_STRING: + ret = PBC_STRING; + break; + case PTYPE_BYTES: + ret = PBC_BYTES; + break; + case PTYPE_ENUM: + ret = PBC_ENUM; + if (type) { + *type = field->type_name.e->key; + } + break; + case PTYPE_MESSAGE: + ret = PBC_MESSAGE; + if (type) { + *type = field->type_name.m->key; + } + break; + default: + return 0; + } + if (field->label == LABEL_REPEATED || + field->label == LABEL_PACKED) { + ret |= PBC_REPEATED; + } + + return ret; +} + +int +pbc_type(struct pbc_env * p, const char * typename , const char * key , const char ** type) { + struct _message *m = _pbcP_get_message(p, typename); + if (m==NULL) { + return 0; + } + if (key == NULL) { + return PBC_NOEXIST; + } + struct _field * field = _pbcM_sp_query(m->name, key); + return _pbcP_type(field, type); +} diff --git a/lua-protobuf/proto.h b/lua-protobuf/proto.h new file mode 100644 index 00000000..c5b93673 --- /dev/null +++ b/lua-protobuf/proto.h @@ -0,0 +1,64 @@ +#ifndef PROTOBUFC_PROTO_H +#define PROTOBUFC_PROTO_H + +#include "pbc.h" +#include "map.h" +#include "array.h" + +#include +#include + +struct map_ip; +struct map_si; +struct map_sp; +struct _message; +struct _enum; + +#define LABEL_OPTIONAL 0 +#define LABEL_REQUIRED 1 +#define LABEL_REPEATED 2 +#define LABEL_PACKED 3 + +struct _field { + int id; + const char *name; + int type; + int label; + pbc_var default_v; + union { + const char * n; + struct _message * m; + struct _enum * e; + } type_name; +}; + +struct _message { + const char * key; + struct map_ip * id; // id -> _field + struct map_sp * name; // string -> _field + struct pbc_rmessage * def; // default message + struct pbc_env * env; +}; + +struct _enum { + const char * key; + struct map_ip * id; + struct map_si * name; + pbc_var default_v; +}; + +struct pbc_env { + struct map_sp * files; // string -> void * + struct map_sp * enums; // string -> _enum + struct map_sp * msgs; // string -> _message + const char * lasterror; +}; + +struct _message * _pbcP_init_message(struct pbc_env * p, const char *name); +void _pbcP_push_message(struct pbc_env * p, const char *name, struct _field *f , pbc_array queue); +struct _enum * _pbcP_push_enum(struct pbc_env * p, const char *name, struct map_kv *table, int sz ); +int _pbcP_message_default(struct _message * m, const char * name, pbc_var defv); +struct _message * _pbcP_get_message(struct pbc_env * p, const char *name); +int _pbcP_type(struct _field * field, const char **type); + +#endif diff --git a/lua-protobuf/register.c b/lua-protobuf/register.c new file mode 100644 index 00000000..a1bf4280 --- /dev/null +++ b/lua-protobuf/register.c @@ -0,0 +1,329 @@ +#include "pbc.h" +#include "proto.h" +#include "alloc.h" +#include "map.h" +#include "bootstrap.h" +#include "context.h" +#include "stringpool.h" + +#include +#include + +static const char * +_concat_name(struct _stringpool *p , const char *prefix , int prefix_sz , const char *name , int name_sz) { + if (prefix_sz == 0) { + return _pbcS_build(p , name, name_sz); + } + char temp[name_sz + prefix_sz + 2]; + memcpy(temp,prefix,prefix_sz); + temp[prefix_sz] = '.'; + memcpy(temp+prefix_sz+1,name,name_sz); + temp[name_sz + prefix_sz + 1] = '\0'; + + return _pbcS_build(p , temp, name_sz + prefix_sz + 1); +} + +static void +_register_enum(struct pbc_env *p, struct _stringpool *pool, struct pbc_rmessage * enum_type, const char *prefix, int prefix_sz) { + int field_count = pbc_rmessage_size(enum_type, "value"); + struct map_kv *table = malloc(field_count * sizeof(struct map_kv)); + int i; + for (i=0;itype == PTYPE_STRING || f->type == PTYPE_BYTES) { + f->default_v->s.str = ""; + f->default_v->s.len = 0; + } else { + f->default_v->integer.low = 0; + f->default_v->integer.hi = 0; + } + return; + } + + switch (f->type) { + case PTYPE_DOUBLE: + case PTYPE_FLOAT: + f->default_v->real = strtod(value,NULL); + break; + case PTYPE_STRING: + f->default_v->s.str = _pbcS_build(pool, value , sz); + f->default_v->s.len = sz; + break; + case PTYPE_ENUM: + // enum default value will be converted to f->default_v->e in bootstrap.c : set_field_one() + f->default_v->s.str = value; + f->default_v->s.len = sz; + break; + case PTYPE_BOOL: + if (strcmp(value,"true") == 0) { + f->default_v->integer.low = 1; + } else { + f->default_v->integer.low = 0; + } + f->default_v->integer.hi = 0; + break; + case PTYPE_UINT64: + case PTYPE_INT64: + case PTYPE_SFIXED64: + case PTYPE_SINT64: { + long long v = strtoll(value, NULL, 10); + f->default_v->integer.low = (long) v; + f->default_v->integer.hi = (long)(v >> 32); + break; + } + case PTYPE_INT32: + case PTYPE_FIXED32: + case PTYPE_SFIXED32: + case PTYPE_SINT32: + f->default_v->integer.low = strtol(value, NULL, 10); + if (f->default_v->integer.low < 0) { + f->default_v->integer.hi = -1; + } else { + f->default_v->integer.hi = 0; + } + break; + case PTYPE_UINT32: + f->default_v->integer.low = strtoul(value, NULL, 10); + f->default_v->integer.hi = 0; + break; + case PTYPE_BYTES: + case PTYPE_MESSAGE: + // bytes and message types have no default value + f->default_v->m.buffer = 0; + f->default_v->m.len = 0; + break; + default: + f->default_v->integer.low = 0; + f->default_v->integer.hi = 0; + break; + } +} + +static void +_register_field(struct pbc_rmessage * field, struct _field * f, struct _stringpool *pool) { + f->id = pbc_rmessage_integer(field, "number", 0 , 0); + f->type = pbc_rmessage_integer(field, "type", 0 , 0); // enum + f->label = pbc_rmessage_integer(field, "label", 0, 0) - 1; // LABEL_OPTIONAL = 0 + if (pbc_rmessage_size(field , "options") > 0) { + struct pbc_rmessage * options = pbc_rmessage_message(field, "options" , 0); + int packed = pbc_rmessage_integer(options , "packed" , 0 , NULL); + if (packed) { + f->label = LABEL_PACKED; + } + } + f->type_name.n = pbc_rmessage_string(field, "type_name", 0 , NULL) +1; // abandon prefix '.' + int vsz; + const char * default_value = pbc_rmessage_string(field, "default_value", 0 , &vsz); + _set_default(pool , f , f->type, default_value , vsz); +} + +static void +_register_extension(struct pbc_env *p, struct _stringpool *pool , const char * prefix, int prefix_sz, struct pbc_rmessage * msg, pbc_array queue) { + int extension_count = pbc_rmessage_size(msg , "extension"); + if (extension_count <= 0) + return; + int i; + + const char * last = NULL; + + for (i=0;ifiles, filename)) { + return CHECK_FILE_EXIST; + } + int sz = pbc_rmessage_size(file, "dependency"); + int i; + for (i=0;ifiles, dname) == NULL) { + return CHECK_FILE_DEPENDENCY; + } + } + + *fname = filename; + + return CHECK_FILE_OK; +} + +static int +_register_no_dependency(struct pbc_env * p,struct pbc_rmessage ** files , int n ) { + int r = 0; + int i; + for (i=0;ifiles , filename, pool); + _register(p,files[i],pool); + files[i] = NULL; + } + break; + } + } + return r; +} + +int +pbc_register(struct pbc_env * p, struct pbc_slice *slice) { + struct pbc_rmessage * message = pbc_rmessage_new(p, "google.protobuf.FileDescriptorSet", slice); + if (message == NULL) { + p->lasterror = "register open google.protobuf.FileDescriptorSet fail"; + return 1; + } + int n = pbc_rmessage_size(message, "file"); + struct pbc_rmessage * files[n]; + int i; + if (n == 0) { + p->lasterror = "register empty"; + goto _error; + } + for (i=0;ilasterror = "register open fail"; + goto _error; + } + } + + int r = n; + do { + int rr = _register_no_dependency(p,files , n); + if (rr == r) { + p->lasterror = "register dependency error"; + goto _error; + } + r = rr; + } while (r>0); + + pbc_rmessage_delete(message); + return 0; +_error: + pbc_rmessage_delete(message); + return 1; +} diff --git a/lua-protobuf/rmessage.c b/lua-protobuf/rmessage.c new file mode 100644 index 00000000..a93146da --- /dev/null +++ b/lua-protobuf/rmessage.c @@ -0,0 +1,455 @@ +#include "pbc.h" +#include "alloc.h" +#include "map.h" +#include "context.h" +#include "proto.h" +#include "pattern.h" +#include "varint.h" + +#include +#include + +struct pbc_rmessage { + struct _message * msg; + struct map_sp * index; // key -> struct value * + struct heap * heap; +}; + +union _var { + pbc_var var; + pbc_array array; + struct pbc_rmessage message; +} ; + +struct value { + struct _field * type; + union _var v; +}; + +int +pbc_rmessage_next(struct pbc_rmessage *m, const char **key) { + struct value * v = _pbcM_sp_next(m->index, key); + if (*key == NULL) { + return 0; + } + return _pbcP_type(v->type, NULL); +} + +#define SIZE_VAR (offsetof(struct value, v) + sizeof(pbc_var)) +#define SIZE_ARRAY (offsetof(struct value, v) + sizeof(pbc_array)) +#define SIZE_MESSAGE (offsetof(struct value, v) + sizeof(struct pbc_rmessage)) + +static struct value * +read_string(struct heap *h, struct atom *a,struct _field *f, uint8_t *buffer) { + const char * temp = (const char *) (buffer + a->v.s.start); + int len = a->v.s.end - a->v.s.start; + + if (len > 0 && temp[len-1] == '\0') { + struct value * v = _pbcH_alloc(h, SIZE_VAR); + v->v.var->s.str = temp; + v->v.var->s.len = len; + return v; + } else { + struct value * v = _pbcH_alloc(h, SIZE_VAR + len + 1); + memcpy(((char *)v) + SIZE_VAR , temp, len); + *(((char *)v) + SIZE_VAR + len) = '\0'; + v->v.var->s.str = ((char *)v) + SIZE_VAR; + v->v.var->s.len = len; + return v; + } +} + +static void +read_string_var(struct heap *h, pbc_var var,struct atom *a,struct _field *f,uint8_t *buffer) { + const char * temp = (const char *) (buffer + a->v.s.start); + int len = a->v.s.end - a->v.s.start; + if (len == 0) { + var->s.str = ""; + var->s.len = 0; + } + else if (temp[len-1] == '\0') { + var->s.str = temp; + var->s.len = len; + } else { + char * temp2 = _pbcH_alloc(h, len + 1); + memcpy(temp2, temp, len); + temp2[len]='\0'; + var->s.str = temp2; + var->s.len = -len; + } +} + +static void _pbc_rmessage_new(struct pbc_rmessage * ret , struct _message * type , void *buffer, int size, struct heap *h); + +static struct value * +read_value(struct heap *h, struct _field *f, struct atom * a, uint8_t *buffer) { + struct value * v; + + switch (f->type) { + case PTYPE_DOUBLE: + CHECK_BIT64(a,NULL); + v = _pbcH_alloc(h, SIZE_VAR); + v->v.var->real = read_double(a); + break; + case PTYPE_FLOAT: + CHECK_BIT32(a,NULL); + v = _pbcH_alloc(h, SIZE_VAR); + v->v.var->real = (double) read_float(a); + break; + case PTYPE_ENUM: + CHECK_VARINT(a,NULL); + v = _pbcH_alloc(h, SIZE_VAR); + v->v.var->e.id = a->v.i.low; + v->v.var->e.name = _pbcM_ip_query(f->type_name.e->id , a->v.i.low); + break; + case PTYPE_INT64: + case PTYPE_UINT64: + case PTYPE_INT32: + case PTYPE_UINT32: + case PTYPE_BOOL: + CHECK_VARINT(a,NULL); + v = _pbcH_alloc(h, SIZE_VAR); + v->v.var->integer = a->v.i; + break; + case PTYPE_FIXED32: + case PTYPE_SFIXED32: + CHECK_BIT32(a,NULL); + v = _pbcH_alloc(h, SIZE_VAR); + v->v.var->integer = a->v.i; + break; + case PTYPE_FIXED64: + case PTYPE_SFIXED64: + CHECK_BIT64(a,NULL); + v = _pbcH_alloc(h, SIZE_VAR); + v->v.var->integer = a->v.i; + break; + case PTYPE_SINT32: + CHECK_VARINT(a,NULL); + v = _pbcH_alloc(h, SIZE_VAR); + v->v.var->integer = a->v.i; + _pbcV_dezigzag32(&(v->v.var->integer)); + break; + case PTYPE_SINT64: + CHECK_VARINT(a,NULL); + v = _pbcH_alloc(h, SIZE_VAR); + v->v.var->integer = a->v.i; + _pbcV_dezigzag64(&(v->v.var->integer)); + break; + case PTYPE_STRING: + CHECK_LEND(a,NULL); + v = read_string(h,a,f,buffer); + break; + case PTYPE_BYTES: + CHECK_LEND(a,NULL); + v = _pbcH_alloc(h, SIZE_VAR); + v->v.var->s.str = (const char *)(buffer + a->v.s.start); + v->v.var->s.len = a->v.s.end - a->v.s.start; + break; + case PTYPE_MESSAGE: + CHECK_LEND(a,NULL); + v = _pbcH_alloc(h, SIZE_MESSAGE); + _pbc_rmessage_new(&(v->v.message), f->type_name.m , + buffer + a->v.s.start , + a->v.s.end - a->v.s.start,h); + break; + default: + return NULL; + } + v->type = f; + return v; +} + +static void +push_value_packed(struct _message * type, pbc_array array, struct _field *f, struct atom * aa, uint8_t *buffer) { + int n = _pbcP_unpack_packed((uint8_t *)buffer + aa->v.s.start, aa->v.s.end - aa->v.s.start, + f->type , array); + if (n<=0) { + // todo : error + type->env->lasterror = "Unpack packed field error"; + return; + } + if (f->type == PTYPE_ENUM) { + int i; + for (i=0;iinteger.low; + v->e.id = id; + v->e.name = _pbcM_ip_query(f->type_name.e->id , id); + } + } +} + +static void +push_value_array(struct heap *h, pbc_array array, struct _field *f, struct atom * a, uint8_t *buffer) { + pbc_var v; + + switch (f->type) { + case PTYPE_DOUBLE: + v->real = read_double(a); + break; + case PTYPE_FLOAT: + v->real = (double) read_float(a); + break; + case PTYPE_ENUM: + v->e.id = a->v.i.low; + v->e.name = _pbcM_ip_query(f->type_name.e->id , a->v.i.low); + break; + case PTYPE_INT64: + case PTYPE_UINT64: + case PTYPE_INT32: + case PTYPE_UINT32: + case PTYPE_FIXED32: + case PTYPE_FIXED64: + case PTYPE_SFIXED32: + case PTYPE_SFIXED64: + case PTYPE_BOOL: + v->integer = a->v.i; + break; + case PTYPE_SINT32: + v->integer = a->v.i; + _pbcV_dezigzag32(&(v->integer)); + break; + case PTYPE_SINT64: + v->integer = a->v.i; + _pbcV_dezigzag64(&(v->integer)); + break; + case PTYPE_STRING: + CHECK_LEND(a, ); + read_string_var(h,v,a,f,buffer); + break; + case PTYPE_BYTES: + CHECK_LEND(a, ); + v->s.str = (const char *)(buffer + a->v.s.start); + v->s.len = a->v.s.end - a->v.s.start; + break; + case PTYPE_MESSAGE: { + CHECK_LEND(a, ); + struct pbc_rmessage message; + _pbc_rmessage_new(&message, f->type_name.m , + buffer + a->v.s.start , + a->v.s.end - a->v.s.start,h); + if (message.msg == NULL) { + return; + } + v->p[0] = message.msg; + v->p[1] = message.index; + break; + } + default: + return; + } + + _pbcA_push(array,v); +} + +static void +_pbc_rmessage_new(struct pbc_rmessage * ret , struct _message * type , void *buffer, int size , struct heap *h) { + if (size == 0) { + ret->msg = type; + ret->index = _pbcM_sp_new(0 , h); + ret->heap = h; + return; + } + pbc_ctx _ctx; + int count = _pbcC_open(_ctx,buffer,size); + if (count <= 0) { + type->env->lasterror = "rmessage decode context error"; + memset(ret , 0, sizeof(*ret)); + return; + } + struct context * ctx = (struct context *)_ctx; + + ret->msg = type; + ret->index = _pbcM_sp_new(count, h); + ret->heap = h; + + int i; + + for (i=0;inumber;i++) { + int id = ctx->a[i].wire_id >> 3; + struct _field * f = _pbcM_ip_query(type->id , id); + if (f) { + if (f->label == LABEL_REPEATED || f->label == LABEL_PACKED) { + struct value * v; + void ** vv = _pbcM_sp_query_insert(ret->index, f->name); + if (*vv == NULL) { + v = _pbcH_alloc(h, SIZE_ARRAY); + v->type = f; + _pbcA_open_heap(v->v.array,ret->heap); + *vv = v; + } else { + v= *vv; + } + if (f->label == LABEL_PACKED) { + push_value_packed(type, v->v.array , f , &(ctx->a[i]), buffer); + if (pbc_array_size(v->v.array) == 0) { + type->env->lasterror = "rmessage decode packed data error"; + *vv = NULL; + } + } else { + push_value_array(h,v->v.array , f, &(ctx->a[i]), buffer); + if (pbc_array_size(v->v.array) == 0) { + type->env->lasterror = "rmessage decode repeated data error"; + *vv = NULL; + } + } + } else { + struct value * v = read_value(h, f, &(ctx->a[i]), buffer); + if (v) { + _pbcM_sp_insert(ret->index, f->name, v); + } else { + type->env->lasterror = "rmessage decode data error"; + } + } + } + } + + _pbcC_close(_ctx); +} + +struct pbc_rmessage * +pbc_rmessage_new(struct pbc_env * env, const char * typename , struct pbc_slice * slice) { + struct _message * msg = _pbcP_get_message(env, typename); + if (msg == NULL) { + env->lasterror = "Proto not found"; + return NULL; + } + struct pbc_rmessage temp; + struct heap * h = _pbcH_new(slice->len); + _pbc_rmessage_new(&temp, msg , slice->buffer, slice->len , h); + if (temp.msg == NULL) { + _pbcH_delete(h); + return NULL; + } + + struct pbc_rmessage *m = _pbcH_alloc(temp.heap, sizeof(*m)); + *m = temp; + return m; +} + +void +pbc_rmessage_delete(struct pbc_rmessage * m) { + if (m) { + _pbcH_delete(m->heap); + } +} + +const char * +pbc_rmessage_string(struct pbc_rmessage * m , const char *key , int index, int *sz) { + struct value * v = _pbcM_sp_query(m->index,key); + int type = 0; + pbc_var var; + if (v == NULL) { + type = _pbcP_message_default(m->msg, key, var); + } else { + if (v->type->label == LABEL_REPEATED || v->type->label == LABEL_PACKED) { + _pbcA_index(v->v.array, index, var); + } else { + var[0] = v->v.var[0]; + } + type = v->type->type; + } + + if (type == PTYPE_ENUM) { + if (sz) { + *sz = strlen(var->e.name); + } + return var->e.name; + } + + if (sz) { + int len = var->s.len; + if (len<0) { + len = - len; + } + *sz = len; + } + return var->s.str; +} + +uint32_t +pbc_rmessage_integer(struct pbc_rmessage *m , const char *key , int index, uint32_t *hi) { + struct value * v = _pbcM_sp_query(m->index,key); + pbc_var var; + int type = 0; + if (v == NULL) { + type = _pbcP_message_default(m->msg, key, var); + } else { + if (v->type->label == LABEL_REPEATED || v->type->label == LABEL_PACKED) { + _pbcA_index(v->v.array, index, var); + } else { + var[0] = v->v.var[0]; + } + type = v->type->type; + } + + if (type == PTYPE_ENUM) { + if (hi) { + *hi = 0; + } + return var->e.id; + } + + if (hi) { + *hi = var->integer.hi; + } + return var->integer.low; +} + +double +pbc_rmessage_real(struct pbc_rmessage * m, const char *key , int index) { + struct value * v = _pbcM_sp_query(m->index,key); + pbc_var var; + if (v == NULL) { + _pbcP_message_default(m->msg, key, var); + } else { + if (v->type->label == LABEL_REPEATED || v->type->label == LABEL_PACKED) { + _pbcA_index(v->v.array, index, var); + } else { + return v->v.var->real; + } + } + return var->real; +} + + +struct pbc_rmessage * +pbc_rmessage_message(struct pbc_rmessage * rm, const char *key, int index) { + struct value * v = _pbcM_sp_query(rm->index,key); + if (v == NULL) { + struct _field * f = _pbcM_sp_query(rm->msg->name, key); + if (f == NULL) { + rm->msg->env->lasterror = "Invalid key for sub-message"; + // invalid key + return NULL; + } + struct _message * m = f->type_name.m; + + if (m->def == NULL) { + // m->def will be free at the end (pbc_delete). + m->def = malloc(sizeof(struct pbc_rmessage)); + m->def->msg = m; + m->def->index = NULL; + } + return m->def; + } else { + if (v->type->label == LABEL_REPEATED) { + return _pbcA_index_p(v->v.array,index); + } else { + return &(v->v.message); + } + } +} + +int +pbc_rmessage_size(struct pbc_rmessage *m, const char *key) { + struct value * v = _pbcM_sp_query(m->index,key); + if (v == NULL) { + return 0; + } + if (v->type->label == LABEL_REPEATED || v->type->label == LABEL_PACKED) { + return pbc_array_size(v->v.array); + } else { + return 1; + } +} \ No newline at end of file diff --git a/lua-protobuf/stringpool.c b/lua-protobuf/stringpool.c new file mode 100644 index 00000000..9e644585 --- /dev/null +++ b/lua-protobuf/stringpool.c @@ -0,0 +1,60 @@ +#include "alloc.h" + +#include +#include + +#define PAGE_SIZE 256 + +struct _stringpool { + char * buffer; + size_t len; + struct _stringpool *next; +}; + +struct _stringpool * +_pbcS_new(void) { + struct _stringpool * ret = (struct _stringpool *)malloc(sizeof(struct _stringpool) + PAGE_SIZE); + ret->buffer = (char *)(ret + 1); + ret->len = 0; + ret->next = NULL; + return ret; +} + +void +_pbcS_delete(struct _stringpool *pool) { + while(pool) { + struct _stringpool *next = pool->next; + free(pool); + pool = next; + } +} + +const char * +_pbcS_build(struct _stringpool *pool, const char * str , int sz) { + size_t s = sz + 1; + if (s < PAGE_SIZE - pool->len) { + char * ret = pool->buffer + pool->len; + memcpy(pool->buffer + pool->len, str, s); + pool->len += s; + return ret; + } + if (s > PAGE_SIZE) { + struct _stringpool * next = (struct _stringpool *)malloc(sizeof(struct _stringpool) + s); + next->buffer = (char *)(next + 1); + memcpy(next->buffer, str, s); + next->len = s; + next->next = pool->next; + pool->next = next; + return next->buffer; + } + struct _stringpool *next = (struct _stringpool *)malloc(sizeof(struct _stringpool) + PAGE_SIZE); + next->buffer = pool->buffer; + next->next = pool->next; + next->len = pool->len; + + pool->next = next; + pool->buffer = (char *)(next + 1); + memcpy(pool->buffer, str, s); + pool->len = s; + return pool->buffer; +} diff --git a/lua-protobuf/stringpool.h b/lua-protobuf/stringpool.h new file mode 100644 index 00000000..394423da --- /dev/null +++ b/lua-protobuf/stringpool.h @@ -0,0 +1,10 @@ +#ifndef PROTOBUF_C_STRINGPOOL_H +#define PROTOBUF_C_STRINGPOOL_H + +struct _stringpool; + +struct _stringpool * _pbcS_new(void); +void _pbcS_delete(struct _stringpool *pool); +const char * _pbcS_build(struct _stringpool *pool, const char * str , int sz); + +#endif diff --git a/lua-protobuf/test.lua b/lua-protobuf/test.lua new file mode 100644 index 00000000..3855e0e6 --- /dev/null +++ b/lua-protobuf/test.lua @@ -0,0 +1,30 @@ +require "protobuf" + +addr = io.open("addressbook.pb","rb") +buffer = addr:read "*a" +addr:close() + +protobuf.register(buffer) + +addressbook = { + name = "Alice", + id = 12345, + phone = { + { number = "1301234567" }, + { number = "87654321", type = "WORK" }, + } +} + +code = protobuf.encode("tutorial.Person", addressbook) + +decode = protobuf.decode("tutorial.Person" , code) + +print(decode.name) +print(decode.id) +for _,v in ipairs(decode.phone) do + print("\t"..v.number, v.type) +end + +phonebuf = protobuf.pack("tutorial.Person.PhoneNumber number","87654321") +buffer = protobuf.pack("tutorial.Person name id phone", "Alice", 123, { phonebuf }) +print(protobuf.unpack("tutorial.Person name id phone", buffer)) diff --git a/lua-protobuf/varint.c b/lua-protobuf/varint.c new file mode 100644 index 00000000..b80a7172 --- /dev/null +++ b/lua-protobuf/varint.c @@ -0,0 +1,110 @@ +#include "varint.h" + +#include "pbc.h" + +#include + +inline int +_pbcV_encode32(uint32_t number, uint8_t buffer[10]) +{ + if (number < 0x80) { + buffer[0] = (uint8_t) number ; + return 1; + } + buffer[0] = (uint8_t) (number | 0x80 ); + if (number < 0x4000) { + buffer[1] = (uint8_t) (number >> 7 ); + return 2; + } + buffer[1] = (uint8_t) ((number >> 7) | 0x80 ); + if (number < 0x200000) { + buffer[2] = (uint8_t) (number >> 14); + return 3; + } + buffer[2] = (uint8_t) ((number >> 14) | 0x80 ); + if (number < 0x10000000) { + buffer[3] = (uint8_t) (number >> 21); + return 4; + } + buffer[3] = (uint8_t) ((number >> 21) | 0x80 ); + buffer[4] = (uint8_t) (number >> 28); + return 5; +} + +int +_pbcV_encode(uint64_t number, uint8_t buffer[10]) +{ + if ((number & 0xffffffff) == number) { + return _pbcV_encode32((uint32_t)number , buffer); + } + int i = 0; + do { + buffer[i] = (uint8_t)(number | 0x80); + number >>= 7; + ++i; + } while (number >= 0x80); + buffer[i] = (uint8_t)number; + return i+1; +} + +int +_pbcV_decode(uint8_t buffer[10], struct longlong *result) { + if (!(buffer[0] & 0x80)) { + result->low = buffer[0]; + result->hi = 0; + return 1; + } + uint32_t r = buffer[0] & 0x7f; + int i; + for (i=1;i<4;i++) { + r |= ((buffer[i]&0x7f) << (7*i)); + if (!(buffer[i] & 0x80)) { + result->low = r; + result->hi = 0; + return i+1; + } + } + uint64_t lr = 0; + for (i=4;i<10;i++) { + lr |= ((buffer[i] & 0x7f) << (7*(i-4))); + if (!(buffer[i] & 0x80)) { + result->hi = (uint32_t)(lr >> 4); + result->low = r | (((uint32_t)lr & 0xf) << 28); + return i+1; + } + } + + result->low = 0; + result->hi = 0; + return 10; +} + +int +_pbcV_zigzag32(int32_t n, uint8_t buffer[10]) +{ + n = (n << 1) ^ (n >> 31); + return _pbcV_encode32(n,buffer); +} + +int +_pbcV_zigzag(int64_t n, uint8_t buffer[10]) +{ + n = (n << 1) ^ (n >> 63); + return _pbcV_encode(n,buffer); +} + +void +_pbcV_dezigzag64(struct longlong *r) +{ + uint32_t low = r->low; + r->low = ((low >> 1) | ((r->hi & 1) << 31)) ^ - (low & 1); + r->hi = (r->hi >> 1) ^ - (low & 1); +} + +void +_pbcV_dezigzag32(struct longlong *r) +{ + uint32_t low = r->low; + r->low = (low >> 1) ^ - (low & 1); + r->hi = -(low >> 31); +} diff --git a/lua-protobuf/varint.h b/lua-protobuf/varint.h new file mode 100644 index 00000000..000882ec --- /dev/null +++ b/lua-protobuf/varint.h @@ -0,0 +1,20 @@ +#ifndef PROTOBUF_C_VARINT_H +#define PROTOBUF_C_VARINT_H + +#include + +struct longlong { + uint32_t low; + uint32_t hi; +}; + +int _pbcV_encode32(uint32_t number, uint8_t buffer[10]); +int _pbcV_encode(uint64_t number, uint8_t buffer[10]); +int _pbcV_zigzag32(int32_t number, uint8_t buffer[10]); +int _pbcV_zigzag(int64_t number, uint8_t buffer[10]); + +int _pbcV_decode(uint8_t buffer[10], struct longlong *result); +void _pbcV_dezigzag64(struct longlong *r); +void _pbcV_dezigzag32(struct longlong *r); + +#endif diff --git a/lua-protobuf/wmessage.c b/lua-protobuf/wmessage.c new file mode 100644 index 00000000..bf81f69f --- /dev/null +++ b/lua-protobuf/wmessage.c @@ -0,0 +1,509 @@ +#include "pbc.h" +#include "context.h" +#include "alloc.h" +#include "varint.h" +#include "map.h" +#include "proto.h" + +#include +#include +#include +#include + +#define WMESSAGE_SIZE 64 + +struct pbc_wmessage { + struct _message *type; + uint8_t * buffer; + uint8_t * ptr; + uint8_t * endptr; + pbc_array sub; + struct map_sp *packed; + struct heap * heap; +}; + +struct _packed { + int id; + int ptype; + pbc_array data; +}; + +static struct pbc_wmessage * +_wmessage_new(struct heap *h, struct _message *msg) { + struct pbc_wmessage * m = _pbcH_alloc(h, sizeof(*m)); + m->type = msg; + m->buffer = _pbcH_alloc(h, WMESSAGE_SIZE); + m->ptr = m->buffer; + m->endptr = m->buffer + WMESSAGE_SIZE; + _pbcA_open_heap(m->sub, h); + m->packed = NULL; + m->heap = h; + + return m; +} + +struct pbc_wmessage * +pbc_wmessage_new(struct pbc_env * env, const char *typename) { + struct _message * msg = _pbcP_get_message(env, typename); + if (msg == NULL) + return NULL; + struct heap *h = _pbcH_new(0); + return _wmessage_new(h, msg); +} + +void +pbc_wmessage_delete(struct pbc_wmessage *m) { + if (m) { + _pbcH_delete(m->heap); + } +} + +static void +_expand(struct pbc_wmessage *m, int sz) { + if (m->ptr + sz > m->endptr) { + int cap = m->endptr - m->buffer; + sz = m->ptr + sz - m->buffer; + do { + cap = cap * 2; + } while ( sz > cap ) ; + int old_size = m->ptr - m->buffer; + uint8_t * buffer = _pbcH_alloc(m->heap, cap); + memcpy(buffer, m->buffer, old_size); + m->ptr = buffer + (m->ptr - m->buffer); + m->endptr = buffer + cap; + m->buffer = buffer; + } +} + +static struct _packed * +_get_packed(struct pbc_wmessage *m , struct _field *f , const char *key) { + if (m->packed == NULL) { + m->packed = _pbcM_sp_new(4, m->heap); + } + void ** v = _pbcM_sp_query_insert(m->packed , key); + if (*v == NULL) { + *v = _pbcH_alloc(m->heap, sizeof(struct _packed)); + struct _packed *p = *v; + p->id = f->id; + p->ptype = f->type; + _pbcA_open_heap(p->data, m->heap); + return p; + } + return *v; +} + +static void +_packed_integer(struct pbc_wmessage *m, struct _field *f, const char *key , uint32_t low, uint32_t hi) { + struct _packed * packed = _get_packed(m,f,key); + pbc_var var; + var->integer.low = low; + var->integer.hi = hi; + _pbcA_push(packed->data , var); +} + +static void +_packed_real(struct pbc_wmessage *m, struct _field *f, const char *key , double v) { + struct _packed * packed = _get_packed(m,f,key); + pbc_var var; + var->real = v; + _pbcA_push(packed->data , var); +} + +static inline void +int64_encode(uint32_t low, uint32_t hi , uint8_t * buffer) { + buffer[0] = (uint8_t)(low & 0xff); + buffer[1] = (uint8_t)(low >> 8 & 0xff); + buffer[2] = (uint8_t)(low >> 16 & 0xff); + buffer[3] = (uint8_t)(low >> 24 & 0xff); + buffer[4] = (uint8_t)(hi & 0xff); + buffer[5] = (uint8_t)(hi >> 8 & 0xff); + buffer[6] = (uint8_t)(hi >> 16 & 0xff); + buffer[7] = (uint8_t)(hi >> 24 & 0xff); +} + +static inline void +int32_encode(uint32_t low, uint8_t * buffer) { + buffer[0] = (uint8_t)(low & 0xff); + buffer[1] = (uint8_t)(low >> 8 & 0xff); + buffer[2] = (uint8_t)(low >> 16 & 0xff); + buffer[3] = (uint8_t)(low >> 24 & 0xff); +} + +int +pbc_wmessage_integer(struct pbc_wmessage *m, const char *key, uint32_t low, uint32_t hi) { + struct _field * f = _pbcM_sp_query(m->type->name,key); + if (f==NULL) { + // todo : error + m->type->env->lasterror = "wmessage_interger query key error"; + return -1; + } + if (f->label == LABEL_PACKED) { + _packed_integer(m , f, key , low, hi); + return 0; + } + if (f->label == LABEL_OPTIONAL) { + if (f->type == PTYPE_ENUM) { + if (low == f->default_v->e.id) + return 0; + } else { + if (low == f->default_v->integer.low && + hi == f->default_v->integer.hi) { + return 0; + } + } + } + int id = f->id << 3; + + _expand(m,20); + switch (f->type) { + case PTYPE_INT64: + case PTYPE_UINT64: + case PTYPE_INT32: + id |= WT_VARINT; + m->ptr += _pbcV_encode32(id, m->ptr); + m->ptr += _pbcV_encode((uint64_t)low | (uint64_t)hi << 32 , m->ptr); + break; + case PTYPE_UINT32: + case PTYPE_ENUM: + case PTYPE_BOOL: + id |= WT_VARINT; + m->ptr += _pbcV_encode32(id, m->ptr); + m->ptr += _pbcV_encode32(low, m->ptr); + break; + case PTYPE_FIXED64: + case PTYPE_SFIXED64: + id |= WT_BIT64; + m->ptr += _pbcV_encode32(id, m->ptr); + int64_encode(low,hi,m->ptr); + m->ptr += 8; + break; + case PTYPE_FIXED32: + case PTYPE_SFIXED32: + id |= WT_BIT32; + m->ptr += _pbcV_encode32(id, m->ptr); + int32_encode(low,m->ptr); + m->ptr += 4; + break; + case PTYPE_SINT32: + id |= WT_VARINT; + m->ptr += _pbcV_encode32(id, m->ptr); + m->ptr += _pbcV_zigzag32(low, m->ptr); + break; + case PTYPE_SINT64: + id |= WT_VARINT; + m->ptr += _pbcV_encode32(id, m->ptr); + m->ptr += _pbcV_zigzag((uint64_t)low | (uint64_t)hi << 32 , m->ptr); + break; + } + + return 0; +} + +int +pbc_wmessage_real(struct pbc_wmessage *m, const char *key, double v) { + struct _field * f = _pbcM_sp_query(m->type->name,key); + if (f == NULL) { + // todo : error + m->type->env->lasterror = "wmessage_real query key error"; + return -1; + } + if (f->label == LABEL_PACKED) { + _packed_real(m , f, key , v); + return 0; + } + + if (f->label == LABEL_OPTIONAL) { + if (v == f->default_v->real) + return 0; + } + int id = f->id << 3; + _expand(m,18); + switch (f->type) { + case PTYPE_FLOAT: { + id |= WT_BIT32; + m->ptr += _pbcV_encode32(id, m->ptr); + float_encode(v , m->ptr); + m->ptr += 4; + break; + } + case PTYPE_DOUBLE: + id |= WT_BIT64; + m->ptr += _pbcV_encode32(id, m->ptr); + double_encode(v , m->ptr); + m->ptr += 8; + break; + } + + return 0; +} + +int +pbc_wmessage_string(struct pbc_wmessage *m, const char *key, const char * v, int len) { + struct _field * f = _pbcM_sp_query(m->type->name,key); + if (f == NULL) { + // todo : error + m->type->env->lasterror = "wmessage_string query key error"; + return -1; + } + + bool varlen = false; + + if (len <=0) { + varlen = true; + // -1 for add '\0' + len = strlen(v) - len; + } + if (f->label == LABEL_PACKED) { + if (f->type == PTYPE_ENUM) { + char temp[len+1]; + if (!varlen || v[len] != '\0') { + memcpy(temp,v,len); + temp[len]='\0'; + v = temp; + } + int enum_id = 0; + int err = _pbcM_si_query(f->type_name.e->name, v , &enum_id); + if (err) { + // todo : error , invalid enum + m->type->env->lasterror = "wmessage_string packed invalid enum"; + return -1; + } + _packed_integer(m , f, key , enum_id , 0); + } + return 0; + } + + if (f->label == LABEL_OPTIONAL) { + if (f->type == PTYPE_ENUM) { + if (strncmp(v , f->default_v->e.name, len) == 0 && f->default_v->e.name[len] =='\0') { + return 0; + } + } else if (f->type == PTYPE_STRING) { + if (len == f->default_v->s.len && + strcmp(v, f->default_v->s.str) == 0) { + return 0; + } + } + } + int id = f->id << 3; + _expand(m,20); + switch (f->type) { + case PTYPE_ENUM : { + char temp[len+1]; + if (!varlen || v[len] != '\0') { + memcpy(temp,v,len); + temp[len]='\0'; + v = temp; + } + int enum_id = 0; + int err = _pbcM_si_query(f->type_name.e->name, v, &enum_id); + if (err) { + // todo : error , enum invalid + m->type->env->lasterror = "wmessage_string invalid enum"; + return -1; + } + id |= WT_VARINT; + m->ptr += _pbcV_encode32(id, m->ptr); + m->ptr += _pbcV_encode32(enum_id, m->ptr); + break; + } + case PTYPE_STRING: + case PTYPE_BYTES: + id |= WT_LEND; + m->ptr += _pbcV_encode32(id, m->ptr); + m->ptr += _pbcV_encode32(len, m->ptr); + _expand(m,len); + memcpy(m->ptr , v , len); + m->ptr += len; + break; + } + + return 0; +} + +struct pbc_wmessage * +pbc_wmessage_message(struct pbc_wmessage *m, const char *key) { + struct _field * f = _pbcM_sp_query(m->type->name,key); + if (f == NULL) { + // todo : error + m->type->env->lasterror = "wmessage_message query key error"; + return NULL; + } + pbc_var var; + var->p[0] = _wmessage_new(m->heap, f->type_name.m); + var->p[1] = f; + _pbcA_push(m->sub , var); + return var->p[0]; +} + +static void +_pack_packed_64(struct _packed *p,struct pbc_wmessage *m) { + int n = pbc_array_size(p->data); + int len = n * 8; + int i; + pbc_var var; + _expand(m,10 + len); + m->ptr += _pbcV_encode32(len, m->ptr); + switch (p->ptype) { + case PTYPE_DOUBLE: + for (i=0;idata, i, var); + double_encode(var->real , m->ptr + i * 8); + } + break; + default: + for (i=0;idata, i, var); + int64_encode(var->integer.low , var->integer.hi, m->ptr + i * 8); + } + break; + } + m->ptr += len; +} + +static void +_pack_packed_32(struct _packed *p,struct pbc_wmessage *m) { + int n = pbc_array_size(p->data); + int len = n * 4; + int i; + pbc_var var; + _expand(m,10 + len); + m->ptr += _pbcV_encode32(len, m->ptr); + switch (p->ptype) { + case PTYPE_FLOAT: + for (i=0;idata, i, var); + float_encode(var->real , m->ptr + i * 8); + } + break; + default: + for (i=0;idata, i, var); + int32_encode(var->integer.low , m->ptr + i * 8); + } + break; + } + m->ptr += len; +} + +static void +_pack_packed_varint(struct _packed *p,struct pbc_wmessage *m) { + int n = pbc_array_size(p->data); + int offset = m->ptr - m->buffer; + int len = n * 2; + if (p->ptype == PTYPE_BOOL) { + len = n; + } + int i; + pbc_var var; + _expand(m,10 + len); + int len_len = _pbcV_encode32(len, m->ptr); + m->ptr += len_len; + + switch (p->ptype) { + case PTYPE_INT64: + case PTYPE_UINT64: + for (i=0;idata, i, var); + _expand(m,10); + m->ptr += _pbcV_encode((uint64_t)var->integer.low | (uint64_t)var->integer.hi << 32 , m->ptr); + } + break; + case PTYPE_INT32: + case PTYPE_BOOL: + case PTYPE_UINT32: + case PTYPE_ENUM: + for (i=0;idata, i, var); + _expand(m,10); + m->ptr += _pbcV_encode32(var->integer.low , m->ptr); + } + break; + case PTYPE_SINT32: + for (i=0;idata, i, var); + _expand(m,10); + m->ptr += _pbcV_zigzag32(var->integer.low, m->ptr); + } + break; + case PTYPE_SINT64: + for (i=0;idata, i, var); + _expand(m,10); + m->ptr += _pbcV_zigzag((uint64_t)var->integer.low | (uint64_t)var->integer.hi << 32 , m->ptr); + } + break; + default: + // error + memset(m->ptr , 0 , n); + m->ptr += n; + m->type->env->lasterror = "wmessage type error when pack packed"; + break; + } + int end_offset = m->ptr - m->buffer; + int end_len = end_offset - (offset + len_len); + if (end_len != len) { + uint8_t temp[10]; + int end_len_len = _pbcV_encode32(end_len, temp); + if (end_len_len != len_len) { + _expand(m, end_len_len); + memmove(m->buffer + offset + end_len_len , + m->buffer + offset + len_len , + end_len); + } + memcpy(m->buffer + offset , temp, end_len_len); + } +} + +static void +_pack_packed(void *p, void *ud) { + struct _packed *packed = p; + struct pbc_wmessage * m = ud; + int id = packed->id << 3 | WT_LEND; + _expand(m,10); + m->ptr += _pbcV_encode32(id, m->ptr); + switch(packed->ptype) { + case PTYPE_DOUBLE: + case PTYPE_FIXED64: + case PTYPE_SFIXED64: + _pack_packed_64(packed,m); + break; + case PTYPE_FLOAT: + case PTYPE_FIXED32: + case PTYPE_SFIXED32: + _pack_packed_32(packed,m); + break; + default: + _pack_packed_varint(packed,m); + break; + } +} + +void * +pbc_wmessage_buffer(struct pbc_wmessage *m, struct pbc_slice *slice) { + if (m->packed) { + _pbcM_sp_foreach_ud(m->packed , _pack_packed, m); + } + int i; + int n = pbc_array_size(m->sub); + for (i=0;isub, i , var); + struct pbc_slice s; + pbc_wmessage_buffer(var->p[0] , &s); + if (s.buffer) { + struct _field * f = var->p[1]; + int id = f->id << 3 | WT_LEND; + _expand(m,20+s.len); + m->ptr += _pbcV_encode32(id, m->ptr); + m->ptr += _pbcV_encode32(s.len, m->ptr); + memcpy(m->ptr, s.buffer, s.len); + m->ptr += s.len; + } + } + slice->buffer = m->buffer; + slice->len = m->ptr - m->buffer; + + return m->buffer; +} + diff --git a/lualib/protobuf.lua b/lualib/protobuf.lua new file mode 100644 index 00000000..cb741b60 --- /dev/null +++ b/lualib/protobuf.lua @@ -0,0 +1,592 @@ +local c = require "protobuf.c" +local setmetatable = setmetatable +local type = type +local table = table +local assert = assert +local pairs = pairs +local ipairs = ipairs +local string = string +local print = print +local io = io +local tinsert = table.insert +local rawget = rawget + +module "protobuf" + +local _pattern_cache = {} + +local function clear_pat() + for _,v in pairs(_pattern_cache) do + c._pattern_delete(v.CObj) + end +end + +local P = c._env_new() +setmetatable(_M , { + __gc = function(t) + clear_pat() + c._env_delete(P) + end }) + +function lasterror() + return c._last_error(P) +end + +local decode_type_cache = {} +local _R_meta = {} + +function _R_meta:__index(key) + local v = decode_type_cache[self._CType][key](self, key) + self[key] = v + return v +end + +local function link_parent(self, parent) + local link = rawget(parent, _Children) + if link == nil then + link = {} + rawset(parent, "_Children" , link) + end + tinsert(link, self) +end + +local function remove_cobj(self) + self._CObj = nil + setmetatable(self, nil) + local children = self._Children + if children then + for _,v in ipairs(children) do + remove_cobj(v) + end + end +end + +local function delete_all(self) + c._rmessage_delete(self._CObj) + remove_cobj(self) +end + +local _reader = {} + +function _reader:int(key) + return c._rmessage_integer(self._CObj , key , 0) +end + +function _reader:real(key) + return c._rmessage_real(self._CObj , key , 0) +end + +function _reader:string(key) + return c._rmessage_string(self._CObj , key , 0) +end + +function _reader:bool(key) + return c._rmessage_integer(self._CObj , key , 0) ~= 0 +end + +function _reader:message(key, message_type) + local rmessage = c._rmessage_message(self._CObj , key , 0) + if rmessage then + local v = { + _CObj = rmessage, + _CType = message_type, + _Parent = self, + } + return setmetatable( v , _R_meta ) + end +end + +function _reader:int32(key) + return c._rmessage_int32(self._CObj , key , 0) +end + +function _reader:int64(key) + return c._rmessage_int64(self._CObj , key , 0) +end + +function _reader:int52(key) + return c._rmessage_int52(self._CObj , key , 0) +end + +function _reader:uint52(key) + return c._rmessage_uint52(self._CObj , key , 0) +end + +function _reader:int_repeated(key) + local cobj = self._CObj + local n = c._rmessage_size(cobj , key) + local ret = {} + for i=0,n-1 do + tinsert(ret, c._rmessage_integer(cobj , key , i)) + end + return ret +end + +function _reader:real_repeated(key) + local cobj = self._CObj + local n = c._rmessage_size(cobj , key) + local ret = {} + for i=0,n-1 do + tinsert(ret, c._rmessage_real(cobj , key , i)) + end + return ret +end + +function _reader:string_repeated(key) + local cobj = self._CObj + local n = c._rmessage_size(cobj , key) + local ret = {} + for i=0,n-1 do + tinsert(ret, c._rmessage_string(cobj , key , i)) + end + return ret +end + +function _reader:bool_repeated(key) + local cobj = self._CObj + local n = c._rmessage_size(cobj , key) + local ret = {} + for i=0,n-1 do + tinsert(ret, c._rmessage_integer(cobj , key , i) ~= 0) + end + return ret +end + +function _reader:message_repeated(key, message_type) + local cobj = self._CObj + local n = c._rmessage_size(cobj , key) + local ret = {} + for i=0,n-1 do + local m = { + _CObj = c._rmessage_message(cobj , key , i), + _CType = message_type, + _Parent = self, + } + tinsert(ret, setmetatable( m , _R_meta )) + end + return ret +end + +function _reader:int32_repeated(key) + local cobj = self._CObj + local n = c._rmessage_size(cobj , key) + local ret = {} + for i=0,n-1 do + tinsert(ret, c._rmessage_int32(cobj , key , i)) + end + return ret +end + +function _reader:int64_repeated(key) + local cobj = self._CObj + local n = c._rmessage_size(cobj , key) + local ret = {} + for i=0,n-1 do + tinsert(ret, c._rmessage_int64(cobj , key , i)) + end + return ret +end + +function _reader:int52_repeated(key) + local cobj = self._CObj + local n = c._rmessage_size(cobj , key) + local ret = {} + for i=0,n-1 do + tinsert(ret, c._rmessage_int52(cobj , key , i)) + end + return ret +end + +function _reader:uint52_repeated(key) + local cobj = self._CObj + local n = c._rmessage_size(cobj , key) + local ret = {} + for i=0,n-1 do + tinsert(ret, c._rmessage_uint52(cobj , key , i)) + end + return ret +end + +_reader[1] = function(msg) return _reader.int end +_reader[2] = function(msg) return _reader.real end +_reader[3] = function(msg) return _reader.bool end +_reader[4] = function(msg) return _reader.string end +_reader[5] = function(msg) return _reader.string end +_reader[6] = function(msg) + local message = _reader.message + return function(self,key) + return message(self, key, msg) + end +end +_reader[7] = function(msg) return _reader.int64 end +_reader[8] = function(msg) return _reader.int32 end +_reader[9] = _reader[5] +_reader[10] = function(msg) return _reader.int52 end +_reader[11] = function(msg) return _reader.uint52 end + +_reader[128+1] = function(msg) return _reader.int_repeated end +_reader[128+2] = function(msg) return _reader.real_repeated end +_reader[128+3] = function(msg) return _reader.bool_repeated end +_reader[128+4] = function(msg) return _reader.string_repeated end +_reader[128+5] = function(msg) return _reader.string_repeated end +_reader[128+6] = function(msg) + local message = _reader.message_repeated + return function(self,key) + return message(self, key, msg) + end +end +_reader[128+7] = function(msg) return _reader.int64_repeated end +_reader[128+8] = function(msg) return _reader.int32_repeated end +_reader[128+9] = _reader[128+5] +_reader[128+10] = function(msg) return _reader.int52_repeated end +_reader[128+11] = function(msg) return _reader.uint52_repeated end + +local _decode_type_meta = {} + +function _decode_type_meta:__index(key) + local t, msg = c._env_type(P, self._CType, key) + local func = assert(_reader[t],key)(msg) + self[key] = func + return func +end + +setmetatable(decode_type_cache , { + __index = function(self, key) + local v = setmetatable({ _CType = key } , _decode_type_meta) + self[key] = v + return v + end +}) + +local _R_metagc = { + __index = _R_meta.__index +} + +_R_metagc.__gc = assert(delete_all) + +local function decode_message( message , buffer, length) + local rmessage = c._rmessage_new(P, message, buffer, length) + if rmessage then + local self = { + _CObj = rmessage, + _CType = message, + } + return setmetatable( self , _R_metagc ) + end +end + +--close_decoder = assert(delete_all) + +function register( buffer) + c._env_register(P, buffer) +end + +function register_file(filename) + local f = assert(io.open(filename , "rb")) + local buffer = f:read "*a" + c._env_register(P, buffer) + f:close() +end + +----------- encode ---------------- + +local encode_type_cache = {} + +local function encode_message(CObj, message_type, t) + local type = encode_type_cache[message_type] + for k,v in pairs(t) do + local func = type[k] + func(CObj, k , v) + end +end + +local _writer = { + int = c._wmessage_integer, + real = c._wmessage_real, + enum = c._wmessage_string, + string = c._wmessage_string, + int64 = c._wmessage_int64, + int32 = c._wmessage_int32, + int52 = c._wmessage_int52, + uint52 = c._wmessage_uint52, +} + +function _writer:bool(k,v) + c._wmessage_integer(self, k, v and 1 or 0) +end + +function _writer:message(k, v , message_type) + local submessage = c._wmessage_message(self, k) + encode_message(submessage, message_type, v) +end + +function _writer:int_repeated(k,v) + for _,v in ipairs(v) do + c._wmessage_integer(self,k,v) + end +end + +function _writer:real_repeated(k,v) + for _,v in ipairs(v) do + c._wmessage_real(self,k,v) + end +end + +function _writer:bool_repeated(k,v) + for _,v in ipairs(v) do + c._wmessage_integer(self, k, v and 1 or 0) + end +end + +function _writer:string_repeated(k,v) + for _,v in ipairs(v) do + c._wmessage_string(self,k,v) + end +end + +function _writer:message_repeated(k,v, message_type) + for _,v in ipairs(v) do + local submessage = c._wmessage_message(self, k) + encode_message(submessage, message_type, v) + end +end + +function _writer:int32_repeated(k,v) + for _,v in ipairs(v) do + c._wmessage_int32(self,k,v) + end +end + +function _writer:int64_repeated(k,v) + for _,v in ipairs(v) do + c._wmessage_int64(self,k,v) + end +end + +function _writer:int52_repeated(k,v) + for _,v in ipairs(v) do + c._wmessage_int52(self,k,v) + end +end + +function _writer:uint52_repeated(k,v) + for _,v in ipairs(v) do + c._wmessage_uint52(self,k,v) + end +end + +_writer[1] = function(msg) return _writer.int end +_writer[2] = function(msg) return _writer.real end +_writer[3] = function(msg) return _writer.bool end +_writer[4] = function(msg) return _writer.string end +_writer[5] = function(msg) return _writer.string end +_writer[6] = function(msg) + local message = _writer.message + return function(self,key , v) + return message(self, key, v, msg) + end +end +_writer[7] = function(msg) return _writer.int64 end +_writer[8] = function(msg) return _writer.int32 end +_writer[9] = _writer[5] +_writer[10] = function(msg) return _writer.int52 end +_writer[11] = function(msg) return _writer.uint52 end + +_writer[128+1] = function(msg) return _writer.int_repeated end +_writer[128+2] = function(msg) return _writer.real_repeated end +_writer[128+3] = function(msg) return _writer.bool_repeated end +_writer[128+4] = function(msg) return _writer.string_repeated end +_writer[128+5] = function(msg) return _writer.string_repeated end +_writer[128+6] = function(msg) + local message = _writer.message_repeated + return function(self,key, v) + return message(self, key, v, msg) + end +end +_writer[128+7] = function(msg) return _writer.int64_repeated end +_writer[128+8] = function(msg) return _writer.int32_repeated end +_writer[128+9] = _writer[128+5] +_writer[128+10] = function(msg) return _writer.int52_repeated end +_writer[128+11] = function(msg) return _writer.uint52_repeated end + +local _encode_type_meta = {} + +function _encode_type_meta:__index(key) + local t, msg = c._env_type(P, self._CType, key) + local func = assert(_writer[t],key)(msg) + self[key] = func + return func +end + +setmetatable(encode_type_cache , { + __index = function(self, key) + local v = setmetatable({ _CType = key } , _encode_type_meta) + self[key] = v + return v + end +}) + +function encode( message, t , func) + local encoder = c._wmessage_new(P, message) + assert(encoder , message) + encode_message(encoder, message, t) + if func then + local buffer, len = c._wmessage_buffer(encoder) + func(buffer, len) + c._wmessage_delete(encoder) + else + local s = c._wmessage_buffer_string(encoder) + c._wmessage_delete(encoder) + return s + end +end + +--------- unpack ---------- + +local _pattern_type = { + [1] = {"%d","i"}, + [2] = {"%F","r"}, + [3] = {"%d","b"}, + [4] = {"%d","i"}, + [5] = {"%s","s"}, + [6] = {"%s","m"}, + [7] = {"%D","x"}, + [8] = {"%d","p"}, + [10] = {"%D","d"}, + [11] = {"%D","u"}, + [128+1] = {"%a","I"}, + [128+2] = {"%a","R"}, + [128+3] = {"%a","B"}, + [128+4] = {"%a","I"}, + [128+5] = {"%a","S"}, + [128+6] = {"%a","M"}, + [128+7] = {"%a","X"}, + [128+8] = {"%a","P"}, + [128+10] = {"%a", "D" }, + [128+11] = {"%a", "U" }, +} + +_pattern_type[9] = _pattern_type[5] +_pattern_type[128+9] = _pattern_type[128+5] + + +local function _pattern_create(pattern) + local iter = string.gmatch(pattern,"[^ ]+") + local message = iter() + local cpat = {} + local lua = {} + for v in iter do + local tidx = c._env_type(P, message, v) + local t = _pattern_type[tidx] + assert(t,tidx) + tinsert(cpat,v .. " " .. t[1]) + tinsert(lua,t[2]) + end + local cobj = c._pattern_new(P, message , "@" .. table.concat(cpat," ")) + if cobj == nil then + return + end + local pat = { + CObj = cobj, + format = table.concat(lua), + size = 0 + } + pat.size = c._pattern_size(pat.format) + + return pat +end + +setmetatable(_pattern_cache, { + __index = function(t, key) + local v = _pattern_create(key) + t[key] = v + return v + end +}) + +function unpack(pattern, buffer, length) + local pat = _pattern_cache[pattern] + return c._pattern_unpack(pat.CObj , pat.format, pat.size, buffer, length) +end + +function pack(pattern, ...) + local pat = _pattern_cache[pattern] + return c._pattern_pack(pat.CObj, pat.format, pat.size , ...) +end + +function check(typename , field) + if field == nil then + return c._env_type(P,typename) + else + return c._env_type(P,typename,field) ~=0 + end +end + +-------------- + +local default_cache = setmetatable({} , {__mode = "kv"}) + +local function default_table(typename) + local v = default_cache[typename] + if v then + return v + end + + v = { __index = assert(decode_message(typename , "")) } + + default_cache[typename] = v + return v +end + +local decode_message_mt = {} + +local function decode_message_cb(typename, buffer) + return setmetatable ( { typename, buffer } , decode_message_mt) +end + +function decode(typename, buffer, length) + local ret = {} + local ok = c._decode(P, decode_message_cb , ret , typename, buffer, length) + if ok then + return setmetatable(ret , default_table(typename)) + else + return false , c._last_error(P) + end +end + +local function expand(tbl) + local typename = rawget(tbl , 1) + local buffer = rawget(tbl , 2) + tbl[1] , tbl[2] = nil , nil + assert(c._decode(P, decode_message_cb , tbl , typename, buffer), typename) + setmetatable(tbl , default_table(typename)) +end + +function decode_message_mt.__index(tbl, key) + expand(tbl) + return tbl[key] +end + +function decode_message_mt.__pairs(tbl) + expand(tbl) + return pairs(tbl) +end + +local function set_default(typename, tbl) + for k,v in pairs(tbl) do + if type(v) == "table" then + local t, msg = c._env_type(P, typename, k) + if t == 6 then + set_default(msg, v) + elseif t == 128+6 then + for _,v in ipairs(v) do + set_default(msg, v) + end + end + end + end + return setmetatable(tbl , default_table(typename)) +end + + +default=set_default diff --git a/lualib/protobuf/parser.lua b/lualib/protobuf/parser.lua new file mode 100644 index 00000000..dbe74b6a --- /dev/null +++ b/lualib/protobuf/parser.lua @@ -0,0 +1,397 @@ +----------------------- +-- simple proto parser +----------------------- + +local lpeg = require "lpeg" +local P = lpeg.P +local S = lpeg.S +local R = lpeg.R +local C = lpeg.C +local Ct = lpeg.Ct +local Cg = lpeg.Cg +local Cc = lpeg.Cc +local V = lpeg.V + +local next = next +local error = error +local tonumber = tonumber +local pairs = pairs +local ipairs = ipairs +local rawset = rawset +local tinsert = table.insert +local smatch = string.match +local sbyte = string.byte + +local internal_type = { + double = "TYPE_DOUBLE", + float = "TYPE_FLOAT", + uint64 = "TYPE_UINT64", + int = "TYPE_INT32", + int32 = "TYPE_INT32", + int64 = "TYPE_INT64", + fixed64 = "TYPE_FIXED64", + fixed32 = "TYPE_FIXED32", + bool = "TYPE_BOOL", + string = "TYPE_STRING", + bytes = "TYPE_BYTES", + uint32 = "TYPE_UINT32", + sfixed32 = "TYPE_SFIXED32", + sfixed64 = "TYPE_SFIXED64", + sint32 = "TYPE_SINT32", + sint64 = "TYPE_SINT64", +} + +local function count_lines(_,pos, parser_state) + if parser_state.pos < pos then + parser_state.line = parser_state.line + 1 + parser_state.pos = pos + end + return pos +end + +local exception = lpeg.Cmt( lpeg.Carg(1) , function ( _ , pos, parser_state) + error( "syntax error at [" .. (parser_state.file or "") .."] (" .. parser_state.line ..")" ) + return pos +end) + +local eof = P(-1) +local newline = lpeg.Cmt((P"\n" + "\r\n") * lpeg.Carg(1) ,count_lines) +local line_comment = "//" * (1 - newline) ^0 * (newline + eof) +local blank = S" \t" + newline + line_comment +local blank0 = blank ^ 0 +local blanks = blank ^ 1 +local alpha = R"az" + R"AZ" + "_" +local alnum = alpha + R"09" +local str_c = (1 - S("\\\"")) + P("\\") * 1 +local str = P"\"" * C(str_c^0) * "\"" +local dotname = ("." * alpha * alnum ^ 0) ^ 0 +local typename = C(alpha * alnum ^ 0 * dotname) +local name = C(alpha * alnum ^ 0) +local filename = C((alnum + "/" + "." + "-")^1) +local id = R"09" ^ 1 / tonumber + "max" * Cc(-1) +local bool = "true" * Cc(true) + "false" * Cc(false) +local value = str + bool + name + id +local patterns = {} + +local enum_item = Cg(name * blank0 * "=" * blank0 * id * blank0 * ";" * blank0) + +local function insert(tbl, k,v) + tinsert(tbl, { name = k , number = v }) + return tbl +end + +patterns.ENUM = Ct(Cg("enum","type") * blanks * Cg(typename,"name") * blank0 * + "{" * blank0 * + Cg(lpeg.Cf(Ct"" * enum_item^1 , insert),"value") + * "}" * blank0) + +local prefix_field = P"required" * Cc"LABEL_REQUIRED" + + P"optional" * Cc"LABEL_OPTIONAL" + + P"repeated" * Cc"LABEL_REPEATED" +local postfix_pair = blank0 * Cg(name * blank0 * "=" * blank0 * value * blank0) +local postfix_pair_2 = blank0 * "," * postfix_pair +local postfix_field = "[" * postfix_pair * postfix_pair_2^0 * blank0 * "]" +local options = lpeg.Cf(Ct"" * postfix_field , rawset) ^ -1 + +local function setoption(t, options) + if next(options) then + t.options = options + end + return t +end + +local message_field = lpeg.Cf ( + Ct( Cg(prefix_field,"label") * blanks * + Cg(typename,"type_name") * blanks * + Cg(name,"name") * blank0 * "=" * blank0 * + Cg(id,"number") + ) * blank0 * options , + setoption) * blank0 * ";" * blank0 + +local extensions = Ct( + Cg("extensions" , "type") * blanks * + Cg(id,"start") * blanks * "to" * blanks * + Cg(id,"end") * blank0 * ";" * blank0 + ) + +patterns.EXTEND = Ct( + Cg("extend", "type") * blanks * + Cg(typename, "name") * blank0 * "{" * blank0 * + Cg(Ct((message_field) ^ 1),"extension") * "}" * blank0 + ) + +patterns.MESSAGE = P { Ct( + Cg("message","type") * blanks * + Cg(typename,"name") * blank0 * "{" * blank0 * + Cg(Ct((message_field + patterns.ENUM + extensions + patterns.EXTEND + V(1)) ^ 1),"items") * "}" * blank0 + ) } + +patterns.OPTION = Ct( + Cg("option" , "type") * blanks * + Cg(name, "name") * blank0 * "=" * blank0 * + Cg(value, "value") + ) * blank0 * ";" * blank0 + +patterns.IMPORT = Ct( Cg("import" , "type") * blanks * Cg(filename, "name") ) * blank0 + +patterns.PACKAGE = Ct( Cg("package", "type") * blanks * Cg(typename, "name") ) * blank0 * ";" * blank0 + +local proto_tbl = { "PROTO" } + +do + local k, v = next(patterns) + local p = V(k) + proto_tbl[k] = v + for k,v in next , patterns , k do + proto_tbl[k] = v + p = p + V(k) + end + proto_tbl.PROTO = Ct(blank0 * p ^ 1) +end + +local proto = P(proto_tbl) + +local deal = {} + +function deal:import(v) + self.dependency = self.dependency or {} + tinsert(self.dependency , v.name) +end + +function deal:package(v) + self.package = v.name +end + +function deal:enum(v) + self.enum_type = self.enum_type or {} + tinsert(self.enum_type , v) +end + +function deal:option(v) + self.options = self.options or {} + self.options[v.name] = v.value +end + +function deal:extend(v) + self.extension = self.extension or {} + local extendee = v.name + for _,v in ipairs(v.extension) do + v.extendee = extendee + v.type = internal_type[v.type_name] + if v.type then + v.type_name = nil + end + tinsert(self.extension , v) + end +end + +function deal:extensions(v) + self.extension_range = self.extension_range or {} + tinsert(self.extension_range, v) +end + +local function _add_nested_message(self, item) + if item.type == nil then + item.type = internal_type[item.type_name] + if item.type then + item.type_name = nil + end + self.field = self.field or {} + tinsert(self.field, item) + else + local f = deal[item.type] + item.type = nil + f(self , item) + end +end + +function deal:message(v) + self.nested_type = self.nested_type or {} + local m = { name = v.name } + tinsert(self.nested_type , m) + for _,v in ipairs(v.items) do + _add_nested_message(m, v) + end +end + +local function fix(r) + local p = {} + for _,v in ipairs(r) do + local f = deal[v.type] + v.type = nil + f(p , v) + end + + p.message_type = p.nested_type + p.nested_type = nil + + return p +end + +--- fix message name + +local NULL = {} + +local function _match_name(namespace , n , all) + if sbyte(n) == 46 then + return n + end + + repeat + local name = namespace .. "." .. n + if all[name] then + return name + end + namespace = smatch(namespace,"(.*)%.[%w_]+$") + until namespace == nil +end + +local function _fix_field(namespace , field, all) + local type_name = field.type_name + if type_name == "" then + field.type_name = nil + return + elseif type_name == nil then + return + end + + local full_name = assert(_match_name(namespace, field.type_name, all) , field.type_name , all) + + field.type_name = full_name + field.type = all[full_name] + + local options = field.options + if options then + if options.default then + field.default_value = tostring(options.default) + options.default = nil + end + if next(options) == nil then + field.options = nil + end + end +end + +local function _fix_extension(namespace, ext, all) + for _,field in ipairs(ext or NULL) do + field.extendee = assert(_match_name(namespace, field.extendee,all),field.extendee) + _fix_field(namespace , field , all) + end +end + +local function _fix_message(msg , all) + for _,field in ipairs(msg.field or NULL) do + _fix_field(assert(all[msg],msg.name) , field , all) + end + for _,nest in ipairs(msg.nested_type or NULL) do + _fix_message(nest , all) + end + _fix_extension(all[msg] , msg.extension , all) +end + +local function _fix_typename(file , all) + for _,message in ipairs(file.message_type or NULL) do + _fix_message(message , all) + end + _fix_extension(file.package , file.extension , all) +end + +--- merge messages + +local function _enum_fullname(prefix, enum , all) + local fullname + if sbyte(enum.name) == 46 then + fullname = enum.name + else + fullname = prefix .. "." .. enum.name + end + all[fullname] = "TYPE_ENUM" + all[enum] = fullname +end + +local function _message_fullname(prefix , msg , all) + local fullname + if sbyte(msg.name) == 46 then + fullname = msg.name + else + fullname = prefix .. "." .. msg.name + end + all[fullname] = "TYPE_MESSAGE" + all[msg] = fullname + for _,nest in ipairs(msg.nested_type or NULL) do + _message_fullname(fullname , nest , all) + end + for _,enum in ipairs(msg.enum_type or NULL) do + _enum_fullname(fullname , enum , all) + end +end + +local function _gen_fullname(file , all) + local prefix = "" + if file.package then + prefix = "." .. file.package + end + for _,message in ipairs(file.message_type or NULL) do + _message_fullname(prefix , message , all) + end + for _,enum in ipairs(file.enum_type or NULL) do + _enum_fullname(prefix , enum , all) + end +end + +--- parser + +local parser = {} + +local function parser_one(text,filename) + local state = { file = filename, pos = 0, line = 1 } + local r = lpeg.match(proto * -1 + exception , text , 1, state ) + local t = fix(r) + return t +end + +function parser.parser(text,filename) + local t = parser_one(text,filename) + local all = {} + _gen_fullname(t,all) + _fix_typename(t , all) + return t +end + +local pb = require "protobuf" + +function parser.register(fileset , path) + local all = {} + local files = {} + if type(fileset) == "string" then + fileset = { fileset } + end + for _, filename in ipairs(fileset) do + local fullname + if path then + fullname = path .. "/" .. filename + else + fullname = filename + end + local f = assert(io.open(fullname , "r")) + local buffer = f:read "*a" + f:close() + local t = parser_one(buffer,filename) + _gen_fullname(t,all) + t.name = filename + tinsert(files , t) + end + for _,file in ipairs(files) do + _fix_typename(file,all) + end + + local pbencode = pb.encode("google.protobuf.FileDescriptorSet" , { file = files }) + + if pbencode == nil then + error(pb.lasterror()) + end + pb.register(pbencode) + return files +end + +return parser \ No newline at end of file