sproto update

lualib-src/sproto/README.md

@@ -140,11 +140,11 @@ The schema text is like this:
 }
 
 .AddressBook {
-    person 0 : *Person
+    person 0 : *Person(id)	# (id) is optional, means Person.id is main index.
 }
 
 foobar 1 {	# define a new protocol (for RPC used) with tag 1
-    request person	# Associate the type person with foobar.request
+    request Person	# Associate the type Person with foobar.request
     response {	# define the foobar.response type
         ok 0 : boolean
     }
@@ -161,6 +161,7 @@ A schema text can be self-described by the sproto schema language.
         type 1 : string
         id 2 : integer
         array 3 : boolean
+        key 4 : integer	# optional tag for map
     }
     name 0 : string
     fields 1 : *field
@@ -183,10 +184,12 @@ Types
 =======
 
 * **string** : binary string
-* **integer** : integer, the max length of a integer is signed 64bit.
+* **integer** : integer, the max length of an integer is signed 64bit.
 * **boolean** : true or false
 
-You can add * before the typename to declare an array.
+You can add * before the typename to declare an array. 
+
+You can also specify a main index, the array whould be encode as an unordered map.
 
 User defined type can be any name in alphanumeric characters except the build-in typenames, and nested types are supported.
 
@@ -319,7 +322,19 @@ struct sproto_type * sproto_type(struct sproto *, const char * typename);
 Query the type object from a sproto object:
 
 ```C
-typedef int (*sproto_callback)(void *ud, const char *tagname, int type, int index, struct sproto_type *, void *value, int length);
+struct sproto_arg {
+	void *ud;
+	const char *tagname;
+	int tagid;
+	int type;
+	struct sproto_type *subtype;
+	void *value;
+	int length;
+	int index;	// array base 1
+	int mainindex;	// for map
+};
+
+typedef int (*sproto_callback)(const struct sproto_arg *args);
 
 int sproto_decode(struct sproto_type *, const void * data, int size, sproto_callback cb, void *ud);
 int sproto_encode(struct sproto_type *, void * buffer, int size, sproto_callback cb, void *ud);
@@ -334,13 +349,9 @@ int sproto_unpack(const void * src, int srcsz, void * buffer, int bufsz);
 
 pack and unpack the message with the 0 packing algorithm.
 
-JIT
+Other Implementions and bindings
 =====
-You may also interest in https://github.com/lvzixun/sproto-JIT
-
-C# version
-=====
-https://github.com/lvzixun/sproto-Csharp
+See Wiki https://github.com/cloudwu/sproto/wiki
 
 Question?
 ==========

lualib-src/sproto/lsproto.c

@@ -9,7 +9,6 @@
 #define MAX_GLOBALSPROTO 16
 #define ENCODE_BUFFERSIZE 2050
 
-//#define ENCODE_BUFFERSIZE 2050
 #define ENCODE_MAXSIZE 0x1000000
 #define ENCODE_DEEPLEVEL 64
 
@@ -94,18 +93,19 @@ struct encode_ud {
 	const char * array_tag;
 	int array_index;
 	int deep;
+	int iter_index;
 };
 
 static int
-encode(void *ud, const char *tagname, int type, int index, struct sproto_type *st, void *value, int length) {
-	struct encode_ud *self = ud;
+encode(const struct sproto_arg *args) {
+	struct encode_ud *self = args->ud;
 	lua_State *L = self->L;
 	if (self->deep >= ENCODE_DEEPLEVEL)
 		return luaL_error(L, "The table is too deep");
-	if (index > 0) {
-		if (tagname != self->array_tag) {
-			self->array_tag = tagname;
-			lua_getfield(L, self->tbl_index, tagname);
+	if (args->index > 0) {
+		if (args->tagname != self->array_tag) {
+			self->array_tag = args->tagname;
+			lua_getfield(L, self->tbl_index, args->tagname);
 			if (lua_isnil(L, -1)) {
 				if (self->array_index) {
 					lua_replace(L, self->array_index);
@@ -119,15 +119,30 @@ encode(void *ud, const char *tagname, int type, int index, struct sproto_type *s
 				self->array_index = lua_gettop(L);
 			}
 		}
-		lua_rawgeti(L, self->array_index, index);
+		if (args->mainindex >= 0) {
+			// use lua_next to iterate the table
+			// todo: check the key is equal to mainindex value
+
+			lua_pushvalue(L,self->iter_index);
+			if (!lua_next(L, self->array_index)) {
+				// iterate end
+				lua_pushnil(L);
+				lua_replace(L, self->iter_index);
+				return 0;
+			}
+			lua_insert(L, -2);
+			lua_replace(L, self->iter_index);
+		} else {
+			lua_rawgeti(L, self->array_index, args->index);
+		}
 	} else {
-		lua_getfield(L, self->tbl_index, tagname);
+		lua_getfield(L, self->tbl_index, args->tagname);
 	}
 	if (lua_isnil(L, -1)) {
 		lua_pop(L,1);
 		return 0;
 	}
-	switch (type) {
+	switch (args->type) {
 	case SPROTO_TINTEGER: {
 		lua_Integer v = luaL_checkinteger(L, -1);
 		lua_Integer vh;
@@ -135,26 +150,26 @@ encode(void *ud, const char *tagname, int type, int index, struct sproto_type *s
 		// notice: in lua 5.2, lua_Integer maybe 52bit
 		vh = v >> 31;
 		if (vh == 0 || vh == -1) {
-			*(uint32_t *)value = (uint32_t)v;
+			*(uint32_t *)args->value = (uint32_t)v;
 			return 4;
 		}
 		else {
-			*(uint64_t *)value = (uint64_t)v;
+			*(uint64_t *)args->value = (uint64_t)v;
 			return 8;
 		}
 	}
 	case SPROTO_TBOOLEAN: {
 		int v = lua_toboolean(L, -1);
-		*(int *)value = v;
+		*(int *)args->value = v;
 		lua_pop(L,1);
 		return 4;
 	}
 	case SPROTO_TSTRING: {
 		size_t sz = 0;
 		const char * str = luaL_checklstring(L, -1, &sz);
-		if (sz > length)
+		if (sz > args->length)
 			return -1;
-		memcpy(value, str, sz);
+		memcpy(args->value, str, sz);
 		lua_pop(L,1);
 		return sz;
 	}
@@ -162,17 +177,19 @@ encode(void *ud, const char *tagname, int type, int index, struct sproto_type *s
 		struct encode_ud sub;
 		int r;
 		sub.L = L;
-		sub.st = st;
+		sub.st = args->subtype;
 		sub.tbl_index = lua_gettop(L);
 		sub.array_tag = NULL;
 		sub.array_index = 0;
 		sub.deep = self->deep + 1;
-		r = sproto_encode(st, value, length, encode, &sub);
-		lua_pop(L,1);
+		lua_pushnil(L);	// prepare an iterator slot
+		sub.iter_index = sub.tbl_index + 1;
+		r = sproto_encode(args->subtype, args->value, args->length, encode, &sub);
+		lua_pop(L,2);	// pop the value and the iterator slot
 		return r;
 	}
 	default:
-		return luaL_error(L, "Invalid field type %d", type);
+		return luaL_error(L, "Invalid field type %d", args->type);
 	}
 }
 
@@ -211,18 +228,23 @@ lencode(lua_State *L) {
 		return luaL_argerror(L, 1, "Need a sproto_type object");
 	}
 	luaL_checktype(L, 2, LUA_TTABLE);
-	luaL_checkstack(L, ENCODE_DEEPLEVEL + 8, NULL);
+	luaL_checkstack(L, ENCODE_DEEPLEVEL*2 + 8, NULL);
 	self.L = L;
 	self.st = st;
 	self.tbl_index = 2;
 	self.array_tag = NULL;
 	self.array_index = 0;
 	self.deep = 0;
+	lua_pushnil(L);	// for iterator (stack 3)
+	self.iter_index = 3;
 	for (;;) {
 		int r = sproto_encode(st, buffer, sz, encode, &self);
 		if (r<0) {
 			buffer = expand_buffer(L, sz, sz*2);
 			sz *= 2;
+			// reset iterator slot
+			lua_pushnil(L);
+			lua_replace(L, 3);
 		} else {
 			lua_pushlstring(L, buffer, r);
 			return 1;
@@ -236,21 +258,23 @@ struct decode_ud {
 	int array_index;
 	int result_index;
 	int deep;
+	int mainindex_tag;
+	int key_index;
 };
 
 static int
-decode(void *ud, const char *tagname, int type, int index, struct sproto_type *st, void *value, int length) {
-	struct decode_ud * self = ud;
+decode(const struct sproto_arg *args) {
+	struct decode_ud * self = args->ud;
 	lua_State *L = self->L;
 	if (self->deep >= ENCODE_DEEPLEVEL)
 		return luaL_error(L, "The table is too deep");
-	if (index > 0) {
+	if (args->index > 0) {
 		// It's array
-		if (tagname != self->array_tag) {
-			self->array_tag = tagname;
+		if (args->tagname != self->array_tag) {
+			self->array_tag = args->tagname;
 			lua_newtable(L);
 			lua_pushvalue(L, -1);
-			lua_setfield(L, self->result_index, tagname);
+			lua_setfield(L, self->result_index, args->tagname);
 			if (self->array_index) {
 				lua_replace(L, self->array_index);
 			} else {
@@ -258,20 +282,20 @@ decode(void *ud, const char *tagname, int type, int index, struct sproto_type *s
 			}
 		}
 	}
-	switch (type) {
+	switch (args->type) {
 	case SPROTO_TINTEGER: {
 		// notice: in lua 5.2, 52bit integer support (not 64)
-		lua_Integer v = *(uint64_t*)value;
+		lua_Integer v = *(uint64_t*)args->value;
 		lua_pushinteger(L, v);
 		break;
 	}
 	case SPROTO_TBOOLEAN: {
-		int v = *(uint64_t*)value;
+		int v = *(uint64_t*)args->value;
 		lua_pushboolean(L,v);
 		break;
 	}
 	case SPROTO_TSTRING: {
-		lua_pushlstring(L, value, length);
+		lua_pushlstring(L, args->value, args->length);
 		break;
 	}
 	case SPROTO_TSTRUCT: {
@@ -283,20 +307,47 @@ decode(void *ud, const char *tagname, int type, int index, struct sproto_type *s
 		sub.deep = self->deep + 1;
 		sub.array_index = 0;
 		sub.array_tag = NULL;
+		if (args->mainindex >= 0) {
+			// This struct will set into a map, so mark the main index tag.
+			sub.mainindex_tag = args->mainindex;
+			lua_pushnil(L);
+			sub.key_index = lua_gettop(L);
 
-		r = sproto_decode(st, value, length, decode, &sub);
-		if (r < 0 || r != length)
-			return r;
-		lua_settop(L, sub.result_index);
-		break;
+			r = sproto_decode(args->subtype, args->value, args->length, decode, &sub);
+			if (r < 0 || r != args->length)
+				return r;
+			// assert(args->index > 0);
+			lua_pushvalue(L, sub.key_index);
+			if (lua_isnil(L, -1)) {
+				luaL_error(L, "Can't find main index (tag=%d) in [%s]", args->mainindex, args->tagname);
+			}
+			lua_pushvalue(L, sub.result_index);
+			lua_rawset(L, self->array_index);
+			lua_settop(L, sub.result_index-1);
+			return 0;
+		} else {
+			sub.mainindex_tag = -1;
+			sub.key_index = 0;
+			r = sproto_decode(args->subtype, args->value, args->length, decode, &sub);
+			if (r < 0 || r != args->length)
+				return r;
+			lua_settop(L, sub.result_index);
+			break;
+		}
 	}
 	default:
 		luaL_error(L, "Invalid type");
 	}
-	if (index > 0) {
-		lua_rawseti(L, self->array_index, index);
+	if (args->index > 0) {
+		lua_rawseti(L, self->array_index, args->index);
 	} else {
-		lua_setfield(L, self->result_index, tagname);
+		if (self->mainindex_tag == args->tagid) {
+			// This tag is marked, save the value to key_index
+			// assert(self->key_index > 0);
+			lua_pushvalue(L,-1);
+			lua_replace(L, self->key_index);
+		}
+		lua_setfield(L, self->result_index, args->tagname);
 	}
 
 	return 0;
@@ -339,12 +390,14 @@ ldecode(lua_State *L) {
 	if (!lua_istable(L, -1)) {
 		lua_newtable(L);
 	}
-	luaL_checkstack(L, ENCODE_DEEPLEVEL*2 + 8, NULL);
+	luaL_checkstack(L, ENCODE_DEEPLEVEL*3 + 8, NULL);
 	self.L = L;
 	self.result_index = lua_gettop(L);
 	self.array_index = 0;
 	self.array_tag = NULL;
 	self.deep = 0;
+	self.mainindex_tag = -1;
+	self.key_index = 0;
 	r = sproto_decode(st, buffer, (int)sz, decode, &self);
 	if (r < 0) {
 		return luaL_error(L, "decode error");

lualib-src/sproto/sproto.c

@@ -17,6 +17,7 @@ struct field {
 	int type;
 	const char * name;
 	struct sproto_type * st;
+	int key;
 };
 
 struct sproto_type {
@@ -188,6 +189,7 @@ import_field(struct sproto *s, struct field *f, const uint8_t * stream) {
 	f->type = -1;
 	f->name = NULL;
 	f->st = NULL;
+	f->key = -1;
 
 	sz = todword(stream);
 	stream += SIZEOF_LENGTH;
@@ -234,6 +236,9 @@ import_field(struct sproto *s, struct field *f, const uint8_t * stream) {
 			if (value)
 				array = SPROTO_TARRAY;
 			break;
+		case 5:	// key
+			f->key = value;
+			break;
 		default:
 			return NULL;
 		}
@@ -485,7 +490,11 @@ sproto_dump(struct sproto *s) {
 					type_name = buildin[t];
 				}
 			}
-			printf("\t%s (%d) %s%s\n", f->name, f->tag, array, type_name);
+			if (f->key >= 0) {
+				printf("\t%s (%d) %s%s(%d)\n", f->name, f->tag, array, type_name, f->key);
+			} else {
+				printf("\t%s (%d) %s%s\n", f->name, f->tag, array, type_name);
+			}
 		}
 	}
 	printf("=== %d protocol ===\n", s->protocol_n);
@@ -637,22 +646,37 @@ encode_uint64(uint64_t v, uint8_t * data, int size) {
 	return fill_size(data, sizeof(v));
 }
 
+/*
+//#define CB(tagname,type,index,subtype,value,length) cb(ud, tagname,type,index,subtype,value,length)
+
 static int
-encode_string(sproto_callback cb, void *ud, struct field *f, uint8_t *data, int size) {
+do_cb(sproto_callback cb, void *ud, const char *tagname, int type, int index, struct sproto_type *subtype, void *value, int length) {
+	if (subtype) {
+		if (type >= 0) {
+			printf("callback: tag=%s[%d], subtype[%s]:%d\n",tagname,index, subtype->name, type);
+		} else {
+			printf("callback: tag=%s[%d], subtype[%s]\n",tagname,index, subtype->name);
+		}
+	} else if (index > 0) {
+		printf("callback: tag=%s[%d]\n",tagname,index);
+	} else if (index == 0) {
+		printf("callback: tag=%s\n",tagname);
+	} else {
+		printf("callback: tag=%s [mainkey]\n",tagname);
+	}
+	return cb(ud, tagname,type,index,subtype,value,length);
+}
+#define CB(tagname,type,index,subtype,value,length) do_cb(cb,ud, tagname,type,index,subtype,value,length)
+*/
+
+static int
+encode_object(sproto_callback cb, struct sproto_arg *args, uint8_t *data, int size) {
 	int sz;
 	if (size < SIZEOF_LENGTH)
 		return -1;
-	sz = cb(ud, f->name, SPROTO_TSTRING, 0, NULL, data+SIZEOF_LENGTH, size-SIZEOF_LENGTH);
-	return fill_size(data, sz);
-}
-
-static int
-encode_struct(sproto_callback cb, void *ud, struct field *f, uint8_t *data, int size) {
-	int sz;
-	if (size < SIZEOF_LENGTH) {
-		return -1;
-	}
-	sz = cb(ud, f->name, SPROTO_TSTRUCT, 0, f->st, data+SIZEOF_LENGTH, size-SIZEOF_LENGTH);
+	args->value = data+SIZEOF_LENGTH;
+	args->length = size-SIZEOF_LENGTH;
+	sz = cb(args);
 	return fill_size(data, sz);
 }
 
@@ -672,7 +696,7 @@ uint32_to_uint64(int negative, uint8_t *buffer) {
 }
 
 static uint8_t *
-encode_integer_array(sproto_callback cb, void *ud, struct field *f, uint8_t *buffer, int size) {
+encode_integer_array(sproto_callback cb, struct sproto_arg *args, uint8_t *buffer, int size) {
 	uint8_t * header = buffer;
 	int intlen;
 	int index;
@@ -688,7 +712,10 @@ encode_integer_array(sproto_callback cb, void *ud, struct field *f, uint8_t *buf
 			uint64_t u64;
 			uint32_t u32;
 		} u;
-		sz = cb(ud, f->name, SPROTO_TINTEGER, index, f->st, &u, sizeof(u));
+		args->value = &u;
+		args->length = sizeof(u);
+		args->index = index;
+		sz = cb(args);
 		if (sz < 0)
 			return NULL;
 		if (sz == 0)
@@ -749,27 +776,26 @@ encode_integer_array(sproto_callback cb, void *ud, struct field *f, uint8_t *buf
 }
 
 static int
-encode_array(sproto_callback cb, void *ud, struct field *f, uint8_t *data, int size) {
+encode_array(sproto_callback cb, struct sproto_arg *args, uint8_t *data, int size) {
 	uint8_t * buffer;
-	int index;
-	int type;
 	int sz;
 	if (size < SIZEOF_LENGTH)
 		return -1;
 	size -= SIZEOF_LENGTH;
-	index = 1;
 	buffer = data + SIZEOF_LENGTH;
-	type = f->type & ~SPROTO_TARRAY;
-	switch (type) {
+	switch (args->type) {
 	case SPROTO_TINTEGER:
-		buffer = encode_integer_array(cb,ud,f,buffer,size);
+		buffer = encode_integer_array(cb,args,buffer,size);
 		if (buffer == NULL)
 			return -1;
 		break;
 	case SPROTO_TBOOLEAN:
+		args->index = 1;
 		for (;;) {
 			int v = 0;
-			int sz = cb(ud, f->name, type, index, f->st, &v, sizeof(v));
+			args->value = &v;
+			args->length = sizeof(v);
+			sz = cb(args);
 			if (sz < 0)
 				return -1;
 			if (sz == 0)
@@ -779,16 +805,18 @@ encode_array(sproto_callback cb, void *ud, struct field *f, uint8_t *data, int s
 			buffer[0] = v ? 1: 0;
 			size -= 1;
 			buffer += 1;
-			index++;
+			++args->index;
 		}
 		break;
 	default:
+		args->index = 1;
 		for (;;) {
-			int sz;
 			if (size < SIZEOF_LENGTH)
 				return -1;
 			size -= SIZEOF_LENGTH;
-			sz = cb(ud, f->name, type, index, f->st, buffer+SIZEOF_LENGTH, size);
+			args->value = buffer+SIZEOF_LENGTH;
+			args->length = size;
+			sz = cb(args);
 			if (sz < 0)
 				return -1;
 			if (sz == 0)
@@ -796,7 +824,7 @@ encode_array(sproto_callback cb, void *ud, struct field *f, uint8_t *data, int s
 			fill_size(buffer, sz);
 			buffer += SIZEOF_LENGTH+sz;
 			size -=sz;
-			index ++;
+			++args->index;
 		}
 		break;
 	}
@@ -808,6 +836,7 @@ encode_array(sproto_callback cb, void *ud, struct field *f, uint8_t *data, int s
 
 int
 sproto_encode(struct sproto_type *st, void * buffer, int size, sproto_callback cb, void *ud) {
+	struct sproto_arg args;
 	uint8_t * header = buffer;
 	uint8_t * data;
 	int header_sz = SIZEOF_HEADER + st->maxn * SIZEOF_FIELD;
@@ -817,6 +846,7 @@ sproto_encode(struct sproto_type *st, void * buffer, int size, sproto_callback c
 	int datasz;
 	if (size < header_sz)
 		return -1;
+	args.ud = ud;
 	data = header + header_sz;
 	size -= header_sz;
 	index = 0;
@@ -826,9 +856,16 @@ sproto_encode(struct sproto_type *st, void * buffer, int size, sproto_callback c
 		int type = f->type;
 		int value = 0;
 		int sz = -1;
+		args.tagname = f->name;
+		args.tagid = f->tag;
+		args.subtype = f->st;
+		args.mainindex = f->key;
 		if (type & SPROTO_TARRAY) {
-			sz = encode_array(cb,ud, f, data, size);
+			args.type = type & ~SPROTO_TARRAY;
+			sz = encode_array(cb, &args, data, size);
 		} else {
+			args.type = type;
+			args.index = 0;
 			switch(type) {
 			case SPROTO_TINTEGER:
 			case SPROTO_TBOOLEAN: {
@@ -836,7 +873,9 @@ sproto_encode(struct sproto_type *st, void * buffer, int size, sproto_callback c
 					uint64_t u64;
 					uint32_t u32;
 				} u;
-				sz = cb(ud, f->name, type, 0, NULL, &u, sizeof(u));
+				args.value = &u;
+				args.length = sizeof(u);
+				sz = cb(&args);
 				if (sz < 0)
 					return -1;
 				if (sz == 0)
@@ -855,12 +894,9 @@ sproto_encode(struct sproto_type *st, void * buffer, int size, sproto_callback c
 				}
 				break;
 			}
+			case SPROTO_TSTRUCT:
 			case SPROTO_TSTRING: {
-				sz = encode_string(cb, ud, f, data, size);
-				break;
-			}
-			case SPROTO_TSTRUCT: {
-				sz = encode_struct(cb, ud, f, data, size);
+				sz = encode_object(cb, &args, data, size);
 				break;
 			}
 			}
@@ -904,9 +940,8 @@ sproto_encode(struct sproto_type *st, void * buffer, int size, sproto_callback c
 }
 
 static int
-decode_array_object(sproto_callback cb, void *ud, struct field *f, uint8_t * stream, int sz) {
+decode_array_object(sproto_callback cb, struct sproto_arg *args, uint8_t * stream, int sz) {
 	uint32_t hsz;
-	int type = f->type & ~SPROTO_TARRAY;
 	int index = 1;
 	while (sz > 0) {
 		if (sz < SIZEOF_LENGTH)
@@ -916,7 +951,10 @@ decode_array_object(sproto_callback cb, void *ud, struct field *f, uint8_t * str
 		sz -= SIZEOF_LENGTH;
 		if (hsz > sz)
 			return -1;
-		if (cb(ud, f->name, type, index, f->st, stream, hsz))
+		args->index = index;
+		args->value = stream;
+		args->length = hsz;
+		if (cb(args))
 			return -1;
 		sz -= hsz;
 		stream += hsz;
@@ -935,9 +973,9 @@ expand64(uint32_t v) {
 }
 
 static int
-decode_array(sproto_callback cb, void *ud, struct field *f, uint8_t * stream) {
+decode_array(sproto_callback cb, struct sproto_arg *args, uint8_t * stream) {
 	uint32_t sz = todword(stream);
-	int type = f->type & ~SPROTO_TARRAY;
+	int type = args->type;
 	int i;
 	stream += SIZEOF_LENGTH;
 	switch (type) {
@@ -953,7 +991,10 @@ decode_array(sproto_callback cb, void *ud, struct field *f, uint8_t * stream) {
 				return -1;
 			for (i=0;i<sz/sizeof(uint32_t);i++) {
 				uint64_t value = expand64(todword(stream + i*sizeof(uint32_t)));
-				cb(ud, f->name, SPROTO_TINTEGER, i+1, NULL, &value, sizeof(value));
+				args->index = i+1;
+				args->value = &value;
+				args->length = sizeof(value);
+				cb(args);
 			}
 		} else if (len == sizeof(uint64_t)) {
 			if (sz % sizeof(uint64_t) != 0)
@@ -962,7 +1003,10 @@ decode_array(sproto_callback cb, void *ud, struct field *f, uint8_t * stream) {
 				uint64_t low = todword(stream + i*sizeof(uint64_t));
 				uint64_t hi = todword(stream + i*sizeof(uint64_t) + sizeof(uint32_t));
 				uint64_t value = low | hi << 32;
-				cb(ud, f->name, SPROTO_TINTEGER, i+1, NULL, &value, sizeof(value));
+				args->index = i+1;
+				args->value = &value;
+				args->length = sizeof(value);
+				cb(args);
 			}
 		} else {
 			return -1;
@@ -972,12 +1016,15 @@ decode_array(sproto_callback cb, void *ud, struct field *f, uint8_t * stream) {
 	case SPROTO_TBOOLEAN:
 		for (i=0;i<sz;i++) {
 			uint64_t value = stream[i];
-			cb(ud, f->name, SPROTO_TBOOLEAN, i+1, NULL, &value, sizeof(value));
+			args->index = i+1;
+			args->value = &value;
+			args->length = sizeof(value);
+			cb(args);
 		}
 		break;
 	case SPROTO_TSTRING:
 	case SPROTO_TSTRUCT:
-		return decode_array_object(cb, ud, f, stream, sz);
+		return decode_array_object(cb, args, stream, sz);
 	default:
 		return -1;
 	}
@@ -986,6 +1033,7 @@ decode_array(sproto_callback cb, void *ud, struct field *f, uint8_t * stream) {
 
 int
 sproto_decode(struct sproto_type *st, const void * data, int size, sproto_callback cb, void *ud) {
+	struct sproto_arg args;
 	int total = size;
 	uint8_t * stream;
 	uint8_t * datastream;
@@ -994,6 +1042,8 @@ sproto_decode(struct sproto_type *st, const void * data, int size, sproto_callba
 	int tag;
 	if (size < SIZEOF_HEADER)
 		return -1;
+	// debug print
+	// printf("sproto_decode[%p] (%s)\n", ud, st->name);
 	stream = (void *)data;
 	fn = toword(stream);
 	stream += SIZEOF_HEADER;
@@ -1001,7 +1051,8 @@ sproto_decode(struct sproto_type *st, const void * data, int size, sproto_callba
 	if (size < fn * SIZEOF_FIELD)
 		return -1;
 	datastream = stream + fn * SIZEOF_FIELD;
-	size -= fn * SIZEOF_FIELD ;
+	size -= fn * SIZEOF_FIELD;
+	args.ud = ud;
 
 	tag = -1;
 	for (i=0;i<fn;i++) {
@@ -1028,9 +1079,15 @@ sproto_decode(struct sproto_type *st, const void * data, int size, sproto_callba
 		f = findtag(st, tag);
 		if (f == NULL)
 			continue;
+		args.tagname = f->name;
+		args.tagid = f->tag;
+		args.type = f->type & ~SPROTO_TARRAY;
+		args.subtype = f->st;
+		args.index = 0;
+		args.mainindex = f->key;
 		if (value < 0) {
 			if (f->type & SPROTO_TARRAY) {
-				if (decode_array(cb, ud, f, currentdata)) {
+				if (decode_array(cb, &args, currentdata)) {
 					return -1;
 				}
 			} else {
@@ -1039,21 +1096,27 @@ sproto_decode(struct sproto_type *st, const void * data, int size, sproto_callba
 					uint32_t sz = todword(currentdata);
 					if (sz == sizeof(uint32_t)) {
 						uint64_t v = expand64(todword(currentdata + SIZEOF_LENGTH));
-						cb(ud, f->name, SPROTO_TINTEGER, 0, NULL, &v, sizeof(v));
+						args.value = &v;
+						args.length = sizeof(v);
+						cb(&args);
 					} else if (sz != sizeof(uint64_t)) {
 						return -1;
 					} else {
 						uint32_t low = todword(currentdata + SIZEOF_LENGTH);
 						uint32_t hi = todword(currentdata + SIZEOF_LENGTH + sizeof(uint32_t));
 						uint64_t v = (uint64_t)low | (uint64_t) hi << 32;
-						cb(ud, f->name, SPROTO_TINTEGER, 0, NULL, &v, sizeof(v));
+						args.value = &v;
+						args.length = sizeof(v);
+						cb(&args);
 					}
 					break;
 				}
 				case SPROTO_TSTRING:
 				case SPROTO_TSTRUCT: {
 					uint32_t sz = todword(currentdata);
-					if (cb(ud, f->name, f->type, 0, f->st, currentdata+SIZEOF_LENGTH, sz))
+					args.value = currentdata+SIZEOF_LENGTH;
+					args.length = sz;
+					if (cb(&args))
 						return -1;
 					break;
 				}
@@ -1065,7 +1128,9 @@ sproto_decode(struct sproto_type *st, const void * data, int size, sproto_callba
 			return -1;
 		} else {
 			uint64_t v = value;
-			cb(ud, f->name, f->type, 0, NULL, &v, sizeof(v));
+			args.value = &v;
+			args.length = sizeof(v);
+			cb(&args);
 		}
 	}
 	return total - size;

lualib-src/sproto/sproto.h

@@ -27,7 +27,19 @@ struct sproto_type * sproto_type(struct sproto *, const char * type_name);
 int sproto_pack(const void * src, int srcsz, void * buffer, int bufsz);
 int sproto_unpack(const void * src, int srcsz, void * buffer, int bufsz);
 
-typedef int (*sproto_callback)(void *ud, const char *tagname, int type, int index, struct sproto_type *, void *value, int length);
+struct sproto_arg {
+	void *ud;
+	const char *tagname;
+	int tagid;
+	int type;
+	struct sproto_type *subtype;
+	void *value;
+	int length;
+	int index;	// array base 1
+	int mainindex;	// for map
+};
+
+typedef int (*sproto_callback)(const struct sproto_arg *args);
 
 int sproto_decode(struct sproto_type *, const void * data, int size, sproto_callback cb, void *ud);
 int sproto_encode(struct sproto_type *, void * buffer, int size, sproto_callback cb, void *ud);

lualib/sprotoparser.lua

@@ -1,6 +1,40 @@
 local lpeg = require "lpeg"
 local table = require "table"
 
+local packbytes
+local packvalue
+
+if _VERSION == "Lua 5.3" then
+	function packbytes(str)
+		return string.pack("<s4",str)
+	end
+
+	function packvalue(id)
+		id = (id + 1) * 2
+		return string.pack("<I2",id)
+	end
+else
+	function packbytes(str)
+		local size = #str
+		local a = size % 256
+		size = math.floor(size / 256)
+		local b = size % 256
+		size = math.floor(size / 256)
+		local c = size % 256
+		size = math.floor(size / 256)
+		local d = size
+		return string.char(a)..string.char(b)..string.char(c)..string.char(d) .. str
+	end
+
+	function packvalue(id)
+		id = (id + 1) * 2
+		assert(id >=0 and id < 65536)
+		local a = id % 256
+		local b = math.floor(id / 256)
+		return string.char(a) .. string.char(b)
+	end
+end
+
 local P = lpeg.P
 local S = lpeg.S
 local R = lpeg.R
@@ -35,6 +69,7 @@ local word = alpha * alnum ^ 0
 local name = C(word)
 local typename = C(word * ("." * word) ^ 0)
 local tag = R"09" ^ 1 / tonumber
+local mainkey = "(" * blank0 * name * blank0 * ")"
 
 local function multipat(pat)
 	return Ct(blank0 * (pat * blanks) ^ 0 * pat^0 * blank0)
@@ -46,7 +81,7 @@ end
 
 local typedef = P {
 	"ALL",
-	FIELD = namedpat("field", (name * blanks * tag * blank0 * ":" * blank0 * (C"*")^0 * typename)),
+	FIELD = namedpat("field", (name * blanks * tag * blank0 * ":" * blank0 * (C"*")^0 * typename * mainkey^0)),
 	STRUCT = P"{" * multipat(V"FIELD" + V"TYPE") * P"}",
 	TYPE = namedpat("type", P"." * name * blank0 * V"STRUCT" ),
 	SUBPROTO = Ct((C"request" + C"response") * blanks * (name + V"STRUCT")),
@@ -97,6 +132,11 @@ function convert.type(all, obj)
 				field.array = true
 				fieldtype = f[4]
 			end
+			local mainkey = f[5]
+			if mainkey then
+				assert(field.array)
+				field.key = mainkey
+			end
 			field.typename = fieldtype
 		else
 			assert(f.type == "type")	-- nest type
@@ -176,6 +216,7 @@ end
 		type 2 : integer
 		tag	3 :	integer
 		array 4	: boolean
+		key 5 : integer # If key exists, array must be true, and it's a map.
 	}
 	name 0 : string
 	fields 1 : *field
@@ -194,25 +235,18 @@ end
 }
 ]]
 
-local function packbytes(str)
-	str = string.pack("<s4",str)
-	return str
-end
-
-local function packvalue(id)
-	id = (id + 1) * 2
-	local str = string.pack("<I2", id)
-	return str
-end
-
 local function packfield(f)
 	local strtbl = {}
 	if f.array then
-		table.insert(strtbl, "\5\0")  -- 5 fields
+		if f.key then
+			table.insert(strtbl, "\6\0")  -- 6 fields
+		else
+			table.insert(strtbl, "\5\0")  -- 5 fields
+		end
 	else
 		table.insert(strtbl, "\4\0")	-- 4 fields
 	end
-	table.insert(strtbl, "\0\0")	-- name	(tag = 0, ref =0)
+	table.insert(strtbl, "\0\0")	-- name	(tag = 0, ref an object)
 	if f.buildin then
 		table.insert(strtbl, packvalue(f.buildin))	-- buildin (tag = 1)
 		table.insert(strtbl, "\1\0")	-- skip (tag = 2)
@@ -225,7 +259,10 @@ local function packfield(f)
 	if f.array then
 		table.insert(strtbl, packvalue(1))	-- array = true (tag = 4)
 	end
-	table.insert(strtbl, packbytes(f.name))
+	if f.key then
+		table.insert(strtbl, packvalue(f.key)) -- key tag (tag = 5)
+	end
+	table.insert(strtbl, packbytes(f.name)) -- external object (name)
 	return packbytes(table.concat(strtbl))
 end
 
@@ -238,11 +275,22 @@ local function packtype(name, t, alltypes)
 		tmp.tag = f.tag
 
 		tmp.buildin = buildin_types[f.typename]
+		local subtype
 		if not tmp.buildin then
-			tmp.type = assert(alltypes[f.typename])
+			subtype = assert(alltypes[f.typename])
+			tmp.type = subtype.id
 		else
 			tmp.type = nil
 		end
+		if f.key then
+			tmp.key = subtype.fields[f.key]
+			if not tmp.key then
+				error("Invalid map index :" .. f.key)
+			end
+		else
+			tmp.key = nil
+		end
+
 		table.insert(fields, packfield(tmp))
 	end
 	local data
@@ -274,6 +322,7 @@ local function packproto(name, p, alltypes)
 		if request == nil then
 			error(string.format("Protocol %s request type %s not found", name, p.request))
 		end
+		request = request.id
 	end
 	local tmp = {
 		"\4\0",	-- 4 fields
@@ -284,12 +333,12 @@ local function packproto(name, p, alltypes)
 		tmp[1] = "\2\0"
 	else
 		if p.request then
-			table.insert(tmp, packvalue(alltypes[p.request])) -- request typename (tag=2)
+			table.insert(tmp, packvalue(alltypes[p.request].id)) -- request typename (tag=2)
 		else
 			table.insert(tmp, "\1\0")
 		end
 		if p.response then
-			table.insert(tmp, packvalue(alltypes[p.response])) -- request typename (tag=3)
+			table.insert(tmp, packvalue(alltypes[p.response].id)) -- request typename (tag=3)
 		else
 			tmp[1] = "\3\0"
 		end
@@ -312,7 +361,13 @@ local function packgroup(t,p)
 	end
 	table.sort(alltypes)	-- make result stable
 	for idx, name in ipairs(alltypes) do
-		alltypes[name] = idx - 1
+		local fields = {}
+		for _, type_fields in ipairs(t[name]) do
+			if buildin_types[type_fields.typename] then
+				fields[type_fields.name] = type_fields.tag
+			end
+		end
+		alltypes[name] = { id = idx - 1, fields = fields }
 	end
 	tt = {}
 	for _,name in ipairs(alltypes) do