httpd don't need readline now

lualib-src/lua-socket.c

@@ -23,7 +23,6 @@ struct buffer_node {
 };
 
 struct socket_buffer {
-	int limit;
 	int size;
 	int offset;
 	struct buffer_node *head;
@@ -66,7 +65,6 @@ lnewpool(lua_State *L, int sz) {
 static int
 lnewbuffer(lua_State *L) {
 	struct socket_buffer * sb = lua_newuserdata(L, sizeof(*sb));	
-	sb->limit = luaL_optint(L,1,BUFFER_LIMIT);
 	sb->size = 0;
 	sb->offset = 0;
 	sb->head = NULL;
@@ -129,9 +127,6 @@ lpushbuffer(lua_State *L) {
 	sb->size += sz;
 
 	lua_pushinteger(L, sb->size);
-	if (sb->limit > 0 && sb->size > sb->limit) {
-		return luaL_error(L, "buffer overflow (limit = %d, size = %d)", sb->limit, sb->size);
-	}
 
 	return 1;
 }

lualib/http/httpd.lua

@@ -1,7 +1,6 @@
 local table = table
 
 local httpd = {}
-local READLIMIT = 8192	-- limit bytes per read
 
 local http_status_msg = {
 	[100] = "Continue",
@@ -46,76 +45,150 @@ local http_status_msg = {
 	[505] = "HTTP Version not supported",
 }
 
-local function recvheader(readline, header)
-	local line = readline()
-	if line == "" then
-		return header
+local function recvheader(readbytes, limit, lines, last)
+	local idx = 1
+	local len = 0
+	local header = last
+	while true do
+		header = header .. readbytes()
+		if #header + len > limit then
+			return
+		end
+		while true do
+			local f,e = header:find("\r\n",1, true)
+			if f then
+				if f == 1 then
+					for i = idx, #lines do
+						lines[i] = nil
+					end
+					return header:sub(3)
+				end
+				lines[idx] = header:sub(1, f-1)
+				idx = idx + 1
+				len = len + f-1
+				header = header:sub(e+1)
+			else
+				break
+			end
+		end
 	end
+end
 
+local function parseheader(lines, from, header)
 	local name, value
-	repeat
+	for i=from,#lines do
+		local line = lines[i]
 		if line:byte(1) == 9 then	-- tab, append last line
+			if name == nil then
+				return
+			end
 			header[name] = header[name] .. line:sub(2)
 		else
 			name, value = line:match "^(.-):%s*(.*)"
-			assert(name and value)
+			if name == nil or value == nil then
+				return
+			end
 			name = name:lower()
 			if header[name] then
 				header[name] = header[name] .. ", " .. value
 			else
 				header[name] = value
 			end
-			line = readline()
 		end
-	until line == ""
-
+	end
 	return header
 end
 
-local function recvbody(readbytes, length)
-	if length < READLIMIT then
-		return readbytes(length)
-	end
-	local tmp = {}
+local function chunksize(readbytes, body)
 	while true do
-		if length <= READLIMIT then
-			table.insert(tmp, readbytes(length))
+		if #body > 128 then
+			return
+		end
+		body = body .. readbytes()
+		local f,e = body:find("\r\n",1,true)
+		if f then
+			return tonumber(body:sub(1,f-1),16), body:sub(e+1)
+		end
+	end
+end
+
+local function readcrln(readbytes, body)
+	if #body > 2 then
+		if body:sub(1,2) ~= "\r\n" then
+			return
+		end
+		return body:sub(3)
+	else
+		body = body .. readbytes(2-#body)
+		if body ~= "\r\n" then
+			return
+		end
+		return ""
+	end
+end
+
+local tmpline = {}
+
+local function recvchunkedbody(readbytes, bodylimit, header, body)
+	local result = ""
+	local size = 0
+
+	while true do
+		local sz
+		sz , body = chunksize(readbytes, body)
+		if not sz then
+			return
+		end
+		if sz == 0 then
 			break
 		end
-		table.insert(tmp, readbytes(READLIMIT))
-		length = length - READLIMIT
+		size = size + sz
+		if bodylimit and size > bodylimit then
+			return
+		end
+		if #body >= sz then
+			result = result .. body:sub(1,sz)
+			body = body:sub(sz+1)
+		else
+			result = result .. body .. readbytes(sz - #body)
+			body = ""
+		end
+		body = readcrln(readbytes, body)
+		if not body then
+			return
+		end
 	end
-	return table.concat(tmp)
+
+	body = readcrln(readbytes, body)
+	if not body then
+		return
+	end
+	body = recvheader(readbytes, 8192, tmpline, body)
+	if not body then
+		return
+	end
+
+	header = parseheader(tmpline,1,header)
+
+	return result, header
 end
 
-local function recvchunkedbody(readline, readbytes, header)
-	local size = assert(tonumber(readline(),16))
-	local body = recvbody(readbytes,size)
-	assert(readbytes(2) == "\r\n")
-	size = assert(tonumber(readline(),16))
-	if size > 0 then
-		local bodys = { body }
-		repeat
-			table.insert(bodys, recvbody(readbytes,size))
-			assert(readbytes(2) == "\r\n")
-			size = assert(tonumber(readline(),16))
-		until size <= 0
-		body = table.concat(bodys)
+local function readall(readbytes, bodylimit)
+	local body = recvheader(readbytes, 8192, tmpline, "")
+	if not body then
+		return 413	-- Request Entity Too Large
 	end
-	assert(readbytes(2) == "\r\n")
-	header = recvheader(readline, header)
-	return body, header
-end
-
-local function readall(readline, readbytes)
-	local request = readline()
+	local request = assert(tmpline[1])
 	local method, url, httpver = request:match "^(%a+)%s+(.-)%s+HTTP/([%d%.]+)$"
 	assert(method and url and httpver)
 	httpver = assert(tonumber(httpver))
 	if httpver < 1.0 or httpver > 1.1 then
 		return 505	-- HTTP Version not supported
 	end
-	local header = recvheader(readline, {})
+	local header = parseheader(tmpline,2,{})
+	if not header then
+		return 400	-- Bad request
+	end
 	local length = header["content-length"]
 	if length then
 		length = tonumber(length)
@@ -127,23 +200,31 @@ local function readall(readline, readbytes)
 		end
 	end
 
-	local body
 	if mode == "chunked" then
-		body, header = recvchunkedbody(readline, readbytes, header)
+		body, header = recvchunkedbody(readbytes, bodylimit, header, body)
+		if not body then
+			return 413
+		end
 	else
 		-- identity mode
 		if length then
-			body = readbody(readbytes, length)
+			if length > bodylimit then
+				return 413
+			end
+			if #body >= length then
+				body = body:sub(1,length)
+			else
+				local padding = readbytes(length - #body)
+				body = body .. padding
+			end
 		end
 	end
 
 	return 200, url, method, header, body
 end
 
-function httpd.read_request(readfunc)
-	local readline = assert(readfunc.readline)
-	local readbytes = assert(readfunc.readbytes)
-	local ok, code, url, method, header, body = pcall(readall, readline, readbytes)
+function httpd.read_request(readbytes, bodylimit)
+	local ok, code, url, method, header, body = pcall(readall, readbytes, bodylimit)
 	if ok then
 		return code, url, method, header, body
 	else

lualib/http/sockethelper.lua

@@ -1,6 +1,5 @@
 local socket = require "socket"
 
-local readline = socket.readline
 local readbytes = socket.read
 local writebytes = socket.write
 
@@ -10,18 +9,7 @@ local socket_error = {}
 sockethelper.socket_error = socket_error
 
 function sockethelper.readfunc(fd)
-	local helper = {}
-
-	function helper.readline()
-		local ret = readline(fd, "\r\n")
-		if ret then
-			return ret
-		else
-			error(socket_error)
-		end
-	end
-
-	function helper.readbytes(sz)
+	return function (sz)
 		local ret = readbytes(fd, sz)
 		if ret then
 			return ret
@@ -29,8 +17,6 @@ function sockethelper.readfunc(fd)
 			error(socket_error)
 		end
 	end
-
-	return helper
 end
 
 function sockethelper.writefunc(fd)

lualib/snax/loginserver.lua

@@ -47,14 +47,15 @@ local function write(fd, text)
 end
 
 local function launch_slave(auth_handler)
-	-- set socket buffer limit (8K)
-	-- If the attacker send large package, close the socket
-	socket.limit(8192)
-
 	local function auth(fd, addr)
 		fd = assert(tonumber(fd))
 		skynet.error(string.format("connect from %s (fd = %d)", addr, fd))
 		socket.start(fd)
+
+		-- set socket buffer limit (8K)
+		-- If the attacker send large package, close the socket
+		socket.limit(fd, 8192)
+
 		local challenge = crypt.randomkey()
 		write(fd, crypt.base64encode(challenge).."\n")
 

lualib/socket.lua

@@ -2,7 +2,6 @@ local driver = require "socketdriver"
 local skynet = require "skynet"
 local assert = assert
 
-local buffer_limit = -1
 local socket = {}	-- api
 local buffer_pool = {}	-- store all message buffer object
 local socket_pool = setmetatable( -- store all socket object
@@ -48,13 +47,7 @@ socket_message[1] = function(id, size, data)
 		return
 	end
 
-	local ok , sz = pcall(driver.push, s.buffer, buffer_pool, data, size)
-	if not ok then
-		skynet.error("socket: error on ", id , sz)
-		driver.clear(s.buffer,buffer_pool)
-		driver.close(id)
-		return
-	end
+	local sz = driver.push(s.buffer, buffer_pool, data, size)
 	local rr = s.read_required
 	local rrt = type(rr)
 	if rrt == "number" then
@@ -63,11 +56,19 @@ socket_message[1] = function(id, size, data)
 			s.read_required = nil
 			wakeup(s)
 		end
-	elseif rrt == "string" then
-		-- read line
-		if driver.readline(s.buffer,nil,rr) then
-			s.read_required = nil
-			wakeup(s)
+	else
+		if s.buffer_limit and sz > s.buffer_limit then
+			skynet.error(string.format("socket buffer overlow: fd=%d size=%d", id , sz))
+			driver.clear(s.buffer,buffer_pool)
+			driver.close(id)
+			return
+		end
+		if rrt == "string" then
+			-- read line
+			if driver.readline(s.buffer,nil,rr) then
+				s.read_required = nil
+				wakeup(s)
+			end
 		end
 	end
 end
@@ -130,7 +131,7 @@ skynet.register_protocol {
 local function connect(id, func)
 	local newbuffer
 	if func == nil then
-		newbuffer = driver.buffer(buffer_limit)
+		newbuffer = driver.buffer()
 	end
 	local s = {
 		id = id,
@@ -206,6 +207,27 @@ end
 function socket.read(id, sz)
 	local s = socket_pool[id]
 	assert(s)
+	if sz == nil then
+		-- read some bytes
+		local ret = driver.readall(s.buffer, buffer_pool)
+		if ret ~= "" then
+			return ret
+		end
+
+		if not s.connected then
+			return false, ret
+		end
+		assert(not s.read_required)
+		s.read_required = 0
+		suspend(s)
+		ret = driver.readall(s.buffer, buffer_pool)
+		if ret ~= "" then
+			return ret
+		else
+			return false, ret
+		end
+	end
+
 	local ret = driver.pop(s.buffer, buffer_pool, sz)
 	if ret then
 		return ret
@@ -325,8 +347,9 @@ function socket.abandon(id)
 	socket_pool[id] = nil
 end
 
-function socket.limit(limit)
-	buffer_limit = limit
+function socket.limit(id, limit)
+	local s = assert(socket_pool[id])
+	s.buffer_limit = limit
 end
 
 return socket

test/testsocket.lua

@@ -7,9 +7,9 @@ local function echo(id)
 	socket.start(id)
 
 	while true do
-		local str = socket.readline(id,"\n")
+		local str = socket.read(id)
 		if str then
-			socket.write(id, str .. "\n")
+			socket.write(id, str)
 		else
 			socket.close(id)
 			return
@@ -21,9 +21,6 @@ if mode == "agent" then
 	id = tonumber(id)
 
 	skynet.start(function()
-		-- A small limit, if socket buffer overflow, close the client
-		socket.limit(64)
-
 		skynet.fork(function()
 			echo(id)
 			skynet.exit()