From 18f20425a098b969f17903b8ba938fa3d30a1d39 Mon Sep 17 00:00:00 2001 From: Cloud Wu Date: Wed, 23 Jul 2014 12:19:48 +0800 Subject: [PATCH] httpd don't need readline now --- lualib-src/lua-socket.c | 5 - lualib/http/httpd.lua | 177 +++++++++++++++++++++++++---------- lualib/http/sockethelper.lua | 16 +--- lualib/snax/loginserver.lua | 9 +- lualib/socket.lua | 55 +++++++---- test/testsocket.lua | 7 +- 6 files changed, 176 insertions(+), 93 deletions(-) diff --git a/lualib-src/lua-socket.c b/lualib-src/lua-socket.c index 8bfefc0e..916fa7a9 100644 --- a/lualib-src/lua-socket.c +++ b/lualib-src/lua-socket.c @@ -23,7 +23,6 @@ struct buffer_node { }; struct socket_buffer { - int limit; int size; int offset; struct buffer_node *head; @@ -66,7 +65,6 @@ lnewpool(lua_State *L, int sz) { static int lnewbuffer(lua_State *L) { struct socket_buffer * sb = lua_newuserdata(L, sizeof(*sb)); - sb->limit = luaL_optint(L,1,BUFFER_LIMIT); sb->size = 0; sb->offset = 0; sb->head = NULL; @@ -129,9 +127,6 @@ lpushbuffer(lua_State *L) { sb->size += sz; lua_pushinteger(L, sb->size); - if (sb->limit > 0 && sb->size > sb->limit) { - return luaL_error(L, "buffer overflow (limit = %d, size = %d)", sb->limit, sb->size); - } return 1; } diff --git a/lualib/http/httpd.lua b/lualib/http/httpd.lua index 3ced98c6..5333bfd5 100644 --- a/lualib/http/httpd.lua +++ b/lualib/http/httpd.lua @@ -1,7 +1,6 @@ local table = table local httpd = {} -local READLIMIT = 8192 -- limit bytes per read local http_status_msg = { [100] = "Continue", @@ -46,76 +45,150 @@ local http_status_msg = { [505] = "HTTP Version not supported", } -local function recvheader(readline, header) - local line = readline() - if line == "" then - return header +local function recvheader(readbytes, limit, lines, last) + local idx = 1 + local len = 0 + local header = last + while true do + header = header .. readbytes() + if #header + len > limit then + return + end + while true do + local f,e = header:find("\r\n",1, true) + if f then + if f == 1 then + for i = idx, #lines do + lines[i] = nil + end + return header:sub(3) + end + lines[idx] = header:sub(1, f-1) + idx = idx + 1 + len = len + f-1 + header = header:sub(e+1) + else + break + end + end end +end +local function parseheader(lines, from, header) local name, value - repeat + for i=from,#lines do + local line = lines[i] if line:byte(1) == 9 then -- tab, append last line + if name == nil then + return + end header[name] = header[name] .. line:sub(2) else name, value = line:match "^(.-):%s*(.*)" - assert(name and value) + if name == nil or value == nil then + return + end name = name:lower() if header[name] then header[name] = header[name] .. ", " .. value else header[name] = value end - line = readline() end - until line == "" - + end return header end -local function recvbody(readbytes, length) - if length < READLIMIT then - return readbytes(length) - end - local tmp = {} +local function chunksize(readbytes, body) while true do - if length <= READLIMIT then - table.insert(tmp, readbytes(length)) + if #body > 128 then + return + end + body = body .. readbytes() + local f,e = body:find("\r\n",1,true) + if f then + return tonumber(body:sub(1,f-1),16), body:sub(e+1) + end + end +end + +local function readcrln(readbytes, body) + if #body > 2 then + if body:sub(1,2) ~= "\r\n" then + return + end + return body:sub(3) + else + body = body .. readbytes(2-#body) + if body ~= "\r\n" then + return + end + return "" + end +end + +local tmpline = {} + +local function recvchunkedbody(readbytes, bodylimit, header, body) + local result = "" + local size = 0 + + while true do + local sz + sz , body = chunksize(readbytes, body) + if not sz then + return + end + if sz == 0 then break end - table.insert(tmp, readbytes(READLIMIT)) - length = length - READLIMIT + size = size + sz + if bodylimit and size > bodylimit then + return + end + if #body >= sz then + result = result .. body:sub(1,sz) + body = body:sub(sz+1) + else + result = result .. body .. readbytes(sz - #body) + body = "" + end + body = readcrln(readbytes, body) + if not body then + return + end end - return table.concat(tmp) + + body = readcrln(readbytes, body) + if not body then + return + end + body = recvheader(readbytes, 8192, tmpline, body) + if not body then + return + end + + header = parseheader(tmpline,1,header) + + return result, header end -local function recvchunkedbody(readline, readbytes, header) - local size = assert(tonumber(readline(),16)) - local body = recvbody(readbytes,size) - assert(readbytes(2) == "\r\n") - size = assert(tonumber(readline(),16)) - if size > 0 then - local bodys = { body } - repeat - table.insert(bodys, recvbody(readbytes,size)) - assert(readbytes(2) == "\r\n") - size = assert(tonumber(readline(),16)) - until size <= 0 - body = table.concat(bodys) +local function readall(readbytes, bodylimit) + local body = recvheader(readbytes, 8192, tmpline, "") + if not body then + return 413 -- Request Entity Too Large end - assert(readbytes(2) == "\r\n") - header = recvheader(readline, header) - return body, header -end - -local function readall(readline, readbytes) - local request = readline() + local request = assert(tmpline[1]) local method, url, httpver = request:match "^(%a+)%s+(.-)%s+HTTP/([%d%.]+)$" assert(method and url and httpver) httpver = assert(tonumber(httpver)) if httpver < 1.0 or httpver > 1.1 then return 505 -- HTTP Version not supported end - local header = recvheader(readline, {}) + local header = parseheader(tmpline,2,{}) + if not header then + return 400 -- Bad request + end local length = header["content-length"] if length then length = tonumber(length) @@ -127,23 +200,31 @@ local function readall(readline, readbytes) end end - local body if mode == "chunked" then - body, header = recvchunkedbody(readline, readbytes, header) + body, header = recvchunkedbody(readbytes, bodylimit, header, body) + if not body then + return 413 + end else -- identity mode if length then - body = readbody(readbytes, length) + if length > bodylimit then + return 413 + end + if #body >= length then + body = body:sub(1,length) + else + local padding = readbytes(length - #body) + body = body .. padding + end end end return 200, url, method, header, body end -function httpd.read_request(readfunc) - local readline = assert(readfunc.readline) - local readbytes = assert(readfunc.readbytes) - local ok, code, url, method, header, body = pcall(readall, readline, readbytes) +function httpd.read_request(readbytes, bodylimit) + local ok, code, url, method, header, body = pcall(readall, readbytes, bodylimit) if ok then return code, url, method, header, body else diff --git a/lualib/http/sockethelper.lua b/lualib/http/sockethelper.lua index 64b82c7e..14668c8b 100644 --- a/lualib/http/sockethelper.lua +++ b/lualib/http/sockethelper.lua @@ -1,6 +1,5 @@ local socket = require "socket" -local readline = socket.readline local readbytes = socket.read local writebytes = socket.write @@ -10,18 +9,7 @@ local socket_error = {} sockethelper.socket_error = socket_error function sockethelper.readfunc(fd) - local helper = {} - - function helper.readline() - local ret = readline(fd, "\r\n") - if ret then - return ret - else - error(socket_error) - end - end - - function helper.readbytes(sz) + return function (sz) local ret = readbytes(fd, sz) if ret then return ret @@ -29,8 +17,6 @@ function sockethelper.readfunc(fd) error(socket_error) end end - - return helper end function sockethelper.writefunc(fd) diff --git a/lualib/snax/loginserver.lua b/lualib/snax/loginserver.lua index 1ad5b70e..c5de769f 100644 --- a/lualib/snax/loginserver.lua +++ b/lualib/snax/loginserver.lua @@ -47,14 +47,15 @@ local function write(fd, text) end local function launch_slave(auth_handler) - -- set socket buffer limit (8K) - -- If the attacker send large package, close the socket - socket.limit(8192) - local function auth(fd, addr) fd = assert(tonumber(fd)) skynet.error(string.format("connect from %s (fd = %d)", addr, fd)) socket.start(fd) + + -- set socket buffer limit (8K) + -- If the attacker send large package, close the socket + socket.limit(fd, 8192) + local challenge = crypt.randomkey() write(fd, crypt.base64encode(challenge).."\n") diff --git a/lualib/socket.lua b/lualib/socket.lua index 28268747..f54d2175 100644 --- a/lualib/socket.lua +++ b/lualib/socket.lua @@ -2,7 +2,6 @@ local driver = require "socketdriver" local skynet = require "skynet" local assert = assert -local buffer_limit = -1 local socket = {} -- api local buffer_pool = {} -- store all message buffer object local socket_pool = setmetatable( -- store all socket object @@ -48,13 +47,7 @@ socket_message[1] = function(id, size, data) return end - local ok , sz = pcall(driver.push, s.buffer, buffer_pool, data, size) - if not ok then - skynet.error("socket: error on ", id , sz) - driver.clear(s.buffer,buffer_pool) - driver.close(id) - return - end + local sz = driver.push(s.buffer, buffer_pool, data, size) local rr = s.read_required local rrt = type(rr) if rrt == "number" then @@ -63,11 +56,19 @@ socket_message[1] = function(id, size, data) s.read_required = nil wakeup(s) end - elseif rrt == "string" then - -- read line - if driver.readline(s.buffer,nil,rr) then - s.read_required = nil - wakeup(s) + else + if s.buffer_limit and sz > s.buffer_limit then + skynet.error(string.format("socket buffer overlow: fd=%d size=%d", id , sz)) + driver.clear(s.buffer,buffer_pool) + driver.close(id) + return + end + if rrt == "string" then + -- read line + if driver.readline(s.buffer,nil,rr) then + s.read_required = nil + wakeup(s) + end end end end @@ -130,7 +131,7 @@ skynet.register_protocol { local function connect(id, func) local newbuffer if func == nil then - newbuffer = driver.buffer(buffer_limit) + newbuffer = driver.buffer() end local s = { id = id, @@ -206,6 +207,27 @@ end function socket.read(id, sz) local s = socket_pool[id] assert(s) + if sz == nil then + -- read some bytes + local ret = driver.readall(s.buffer, buffer_pool) + if ret ~= "" then + return ret + end + + if not s.connected then + return false, ret + end + assert(not s.read_required) + s.read_required = 0 + suspend(s) + ret = driver.readall(s.buffer, buffer_pool) + if ret ~= "" then + return ret + else + return false, ret + end + end + local ret = driver.pop(s.buffer, buffer_pool, sz) if ret then return ret @@ -325,8 +347,9 @@ function socket.abandon(id) socket_pool[id] = nil end -function socket.limit(limit) - buffer_limit = limit +function socket.limit(id, limit) + local s = assert(socket_pool[id]) + s.buffer_limit = limit end return socket diff --git a/test/testsocket.lua b/test/testsocket.lua index 05313a11..3893cd61 100644 --- a/test/testsocket.lua +++ b/test/testsocket.lua @@ -7,9 +7,9 @@ local function echo(id) socket.start(id) while true do - local str = socket.readline(id,"\n") + local str = socket.read(id) if str then - socket.write(id, str .. "\n") + socket.write(id, str) else socket.close(id) return @@ -21,9 +21,6 @@ if mode == "agent" then id = tonumber(id) skynet.start(function() - -- A small limit, if socket buffer overflow, close the client - socket.limit(64) - skynet.fork(function() echo(id) skynet.exit()