From 7beed39b1d60854f305a19320f05e366e58fc61c Mon Sep 17 00:00:00 2001 From: Cloud Wu Date: Mon, 4 Aug 2014 16:34:29 +0800 Subject: [PATCH 1/5] bugfix: use temp array for each request --- lualib/http/httpd.lua | 11 +++-------- lualib/http/sockethelper.lua | 12 ++++++++++++ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/lualib/http/httpd.lua b/lualib/http/httpd.lua index 1fe507de..190b6b93 100644 --- a/lualib/http/httpd.lua +++ b/lualib/http/httpd.lua @@ -64,16 +64,11 @@ local function recvheader(readbytes, limit, lines, header) end end end - local idx = 1 for v in header:gmatch("(.-)\r\n") do if v == "" then break end - lines[idx] = v - idx = idx + 1 - end - for i = idx, #lines do - lines[i] = nil + table.insert(lines, v) end return result end @@ -131,8 +126,6 @@ local function readcrln(readbytes, body) end end -local tmpline = {} - local function recvchunkedbody(readbytes, bodylimit, header, body) local result = "" local size = 0 @@ -167,6 +160,7 @@ local function recvchunkedbody(readbytes, bodylimit, header, body) if not body then return end + local tmpline = {} body = recvheader(readbytes, 8192, tmpline, body) if not body then return @@ -178,6 +172,7 @@ local function recvchunkedbody(readbytes, bodylimit, header, body) end local function readall(readbytes, bodylimit) + local tmpline = {} local body = recvheader(readbytes, 8192, tmpline, "") if not body then return 413 -- Request Entity Too Large diff --git a/lualib/http/sockethelper.lua b/lualib/http/sockethelper.lua index 89c7ec93..febe4cea 100644 --- a/lualib/http/sockethelper.lua +++ b/lualib/http/sockethelper.lua @@ -28,4 +28,16 @@ function sockethelper.writefunc(fd) end end +function sockethelper.connect(host, port) + local fd = socket.open(host, port) + if fd then + return fd + end + error(socket_error) +end + +function sockethelper.close(fd) + socket.close(fd) +end + return sockethelper From dec2e6fe4c1fd0d21a1bfe82f9649b4073395d5d Mon Sep 17 00:00:00 2001 From: Cloud Wu Date: Mon, 4 Aug 2014 18:06:16 +0800 Subject: [PATCH 2/5] add httpc --- examples/simpleweb.lua | 5 ++ lualib/http/httpc.lua | 114 ++++++++++++++++++++++++++++++++ lualib/http/httpd.lua | 136 ++------------------------------------- lualib/http/internal.lua | 135 ++++++++++++++++++++++++++++++++++++++ test/testhttp.lua | 16 +++++ 5 files changed, 276 insertions(+), 130 deletions(-) create mode 100644 lualib/http/httpc.lua create mode 100644 lualib/http/internal.lua create mode 100644 test/testhttp.lua diff --git a/examples/simpleweb.lua b/examples/simpleweb.lua index 19870aba..234c2d08 100644 --- a/examples/simpleweb.lua +++ b/examples/simpleweb.lua @@ -39,6 +39,11 @@ skynet.start(function() table.insert(tmp, string.format("query: %s= %s", k,v)) end end + table.insert(tmp, "-----header----") + for k,v in pairs(header) do + table.insert(tmp, string.format("%s = %s",k,v)) + end + table.insert(tmp, "-----body----\n" .. body) response(id, code, table.concat(tmp,"\n")) end else diff --git a/lualib/http/httpc.lua b/lualib/http/httpc.lua new file mode 100644 index 00000000..784b186b --- /dev/null +++ b/lualib/http/httpc.lua @@ -0,0 +1,114 @@ +local socket = require "http.sockethelper" +local url = require "http.url" +local internal = require "http.internal" +local string = string +local table = table + +local httpc = {} + +local function request(fd, method, host, url, recvheader, header, content) + local read = socket.readfunc(fd) + local write = socket.writefunc(fd) + local header_content = "" + if header then + for k,v in pairs(header) do + header_content = string.format("%s%s:%s\r\n", header_content, k, v) + end + end + + if content then + local data = string.format("%s %s HTTP/1.1\r\nhost:%s\r\ncontent-length:%d\r\n%s\r\n%s", method, url, host, #content, header_content, content) + write(data) + else + local request_header = string.format("%s %s HTTP/1.1\r\nhost:%s\r\ncontent-length:0\r\n%s\r\n", method, url, host, header_content) + write(request_header) + end + + local tmpline = {} + local body = internal.recvheader(read, tmpline, "") + if not body then + error(socket.socket_error) + end + + local statusline = tmpline[1] + local code, info = statusline:match "HTTP/[%d%.]+%s+([%d]+)%s+(.*)$" + code = assert(tonumber(code)) + + local header = internal.parseheader(tmpline,2,recvheader or {}) + if not header then + error("Invalid HTTP response header") + end + + local length = header["content-length"] + if length then + length = tonumber(length) + end + local mode = header["transfer-encoding"] + if mode then + if mode ~= "identity" and mode ~= "chunked" then + error ("Unsupport transfer-encoding") + end + end + + if mode == "chunked" then + body, header = internal.recvchunkedbody(read, nil, header, body) + if not body then + error("Invalid response body") + end + else + -- identity mode + if length then + if #body >= length then + body = body:sub(1,length) + else + local padding = read(length - #body) + body = body .. padding + end + else + body = nil + end + end + + return code, body +end + +function httpc.request(method, host, url, recvheader, header, content) + local hostname, port = host:match"([^:]+):?(%d*)$" + if port == "" then + port = 80 + else + port = tonumber(port) + end + local fd = socket.connect(hostname, port) + local ok , statuscode, body = pcall(request, fd,method, host, url, recvheader, header, content) + if ok then + return statuscode, body + else + socket.close(fd) + error(statuscode) + end +end + +function httpc.get(...) + return httpc.request("GET", ...) +end + +local function escape(s) + return (string.gsub(s, "([^A-Za-z0-9_])", function(c) + return string.format("%%%02X", string.byte(c)) + end)) +end + +function httpc.post(host, url, form, recvheader) + local header = { + ["content-type"] = "application/x-www-form-urlencoded" + } + local body = {} + for k,v in pairs(form) do + table.insert(body, string.format("%s=%s",escape(k),escape(v))) + end + + return httpc.request("POST", host, url, recvheader, header, table.concat(body , "&")) +end + +return httpc diff --git a/lualib/http/httpd.lua b/lualib/http/httpd.lua index 190b6b93..5f8f6de8 100644 --- a/lualib/http/httpd.lua +++ b/lualib/http/httpd.lua @@ -1,3 +1,5 @@ +local internal = require "http.internal" + local table = table local httpd = {} @@ -45,135 +47,9 @@ local http_status_msg = { [505] = "HTTP Version not supported", } -local function recvheader(readbytes, limit, lines, header) - local result - local e = header:find("\r\n\r\n", 1, true) - if e then - result = header:sub(e+4) - else - while true do - local bytes = readbytes() - header = header .. bytes - if #header > limit then - return - end - e = header:find("\r\n\r\n", -#bytes-3, true) - if e then - result = header:sub(e+4) - break - end - end - end - for v in header:gmatch("(.-)\r\n") do - if v == "" then - break - end - table.insert(lines, v) - end - return result -end - -local function parseheader(lines, from, header) - local name, value - for i=from,#lines do - local line = lines[i] - if line:byte(1) == 9 then -- tab, append last line - if name == nil then - return - end - header[name] = header[name] .. line:sub(2) - else - name, value = line:match "^(.-):%s*(.*)" - if name == nil or value == nil then - return - end - name = name:lower() - if header[name] then - header[name] = header[name] .. ", " .. value - else - header[name] = value - end - end - end - return header -end - -local function chunksize(readbytes, body) - while true do - if #body > 128 then - return - end - body = body .. readbytes() - local f,e = body:find("\r\n",1,true) - if f then - return tonumber(body:sub(1,f-1),16), body:sub(e+1) - end - end -end - -local function readcrln(readbytes, body) - if #body > 2 then - if body:sub(1,2) ~= "\r\n" then - return - end - return body:sub(3) - else - body = body .. readbytes(2-#body) - if body ~= "\r\n" then - return - end - return "" - end -end - -local function recvchunkedbody(readbytes, bodylimit, header, body) - local result = "" - local size = 0 - - while true do - local sz - sz , body = chunksize(readbytes, body) - if not sz then - return - end - if sz == 0 then - break - end - size = size + sz - if bodylimit and size > bodylimit then - return - end - if #body >= sz then - result = result .. body:sub(1,sz) - body = body:sub(sz+1) - else - result = result .. body .. readbytes(sz - #body) - body = "" - end - body = readcrln(readbytes, body) - if not body then - return - end - end - - body = readcrln(readbytes, body) - if not body then - return - end - local tmpline = {} - body = recvheader(readbytes, 8192, tmpline, body) - if not body then - return - end - - header = parseheader(tmpline,1,header) - - return result, header -end - local function readall(readbytes, bodylimit) local tmpline = {} - local body = recvheader(readbytes, 8192, tmpline, "") + local body = internal.recvheader(readbytes, tmpline, "") if not body then return 413 -- Request Entity Too Large end @@ -184,7 +60,7 @@ local function readall(readbytes, bodylimit) if httpver < 1.0 or httpver > 1.1 then return 505 -- HTTP Version not supported end - local header = parseheader(tmpline,2,{}) + local header = internal.parseheader(tmpline,2,{}) if not header then return 400 -- Bad request end @@ -194,13 +70,13 @@ local function readall(readbytes, bodylimit) end local mode = header["transfer-encoding"] if mode then - if mode ~= "identity" or mode ~= "chunked" then + if mode ~= "identity" and mode ~= "chunked" then return 501 -- Not Implemented end end if mode == "chunked" then - body, header = recvchunkedbody(readbytes, bodylimit, header, body) + body, header = internal.recvchunkedbody(readbytes, bodylimit, header, body) if not body then return 413 end diff --git a/lualib/http/internal.lua b/lualib/http/internal.lua new file mode 100644 index 00000000..067152ad --- /dev/null +++ b/lualib/http/internal.lua @@ -0,0 +1,135 @@ +local M = {} + +local LIMIT = 8192 + +local function chunksize(readbytes, body) + while true do + if #body > 128 then + return + end + body = body .. readbytes() + local f,e = body:find("\r\n",1,true) + if f then + return tonumber(body:sub(1,f-1),16), body:sub(e+1) + end + end +end + +local function readcrln(readbytes, body) + if #body >= 2 then + if body:sub(1,2) ~= "\r\n" then + return + end + return body:sub(3) + else + body = body .. readbytes(2-#body) + if body ~= "\r\n" then + return + end + return "" + end +end + +function M.recvheader(readbytes, lines, header) + if #header >= 2 then + if header:find "^\r\n" then + return header:sub(3) + end + end + local result + local e = header:find("\r\n\r\n", 1, true) + if e then + result = header:sub(e+4) + else + while true do + local bytes = readbytes() + header = header .. bytes + if #header > LIMIT then + return + end + e = header:find("\r\n\r\n", -#bytes-3, true) + if e then + result = header:sub(e+4) + break + end + if header:find "^\r\n" then + return header:sub(3) + end + end + end + for v in header:gmatch("(.-)\r\n") do + if v == "" then + break + end + table.insert(lines, v) + end + return result +end + +function M.parseheader(lines, from, header) + local name, value + for i=from,#lines do + local line = lines[i] + if line:byte(1) == 9 then -- tab, append last line + if name == nil then + return + end + header[name] = header[name] .. line:sub(2) + else + name, value = line:match "^(.-):%s*(.*)" + if name == nil or value == nil then + return + end + name = name:lower() + if header[name] then + header[name] = header[name] .. ", " .. value + else + header[name] = value + end + end + end + return header +end + +function M.recvchunkedbody(readbytes, bodylimit, header, body) + local result = "" + local size = 0 + + while true do + local sz + sz , body = chunksize(readbytes, body) + if not sz then + return + end + if sz == 0 then + break + end + size = size + sz + if bodylimit and size > bodylimit then + return + end + if #body >= sz then + result = result .. body:sub(1,sz) + body = body:sub(sz+1) + else + result = result .. body .. readbytes(sz - #body) + body = "" + end + body = readcrln(readbytes, body) + if not body then + return + end + end + + local tmpline = {} + body = M.recvheader(readbytes, tmpline, body) + if not body then + return + end + + header = M.parseheader(tmpline,1,header) + + return result, header +end + +return M diff --git a/test/testhttp.lua b/test/testhttp.lua new file mode 100644 index 00000000..52abbe4b --- /dev/null +++ b/test/testhttp.lua @@ -0,0 +1,16 @@ +local skynet = require "skynet" +local httpc = require "http.httpc" + +skynet.start(function() + print("GET www.baidu.com") + local header = {} + local status, body = httpc.get("baidu.com", "/", header) + print("[header] =====>") + for k,v in pairs(header) do + print(k,v) + end + print("[body] =====>", status) + print(body) + + skynet.exit() +end) From b18962929b6f54d342c84e0ea9be38c29b60a513 Mon Sep 17 00:00:00 2001 From: Cloud Wu Date: Mon, 4 Aug 2014 18:08:05 +0800 Subject: [PATCH 3/5] bugfix: chunked mode --- lualib/http/httpd.lua | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lualib/http/httpd.lua b/lualib/http/httpd.lua index 190b6b93..a0c027f2 100644 --- a/lualib/http/httpd.lua +++ b/lualib/http/httpd.lua @@ -46,6 +46,11 @@ local http_status_msg = { } local function recvheader(readbytes, limit, lines, header) + if #header >= 2 then + if header:find "^\r\n" then + return header:sub(3) + end + end local result local e = header:find("\r\n\r\n", 1, true) if e then @@ -62,6 +67,9 @@ local function recvheader(readbytes, limit, lines, header) result = header:sub(e+4) break end + if header:find "^\r\n" then + return header:sub(3) + end end end for v in header:gmatch("(.-)\r\n") do @@ -156,10 +164,6 @@ local function recvchunkedbody(readbytes, bodylimit, header, body) end end - body = readcrln(readbytes, body) - if not body then - return - end local tmpline = {} body = recvheader(readbytes, 8192, tmpline, body) if not body then @@ -194,7 +198,7 @@ local function readall(readbytes, bodylimit) end local mode = header["transfer-encoding"] if mode then - if mode ~= "identity" or mode ~= "chunked" then + if mode ~= "identity" and mode ~= "chunked" then return 501 -- Not Implemented end end From f6fa6c7dedbfba6160067e2c0e2cc176026a3e1e Mon Sep 17 00:00:00 2001 From: Cloud Wu Date: Fri, 8 Aug 2014 20:33:38 +0800 Subject: [PATCH 4/5] timer support more than 497 days --- HISTORY.md | 6 ++++ skynet-src/skynet_main.c | 1 - skynet-src/skynet_timer.c | 62 +++++++++++++++++++++++---------------- 3 files changed, 43 insertions(+), 26 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 81718ed3..8ad3a0f8 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,9 @@ +v0.5.1 (2014-8-4) +----------- +* Bugfix : http module +* Bugfix : multicast local channel delete +* Bugfix : socket.read(fd) + v0.5.0 (2014-7-28) ----------- * skynet.exit will quit service immediately. diff --git a/skynet-src/skynet_main.c b/skynet-src/skynet_main.c index 4eb18a59..fce5debb 100644 --- a/skynet-src/skynet_main.c +++ b/skynet-src/skynet_main.c @@ -89,7 +89,6 @@ static const char * load_config = "\ local code = assert(f:read \'*a\')\ local function getenv(name) return assert(os.getenv(name), name) end\ code = string.gsub(code, \'%$([%w_%d]+)\', getenv)\ - print(code)\ f:close()\ local result = {}\ assert(load(code,\'=(load)\',\'t\',result))()\ diff --git a/skynet-src/skynet_timer.c b/skynet-src/skynet_timer.c index 3cd0c51d..3d0421a9 100644 --- a/skynet-src/skynet_timer.c +++ b/skynet-src/skynet_timer.c @@ -9,6 +9,7 @@ #include #include #include +#include #if defined(__APPLE__) #include @@ -33,7 +34,7 @@ struct timer_event { struct timer_node { struct timer_node *next; - int expire; + uint32_t expire; }; struct link_list { @@ -43,9 +44,9 @@ struct link_list { struct timer { struct link_list near[TIME_NEAR]; - struct link_list t[4][TIME_LEVEL-1]; + struct link_list t[4][TIME_LEVEL]; int lock; - int time; + uint32_t time; uint32_t current; uint32_t starttime; uint64_t current_point; @@ -72,21 +73,22 @@ link(struct link_list *list,struct timer_node *node) { static void add_node(struct timer *T,struct timer_node *node) { - int time=node->expire; - int current_time=T->time; + uint32_t time=node->expire; + uint32_t current_time=T->time; if ((time|TIME_NEAR_MASK)==(current_time|TIME_NEAR_MASK)) { link(&T->near[time&TIME_NEAR_MASK],node); } else { int i; - int mask=TIME_NEAR << TIME_LEVEL_SHIFT; + uint32_t mask=TIME_NEAR << TIME_LEVEL_SHIFT; for (i=0;i<3;i++) { if ((time|(mask-1))==(current_time|(mask-1))) { break; } mask <<= TIME_LEVEL_SHIFT; } - link(&T->t[i][((time>>(TIME_NEAR_SHIFT + i*TIME_LEVEL_SHIFT)) & TIME_LEVEL_MASK)-1],node); + + link(&T->t[i][((time>>(TIME_NEAR_SHIFT + i*TIME_LEVEL_SHIFT)) & TIME_LEVEL_MASK)],node); } } @@ -103,29 +105,38 @@ timer_add(struct timer *T,void *arg,size_t sz,int time) { UNLOCK(T); } +static void +move_list(struct timer *T, int level, int idx) { + struct timer_node *current = link_clear(&T->t[level][idx]); + while (current) { + struct timer_node *temp=current->next; + add_node(T,current); + current=temp; + } +} + static void timer_shift(struct timer *T) { LOCK(T); int mask = TIME_NEAR; - int time = (++T->time) >> TIME_NEAR_SHIFT; - int i=0; - - while ((T->time & (mask-1))==0) { - int idx=time & TIME_LEVEL_MASK; - if (idx!=0) { - --idx; - struct timer_node *current = link_clear(&T->t[i][idx]); - while (current) { - struct timer_node *temp=current->next; - add_node(T,current); - current=temp; + uint32_t ct = ++T->time; + if (ct == 0) { + move_list(T, 3, 0); + } else { + uint32_t time = ct >> TIME_NEAR_SHIFT; + int i=0; + + while ((ct & (mask-1))==0) { + int idx=time & TIME_LEVEL_MASK; + if (idx!=0) { + move_list(T, i, idx); + break; } - break; + mask <<= TIME_LEVEL_SHIFT; + time >>= TIME_LEVEL_SHIFT; + ++i; } - mask <<= TIME_LEVEL_SHIFT; - time >>= TIME_LEVEL_SHIFT; - ++i; - } + } UNLOCK(T); } @@ -187,7 +198,7 @@ timer_create_timer() { } for (i=0;i<4;i++) { - for (j=0;jt[i][j]); } } @@ -265,6 +276,7 @@ skynet_updatetime(void) { uint64_t cp = gettime(); if(cp < TI->current_point) { skynet_error(NULL, "time diff error: change from %lld to %lld", cp, TI->current_point); + TI->current_point = cp; } else if (cp != TI->current_point) { uint32_t diff = (uint32_t)(cp - TI->current_point); TI->current_point = cp; From 77d53cf5f1a37d3c11ab8f338b4b98ac49d40970 Mon Sep 17 00:00:00 2001 From: Cloud Wu Date: Mon, 11 Aug 2014 09:45:37 +0800 Subject: [PATCH 5/5] release v0.5.2 --- HISTORY.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 8ad3a0f8..375687fd 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,10 @@ +v0.5.2 (2014-8-11) +----------- +* Bugfix : httpd request +* Bugifx : http chunked mode +* Add : httpc +* timer support more than 497 days + v0.5.1 (2014-8-4) ----------- * Bugfix : http module