diff --git a/Makefile b/Makefile index 7ea2bd25..bffc4307 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ update3rd : CSERVICE = snlua logger gate harbor LUA_CLIB = skynet \ client \ - bson md5 sproto lpeg + bson md5 sproto lpeg ltls LUA_CLIB_SKYNET = \ lua-skynet.c lua-seri.c \ @@ -106,6 +106,9 @@ $(LUA_CLIB_PATH)/client.so : lualib-src/lua-clientsocket.c lualib-src/lua-crypt. $(LUA_CLIB_PATH)/sproto.so : lualib-src/sproto/sproto.c lualib-src/sproto/lsproto.c | $(LUA_CLIB_PATH) $(CC) $(CFLAGS) $(SHARED) -Ilualib-src/sproto $^ -o $@ +$(LUA_CLIB_PATH)/ltls.so : lualib-src/ltls.c | $(LUA_CLIB_PATH) + $(CC) $(CFLAGS) $(SHARED) -Iskynet-src -L/usr/local/opt/openssl/lib -I/usr/local/opt/openssl/include $^ -o $@ -lssl + $(LUA_CLIB_PATH)/lpeg.so : 3rd/lpeg/lpcap.c 3rd/lpeg/lpcode.c 3rd/lpeg/lpprint.c 3rd/lpeg/lptree.c 3rd/lpeg/lpvm.c | $(LUA_CLIB_PATH) $(CC) $(CFLAGS) $(SHARED) -I3rd/lpeg $^ -o $@ diff --git a/examples/simpleweb.lua b/examples/simpleweb.lua index 283b7e91..6c70efae 100644 --- a/examples/simpleweb.lua +++ b/examples/simpleweb.lua @@ -6,26 +6,64 @@ local urllib = require "http.url" local table = table local string = string -local mode = ... +local mode, protocol = ... +protocol = protocol or "http" if mode == "agent" then -local function response(id, ...) - local ok, err = httpd.write_response(sockethelper.writefunc(id), ...) +local function response(id, write, ...) + local ok, err = httpd.write_response(write, ...) if not ok then -- if err == sockethelper.socket_error , that means socket closed. skynet.error(string.format("fd = %d, %s", id, err)) end end + +local SSLCTX_SERVER = nil +local function gen_interface(protocol, fd) + if protocol == "http" then + return { + init = nil, + close = nil, + read = sockethelper.readfunc(fd), + write = sockethelper.writefunc(fd), + } + elseif protocol == "https" then + local tls = require "http.tlshelper" + if not SSLCTX_SERVER then + SSLCTX_SERVER = tls.newctx() + -- gen cert and key + -- openssl req -x509 -newkey rsa:2048 -days 3650 -nodes -keyout server-key.pem -out server-cert.pem + local certfile = skynet.getenv("certfile") or "./server-cert.pem" + local keyfile = skynet.getenv("keyfile") or "./server-key.pem" + print(certfile, keyfile) + SSLCTX_SERVER:set_cert(certfile, keyfile) + end + local tls_ctx = tls.newtls("server", SSLCTX_SERVER) + return { + init = tls.init_responsefunc(fd, tls_ctx), + close = tls.closefunc(tls_ctx), + read = tls.readfunc(fd, tls_ctx), + write = tls.writefunc(fd, tls_ctx), + } + else + error(string.format("Invalid protocol: %s", protocol)) + end +end + skynet.start(function() skynet.dispatch("lua", function (_,_,id) socket.start(id) + local interface = gen_interface(protocol, id) + if interface.init then + interface.init() + end -- limit request body size to 8192 (you can pass nil to unlimit) - local code, url, method, header, body = httpd.read_request(sockethelper.readfunc(id), 8192) + local code, url, method, header, body = httpd.read_request(interface.read, 8192) if code then if code ~= 200 then - response(id, code) + response(id, interface.write, code) else local tmp = {} if header.host then @@ -44,7 +82,7 @@ skynet.start(function() table.insert(tmp, string.format("%s = %s",k,v)) end table.insert(tmp, "-----body----\n" .. body) - response(id, code, table.concat(tmp,"\n")) + response(id, interface.write, code, table.concat(tmp,"\n")) end else if url == sockethelper.socket_error then @@ -54,6 +92,9 @@ skynet.start(function() end end socket.close(id) + if interface.close then + interface.close() + end end) end) @@ -61,12 +102,13 @@ else skynet.start(function() local agent = {} + local protocol = "http" for i= 1, 20 do - agent[i] = skynet.newservice(SERVICE_NAME, "agent") + agent[i] = skynet.newservice(SERVICE_NAME, "agent", protocol) end local balance = 1 local id = socket.listen("0.0.0.0", 8001) - skynet.error("Listen web port 8001") + skynet.error(string.format("Listen web port 8001 protocol:%s", protocol)) socket.start(id , function(id, addr) skynet.error(string.format("%s connected, pass it to agent :%08x", addr, agent[balance])) skynet.send(agent[balance], "lua", id) diff --git a/lualib-src/ltls.c b/lualib-src/ltls.c new file mode 100644 index 00000000..7494e64a --- /dev/null +++ b/lualib-src/ltls.c @@ -0,0 +1,405 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +static bool TLS_IS_INIT = false; + +struct tls_context { + SSL* ssl; + BIO* in_bio; + BIO* out_bio; + bool is_server; + bool is_close; +}; + +struct ssl_ctx { + SSL_CTX* ctx; +}; + +// static int +// _ssl_verify_peer(int ok, X509_STORE_CTX* ctx) { +// return 1; +// } + + +static void +_init_bio(lua_State* L, struct tls_context* tls_p, struct ssl_ctx* ctx_p) { + tls_p->ssl = SSL_new(ctx_p->ctx); + if(!tls_p->ssl) { + luaL_error(L, "SSL_new faild"); + } + + tls_p->in_bio = BIO_new(BIO_s_mem()); + if(!tls_p->in_bio) { + luaL_error(L, "new in bio faild"); + } + BIO_set_mem_eof_return(tls_p->in_bio, -1); /* see: https://www.openssl.org/docs/crypto/BIO_s_mem.html */ + + tls_p->out_bio = BIO_new(BIO_s_mem()); + if(!tls_p->out_bio) { + luaL_error(L, "new out bio faild"); + } + BIO_set_mem_eof_return(tls_p->out_bio, -1); /* see: https://www.openssl.org/docs/crypto/BIO_s_mem.html */ + + SSL_set_bio(tls_p->ssl, tls_p->in_bio, tls_p->out_bio); +} + + +static void +_init_client_context(lua_State* L, struct tls_context* tls_p, struct ssl_ctx* ctx_p) { + tls_p->is_server = false; + _init_bio(L, tls_p, ctx_p); + SSL_set_connect_state(tls_p->ssl); +} + +static void +_init_server_context(lua_State* L, struct tls_context* tls_p, struct ssl_ctx* ctx_p) { + tls_p->is_server = true; + _init_bio(L, tls_p, ctx_p); + SSL_set_accept_state(tls_p->ssl); +} + +static struct tls_context * +_check_context(lua_State* L, int idx) { + struct tls_context* tls_p = (struct tls_context*)lua_touserdata(L, idx); + if(!tls_p) { + luaL_error(L, "need tls context"); + } + if(tls_p->is_close) { + luaL_error(L, "context is closed"); + } + return tls_p; +} + +static struct ssl_ctx * +_check_sslctx(lua_State* L, int idx) { + struct ssl_ctx* ctx_p = (struct ssl_ctx*)lua_touserdata(L, idx); + if(!ctx_p) { + luaL_error(L, "need sslctx"); + } + return ctx_p; +} + +static int +_ltls_context_finished(lua_State* L) { + struct tls_context* tls_p = _check_context(L, 1); + int b = SSL_is_init_finished(tls_p->ssl); + lua_pushboolean(L, b); + return 1; +} + +static int +_ltls_context_close(lua_State* L) { + struct tls_context* tls_p = lua_touserdata(L, 1); + if(!tls_p->is_close) { + SSL_free(tls_p->ssl); + tls_p->ssl = NULL; + tls_p->in_bio = NULL; //in_bio and out_bio will be free when SSL_free is called + tls_p->out_bio = NULL; + tls_p->is_close = true; + } + return 0; +} + +static int +_bio_read(lua_State* L, struct tls_context* tls_p) { + char outbuff[4096]; + int all_read = 0; + int read = 0; + int pending = BIO_ctrl_pending(tls_p->out_bio); + if(pending >0) { + luaL_Buffer b; + luaL_buffinit(L, &b); + do { + read = BIO_read(tls_p->out_bio, outbuff, sizeof(outbuff)); + // printf("_bio_read:%d pending:%d\n", read, pending); + if(read > sizeof(outbuff)) { + luaL_error(L, "invalid BIO_read:%d", read); + }else if(read == -2) { + luaL_error(L, "BIO_read not implemented in the specific BIO type"); + }else if (read > 0) { + all_read += read; + luaL_addlstring(&b, (const char*)outbuff, read); + } + }while(read == sizeof(outbuff)); + if(all_read>0){ + luaL_pushresult(&b); + } + } + return all_read; +} + + +static void +_bio_write(lua_State* L, struct tls_context* tls_p, const char* s, size_t len) { + char* p = (char*)s; + size_t sz = len; + while(sz > 0) { + int written = BIO_write(tls_p->in_bio, p, sz); + if(written > sz) { + luaL_error(L, "invalid BIO_write"); + }else if(written > 0) { + p += written; + sz -= written; + }else if (written == -2) { + luaL_error(L, "BIO_write not implemented in the specific BIO type"); + } + } +} + + +static int +_ltls_context_handshake(lua_State* L) { + struct tls_context* tls_p = _check_context(L, 1); + size_t slen = 0; + const char* exchange = lua_tolstring(L, 2, &slen); + + // check handshake is finished + if(SSL_is_init_finished(tls_p->ssl)) { + luaL_error(L, "handshake is finished"); + } + + // handshake exchange + if(slen > 0 && exchange != NULL) { + _bio_write(L, tls_p, exchange, slen); + } + + // first handshake; initiated by client + if(!SSL_is_init_finished(tls_p->ssl)) { + int ret = SSL_do_handshake(tls_p->ssl); + if(ret == 1) { + return 0; + } else if (ret == -1) { + int all_read = _bio_read(L, tls_p); + if(all_read>0) { + return 1; + } + } else { + int err = SSL_get_error(tls_p->ssl, ret); + luaL_error(L, "SSL_do_handshake error:%d ret:%d", err, ret); + } + } + return 0; +} + + +static int +_ltls_context_read(lua_State* L) { + struct tls_context* tls_p = _check_context(L, 1); + size_t slen = 0; + const char* encrypted_data = lua_tolstring(L, 2, &slen); + + // write encrypted data + if(slen>0 && encrypted_data) { + _bio_write(L, tls_p, encrypted_data, slen); + } + + char outbuff[4096]; + int read = 0; + luaL_Buffer b; + luaL_buffinit(L, &b); + + do { + read = SSL_read(tls_p->ssl, outbuff, sizeof(outbuff)); + if(read < 0) { + int err = SSL_get_error(tls_p->ssl, read); + if(err == SSL_ERROR_WANT_READ) { + break; + } + luaL_error(L, "SSL_read error:%d", err); + }else if(read > sizeof(outbuff)) { + luaL_error(L, "invalid SSL_read"); + }else if (read > 0) { + luaL_addlstring(&b, outbuff, read); + } + }while(read == sizeof(outbuff)); + luaL_pushresult(&b); + return 1; +} + + +static int +_ltls_context_write(lua_State* L) { + struct tls_context* tls_p = _check_context(L, 1); + size_t slen = 0; + char* unencrypted_data = (char*)lua_tolstring(L, 2, &slen); + + while(slen >0) { + int written = SSL_write(tls_p->ssl, unencrypted_data, slen); + if(written < 0) { + int err = SSL_get_error(tls_p->ssl, written); + luaL_error(L, "SSL_write error:%d", err); + }else if(written > slen) { + luaL_error(L, "invalid SSL_write"); + }else if(written>0) { + unencrypted_data += written; + slen -= written; + } + } + + int all_read = _bio_read(L, tls_p); + if(all_read <= 0) { + luaL_error(L, "bio_read error when _ltls_context_write"); + } + return 1; +} + + +static int +_lctx_gc(lua_State* L) { + struct ssl_ctx* ctx_p = _check_sslctx(L, 1); + if(ctx_p->ctx) { + SSL_CTX_free(ctx_p->ctx); + ctx_p->ctx = NULL; + } + return 0; +} + +static int +_lctx_cert(lua_State* L) { + struct ssl_ctx* ctx_p = _check_sslctx(L, 1); + const char* certfile = lua_tostring(L, 2); + const char* key = lua_tostring(L, 3); + if(!certfile) { + luaL_error(L, "need certfile"); + } + + if(!key) { + luaL_error(L, "need private key"); + } + + int ret = SSL_CTX_use_certificate_file(ctx_p->ctx, certfile, SSL_FILETYPE_PEM); + if(ret != 1) { + luaL_error(L, "SSL_CTX_use_certificate_file error:%d", ret); + } + + ret = SSL_CTX_use_PrivateKey_file(ctx_p->ctx, key, SSL_FILETYPE_PEM); + if(ret != 1) { + luaL_error(L, "SSL_CTX_use_PrivateKey_file error:%d", ret); + } + ret = SSL_CTX_check_private_key(ctx_p->ctx); + if(ret != 1) { + luaL_error(L, "SSL_CTX_check_private_key error:%d", ret); + } + return 0; +} + +static int +_lctx_ciphers(lua_State* L) { + struct ssl_ctx* ctx_p = _check_sslctx(L, 1); + const char* s = lua_tostring(L, 2); + if(!s) { + luaL_error(L, "need cipher list"); + } + int ret = SSL_CTX_set_tlsext_use_srtp(ctx_p->ctx, s); + if(ret != 0) { + luaL_error(L, "SSL_CTX_set_tlsext_use_srtp error:%d", ret); + } + return 0; +} + + +static int +lnew_ctx(lua_State* L) { + struct ssl_ctx* ctx_p = (struct ssl_ctx*)lua_newuserdata(L, sizeof(*ctx_p)); + ctx_p->ctx = SSL_CTX_new(SSLv23_method()); + if(!ctx_p->ctx) { + luaL_error(L, "SSL_CTX_new client faild."); + } + + if(luaL_newmetatable(L, "_TLS_SSLCTX_METATABLE_")) { + luaL_Reg l[] = { + {"set_ciphers", _lctx_ciphers}, + {"set_cert", _lctx_cert}, + {NULL, NULL}, + }; + + luaL_newlib(L, l); + lua_setfield(L, -2, "__index"); + lua_pushcfunction(L, _lctx_gc); + lua_setfield(L, -2, "__gc"); + } + lua_setmetatable(L, -2); + return 1; +} + + +static int +lnew_tls(lua_State* L) { + struct tls_context* tls_p = (struct tls_context*)lua_newuserdata(L, sizeof(*tls_p)); + tls_p->is_close = false; + const char* method = luaL_optstring(L, 1, "nil"); + struct ssl_ctx* ctx_p = _check_sslctx(L, 2); + lua_pushvalue(L, 2); + lua_setuservalue(L, -2); // set ssl_ctx associated to tls_context + + if(strcmp(method, "client") == 0) { + _init_client_context(L, tls_p, ctx_p); + }else if(strcmp(method, "server") == 0) { + _init_server_context(L, tls_p, ctx_p); + } else { + luaL_error(L, "invalid method:%s e.g[server, client]", method); + } + + if(luaL_newmetatable(L, "_TLS_CONTEXT_METATABLE_")) { + luaL_Reg l[] = { + {"close", _ltls_context_close}, + {"finished", _ltls_context_finished}, + {"handshake", _ltls_context_handshake}, + {"read", _ltls_context_read}, + {"write", _ltls_context_write}, + {NULL, NULL}, + }; + luaL_newlib(L, l); + lua_setfield(L, -2, "__index"); + lua_pushcfunction(L, _ltls_context_close); + lua_setfield(L, -2, "__gc"); + } + lua_setmetatable(L, -2); + return 1; +} + + +int +luaopen_ltls_c(lua_State* L) { + luaL_Reg l[] = { + {"newctx", lnew_ctx}, + {"newtls", lnew_tls}, + {NULL, NULL}, + }; + luaL_checkversion(L); + luaL_newlib(L, l); + return 1; +} + +void __attribute__((constructor)) ltls_init(void) { +#ifndef OPENSSL_EXTERNAL_INITIALIZATION + SSL_library_init(); + SSL_load_error_strings(); + ERR_load_BIO_strings(); + OpenSSL_add_all_algorithms(); + TLS_IS_INIT = true; +#endif +} + + +void __attribute__((destructor)) ltls_destory(void) { + if(TLS_IS_INIT) { + ERR_remove_state(0); + ENGINE_cleanup(); + CONF_modules_unload(1); + ERR_free_strings(); + EVP_cleanup(); + sk_SSL_COMP_free(SSL_COMP_get_compression_methods()); + CRYPTO_cleanup_all_ex_data(); + } +} diff --git a/lualib/http/httpc.lua b/lualib/http/httpc.lua index 975ccaa9..4f5880bf 100644 --- a/lualib/http/httpc.lua +++ b/lualib/http/httpc.lua @@ -8,9 +8,9 @@ local table = table local httpc = {} -local function request(fd, method, host, url, recvheader, header, content) - local read = socket.readfunc(fd) - local write = socket.writefunc(fd) +local function request(interface, method, host, url, recvheader, header, content) + local read = interface.read + local write = interface.write local header_content = "" if header then if not header.host then @@ -74,7 +74,7 @@ local function request(fd, method, host, url, recvheader, header, content) end else -- no content-length, read all - body = body .. socket.readall(fd) + body = body .. interface.readall() end end @@ -88,11 +88,60 @@ function httpc.dns(server,port) dns.server(server,port) end + +local function check_protocol(host) + local protocol = host:match("^[Hh][Tt][Tt][Pp][Ss]?://") + if protocol then + host = string.gsub(host, "^"..protocol, "") + protocol = string.lower(protocol) + if protocol == "https://" then + return "https", host + elseif protocol == "http://" then + return "http", host + else + error(string.format("Invalid protocol: %s", protocol)) + end + else + return "http", host + end +end + +local SSLCTX_CLIENT = nil +local function gen_interface(protocol, fd) + if protocol == "http" then + return { + init = nil, + close = nil, + read = socket.readfunc(fd), + write = socket.writefunc(fd), + readall = function () + return socket.readall(fd) + end, + } + elseif protocol == "https" then + local tls = require "http.tlshelper" + SSLCTX_CLIENT = SSLCTX_CLIENT or tls.newctx() + local tls_ctx = tls.newtls("client", SSLCTX_CLIENT) + return { + init = tls.init_requestfunc(fd, tls_ctx), + close = tls.closefunc(tls_ctx), + read = tls.readfunc(fd, tls_ctx), + write = tls.writefunc(fd, tls_ctx), + readall = tls.readallfunc(fd, tls_ctx), + } + else + error(string.format("Invalid protocol: %s", protocol)) + end +end + + function httpc.request(method, host, url, recvheader, header, content) + local protocol local timeout = httpc.timeout -- get httpc.timeout before any blocked api + protocol, host = check_protocol(host) local hostname, port = host:match"([^:]+):?(%d*)$" if port == "" then - port = 80 + port = protocol=="http" and 80 or protocol=="https" and 443 else port = tonumber(port) end @@ -101,20 +150,31 @@ function httpc.request(method, host, url, recvheader, header, content) end local fd = socket.connect(hostname, port, timeout) if not fd then - error(string.format("http connect error host:%s, port:%s, timeout:%s", hostname, port, timeout)) + error(string.format("%s connect error host:%s, port:%s, timeout:%s", protocol, hostname, port, timeout)) return end + -- print("protocol hostname port", protocol, hostname, port) + local interface = gen_interface(protocol, fd) local finish if timeout then skynet.timeout(timeout, function() if not finish then socket.shutdown(fd) -- shutdown the socket fd, need close later. + if interface.close then + interface.close() + end end end) end - local ok , statuscode, body = pcall(request, fd,method, host, url, recvheader, header, content) + if interface.init then + interface.init() + end + local ok , statuscode, body = pcall(request, interface, method, host, url, recvheader, header, content) finish = true socket.close(fd) + if interface.close then + interface.close() + end if ok then return statuscode, body else diff --git a/lualib/http/tlshelper.lua b/lualib/http/tlshelper.lua new file mode 100644 index 00000000..553acd9e --- /dev/null +++ b/lualib/http/tlshelper.lua @@ -0,0 +1,94 @@ +local socket = require "http.sockethelper" +local c = require "ltls.c" + +local tlshelper = {} + +function tlshelper.init_requestfunc(fd, tls_ctx) + local readfunc = socket.readfunc(fd) + local writefunc = socket.writefunc(fd) + return function () + local ds1 = tls_ctx:handshake() + writefunc(ds1) + while not tls_ctx:finished() do + local ds2 = readfunc() + local ds3 = tls_ctx:handshake(ds2) + if ds3 then + writefunc(ds3) + end + end + end +end + + +function tlshelper.init_responsefunc(fd, tls_ctx) + local readfunc = socket.readfunc(fd) + local writefunc = socket.writefunc(fd) + return function () + while not tls_ctx:finished() do + local ds1 = readfunc() + local ds2 = tls_ctx:handshake(ds1) + if ds2 then + writefunc(ds2) + end + end + local ds3 = tls_ctx:write() + writefunc(ds3) + end +end + +function tlshelper.closefunc(tls_ctx) + return function () + tls_ctx:close() + end +end + +function tlshelper.readfunc(fd, tls_ctx) + local readfunc = socket.readfunc(fd) + local read_buff = "" + return function (sz) + if not sz then + local s = "" + if #read_buff == 0 then + local ds = readfunc(sz) + s = tls_ctx:read(ds) + end + return read_buff .. s + else + while #read_buff < sz do + local ds = readfunc() + local s = tls_ctx:read(ds) + read_buff = read_buff .. s + end + local s = string.sub(read_buff, 1, sz) + read_buff = string.sub(read_buff, sz+1, #read_buff) + return s + end + end +end + +function tlshelper.writefunc(fd, tls_ctx) + local writefunc = socket.writefunc(fd) + return function (s) + local ds = tls_ctx:write(s) + return writefunc(ds) + end +end + +function tlshelper.readallfunc(fd, tls_ctx) + local readfunc = socket.readfunc(fd) + return function () + local ds = socket.readall(fd) + local s = tls_ctx:read(ds) + return s + end +end + +function tlshelper.newctx() + return c.newctx() +end + +function tlshelper.newtls(method, ssl_ctx) + return c.newtls(method, ssl_ctx) +end + +return tlshelper \ No newline at end of file diff --git a/test/testhttp.lua b/test/testhttp.lua index 835752c0..e4a4b45f 100644 --- a/test/testhttp.lua +++ b/test/testhttp.lua @@ -2,12 +2,15 @@ local skynet = require "skynet" local httpc = require "http.httpc" local dns = require "skynet.dns" -local function main() +local function http_test(protocol) --httpc.dns() -- set dns server httpc.timeout = 100 -- set timeout 1 second print("GET baidu.com") + protocol = protocol or "http" local respheader = {} - local status, body = httpc.get("baidu.com", "/", respheader) + local host = string.format("%s://baidu.com", protocol) + print("geting... ".. host) + local status, body = httpc.get(host, "/", respheader) print("[header] =====>") for k,v in pairs(respheader) do print(k,v) @@ -16,13 +19,20 @@ local function main() print(body) local respheader = {} - dns.server() local ip = dns.resolve "baidu.com" print(string.format("GET %s (baidu.com)", ip)) - local status, body = httpc.get("baidu.com", "/", respheader, { host = "baidu.com" }) + local status, body = httpc.get(host, "/", respheader, { host = "baidu.com" }) print(status) end + +local function main() + dns.server() + http_test("http") + print("---------") + http_test("https") +end + skynet.start(function() print(pcall(main)) skynet.exit()