add https client and server support by bios

This commit is contained in:
zixun
2019-03-14 10:36:02 +08:00
parent 1422cae7fb
commit 7746193b5f
6 changed files with 634 additions and 20 deletions

View File

@@ -47,7 +47,7 @@ update3rd :
CSERVICE = snlua logger gate harbor
LUA_CLIB = skynet \
client \
bson md5 sproto lpeg
bson md5 sproto lpeg ltls
LUA_CLIB_SKYNET = \
lua-skynet.c lua-seri.c \
@@ -106,6 +106,9 @@ $(LUA_CLIB_PATH)/client.so : lualib-src/lua-clientsocket.c lualib-src/lua-crypt.
$(LUA_CLIB_PATH)/sproto.so : lualib-src/sproto/sproto.c lualib-src/sproto/lsproto.c | $(LUA_CLIB_PATH)
$(CC) $(CFLAGS) $(SHARED) -Ilualib-src/sproto $^ -o $@
$(LUA_CLIB_PATH)/ltls.so : lualib-src/ltls.c | $(LUA_CLIB_PATH)
$(CC) $(CFLAGS) $(SHARED) -Iskynet-src -L/usr/local/opt/openssl/lib -I/usr/local/opt/openssl/include $^ -o $@ -lssl
$(LUA_CLIB_PATH)/lpeg.so : 3rd/lpeg/lpcap.c 3rd/lpeg/lpcode.c 3rd/lpeg/lpprint.c 3rd/lpeg/lptree.c 3rd/lpeg/lpvm.c | $(LUA_CLIB_PATH)
$(CC) $(CFLAGS) $(SHARED) -I3rd/lpeg $^ -o $@

View File

@@ -6,26 +6,64 @@ local urllib = require "http.url"
local table = table
local string = string
local mode = ...
local mode, protocol = ...
protocol = protocol or "http"
if mode == "agent" then
local function response(id, ...)
local ok, err = httpd.write_response(sockethelper.writefunc(id), ...)
local function response(id, write, ...)
local ok, err = httpd.write_response(write, ...)
if not ok then
-- if err == sockethelper.socket_error , that means socket closed.
skynet.error(string.format("fd = %d, %s", id, err))
end
end
local SSLCTX_SERVER = nil
local function gen_interface(protocol, fd)
if protocol == "http" then
return {
init = nil,
close = nil,
read = sockethelper.readfunc(fd),
write = sockethelper.writefunc(fd),
}
elseif protocol == "https" then
local tls = require "http.tlshelper"
if not SSLCTX_SERVER then
SSLCTX_SERVER = tls.newctx()
-- gen cert and key
-- openssl req -x509 -newkey rsa:2048 -days 3650 -nodes -keyout server-key.pem -out server-cert.pem
local certfile = skynet.getenv("certfile") or "./server-cert.pem"
local keyfile = skynet.getenv("keyfile") or "./server-key.pem"
print(certfile, keyfile)
SSLCTX_SERVER:set_cert(certfile, keyfile)
end
local tls_ctx = tls.newtls("server", SSLCTX_SERVER)
return {
init = tls.init_responsefunc(fd, tls_ctx),
close = tls.closefunc(tls_ctx),
read = tls.readfunc(fd, tls_ctx),
write = tls.writefunc(fd, tls_ctx),
}
else
error(string.format("Invalid protocol: %s", protocol))
end
end
skynet.start(function()
skynet.dispatch("lua", function (_,_,id)
socket.start(id)
local interface = gen_interface(protocol, id)
if interface.init then
interface.init()
end
-- limit request body size to 8192 (you can pass nil to unlimit)
local code, url, method, header, body = httpd.read_request(sockethelper.readfunc(id), 8192)
local code, url, method, header, body = httpd.read_request(interface.read, 8192)
if code then
if code ~= 200 then
response(id, code)
response(id, interface.write, code)
else
local tmp = {}
if header.host then
@@ -44,7 +82,7 @@ skynet.start(function()
table.insert(tmp, string.format("%s = %s",k,v))
end
table.insert(tmp, "-----body----\n" .. body)
response(id, code, table.concat(tmp,"\n"))
response(id, interface.write, code, table.concat(tmp,"\n"))
end
else
if url == sockethelper.socket_error then
@@ -54,6 +92,9 @@ skynet.start(function()
end
end
socket.close(id)
if interface.close then
interface.close()
end
end)
end)
@@ -61,12 +102,13 @@ else
skynet.start(function()
local agent = {}
local protocol = "http"
for i= 1, 20 do
agent[i] = skynet.newservice(SERVICE_NAME, "agent")
agent[i] = skynet.newservice(SERVICE_NAME, "agent", protocol)
end
local balance = 1
local id = socket.listen("0.0.0.0", 8001)
skynet.error("Listen web port 8001")
skynet.error(string.format("Listen web port 8001 protocol:%s", protocol))
socket.start(id , function(id, addr)
skynet.error(string.format("%s connected, pass it to agent :%08x", addr, agent[balance]))
skynet.send(agent[balance], "lua", id)

405
lualib-src/ltls.c Normal file
View File

@@ -0,0 +1,405 @@
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <string.h>
#include <openssl/err.h>
#include <openssl/dh.h>
#include <openssl/ssl.h>
#include <openssl/conf.h>
#include <openssl/engine.h>
#include <lua.h>
#include <lauxlib.h>
static bool TLS_IS_INIT = false;
struct tls_context {
SSL* ssl;
BIO* in_bio;
BIO* out_bio;
bool is_server;
bool is_close;
};
struct ssl_ctx {
SSL_CTX* ctx;
};
// static int
// _ssl_verify_peer(int ok, X509_STORE_CTX* ctx) {
// return 1;
// }
static void
_init_bio(lua_State* L, struct tls_context* tls_p, struct ssl_ctx* ctx_p) {
tls_p->ssl = SSL_new(ctx_p->ctx);
if(!tls_p->ssl) {
luaL_error(L, "SSL_new faild");
}
tls_p->in_bio = BIO_new(BIO_s_mem());
if(!tls_p->in_bio) {
luaL_error(L, "new in bio faild");
}
BIO_set_mem_eof_return(tls_p->in_bio, -1); /* see: https://www.openssl.org/docs/crypto/BIO_s_mem.html */
tls_p->out_bio = BIO_new(BIO_s_mem());
if(!tls_p->out_bio) {
luaL_error(L, "new out bio faild");
}
BIO_set_mem_eof_return(tls_p->out_bio, -1); /* see: https://www.openssl.org/docs/crypto/BIO_s_mem.html */
SSL_set_bio(tls_p->ssl, tls_p->in_bio, tls_p->out_bio);
}
static void
_init_client_context(lua_State* L, struct tls_context* tls_p, struct ssl_ctx* ctx_p) {
tls_p->is_server = false;
_init_bio(L, tls_p, ctx_p);
SSL_set_connect_state(tls_p->ssl);
}
static void
_init_server_context(lua_State* L, struct tls_context* tls_p, struct ssl_ctx* ctx_p) {
tls_p->is_server = true;
_init_bio(L, tls_p, ctx_p);
SSL_set_accept_state(tls_p->ssl);
}
static struct tls_context *
_check_context(lua_State* L, int idx) {
struct tls_context* tls_p = (struct tls_context*)lua_touserdata(L, idx);
if(!tls_p) {
luaL_error(L, "need tls context");
}
if(tls_p->is_close) {
luaL_error(L, "context is closed");
}
return tls_p;
}
static struct ssl_ctx *
_check_sslctx(lua_State* L, int idx) {
struct ssl_ctx* ctx_p = (struct ssl_ctx*)lua_touserdata(L, idx);
if(!ctx_p) {
luaL_error(L, "need sslctx");
}
return ctx_p;
}
static int
_ltls_context_finished(lua_State* L) {
struct tls_context* tls_p = _check_context(L, 1);
int b = SSL_is_init_finished(tls_p->ssl);
lua_pushboolean(L, b);
return 1;
}
static int
_ltls_context_close(lua_State* L) {
struct tls_context* tls_p = lua_touserdata(L, 1);
if(!tls_p->is_close) {
SSL_free(tls_p->ssl);
tls_p->ssl = NULL;
tls_p->in_bio = NULL; //in_bio and out_bio will be free when SSL_free is called
tls_p->out_bio = NULL;
tls_p->is_close = true;
}
return 0;
}
static int
_bio_read(lua_State* L, struct tls_context* tls_p) {
char outbuff[4096];
int all_read = 0;
int read = 0;
int pending = BIO_ctrl_pending(tls_p->out_bio);
if(pending >0) {
luaL_Buffer b;
luaL_buffinit(L, &b);
do {
read = BIO_read(tls_p->out_bio, outbuff, sizeof(outbuff));
// printf("_bio_read:%d pending:%d\n", read, pending);
if(read > sizeof(outbuff)) {
luaL_error(L, "invalid BIO_read:%d", read);
}else if(read == -2) {
luaL_error(L, "BIO_read not implemented in the specific BIO type");
}else if (read > 0) {
all_read += read;
luaL_addlstring(&b, (const char*)outbuff, read);
}
}while(read == sizeof(outbuff));
if(all_read>0){
luaL_pushresult(&b);
}
}
return all_read;
}
static void
_bio_write(lua_State* L, struct tls_context* tls_p, const char* s, size_t len) {
char* p = (char*)s;
size_t sz = len;
while(sz > 0) {
int written = BIO_write(tls_p->in_bio, p, sz);
if(written > sz) {
luaL_error(L, "invalid BIO_write");
}else if(written > 0) {
p += written;
sz -= written;
}else if (written == -2) {
luaL_error(L, "BIO_write not implemented in the specific BIO type");
}
}
}
static int
_ltls_context_handshake(lua_State* L) {
struct tls_context* tls_p = _check_context(L, 1);
size_t slen = 0;
const char* exchange = lua_tolstring(L, 2, &slen);
// check handshake is finished
if(SSL_is_init_finished(tls_p->ssl)) {
luaL_error(L, "handshake is finished");
}
// handshake exchange
if(slen > 0 && exchange != NULL) {
_bio_write(L, tls_p, exchange, slen);
}
// first handshake; initiated by client
if(!SSL_is_init_finished(tls_p->ssl)) {
int ret = SSL_do_handshake(tls_p->ssl);
if(ret == 1) {
return 0;
} else if (ret == -1) {
int all_read = _bio_read(L, tls_p);
if(all_read>0) {
return 1;
}
} else {
int err = SSL_get_error(tls_p->ssl, ret);
luaL_error(L, "SSL_do_handshake error:%d ret:%d", err, ret);
}
}
return 0;
}
static int
_ltls_context_read(lua_State* L) {
struct tls_context* tls_p = _check_context(L, 1);
size_t slen = 0;
const char* encrypted_data = lua_tolstring(L, 2, &slen);
// write encrypted data
if(slen>0 && encrypted_data) {
_bio_write(L, tls_p, encrypted_data, slen);
}
char outbuff[4096];
int read = 0;
luaL_Buffer b;
luaL_buffinit(L, &b);
do {
read = SSL_read(tls_p->ssl, outbuff, sizeof(outbuff));
if(read < 0) {
int err = SSL_get_error(tls_p->ssl, read);
if(err == SSL_ERROR_WANT_READ) {
break;
}
luaL_error(L, "SSL_read error:%d", err);
}else if(read > sizeof(outbuff)) {
luaL_error(L, "invalid SSL_read");
}else if (read > 0) {
luaL_addlstring(&b, outbuff, read);
}
}while(read == sizeof(outbuff));
luaL_pushresult(&b);
return 1;
}
static int
_ltls_context_write(lua_State* L) {
struct tls_context* tls_p = _check_context(L, 1);
size_t slen = 0;
char* unencrypted_data = (char*)lua_tolstring(L, 2, &slen);
while(slen >0) {
int written = SSL_write(tls_p->ssl, unencrypted_data, slen);
if(written < 0) {
int err = SSL_get_error(tls_p->ssl, written);
luaL_error(L, "SSL_write error:%d", err);
}else if(written > slen) {
luaL_error(L, "invalid SSL_write");
}else if(written>0) {
unencrypted_data += written;
slen -= written;
}
}
int all_read = _bio_read(L, tls_p);
if(all_read <= 0) {
luaL_error(L, "bio_read error when _ltls_context_write");
}
return 1;
}
static int
_lctx_gc(lua_State* L) {
struct ssl_ctx* ctx_p = _check_sslctx(L, 1);
if(ctx_p->ctx) {
SSL_CTX_free(ctx_p->ctx);
ctx_p->ctx = NULL;
}
return 0;
}
static int
_lctx_cert(lua_State* L) {
struct ssl_ctx* ctx_p = _check_sslctx(L, 1);
const char* certfile = lua_tostring(L, 2);
const char* key = lua_tostring(L, 3);
if(!certfile) {
luaL_error(L, "need certfile");
}
if(!key) {
luaL_error(L, "need private key");
}
int ret = SSL_CTX_use_certificate_file(ctx_p->ctx, certfile, SSL_FILETYPE_PEM);
if(ret != 1) {
luaL_error(L, "SSL_CTX_use_certificate_file error:%d", ret);
}
ret = SSL_CTX_use_PrivateKey_file(ctx_p->ctx, key, SSL_FILETYPE_PEM);
if(ret != 1) {
luaL_error(L, "SSL_CTX_use_PrivateKey_file error:%d", ret);
}
ret = SSL_CTX_check_private_key(ctx_p->ctx);
if(ret != 1) {
luaL_error(L, "SSL_CTX_check_private_key error:%d", ret);
}
return 0;
}
static int
_lctx_ciphers(lua_State* L) {
struct ssl_ctx* ctx_p = _check_sslctx(L, 1);
const char* s = lua_tostring(L, 2);
if(!s) {
luaL_error(L, "need cipher list");
}
int ret = SSL_CTX_set_tlsext_use_srtp(ctx_p->ctx, s);
if(ret != 0) {
luaL_error(L, "SSL_CTX_set_tlsext_use_srtp error:%d", ret);
}
return 0;
}
static int
lnew_ctx(lua_State* L) {
struct ssl_ctx* ctx_p = (struct ssl_ctx*)lua_newuserdata(L, sizeof(*ctx_p));
ctx_p->ctx = SSL_CTX_new(SSLv23_method());
if(!ctx_p->ctx) {
luaL_error(L, "SSL_CTX_new client faild.");
}
if(luaL_newmetatable(L, "_TLS_SSLCTX_METATABLE_")) {
luaL_Reg l[] = {
{"set_ciphers", _lctx_ciphers},
{"set_cert", _lctx_cert},
{NULL, NULL},
};
luaL_newlib(L, l);
lua_setfield(L, -2, "__index");
lua_pushcfunction(L, _lctx_gc);
lua_setfield(L, -2, "__gc");
}
lua_setmetatable(L, -2);
return 1;
}
static int
lnew_tls(lua_State* L) {
struct tls_context* tls_p = (struct tls_context*)lua_newuserdata(L, sizeof(*tls_p));
tls_p->is_close = false;
const char* method = luaL_optstring(L, 1, "nil");
struct ssl_ctx* ctx_p = _check_sslctx(L, 2);
lua_pushvalue(L, 2);
lua_setuservalue(L, -2); // set ssl_ctx associated to tls_context
if(strcmp(method, "client") == 0) {
_init_client_context(L, tls_p, ctx_p);
}else if(strcmp(method, "server") == 0) {
_init_server_context(L, tls_p, ctx_p);
} else {
luaL_error(L, "invalid method:%s e.g[server, client]", method);
}
if(luaL_newmetatable(L, "_TLS_CONTEXT_METATABLE_")) {
luaL_Reg l[] = {
{"close", _ltls_context_close},
{"finished", _ltls_context_finished},
{"handshake", _ltls_context_handshake},
{"read", _ltls_context_read},
{"write", _ltls_context_write},
{NULL, NULL},
};
luaL_newlib(L, l);
lua_setfield(L, -2, "__index");
lua_pushcfunction(L, _ltls_context_close);
lua_setfield(L, -2, "__gc");
}
lua_setmetatable(L, -2);
return 1;
}
int
luaopen_ltls_c(lua_State* L) {
luaL_Reg l[] = {
{"newctx", lnew_ctx},
{"newtls", lnew_tls},
{NULL, NULL},
};
luaL_checkversion(L);
luaL_newlib(L, l);
return 1;
}
void __attribute__((constructor)) ltls_init(void) {
#ifndef OPENSSL_EXTERNAL_INITIALIZATION
SSL_library_init();
SSL_load_error_strings();
ERR_load_BIO_strings();
OpenSSL_add_all_algorithms();
TLS_IS_INIT = true;
#endif
}
void __attribute__((destructor)) ltls_destory(void) {
if(TLS_IS_INIT) {
ERR_remove_state(0);
ENGINE_cleanup();
CONF_modules_unload(1);
ERR_free_strings();
EVP_cleanup();
sk_SSL_COMP_free(SSL_COMP_get_compression_methods());
CRYPTO_cleanup_all_ex_data();
}
}

View File

@@ -8,9 +8,9 @@ local table = table
local httpc = {}
local function request(fd, method, host, url, recvheader, header, content)
local read = socket.readfunc(fd)
local write = socket.writefunc(fd)
local function request(interface, method, host, url, recvheader, header, content)
local read = interface.read
local write = interface.write
local header_content = ""
if header then
if not header.host then
@@ -74,7 +74,7 @@ local function request(fd, method, host, url, recvheader, header, content)
end
else
-- no content-length, read all
body = body .. socket.readall(fd)
body = body .. interface.readall()
end
end
@@ -88,11 +88,60 @@ function httpc.dns(server,port)
dns.server(server,port)
end
local function check_protocol(host)
local protocol = host:match("^[Hh][Tt][Tt][Pp][Ss]?://")
if protocol then
host = string.gsub(host, "^"..protocol, "")
protocol = string.lower(protocol)
if protocol == "https://" then
return "https", host
elseif protocol == "http://" then
return "http", host
else
error(string.format("Invalid protocol: %s", protocol))
end
else
return "http", host
end
end
local SSLCTX_CLIENT = nil
local function gen_interface(protocol, fd)
if protocol == "http" then
return {
init = nil,
close = nil,
read = socket.readfunc(fd),
write = socket.writefunc(fd),
readall = function ()
return socket.readall(fd)
end,
}
elseif protocol == "https" then
local tls = require "http.tlshelper"
SSLCTX_CLIENT = SSLCTX_CLIENT or tls.newctx()
local tls_ctx = tls.newtls("client", SSLCTX_CLIENT)
return {
init = tls.init_requestfunc(fd, tls_ctx),
close = tls.closefunc(tls_ctx),
read = tls.readfunc(fd, tls_ctx),
write = tls.writefunc(fd, tls_ctx),
readall = tls.readallfunc(fd, tls_ctx),
}
else
error(string.format("Invalid protocol: %s", protocol))
end
end
function httpc.request(method, host, url, recvheader, header, content)
local protocol
local timeout = httpc.timeout -- get httpc.timeout before any blocked api
protocol, host = check_protocol(host)
local hostname, port = host:match"([^:]+):?(%d*)$"
if port == "" then
port = 80
port = protocol=="http" and 80 or protocol=="https" and 443
else
port = tonumber(port)
end
@@ -101,20 +150,31 @@ function httpc.request(method, host, url, recvheader, header, content)
end
local fd = socket.connect(hostname, port, timeout)
if not fd then
error(string.format("http connect error host:%s, port:%s, timeout:%s", hostname, port, timeout))
error(string.format("%s connect error host:%s, port:%s, timeout:%s", protocol, hostname, port, timeout))
return
end
-- print("protocol hostname port", protocol, hostname, port)
local interface = gen_interface(protocol, fd)
local finish
if timeout then
skynet.timeout(timeout, function()
if not finish then
socket.shutdown(fd) -- shutdown the socket fd, need close later.
if interface.close then
interface.close()
end
end
end)
end
local ok , statuscode, body = pcall(request, fd,method, host, url, recvheader, header, content)
if interface.init then
interface.init()
end
local ok , statuscode, body = pcall(request, interface, method, host, url, recvheader, header, content)
finish = true
socket.close(fd)
if interface.close then
interface.close()
end
if ok then
return statuscode, body
else

94
lualib/http/tlshelper.lua Normal file
View File

@@ -0,0 +1,94 @@
local socket = require "http.sockethelper"
local c = require "ltls.c"
local tlshelper = {}
function tlshelper.init_requestfunc(fd, tls_ctx)
local readfunc = socket.readfunc(fd)
local writefunc = socket.writefunc(fd)
return function ()
local ds1 = tls_ctx:handshake()
writefunc(ds1)
while not tls_ctx:finished() do
local ds2 = readfunc()
local ds3 = tls_ctx:handshake(ds2)
if ds3 then
writefunc(ds3)
end
end
end
end
function tlshelper.init_responsefunc(fd, tls_ctx)
local readfunc = socket.readfunc(fd)
local writefunc = socket.writefunc(fd)
return function ()
while not tls_ctx:finished() do
local ds1 = readfunc()
local ds2 = tls_ctx:handshake(ds1)
if ds2 then
writefunc(ds2)
end
end
local ds3 = tls_ctx:write()
writefunc(ds3)
end
end
function tlshelper.closefunc(tls_ctx)
return function ()
tls_ctx:close()
end
end
function tlshelper.readfunc(fd, tls_ctx)
local readfunc = socket.readfunc(fd)
local read_buff = ""
return function (sz)
if not sz then
local s = ""
if #read_buff == 0 then
local ds = readfunc(sz)
s = tls_ctx:read(ds)
end
return read_buff .. s
else
while #read_buff < sz do
local ds = readfunc()
local s = tls_ctx:read(ds)
read_buff = read_buff .. s
end
local s = string.sub(read_buff, 1, sz)
read_buff = string.sub(read_buff, sz+1, #read_buff)
return s
end
end
end
function tlshelper.writefunc(fd, tls_ctx)
local writefunc = socket.writefunc(fd)
return function (s)
local ds = tls_ctx:write(s)
return writefunc(ds)
end
end
function tlshelper.readallfunc(fd, tls_ctx)
local readfunc = socket.readfunc(fd)
return function ()
local ds = socket.readall(fd)
local s = tls_ctx:read(ds)
return s
end
end
function tlshelper.newctx()
return c.newctx()
end
function tlshelper.newtls(method, ssl_ctx)
return c.newtls(method, ssl_ctx)
end
return tlshelper

View File

@@ -2,12 +2,15 @@ local skynet = require "skynet"
local httpc = require "http.httpc"
local dns = require "skynet.dns"
local function main()
local function http_test(protocol)
--httpc.dns() -- set dns server
httpc.timeout = 100 -- set timeout 1 second
print("GET baidu.com")
protocol = protocol or "http"
local respheader = {}
local status, body = httpc.get("baidu.com", "/", respheader)
local host = string.format("%s://baidu.com", protocol)
print("geting... ".. host)
local status, body = httpc.get(host, "/", respheader)
print("[header] =====>")
for k,v in pairs(respheader) do
print(k,v)
@@ -16,13 +19,20 @@ local function main()
print(body)
local respheader = {}
dns.server()
local ip = dns.resolve "baidu.com"
print(string.format("GET %s (baidu.com)", ip))
local status, body = httpc.get("baidu.com", "/", respheader, { host = "baidu.com" })
local status, body = httpc.get(host, "/", respheader, { host = "baidu.com" })
print(status)
end
local function main()
dns.server()
http_test("http")
print("---------")
http_test("https")
end
skynet.start(function()
print(pcall(main))
skynet.exit()