Rewrite harbor and don't need zeromq :)

This commit is contained in:
云风
2012-08-17 11:48:22 +08:00
parent c3260cba72
commit b06d380219
7 changed files with 856 additions and 400 deletions

View File

@@ -1,10 +0,0 @@
all : master test
master : master.c main.c
gcc -g -Wall -o $@ $^ -lzmq
test : test.c
gcc -g -Wall -o $@ $^ -lzmq
clean :
rm master test

View File

@@ -1,17 +0,0 @@
#include "skynet_master.h"
#include <zmq.h>
int
main (int argc, char * argv[]) {
const char * default_port = "tcp://127.0.0.1:2012";
if (argc > 1) {
default_port = argv[2];
}
void *context = zmq_init (1);
skynet_master(context, default_port);
zmq_term (context);
return 0;
}

View File

@@ -1,243 +0,0 @@
#include <zmq.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <stdint.h>
#define HASH_SIZE 4096
#define MAX_SLAVE 255
static uint32_t
calc_hash(const char *name) {
int i;
uint32_t h = 0;
for (i=0;name[i];i++) {
h = h ^ ((h<<5)+(h>>2)+(uint8_t)name[i]);
}
h ^= i;
return h;
}
struct keyvalue {
struct keyvalue * next;
uint32_t hash;
char * key;
size_t value_size;
char * value;
};
struct hashmap {
struct keyvalue *node[HASH_SIZE];
void * zmq;
void * slave[MAX_SLAVE];
};
static struct hashmap *
_hash_new(void *zmq) {
struct hashmap * hash = malloc(sizeof(*hash));
memset(hash, 0, sizeof(*hash));
hash->zmq = zmq;
return hash;
}
static struct keyvalue *
_hash_search(struct hashmap * hash, const char * key) {
uint32_t h = calc_hash(key);
struct keyvalue * n = hash->node[h & (HASH_SIZE-1)];
while (n) {
if (n->hash == h && strcmp(n->key, key) == 0) {
return n;
}
n = n->next;
}
return NULL;
}
static void
_hash_insert(struct hashmap * hash, const char * key, const char *value) {
uint32_t h = calc_hash(key);
struct keyvalue * node = malloc(sizeof(*node));
node->next = hash->node[h & (HASH_SIZE-1)];
node->hash = h;
node->key = strdup(key);
node->value_size = strlen(value);
node->value = malloc(node->value_size+1);
memcpy(node->value, value, node->value_size+1);
hash->node[h & (HASH_SIZE-1)] = node;
}
static void
_hash_delete(struct hashmap * hash, const char * key) {
uint32_t h = calc_hash(key);
struct keyvalue ** ptr = &hash->node[h & (HASH_SIZE-1)];
while(*ptr) {
struct keyvalue *n = *ptr;
if (n->hash == h && strcmp(n->key, key) == 0) {
*ptr = n->next;
free(n->key);
free(n->value);
free(n);
return ;
}
ptr = &(n->next);
}
}
static int
_hash_bind(struct hashmap * hash, int slave, const char *addr) {
void *pub = NULL;
if (addr) {
pub = zmq_socket(hash->zmq, ZMQ_PUSH);
int r = zmq_connect(pub, addr);
if (r<0) {
fprintf(stderr,"Can't connect to [%d] %s\n",slave,addr);
return -1;
}
printf("Connect to [%d] %s\n",slave,addr);
}
void * old_pub = hash->slave[slave-1];
hash->slave[slave-1] = pub;
if (old_pub) {
zmq_close(old_pub);
}
return 0;
}
static void
broadcast(struct hashmap * hash, zmq_msg_t * msg) {
int i;
for (i=0;i<MAX_SLAVE;i++) {
void * pub = hash->slave[i];
if (pub) {
zmq_msg_t part;
zmq_msg_init(&part);
int rc = zmq_send(pub, &part, ZMQ_SNDMORE);
if (rc != 0) {
fprintf(stderr,"Can't publish to %d : %s",i+1,zmq_strerror(errno));
}
zmq_msg_close(&part);
zmq_msg_init(&part);
zmq_msg_copy(&part,msg);
rc = zmq_send(pub, &part, 0);
if (rc != 0) {
fprintf(stderr,"Can't publish to %d : %s",i+1,zmq_strerror(errno));
}
zmq_msg_close(&part);
}
}
}
static void
replace(void * responder, struct hashmap * map, const char *key, const char *value) {
// printf("Replace %s %s\n",key,value);
struct keyvalue * node = _hash_search(map, key);
zmq_msg_t reply;
if (node == NULL) {
_hash_insert(map, key, value);
zmq_msg_init_size (&reply, 0);
} else {
zmq_msg_init_size (&reply, node->value_size);
memcpy (zmq_msg_data (&reply), node->value, node->value_size);
free(node->value);
node->value_size = strlen(value);
node->value = malloc(node->value_size + 1);
memcpy(node->value, value, node->value_size +1);
}
zmq_send (responder, &reply, 0);
zmq_msg_close (&reply);
}
static void
erase(void * responder, struct hashmap *map, const char *key) {
// printf("Erase %s\n",key);
struct keyvalue * node = _hash_search(map, key);
zmq_msg_t reply;
if (node == NULL) {
zmq_msg_init_size (&reply, 0);
} else {
zmq_msg_init_size (&reply, node->value_size);
memcpy (zmq_msg_data (&reply), node->value, node->value_size);
_hash_delete(map, key);
}
zmq_send (responder, &reply, 0);
zmq_msg_close (&reply);
}
static void
query(void * responder, struct hashmap *map, const char * key) {
struct keyvalue * node = _hash_search(map, key);
zmq_msg_t reply;
if (node == NULL) {
zmq_msg_init_size (&reply, 0);
} else {
zmq_msg_init_size (&reply, node->value_size);
memcpy (zmq_msg_data (&reply), node->value, node->value_size);
}
zmq_send (responder, &reply, 0);
zmq_msg_close (&reply);
}
static void
update(void * responder, struct hashmap *map, zmq_msg_t * msg) {
size_t sz = zmq_msg_size (msg);
const char * command = zmq_msg_data (msg);
int i;
for (i=0;i<sz;i++) {
if (command[i] == '=') {
char key[i+1];
memcpy(key, command, i);
key[i] = '\0';
int slave = strtol(key, NULL, 10);
if (sz-i == 1) {
if (slave > 0 && slave <= MAX_SLAVE) {
_hash_bind(map, slave, NULL);
}
erase(responder, map,key);
broadcast(map, msg);
} else {
char value[sz-i];
memcpy(value, command+i+1, sz-i-1);
value[sz-i-1] = '\0';
if (slave > 0 && slave <= MAX_SLAVE) {
_hash_bind(map, slave, value);
}
replace(responder, map, key,value);
broadcast(map, msg);
}
return;
}
}
char key[sz+1];
memcpy(key, command, sz);
key[sz] = '\0';
query(responder, map, key);
}
void
skynet_master(void * context, const char * port) {
void *responder = zmq_socket (context, ZMQ_REP);
int r = zmq_bind(responder, port);
if (r < 0) {
fprintf(stderr, "Can't bind to %s\n",port);
exit(1);
}
printf("Start master on %s\n", port);
struct hashmap *map = _hash_new(context);
for (;;) {
zmq_msg_t request;
zmq_msg_init (&request);
zmq_recv (responder, &request, 0);
update(responder, map, &request);
zmq_msg_close (&request);
}
zmq_close (responder);
}

View File

@@ -1,6 +0,0 @@
#ifndef SKYNET_MASTER_H
#define SKYNET_MASTER_H
void skynet_master(void * context, const char * port);
#endif

View File

@@ -1,124 +0,0 @@
#include <zmq.h>
#include <unistd.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <stdint.h>
#include <assert.h>
static char *
send_command(void * request, const char * command) {
size_t size = strlen(command);
zmq_msg_t req;
zmq_msg_init_size(&req, size);
memcpy(zmq_msg_data(&req) , command, size);
zmq_send (request, &req, 0);
zmq_msg_close (&req);
zmq_msg_t reply;
zmq_msg_init (&reply);
zmq_recv (request, &reply, 0);
size = zmq_msg_size (&reply);
char * ret = malloc(size+1);
memcpy(ret,zmq_msg_data(&reply),size);
ret[size]='\0';
zmq_msg_close (&reply);
return ret;
}
static void
pull_sub(void * local) {
zmq_msg_t part;
zmq_msg_init (&part);
zmq_recv (local, &part, 0);
zmq_msg_close(&part);
zmq_msg_init (&part);
zmq_recv (local, &part, 0);
size_t sz = zmq_msg_size(&part);
char tmp[sz+1];
memcpy(tmp,zmq_msg_data(&part),sz);
tmp[sz]='\0';
printf("%s\n",tmp);
zmq_msg_close(&part);
}
int
main (int argc, char * argv[]) {
const char * default_master = "tcp://127.0.0.1:2012";
int slave = 100;
const char * default_local = "tcp://127.0.0.1:5001";
if (argc > 1) {
default_master = argv[1];
if (argc > 2) {
slave = strtol(argv[2],NULL,10);
if (slave < 1 || slave> 255) {
fprintf(stderr,"Slave id must be in [1,255]\n");
return 1;
}
if (argc > 3) {
default_local = argv[3];
}
}
}
printf("master = %s\n",default_master);
printf("local = [%d]%s\n",slave,default_local);
void *context = zmq_init (1);
void *request = zmq_socket (context, ZMQ_REQ);
void *local = zmq_socket( context, ZMQ_PULL);
int r = zmq_connect(request, default_master);
if (r < 0) {
fprintf(stderr, "Can't connect to %s\n",default_master);
return 1;
}
r = zmq_bind(local, default_local);
if (r < 0) {
fprintf(stderr, "Can't bind to %s\n",default_local);
return 1;
}
char tmp[1024];
sprintf(tmp,"%d=%s",slave,default_local);
char * result = send_command(request,tmp);
free(result);
zmq_pollitem_t items[2];
items[0].socket = local;
items[0].events = ZMQ_POLLIN;
items[1].socket = NULL;
items[1].fd = STDIN_FILENO;
items[1].events = ZMQ_POLLIN;
for (;;) {
int rc = zmq_poll(items,2,-1);
assert (rc >= 0);
if (items[0].revents) {
pull_sub(local);
}
if (items[1].revents) {
char tmp[1024];
fgets(tmp,sizeof(tmp),stdin);
char * cr = strchr(tmp,'\r');
if (cr) {
*cr = '\0';
}
cr = strchr(tmp,'\n');
if (cr) {
*cr = '\0';
}
result = send_command(request, tmp);
printf("%s\n",result);
free(result);
}
}
zmq_close (request);
zmq_term (context);
return 0;
}

View File

@@ -0,0 +1,584 @@
#include "skynet.h"
#include "skynet_harbor.h"
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <sys/uio.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <stdint.h>
#include <errno.h>
#define HASH_SIZE 4096
#define DEFAULT_QUEUE_SIZE 1024
struct msg {
char * buffer;
size_t size;
};
struct msg_queue {
int size;
int head;
int tail;
struct msg * data;
};
struct keyvalue {
struct keyvalue * next;
char key[GLOBALNAME_LENGTH];
uint32_t hash;
uint32_t value;
struct msg_queue * queue;
};
struct hashmap {
struct keyvalue *node[HASH_SIZE];
};
struct remote_message_header {
uint32_t source;
uint32_t destination;
uint32_t session;
};
struct harbor {
int id;
struct hashmap * map;
int master_fd;
char * master_addr;
int remote_fd[REMOTE_MAX];
char * remote_addr[REMOTE_MAX];
};
// hash table
static void
_push_queue(struct msg_queue * queue, const void * buffer, size_t sz, struct remote_message_header * header) {
struct msg * slot = &queue->data[queue->tail];
queue->tail = (queue->tail + 1) % queue->size;
if (queue->tail == queue->head) {
struct msg * new_buffer = malloc(queue->size * 2 * sizeof(struct msg));
int i;
for (i=0;i<queue->size;i++) {
new_buffer[i] = queue->data[(i+queue->head) % queue->size];
}
free(queue->data);
queue->data = new_buffer;
queue->head = 0;
queue->tail = queue->size;
queue->size *= 2;
slot = &queue->data[queue->tail];
}
slot->buffer = malloc(sz + sizeof(*header));
memcpy(slot->buffer, buffer, sz);
memcpy(slot->buffer + sz, header, sizeof(*header));
slot->size = sz + sizeof(*header);
}
static struct msg *
_pop_queue(struct msg_queue * queue) {
if (queue->head == queue->tail) {
return NULL;
}
struct msg * slot = &queue->data[queue->head];
queue->head = (queue->head + 1) % queue->size;
return slot;
}
static struct msg_queue *
_new_queue() {
struct msg_queue * queue = malloc(sizeof(*queue));
queue->size = DEFAULT_QUEUE_SIZE;
queue->head = 0;
queue->tail = 0;
queue->data = malloc(DEFAULT_QUEUE_SIZE * sizeof(struct msg));
return queue;
}
static void
_release_queue(struct msg_queue *queue) {
if (queue == NULL)
return;
struct msg * m = _pop_queue(queue);
while (m) {
free(m->buffer);
m = _pop_queue(queue);
}
free(queue->data);
free(queue);
}
static struct keyvalue *
_hash_search(struct hashmap * hash, const char name[GLOBALNAME_LENGTH]) {
uint32_t *ptr = (uint32_t*) name;
uint32_t h = ptr[0] ^ ptr[1] ^ ptr[2] ^ ptr[3];
struct keyvalue * node = hash->node[h % HASH_SIZE];
while (node) {
if (node->hash == h && strncmp(node->key, name, GLOBALNAME_LENGTH) == 0) {
return node;
}
node = node->next;
}
return NULL;
}
/*
// Don't support erase name yet
static struct void
_hash_erase(struct hashmap * hash, char name[GLOBALNAME_LENGTH) {
uint32_t *ptr = name;
uint32_t h = ptr[0] ^ ptr[1] ^ ptr[2] ^ ptr[3];
struct keyvalue ** ptr = &hash->node[h % HASH_SIZE];
while (*ptr) {
struct keyvalue * node = *ptr;
if (node->hash == h && strncmp(node->key, name, GLOBALNAME_LENGTH) == 0) {
_release_queue(node->queue);
*ptr->next = node->next;
free(node);
return;
}
*ptr = &(node->next);
}
}
*/
static struct keyvalue *
_hash_insert(struct hashmap * hash, const char name[GLOBALNAME_LENGTH]) {
uint32_t *ptr = (uint32_t *)name;
uint32_t h = ptr[0] ^ ptr[1] ^ ptr[2] ^ ptr[3];
struct keyvalue ** pkv = &hash->node[h % HASH_SIZE];
struct keyvalue * node = malloc(sizeof(*node));
memcpy(node->key, name, GLOBALNAME_LENGTH);
node->next = *pkv;
node->queue = NULL;
node->hash = h;
node->value = 0;
*pkv = node;
return node;
}
static struct hashmap *
_hash_new() {
struct hashmap * h = malloc(sizeof(struct hashmap));
memset(h,0,sizeof(*h));
return h;
}
static void
_hash_delete(struct hashmap *hash) {
int i;
for (i=0;i<HASH_SIZE;i++) {
struct keyvalue ** ptr = &hash->node[i];
while (*ptr) {
struct keyvalue * node = *ptr;
ptr = &node->next;
_release_queue(node->queue);
free(node);
}
}
free(hash);
}
///////////////
struct harbor *
harbor_create(void) {
struct harbor * h = malloc(sizeof(*h));
h->id = 0;
h->master_fd = -1;
h->master_addr = NULL;
int i;
for (i=0;i<REMOTE_MAX;i++) {
h->remote_fd[i] = -1;
h->remote_addr[i] = NULL;
}
h->map = _hash_new();
return h;
}
void
harbor_release(struct harbor *h) {
if (h->master_fd >= 0) {
close(h->master_fd);
}
free(h->master_addr);
int i;
for (i=0;i<REMOTE_MAX;i++) {
if (h->remote_fd[i] >= 0) {
close(h->remote_fd[i]);
free(h->remote_addr[i]);
}
}
_hash_delete(h->map);
free(h);
}
static int
_connect_to(struct skynet_context *ctx, const char *ipaddress) {
int fd = socket(AF_INET,SOCK_STREAM,0);
struct sockaddr_in my_addr;
char * port = strchr(ipaddress,':');
if (port==NULL) {
return -1;
}
int sz = port - ipaddress;
char tmp[sz + 1];
memcpy(tmp,ipaddress,sz);
tmp[sz] = '\0';
my_addr.sin_addr.s_addr=inet_addr(tmp);
my_addr.sin_family=AF_INET;
my_addr.sin_port=htons(strtol(port+1,NULL,10));
int r = connect(fd,(struct sockaddr *)&my_addr,sizeof(struct sockaddr_in));
if (r == -1) {
close(fd);
skynet_error(ctx, "Connect to %s error", ipaddress);
return -1;
}
return fd;
}
static inline void
_header_to_message(const struct remote_message_header * header, uint32_t * message) {
message[0] = htonl(header->source);
message[1] = htonl(header->destination);
message[2] = htonl(header->session);
}
static inline void
_message_to_header(const uint32_t *message, struct remote_message_header *header) {
header->source = ntohl(message[0]);
header->destination = ntohl(message[1]);
header->session = ntohl(message[2]);
}
static int
_send_package(int fd, const void * buffer, size_t sz) {
uint16_t header = htons(sz);
struct iovec part[2];
part[0].iov_base = &header;
part[0].iov_len = 2;
part[1].iov_base = (void*)buffer;
part[1].iov_len = sz;
for (;;) {
int err = writev(fd, part, 2);
if (err < 0) {
switch (errno) {
case EAGAIN:
case EINTR:
continue;
}
}
if (err != sz+2) {
return 1;
}
return 0;
}
}
static int
_send_remote(int fd, const char * buffer, size_t sz, struct remote_message_header * cookie) {
struct iovec part[2];
part[0].iov_base = (char *)buffer;
part[0].iov_len = sz;
uint32_t header[3];
_header_to_message(cookie, header);
part[1].iov_base = header;
part[1].iov_len = sizeof(header);
for (;;) {
int err = writev(fd, part, 2);
if (err < 0) {
switch (errno) {
case EAGAIN:
case EINTR:
continue;
}
}
if (err != sz+sizeof(*cookie)) {
return 1;
}
return 0;
}
}
static void
_update_remote_address(struct skynet_context * context, struct harbor *h, int harbor_id, const char * ipaddr) {
if (harbor_id == h->id) {
return;
}
assert(harbor_id > 0 && harbor_id< REMOTE_MAX);
if (h->remote_fd[harbor_id] >=0) {
close(h->remote_fd[harbor_id]);
free(h->remote_addr[harbor_id]);
h->remote_addr[harbor_id] = NULL;
}
h->remote_fd[harbor_id] = _connect_to(context, ipaddr);
if (h->remote_fd[harbor_id] >= 0) {
free(h->remote_addr[harbor_id]);
h->remote_addr[harbor_id] = strdup(ipaddr);
}
}
static void
_dispatch_queue(struct harbor *h, struct skynet_context * context, struct msg_queue * queue, uint32_t handle, const char name[GLOBALNAME_LENGTH] ) {
int harbor_id = handle >> HANDLE_REMOTE_SHIFT;
assert(harbor_id != 0);
int fd = h->remote_fd[harbor_id];
if (fd < 0) {
char tmp [GLOBALNAME_LENGTH+1];
memcpy(tmp, name , GLOBALNAME_LENGTH);
tmp[GLOBALNAME_LENGTH] = '\0';
skynet_error(context, "Drop message to %s (in harbor %d)",tmp,harbor_id);
return;
}
struct msg * m = _pop_queue(queue);
while (m) {
struct remote_message_header * cookie = (struct remote_message_header *)(m->buffer + m->size - sizeof(*cookie));
cookie->destination = handle;
_header_to_message(cookie, (uint32_t *)cookie);
int err = _send_package(fd, m->buffer, m->size);
if (err) {
close(fd);
h->remote_fd[harbor_id] = _connect_to(context, h->remote_addr[harbor_id]);
if (h->remote_fd[harbor_id] < 0) {
skynet_error(context, "Reconnect to harbor %d %s failed",harbor_id, h->remote_addr[harbor_id]);
return;
}
}
m = _pop_queue(queue);
}
}
static void
_update_remote_name(struct harbor *h, struct skynet_context * context, const char name[GLOBALNAME_LENGTH], uint32_t handle) {
struct keyvalue * node = _hash_search(h->map, name);
if (node == NULL) {
node = _hash_insert(h->map, name);
}
node->value = handle;
if (node->queue) {
_dispatch_queue(h, context, node->queue, handle, name);
_release_queue(node->queue);
node->queue = NULL;
}
}
static void
_request_master(struct harbor *h, struct skynet_context * context, const char name[GLOBALNAME_LENGTH], size_t i, uint32_t handle) {
char buffer[4+i];
handle = htonl(handle);
memcpy(buffer, &handle, 4);
memcpy(buffer+4,name,i);
int err = _send_package(h->master_fd, buffer, 4+i);
if (err) {
close(h->master_fd);
h->master_fd = _connect_to(context, h->master_addr);
if (h->master_fd < 0) {
skynet_error(context, "Reconnect to master server %s failed", h->master_addr);
return;
}
_send_package(h->master_fd, buffer, 4+i);
}
}
/*
update global name to master
2 bytes (size)
4 bytes (handle) (handle == 0 for request)
n bytes string (name)
*/
static void
_id_to_hex(char *str, uint32_t id) {
int i;
static char hex[16] = { '0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F' };
str[0] = ':';
for (i=0;i<8;i++) {
str[i+1] = hex[(id >> ((7-i) * 4))&0xf];
}
str[9] = '\0';
}
static int
_remote_send_handle(struct harbor *h, struct skynet_context * context, uint32_t source, uint32_t destination, int session, const char * msg, size_t sz) {
int harbor_id = destination >> HANDLE_REMOTE_SHIFT;
assert(harbor_id != 0);
if (harbor_id == h->id) {
// local message
char srcstr[10];
char desstr[10];
_id_to_hex(srcstr, source);
_id_to_hex(desstr, destination);
skynet_send(context, srcstr, desstr , session, (void *)msg, sz, DONTCOPY);
return 1;
}
int fd = h->remote_fd[harbor_id];
if (fd >= 0) {
struct remote_message_header cookie;
cookie.source = source;
cookie.destination = destination;
cookie.session = (uint32_t)session;
int err = _send_remote(fd, msg,sz,&cookie);
if (err) {
close(fd);
h->remote_fd[harbor_id] = _connect_to(context, h->remote_addr[harbor_id]);
if (h->remote_fd[harbor_id] < 0) {
skynet_error(context, "Reconnect to harbor %d : %s failed", harbor_id, h->remote_addr[harbor_id]);
return 0;
}
}
} else {
_request_master(h, context, NULL, 0, harbor_id);
skynet_error(context, "Drop message to harbor %d from %x to %x (session = %d, msgsz = %d)",harbor_id, source, destination,session,(int)sz);
}
return 0;
}
static int
_remote_send_name(struct harbor *h, struct skynet_context * context, uint32_t source, const char name[GLOBALNAME_LENGTH], int session, const char * msg, size_t sz) {
struct keyvalue * node = _hash_search(h->map, name);
if (node == NULL) {
node = _hash_insert(h->map, name);
}
if (node->value == 0) {
if (node->queue == NULL) {
node->queue = _new_queue();
}
struct remote_message_header header;
header.source = source;
header.destination = 0;
header.session = (uint32_t)session;
_push_queue(node->queue, msg, sz, &header);
return 1;
} else {
return _remote_send_handle(h, context, source, node->value, session, msg, sz);
}
}
static void
_remote_register_name(struct harbor *h, struct skynet_context * context, const char name[GLOBALNAME_LENGTH], uint32_t handle) {
int i;
for (i=0;i<GLOBALNAME_LENGTH;i++) {
if (name[i] == '\0')
break;
}
_update_remote_name(h, context, name, handle);
_request_master(h,context,name,i,handle);
}
static void
_report_local_address(struct harbor *h, struct skynet_context * context, const char * local_address, int harbor_id) {
size_t sz = strlen(local_address);
_request_master(h, context, local_address, sz, harbor_id);
}
static int
_mainloop(struct skynet_context * context, void * ud, int session, const char * addr, const void * msg, size_t sz) {
struct harbor * h = ud;
if (session == SESSION_CLIENT) {
const char * cookie = msg;
cookie += sz - 12;
struct remote_message_header header;
_message_to_header((const uint32_t *)cookie, &header);
if (header.source == 0) {
if (header.destination < REMOTE_MAX) {
// 1 byte harbor id (0~255)
// update remote harbor address
char ip [sz - 11];
memcpy(ip, msg, sz-12);
ip[sz-11] = '\0';
_update_remote_address(context, h, header.destination, ip);
} else {
// update global name
if (sz - 12 > GLOBALNAME_LENGTH) {
char name[sz-11];
memcpy(name, msg, sz-12);
name[sz-11] = '\0';
skynet_error(context, "Global name is too long %s", name);
}
_update_remote_name(h, context, msg, header.destination);
}
} else {
char srcstr[10];
char desstr[10];
_id_to_hex(srcstr, header.source);
_id_to_hex(desstr, header.destination);
skynet_send(context, srcstr, desstr, (int)header.session, (void *)msg, sz-12, DONTCOPY);
return 1;
}
} else {
const struct remote_message *rmsg = msg;
if (sz == sizeof(rmsg->destination)) {
_remote_register_name(h, context, rmsg->destination.name, rmsg->destination.handle);
return 0;
}
assert(sz == sizeof(*rmsg));
uint32_t source_handle = strtoul(addr+1, NULL, 16);
if (rmsg->destination.handle == 0) {
if (_remote_send_name(h, context, source_handle , rmsg->destination.name, session, rmsg->message, rmsg->sz)) {
return 0;
}
} else {
if (_remote_send_handle(h, context, source_handle , rmsg->destination.handle, session, rmsg->message, rmsg->sz)) {
return 0;
}
}
free((void *)rmsg->message);
}
return 0;
}
int
harbor_init(struct harbor *h, struct skynet_context *ctx, const char * args) {
int sz = strlen(args)+1;
char master_addr[sz];
char local_addr[sz];
int harbor_id = 0;
sscanf(args,"%s %s %d",master_addr, local_addr, &harbor_id);
int master_fd = _connect_to(ctx, master_addr);
if (master_fd < 0) {
skynet_error(ctx, "Harbor : Connect to master %s faild",master_addr);
return 1;
}
printf("Connect to master %s\n",master_addr);
h->master_addr = strdup(master_addr);
h->master_fd = master_fd;
const char * self_addr = skynet_command(ctx, "REG", NULL);
char tmp[128];
sprintf(tmp,"gate ! %s %d 0",local_addr,REMOTE_MAX);
const char * gate_addr = skynet_command(ctx, "LAUNCH", tmp);
if (gate_addr == NULL) {
skynet_error(ctx, "Harbor : launch gate failed");
return 1;
}
int n = sprintf(tmp,"broker %s",self_addr);
skynet_send(ctx, NULL, gate_addr, 0, tmp, n, 0);
skynet_send(ctx, NULL, gate_addr, 0, "start", 5, 0);
h->id = harbor_id;
skynet_callback(ctx, h, _mainloop);
_report_local_address(h, ctx, local_addr, harbor_id);
return 0;
}

View File

@@ -0,0 +1,272 @@
#include "skynet.h"
#include "skynet_harbor.h"
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <stdint.h>
#include <errno.h>
#define HASH_SIZE 4096
struct name {
struct name * next;
char key[GLOBALNAME_LENGTH];
uint32_t hash;
uint32_t value;
};
struct namemap {
struct name *node[HASH_SIZE];
};
struct master {
int remote_fd[REMOTE_MAX];
char * remote_addr[REMOTE_MAX];
struct namemap map;
};
struct master *
master_create() {
struct master *m = malloc(sizeof(*m));
int i;
for (i=0;i<REMOTE_MAX;i++) {
m->remote_fd[i] = -1;
m->remote_addr[i] = NULL;
}
memset(&m->map, 0, sizeof(m->map));
return m;
}
void
master_release(struct master * m) {
int i;
for (i=0;i<REMOTE_MAX;i++) {
int fd = m->remote_fd[i];
if (fd < 0) {
close(fd);
}
free(m->remote_addr[i]);
}
for (i=0;i<HASH_SIZE;i++) {
struct name ** ptr = &m->map.node[i];
while (*ptr) {
struct name * node = *ptr;
ptr = &node->next;
free(node);
}
}
free(m);
}
static struct name *
_search_name(struct master *m, char name[GLOBALNAME_LENGTH]) {
uint32_t *ptr = (uint32_t *) name;
uint32_t h = ptr[0] ^ ptr[1] ^ ptr[2] ^ ptr[3];
struct name * node = m->map.node[h % HASH_SIZE];
while (node) {
if (node->hash == h && strncmp(node->key, name, GLOBALNAME_LENGTH) == 0) {
return node;
}
node = node->next;
}
return NULL;
}
static struct name *
_insert_name(struct master *m, char name[GLOBALNAME_LENGTH]) {
uint32_t *ptr = (uint32_t *)name;
uint32_t h = ptr[0] ^ ptr[1] ^ ptr[2] ^ ptr[3];
struct name **pname = &m->map.node[h % HASH_SIZE];
struct name * node = malloc(sizeof(*node));
memcpy(node->key, name, GLOBALNAME_LENGTH);
node->next = *pname;
node->hash = h;
node->value = 0;
*pname = node;
return node;
}
static void
_copy_name(char *name, const char * buffer, size_t sz) {
if (sz < GLOBALNAME_LENGTH) {
memcpy(name, buffer, sz);
memset(name+sz, 0 , GLOBALNAME_LENGTH - sz);
} else {
memcpy(name, buffer, GLOBALNAME_LENGTH);
}
}
static int
_connect_to(const char *ipaddress) {
int fd = socket(AF_INET,SOCK_STREAM,0);
struct sockaddr_in my_addr;
char * port = strchr(ipaddress,':');
if (port==NULL) {
return -1;
}
int sz = port - ipaddress;
char tmp[sz + 1];
memcpy(tmp,ipaddress,sz);
tmp[sz] = '\0';
my_addr.sin_addr.s_addr=inet_addr(tmp);
my_addr.sin_family=AF_INET;
my_addr.sin_port=htons(strtol(port+1,NULL,10));
int r = connect(fd,(struct sockaddr *)&my_addr,sizeof(struct sockaddr_in));
if (r == -1) {
close(fd);
return -1;
}
return fd;
}
static int
_send_to(int fd, const void * buf, size_t sz, uint32_t handle) {
char buffer[2 + sz + 12];
uint16_t header = htons(sz+12);
memcpy(buffer, &header, 2);
memcpy(buffer+2, buf, sz);
uint32_t u32 = 0;
memcpy(buffer+2+sz,&u32,4);
u32 = htonl(handle);
memcpy(buffer+2+sz+4,&u32,4);
u32 = 0;
memcpy(buffer+2+sz+8,&u32,4);
sz += 2 + 12;
for (;;) {
int err = send(fd, buffer, sz, 0);
if (err < 0) {
switch (errno) {
case EAGAIN:
case EINTR:
continue;
}
}
if (err != sz) {
return 1;
}
return 0;
}
}
static void
_broadcast(struct skynet_context * context, struct master *m, const char *name, size_t sz, uint32_t handle) {
int i;
for (i=1;i<REMOTE_MAX;i++) {
int fd = m->remote_fd[i];
if (fd < 0)
continue;
int err = _send_to(fd, name, sz, handle);
if (err) {
close(fd);
fd = _connect_to(m->remote_addr[i]);
if (fd < 0) {
m->remote_fd[i] = -1;
skynet_error(context, "Reconnect to harbor %d : %s faild", i, m->remote_addr[i]);
}
}
}
}
static void
_request_name(struct skynet_context * context, struct master *m, const char * buffer, size_t sz) {
char name[GLOBALNAME_LENGTH];
_copy_name(name, buffer, sz);
struct name * n = _search_name(m, name);
if (n == NULL) {
return;
}
_broadcast(context, m, name, GLOBALNAME_LENGTH, n->value);
}
static void
_update_name(struct skynet_context * context, struct master *m, uint32_t handle, const char * buffer, size_t sz) {
char name[GLOBALNAME_LENGTH];
_copy_name(name, buffer, sz);
struct name * n = _search_name(m, name);
if (n==NULL) {
n = _insert_name(m,name);
}
n->value = handle;
_broadcast(context, m,name,GLOBALNAME_LENGTH, handle);
}
static void
_update_address(struct skynet_context * context, struct master *m, int harbor_id, const char * buffer, size_t sz) {
if (m->remote_fd[harbor_id] >= 0) {
close(m->remote_fd[harbor_id]);
m->remote_fd[harbor_id] = -1;
}
free(m->remote_addr[harbor_id]);
char * addr = malloc(sz+1);
memcpy(addr, buffer, sz);
addr[sz] = '\0';
m->remote_addr[harbor_id] = addr;
int fd = _connect_to(addr);
if (fd<0) {
skynet_error(context, "Can't connect to harbor %d : %s", harbor_id, addr);
return;
}
m->remote_fd[harbor_id] = fd;
_broadcast(context, m, addr, sz, harbor_id);
}
/*
update global name to master
4 bytes (handle) (handle == 0 for request)
n bytes string (name)
*/
static int
_mainloop(struct skynet_context * context, void * ud, int session, const char * addr, const void * msg, size_t sz) {
struct master *m = ud;
assert(session == SESSION_CLIENT);
uint32_t handle = 0;
memcpy(&handle, msg, 4);
handle = ntohl(handle);
sz -= 4;
const char * name = msg;
name += 4;
if (handle == 0) {
_request_name(context, m , name, sz);
} else if (handle < REMOTE_MAX) {
_update_address(context, m , handle, name, sz);
} else {
_update_name(context, m , handle, name, sz);
}
return 0;
}
int
master_init(struct master *m, struct skynet_context *ctx, const char * args) {
char tmp[strlen(args) + 32];
sprintf(tmp,"gate ! %s %d 0",args,REMOTE_MAX);
const char * self_addr = skynet_command(ctx, "REG", NULL);
const char * gate_addr = skynet_command(ctx, "LAUNCH", tmp);
if (gate_addr == NULL) {
skynet_error(ctx, "Master : launch gate failed");
return 1;
}
int n = sprintf(tmp,"broker %s",self_addr);
skynet_send(ctx, NULL, gate_addr, 0, tmp, n, 0);
skynet_send(ctx, NULL, gate_addr, 0, "start", 5, 0);
skynet_callback(ctx, m, _mainloop);
return 0;
}