func (server *Server) InitWebsocket() (err error) {
var (
bind string
listener *net.TCPListener
addr *net.TCPAddr
httpServeMux = http.NewServeMux()
)
httpServeMux.Handle("/subscribe", websocket.Handler(server.subscribeWS))
for _, bind = range server.Config.WebSocket.Bind {
if addr, err = net.ResolveTCPAddr("tcp4", bind); err != nil {
log.Errorf("net.ResolveTCPAddr(tcp4, %s) failed : %v", bind, err)
return
}
if listener, err = net.ListenTCP("tcp4", addr); err != nil {
log.Errorf("net.ListenTCP(tcp4, %s) failed : %v", bind, err)
return
}
httpserver := &http.Server{Handler: httpServeMux}
log.Infof("comet server start websocket listen: %s", bind)
go func() {
if err = httpserver.Serve(listener); err != nil {
log.Errorf("httpserver.Serve(%s) failed : %v", bind, err)
panic(err)
}
}()
}
return
}
// if comet crash : call http://admin/server?id=xx as DELETE method, and restart comet
// keep rpc available
func (r *AdminRpc) Keepalive(quit chan struct{}, network, address string) {
if r.Client == nil {
log.Fatalf("keepalive rpc with admin : rpc.Client is nil")
panic(ErrAdminRpc)
}
var (
err error
call *rpc.Call
done = make(chan *rpc.Call, 1)
args = proto.NoREQ{}
reply = proto.PingRES{}
firstPingTime int64
)
for {
select {
case <-quit:
return
default:
if r.Client != nil {
call = <-r.Client.Go(adminRpcPing, &args, &reply, done).Done
if call.Error != nil {
log.Errorf("adminRpc ping %s failed : %v", address, call.Error)
} else {
if firstPingTime == 0 {
firstPingTime = reply.TimeUnix
}
// if firstPingTime difference : admin is crash. lost all user infomation
if firstPingTime != reply.TimeUnix {
// @TODO client maybe disconnect while doing ReSubAll,
// so, comet need to send all online users to admin later?
err := CometServer.ReSubAll()
if err == nil {
firstPingTime = reply.TimeUnix
}
}
}
}
if r.Client == nil || (call != nil && call.Error == rpc.ErrShutdown) {
//if isDebug {
// log.Debugf("rpc.Dial (%s@%s) failed : %v", network, address, err)
//}
if r.Client, err = rpc.Dial(network, address); err == nil {
} else {
log.Errorf("adminRpc dial error : %v", err)
}
}
}
// @TODO : configuration heartbeat between comet and admin
time.Sleep(5 * time.Second)
}
}
func (c *CometRpc) MPush(msg *proto.PushMMsgREQ) {
var (
reply = &proto.NoRES{}
cc *CometRpcClient
err error
)
cc, err = c.getCometByServerId(msg.ServerId)
if err != nil || cc == nil || cc.Client == nil {
log.Errorf("getCometByServerId(%d) failed : %v", msg.ServerId, err)
return
}
if err = cc.Client.Call(CometServiceMPushMsg, msg, reply); err != nil {
log.Errorf("c.Call(%s, %v, reply) failed : %v", CometServiceMPushMsg, msg, err)
}
}
func (c *CometRpc) processPushMsg(ch chan *proto.MQMsg) {
for {
msg := <-ch
switch msg.OP {
case define.MQ_MESSAGE:
if len(msg.ServerId) == 1 {
pmsg := &proto.PushSMsgREQ{msg.ServerId[0], msg.Msg}
c.SPush(pmsg)
}
case define.MQ_MESSAGE_MULTI:
if len(msg.ServerId) == 1 {
pmsg := &proto.PushMMsgREQ{msg.ServerId[0], msg.UserId, msg.Msg}
c.MPush(pmsg)
}
case define.MQ_MESSAGE_BROADCAST:
pmsg := &proto.PushBroadcastREQ{msg.Msg}
c.broadcast(pmsg)
case define.MQ_MESSAGE_BROADCAST_TOPIC:
pmsg := &proto.PushBroadcastTopicREQ{msg.Topic, msg.ServerId, msg.Msg}
c.broadcastTopic(pmsg)
default:
log.Errorf("unknown operation:%s", msg.OP)
}
}
}
func (server *Server) InitTlsWebsocket(addrs []string, cert, priv string) (err error) {
var (
httpServeMux = http.NewServeMux()
)
httpServeMux.Handle("/subscribe", websocket.Handler(server.subscribeWS))
config := &tls.Config{}
config.Certificates = make([]tls.Certificate, 1)
if config.Certificates[0], err = tls.LoadX509KeyPair(cert, priv); err != nil {
return
}
for _, bind := range addrs {
server := &http.Server{Addr: bind, Handler: httpServeMux}
server.SetKeepAlivesEnabled(true)
log.Infof("comet server start websocket TLS listen: %s", bind)
go func() {
ln, err := net.Listen("tcp", bind)
if err != nil {
return
}
tlsListener := tls.NewListener(ln, config)
if err = server.Serve(tlsListener); err != nil {
log.Errorf("server.Serve(%s) failed : %v", bind, err)
return
}
}()
}
return
}
func reload() {
err := ReloadConfig()
if err != nil {
log.Errorf("ReloadConfig failed : %v", err)
return
}
}
func (r *AdminRpcs) UnSubscribe(a *proto.UnSubREQ) (err error) {
var (
uid string
n string
)
uid = strconv.FormatInt(a.UserId, 10)
n, err = r.CHash.Get(uid)
if err != nil {
return
}
c, ok := r.adminRpc[n]
if !ok {
err = ErrInternal
return
}
var has bool
if has, err = c.UnSubscribe(a); err != nil {
log.Errorf("AdminRpc.UnSubscribe user(%d) to server(%d) error : %v", a.UserId, a.Server, err)
return
}
if !has {
log.Warnf("AdminRpc.UnSubscribe user: %v not exists", a.UserId)
}
return
}
// push
// does invalidate msg, if it is invalid, ignore it
func (c *CometRpc) PushMQ(msg []byte) (err error) {
m := &proto.MQMsg{}
if err = ffjson.Unmarshal(msg, m); err != nil {
log.Errorf("json.Unmarshal(%s) error(%s)", msg, err)
return
}
c.push(m)
return
}
// broadcast broadcast a message to all
func (c *CometRpc) broadcast(msg *proto.PushBroadcastREQ) {
for serverId, cc := range c.Clients {
if cc != nil {
go c.broadcastComet(cc, msg)
} else {
log.Errorf("broadcast to server(%d) failed : rpc.client is nil", serverId)
}
}
}
// broadcast topic
func (c *CometRpc) broadcastTopic(msg *proto.PushBroadcastTopicREQ) {
var (
reply = &proto.NoRES{}
cc *CometRpcClient
err error
)
for _, server := range msg.ServerId {
go func(server int32) {
cc, err = c.getCometByServerId(server)
if err != nil || cc == nil || cc.Client == nil {
log.Errorf("getCometByServerId(%d) failed : %v", msg.ServerId, err)
return
}
if err = cc.Client.Call(CometServiceBroadcastTopic, msg, &reply); err != nil {
log.Errorf("c.Call(%s, %v, reply) failed : %v", CometServiceBroadcastTopic, msg, err)
}
}(server)
}
}
// broadcastComet a message to specified comet
func (c *CometRpc) broadcastComet(cc *CometRpcClient, msg *proto.PushBroadcastREQ) (err error) {
var reply = proto.NoRES{}
if cc == nil || cc.Client == nil {
log.Errorln("rpc client is nil")
return
}
if err = cc.Client.Call(CometServiceBroadcast, msg, &reply); err != nil {
log.Errorf("c.Call(%s, %v, reply) failed : %v", CometServiceBroadcast, msg, err)
}
return
}
func (r *PushRpc) Keepalive(quit chan struct{}, network, address string) {
//if r.Client == nil {
// log.Errorf("keepalive rpc with push : rpc.Client is nil")
// //panic(ErrPushRpc)
//}
var (
err error
call *rpc.Call
done = make(chan *rpc.Call, 1)
args = proto.NoREQ{}
reply = proto.PingRES{}
//firstPingTime int64
)
for {
select {
case <-quit:
return
default:
if r.Client != nil {
call = <-r.Client.Go(pushRpcPing, &args, &reply, done).Done
if call.Error != nil {
log.Errorf("pushRpc ping %s failed : %v", address, call.Error)
}
}
if (call != nil && call.Error == rpc.ErrShutdown) || r.Client == nil {
//if isDebug {
// log.Debugf("rpc.Dial (%s@%s) failed : %v", network, address, err)
//}
if r.Client, err = rpc.Dial(network, address); err == nil {
// @TODO Dial other pushRpc server : but must to avoid avalanche effect
} else {
log.Errorf("pushRpc dial error : %v", err)
}
}
}
// @TODO : configuration heartbeat between comet and push
time.Sleep(5 * time.Second)
}
}
// unsubscribe to admin
func (r *AdminRpc) UnSubscribe(arg *proto.UnSubREQ) (has bool, err error) {
if r.Client == nil {
err = ErrAdminRpc
return
}
reply := &proto.UnSubRES{}
if err = r.Client.Call(adminRpcUnSub, arg, reply); err != nil {
log.Errorf("adminRpc.Call(%s, %v) failed : %v", adminRpcUnSub, arg, err)
return
}
has = reply.Has
return
}
func (r *PushRpc) Init(conf ConfigRpcPush) (err error) {
network := conf.Network
addr := conf.Addr
r.Client, err = rpc.Dial(network, addr)
if err != nil || r.Client == nil {
log.Errorf("pushRpc.Dial(%s@%s) failed : %s", network, addr, err)
//panic(err)
}
r.rpcQuit = make(chan struct{}, 1)
go r.Keepalive(r.rpcQuit, network, addr)
log.Infof("Init push Rpc (%s@%s) sub successfuls", network, addr)
return
}
// keep alive with comet servers
func (c *CometRpc) Keepalive(cometId int32, quit chan struct{}, network, address string) {
var (
client *CometRpcClient
ok bool
err error
call *rpc.Call
done = make(chan *rpc.Call, 1)
args = proto.NoREQ{}
reply = proto.NoRES{}
)
if client, ok = c.Clients[cometId]; !ok {
log.Errorf("keepalive : rpc connect to comet(id : %d) failed", cometId)
panic(ErrConnectComet)
}
for {
select {
case <-quit:
return
default:
if client.Client != nil {
call = <-client.Client.Go(CometServicePing, &args, &reply, done).Done
if call.Error != nil {
log.Errorf("rpc ping %s error(%v)", address, call.Error)
}
}
if client.Client == nil || (call != nil && call.Error == rpc.ErrShutdown) {
if client.Client, err = rpc.Dial(network, address); err == nil {
// @TODO Dial other adminRpc server
}
}
}
time.Sleep(5 * time.Second)
}
}
func rpcListen(network, addr string) {
l, err := net.Listen(network, addr)
if err != nil {
log.Fatalf("Rpc listen(%s, %s) failed(%v)", network, addr, err)
panic(err)
}
log.Infof("Rpc listen(%s@%s) successful", network, addr)
defer func() {
log.Infof("net.Listen(%s, %s) closed", network, addr)
if err := l.Close(); err != nil {
log.Errorf("Rpc Close() failed(%v)", err)
}
}()
rpc.Accept(l)
}
func (r *PushRpcs) Init() (err error) {
r.Config = Conf.Server.RpcPush
r.pushRpc = make([]*PushRpc, 0)
for _, node := range r.Config {
rpc := NewPushRpc()
if err = rpc.Init(node); err != nil {
log.Errorf("init push rpc failed : %v", err)
panic(err)
} else {
r.pushRpc = append(r.pushRpc, rpc)
}
}
// if len(r.pushRpc) == 0 {
// log.Errorln(r.Config)
// return ErrPushRpc
// }
return nil
}
// subscribe to admin
func (r *AdminRpc) Subscribe(arg *proto.SubREQ) (err error) {
if r.Client == nil {
err = ErrAdminRpc
return
}
// @TODO memory pool
reply := &proto.SubRES{}
if err = r.Client.Call(adminRpcSub, arg, reply); err != nil {
if isDebug {
log.Errorf("adminRpc.Call(%s, %v) failed : %v", adminRpcSub, arg, err)
}
return
}
if !reply.Pass {
err = ErrAuthFailed
return
}
return
}
func (server *Server) subscribeWS(conn *websocket.Conn) {
var (
err error
userid int64
p *proto.Proto
b *Bucket
trd *itime.TimerData
ch = NewChannel(server.Config.Bucket.Channel.RingBufferSize, server.Config.Bucket.Channel.PushBufferSize)
topics []int32
tr = server.round.Timer(rand.Int())
token string
)
//
if p, err = ch.ProtoRing.Set(); err == nil {
if userid, topics, token, err = server.wsSubscribe(conn, p); err == nil {
b = server.Bucket(userid)
if tch := b.Channel(userid); tch != nil {
conn.Close()
return
}
err = b.Put(userid, ch)
} else {
log.Errorf("wsSubscribe failed user(%d) failed : %v", userid, err)
}
}
if err != nil {
conn.Close()
log.Errorf("handshake failed : %v", err)
return
}
//
trd = tr.Add(time.Duration(server.Config.HeartBeatTimeout), func() {
conn.Close()
})
ch.SetTopics(topics)
ch.SetToken(token)
trd.Key = userid
tr.Set(trd, time.Duration(server.Config.HeartBeatTimeout))
go server.processWriteWebsocket(userid, conn, ch)
for {
if p, err = ch.ProtoRing.Set(); err != nil {
break
}
if err = p.ReadWebsocket(conn); err != nil {
break
}
if p.OpCode == define.OP_HEARTBEAT_REQ {
// @TODO heartbeat timeout need to close connection
tr.Set(trd, time.Duration(server.Config.HeartBeatTimeout))
p.Body = nil
p.OpCode = define.OP_HEARTBEAT_RES
}
ch.ProtoRing.SetAdv()
ch.Ready()
}
conn.Close()
ch.Close()
b.DelSafe(userid, ch)
disArg := &proto.UnSubREQ{userid, server.Config.Id}
if err = server.UnSubscribe(disArg); err != nil {
log.Errorf("UnSubscribe user(%d) failed : %v", userid, err)
}
if isDebug {
//log.Debugf("userid: %d server websocket goroutine exit", userid)
}
return
}
func initWebsocket(userid int64, topics []int32) {
defer func() {
if err := recover(); err != nil {
log.Errorf("%v\n", err)
}
}()
var (
err error
)
origin := "http://" + Conf.Protocol.Addr + "/subscribe"
url := "ws://" + Conf.Protocol.Addr + "/subscribe"
conn, err := websocket.Dial(url, "", origin)
//log.Debugf("dial : %v", url)
if err != nil {
log.Errorf("websocket.Dial(\"%s\") error(%v)", Conf.Protocol.Addr, err)
return
}
proto := new(Proto)
proto.Ver = 1
proto.Operation = OP_SUB_REQ
msgId := int64(0)
proto.MsgId = msgId
topicstring := ""
for i, t := range topics {
if i != 0 {
topicstring = fmt.Sprintf("%s,%d", topicstring, t)
} else {
topicstring = fmt.Sprintf("%d", t)
}
}
bodystr := fmt.Sprintf(`{"userid":%v, "token":"%v", "topics":[%v]}`, userid, userid, topicstring)
proto.Body = []byte(bodystr)
//log.Debugf("body : %s\n", string(proto.Body))
if err = websocketWriteProto(conn, proto); err != nil {
log.Errorf("websocketWriteProto() error(%v)", err)
return
}
if err = websocketReadProto(conn, proto); err != nil {
log.Errorf("websocketReadProto() error(%v)", err)
return
}
//log.Debugf("auth ok, proto: %v", proto)
msgId++
// writer
go func() {
proto1 := new(Proto)
for {
// heartbeat
proto1.Operation = OP_HEARTBEAT_REQ
proto1.MsgId = msgId
proto1.Body = nil
if err = websocketWriteProto(conn, proto1); err != nil {
log.Errorf("tcpWriteProto() error(%v)", err)
return
}
msgId++
time.Sleep(100 * time.Second)
}
}()
// reader
st := time.Now()
et := time.Now()
for {
if err = websocketReadProto(conn, proto); err != nil {
//if errCount % 100 == 0 {
log.Errorf("wsReadProto(%d) error(%v)", errCount, err)
//}
atomic.AddInt64(&errCount, 1)
return
}
if proto.Operation == OP_HEARTBEAT_RES {
if err = conn.SetReadDeadline(time.Now().Add(10 * time.Hour)); err != nil {
log.Errorf("conn.SetReadDeadline() error(%v)", err)
return
}
} else {
if msgCount%10000 == 0 {
et = time.Now()
log.Debugf("-->%d----->%v", msgCount, et.Sub(st))
st = time.Now()
}
atomic.AddInt64(&msgCount, 1)
}
}
}