func (server *Server) InitWebsocket() (err error) { var ( bind string listener *net.TCPListener addr *net.TCPAddr httpServeMux = http.NewServeMux() ) httpServeMux.Handle("/subscribe", websocket.Handler(server.subscribeWS)) for _, bind = range server.Config.WebSocket.Bind { if addr, err = net.ResolveTCPAddr("tcp4", bind); err != nil { log.Errorf("net.ResolveTCPAddr(tcp4, %s) failed : %v", bind, err) return } if listener, err = net.ListenTCP("tcp4", addr); err != nil { log.Errorf("net.ListenTCP(tcp4, %s) failed : %v", bind, err) return } httpserver := &http.Server{Handler: httpServeMux} log.Infof("comet server start websocket listen: %s", bind) go func() { if err = httpserver.Serve(listener); err != nil { log.Errorf("httpserver.Serve(%s) failed : %v", bind, err) panic(err) } }() } return }
// if comet crash : call http://admin/server?id=xx as DELETE method, and restart comet // keep rpc available func (r *AdminRpc) Keepalive(quit chan struct{}, network, address string) { if r.Client == nil { log.Fatalf("keepalive rpc with admin : rpc.Client is nil") panic(ErrAdminRpc) } var ( err error call *rpc.Call done = make(chan *rpc.Call, 1) args = proto.NoREQ{} reply = proto.PingRES{} firstPingTime int64 ) for { select { case <-quit: return default: if r.Client != nil { call = <-r.Client.Go(adminRpcPing, &args, &reply, done).Done if call.Error != nil { log.Errorf("adminRpc ping %s failed : %v", address, call.Error) } else { if firstPingTime == 0 { firstPingTime = reply.TimeUnix } // if firstPingTime difference : admin is crash. lost all user infomation if firstPingTime != reply.TimeUnix { // @TODO client maybe disconnect while doing ReSubAll, // so, comet need to send all online users to admin later? err := CometServer.ReSubAll() if err == nil { firstPingTime = reply.TimeUnix } } } } if r.Client == nil || (call != nil && call.Error == rpc.ErrShutdown) { //if isDebug { // log.Debugf("rpc.Dial (%s@%s) failed : %v", network, address, err) //} if r.Client, err = rpc.Dial(network, address); err == nil { } else { log.Errorf("adminRpc dial error : %v", err) } } } // @TODO : configuration heartbeat between comet and admin time.Sleep(5 * time.Second) } }
func (c *CometRpc) MPush(msg *proto.PushMMsgREQ) { var ( reply = &proto.NoRES{} cc *CometRpcClient err error ) cc, err = c.getCometByServerId(msg.ServerId) if err != nil || cc == nil || cc.Client == nil { log.Errorf("getCometByServerId(%d) failed : %v", msg.ServerId, err) return } if err = cc.Client.Call(CometServiceMPushMsg, msg, reply); err != nil { log.Errorf("c.Call(%s, %v, reply) failed : %v", CometServiceMPushMsg, msg, err) } }
func (c *CometRpc) processPushMsg(ch chan *proto.MQMsg) { for { msg := <-ch switch msg.OP { case define.MQ_MESSAGE: if len(msg.ServerId) == 1 { pmsg := &proto.PushSMsgREQ{msg.ServerId[0], msg.Msg} c.SPush(pmsg) } case define.MQ_MESSAGE_MULTI: if len(msg.ServerId) == 1 { pmsg := &proto.PushMMsgREQ{msg.ServerId[0], msg.UserId, msg.Msg} c.MPush(pmsg) } case define.MQ_MESSAGE_BROADCAST: pmsg := &proto.PushBroadcastREQ{msg.Msg} c.broadcast(pmsg) case define.MQ_MESSAGE_BROADCAST_TOPIC: pmsg := &proto.PushBroadcastTopicREQ{msg.Topic, msg.ServerId, msg.Msg} c.broadcastTopic(pmsg) default: log.Errorf("unknown operation:%s", msg.OP) } } }
func (server *Server) InitTlsWebsocket(addrs []string, cert, priv string) (err error) { var ( httpServeMux = http.NewServeMux() ) httpServeMux.Handle("/subscribe", websocket.Handler(server.subscribeWS)) config := &tls.Config{} config.Certificates = make([]tls.Certificate, 1) if config.Certificates[0], err = tls.LoadX509KeyPair(cert, priv); err != nil { return } for _, bind := range addrs { server := &http.Server{Addr: bind, Handler: httpServeMux} server.SetKeepAlivesEnabled(true) log.Infof("comet server start websocket TLS listen: %s", bind) go func() { ln, err := net.Listen("tcp", bind) if err != nil { return } tlsListener := tls.NewListener(ln, config) if err = server.Serve(tlsListener); err != nil { log.Errorf("server.Serve(%s) failed : %v", bind, err) return } }() } return }
func reload() { err := ReloadConfig() if err != nil { log.Errorf("ReloadConfig failed : %v", err) return } }
func (r *AdminRpcs) UnSubscribe(a *proto.UnSubREQ) (err error) { var ( uid string n string ) uid = strconv.FormatInt(a.UserId, 10) n, err = r.CHash.Get(uid) if err != nil { return } c, ok := r.adminRpc[n] if !ok { err = ErrInternal return } var has bool if has, err = c.UnSubscribe(a); err != nil { log.Errorf("AdminRpc.UnSubscribe user(%d) to server(%d) error : %v", a.UserId, a.Server, err) return } if !has { log.Warnf("AdminRpc.UnSubscribe user: %v not exists", a.UserId) } return }
// push // does invalidate msg, if it is invalid, ignore it func (c *CometRpc) PushMQ(msg []byte) (err error) { m := &proto.MQMsg{} if err = ffjson.Unmarshal(msg, m); err != nil { log.Errorf("json.Unmarshal(%s) error(%s)", msg, err) return } c.push(m) return }
// broadcast broadcast a message to all func (c *CometRpc) broadcast(msg *proto.PushBroadcastREQ) { for serverId, cc := range c.Clients { if cc != nil { go c.broadcastComet(cc, msg) } else { log.Errorf("broadcast to server(%d) failed : rpc.client is nil", serverId) } } }
// broadcast topic func (c *CometRpc) broadcastTopic(msg *proto.PushBroadcastTopicREQ) { var ( reply = &proto.NoRES{} cc *CometRpcClient err error ) for _, server := range msg.ServerId { go func(server int32) { cc, err = c.getCometByServerId(server) if err != nil || cc == nil || cc.Client == nil { log.Errorf("getCometByServerId(%d) failed : %v", msg.ServerId, err) return } if err = cc.Client.Call(CometServiceBroadcastTopic, msg, &reply); err != nil { log.Errorf("c.Call(%s, %v, reply) failed : %v", CometServiceBroadcastTopic, msg, err) } }(server) } }
// broadcastComet a message to specified comet func (c *CometRpc) broadcastComet(cc *CometRpcClient, msg *proto.PushBroadcastREQ) (err error) { var reply = proto.NoRES{} if cc == nil || cc.Client == nil { log.Errorln("rpc client is nil") return } if err = cc.Client.Call(CometServiceBroadcast, msg, &reply); err != nil { log.Errorf("c.Call(%s, %v, reply) failed : %v", CometServiceBroadcast, msg, err) } return }
func (r *PushRpc) Keepalive(quit chan struct{}, network, address string) { //if r.Client == nil { // log.Errorf("keepalive rpc with push : rpc.Client is nil") // //panic(ErrPushRpc) //} var ( err error call *rpc.Call done = make(chan *rpc.Call, 1) args = proto.NoREQ{} reply = proto.PingRES{} //firstPingTime int64 ) for { select { case <-quit: return default: if r.Client != nil { call = <-r.Client.Go(pushRpcPing, &args, &reply, done).Done if call.Error != nil { log.Errorf("pushRpc ping %s failed : %v", address, call.Error) } } if (call != nil && call.Error == rpc.ErrShutdown) || r.Client == nil { //if isDebug { // log.Debugf("rpc.Dial (%s@%s) failed : %v", network, address, err) //} if r.Client, err = rpc.Dial(network, address); err == nil { // @TODO Dial other pushRpc server : but must to avoid avalanche effect } else { log.Errorf("pushRpc dial error : %v", err) } } } // @TODO : configuration heartbeat between comet and push time.Sleep(5 * time.Second) } }
// unsubscribe to admin func (r *AdminRpc) UnSubscribe(arg *proto.UnSubREQ) (has bool, err error) { if r.Client == nil { err = ErrAdminRpc return } reply := &proto.UnSubRES{} if err = r.Client.Call(adminRpcUnSub, arg, reply); err != nil { log.Errorf("adminRpc.Call(%s, %v) failed : %v", adminRpcUnSub, arg, err) return } has = reply.Has return }
func (r *PushRpc) Init(conf ConfigRpcPush) (err error) { network := conf.Network addr := conf.Addr r.Client, err = rpc.Dial(network, addr) if err != nil || r.Client == nil { log.Errorf("pushRpc.Dial(%s@%s) failed : %s", network, addr, err) //panic(err) } r.rpcQuit = make(chan struct{}, 1) go r.Keepalive(r.rpcQuit, network, addr) log.Infof("Init push Rpc (%s@%s) sub successfuls", network, addr) return }
// keep alive with comet servers func (c *CometRpc) Keepalive(cometId int32, quit chan struct{}, network, address string) { var ( client *CometRpcClient ok bool err error call *rpc.Call done = make(chan *rpc.Call, 1) args = proto.NoREQ{} reply = proto.NoRES{} ) if client, ok = c.Clients[cometId]; !ok { log.Errorf("keepalive : rpc connect to comet(id : %d) failed", cometId) panic(ErrConnectComet) } for { select { case <-quit: return default: if client.Client != nil { call = <-client.Client.Go(CometServicePing, &args, &reply, done).Done if call.Error != nil { log.Errorf("rpc ping %s error(%v)", address, call.Error) } } if client.Client == nil || (call != nil && call.Error == rpc.ErrShutdown) { if client.Client, err = rpc.Dial(network, address); err == nil { // @TODO Dial other adminRpc server } } } time.Sleep(5 * time.Second) } }
func rpcListen(network, addr string) { l, err := net.Listen(network, addr) if err != nil { log.Fatalf("Rpc listen(%s, %s) failed(%v)", network, addr, err) panic(err) } log.Infof("Rpc listen(%s@%s) successful", network, addr) defer func() { log.Infof("net.Listen(%s, %s) closed", network, addr) if err := l.Close(); err != nil { log.Errorf("Rpc Close() failed(%v)", err) } }() rpc.Accept(l) }
func (r *PushRpcs) Init() (err error) { r.Config = Conf.Server.RpcPush r.pushRpc = make([]*PushRpc, 0) for _, node := range r.Config { rpc := NewPushRpc() if err = rpc.Init(node); err != nil { log.Errorf("init push rpc failed : %v", err) panic(err) } else { r.pushRpc = append(r.pushRpc, rpc) } } // if len(r.pushRpc) == 0 { // log.Errorln(r.Config) // return ErrPushRpc // } return nil }
// subscribe to admin func (r *AdminRpc) Subscribe(arg *proto.SubREQ) (err error) { if r.Client == nil { err = ErrAdminRpc return } // @TODO memory pool reply := &proto.SubRES{} if err = r.Client.Call(adminRpcSub, arg, reply); err != nil { if isDebug { log.Errorf("adminRpc.Call(%s, %v) failed : %v", adminRpcSub, arg, err) } return } if !reply.Pass { err = ErrAuthFailed return } return }
func (server *Server) subscribeWS(conn *websocket.Conn) { var ( err error userid int64 p *proto.Proto b *Bucket trd *itime.TimerData ch = NewChannel(server.Config.Bucket.Channel.RingBufferSize, server.Config.Bucket.Channel.PushBufferSize) topics []int32 tr = server.round.Timer(rand.Int()) token string ) // if p, err = ch.ProtoRing.Set(); err == nil { if userid, topics, token, err = server.wsSubscribe(conn, p); err == nil { b = server.Bucket(userid) if tch := b.Channel(userid); tch != nil { conn.Close() return } err = b.Put(userid, ch) } else { log.Errorf("wsSubscribe failed user(%d) failed : %v", userid, err) } } if err != nil { conn.Close() log.Errorf("handshake failed : %v", err) return } // trd = tr.Add(time.Duration(server.Config.HeartBeatTimeout), func() { conn.Close() }) ch.SetTopics(topics) ch.SetToken(token) trd.Key = userid tr.Set(trd, time.Duration(server.Config.HeartBeatTimeout)) go server.processWriteWebsocket(userid, conn, ch) for { if p, err = ch.ProtoRing.Set(); err != nil { break } if err = p.ReadWebsocket(conn); err != nil { break } if p.OpCode == define.OP_HEARTBEAT_REQ { // @TODO heartbeat timeout need to close connection tr.Set(trd, time.Duration(server.Config.HeartBeatTimeout)) p.Body = nil p.OpCode = define.OP_HEARTBEAT_RES } ch.ProtoRing.SetAdv() ch.Ready() } conn.Close() ch.Close() b.DelSafe(userid, ch) disArg := &proto.UnSubREQ{userid, server.Config.Id} if err = server.UnSubscribe(disArg); err != nil { log.Errorf("UnSubscribe user(%d) failed : %v", userid, err) } if isDebug { //log.Debugf("userid: %d server websocket goroutine exit", userid) } return }
func initWebsocket(userid int64, topics []int32) { defer func() { if err := recover(); err != nil { log.Errorf("%v\n", err) } }() var ( err error ) origin := "http://" + Conf.Protocol.Addr + "/subscribe" url := "ws://" + Conf.Protocol.Addr + "/subscribe" conn, err := websocket.Dial(url, "", origin) //log.Debugf("dial : %v", url) if err != nil { log.Errorf("websocket.Dial(\"%s\") error(%v)", Conf.Protocol.Addr, err) return } proto := new(Proto) proto.Ver = 1 proto.Operation = OP_SUB_REQ msgId := int64(0) proto.MsgId = msgId topicstring := "" for i, t := range topics { if i != 0 { topicstring = fmt.Sprintf("%s,%d", topicstring, t) } else { topicstring = fmt.Sprintf("%d", t) } } bodystr := fmt.Sprintf(`{"userid":%v, "token":"%v", "topics":[%v]}`, userid, userid, topicstring) proto.Body = []byte(bodystr) //log.Debugf("body : %s\n", string(proto.Body)) if err = websocketWriteProto(conn, proto); err != nil { log.Errorf("websocketWriteProto() error(%v)", err) return } if err = websocketReadProto(conn, proto); err != nil { log.Errorf("websocketReadProto() error(%v)", err) return } //log.Debugf("auth ok, proto: %v", proto) msgId++ // writer go func() { proto1 := new(Proto) for { // heartbeat proto1.Operation = OP_HEARTBEAT_REQ proto1.MsgId = msgId proto1.Body = nil if err = websocketWriteProto(conn, proto1); err != nil { log.Errorf("tcpWriteProto() error(%v)", err) return } msgId++ time.Sleep(100 * time.Second) } }() // reader st := time.Now() et := time.Now() for { if err = websocketReadProto(conn, proto); err != nil { //if errCount % 100 == 0 { log.Errorf("wsReadProto(%d) error(%v)", errCount, err) //} atomic.AddInt64(&errCount, 1) return } if proto.Operation == OP_HEARTBEAT_RES { if err = conn.SetReadDeadline(time.Now().Add(10 * time.Hour)); err != nil { log.Errorf("conn.SetReadDeadline() error(%v)", err) return } } else { if msgCount%10000 == 0 { et = time.Now() log.Debugf("-->%d----->%v", msgCount, et.Sub(st)) st = time.Now() } atomic.AddInt64(&msgCount, 1) } } }