Beispiel #1
0
func (server *Server) InitWebsocket() (err error) {
	var (
		bind         string
		listener     *net.TCPListener
		addr         *net.TCPAddr
		httpServeMux = http.NewServeMux()
	)
	httpServeMux.Handle("/subscribe", websocket.Handler(server.subscribeWS))
	for _, bind = range server.Config.WebSocket.Bind {
		if addr, err = net.ResolveTCPAddr("tcp4", bind); err != nil {
			log.Errorf("net.ResolveTCPAddr(tcp4, %s) failed : %v", bind, err)
			return
		}
		if listener, err = net.ListenTCP("tcp4", addr); err != nil {
			log.Errorf("net.ListenTCP(tcp4, %s) failed : %v", bind, err)
			return
		}
		httpserver := &http.Server{Handler: httpServeMux}
		log.Infof("comet server start websocket listen: %s", bind)

		go func() {
			if err = httpserver.Serve(listener); err != nil {
				log.Errorf("httpserver.Serve(%s) failed : %v", bind, err)
				panic(err)
			}
		}()
	}
	return
}
Beispiel #2
0
// if comet crash : call http://admin/server?id=xx as DELETE method, and restart comet
// keep rpc available
func (r *AdminRpc) Keepalive(quit chan struct{}, network, address string) {
	if r.Client == nil {
		log.Fatalf("keepalive rpc with admin : rpc.Client is nil")
		panic(ErrAdminRpc)
	}
	var (
		err           error
		call          *rpc.Call
		done          = make(chan *rpc.Call, 1)
		args          = proto.NoREQ{}
		reply         = proto.PingRES{}
		firstPingTime int64
	)
	for {
		select {
		case <-quit:
			return
		default:
			if r.Client != nil {
				call = <-r.Client.Go(adminRpcPing, &args, &reply, done).Done
				if call.Error != nil {
					log.Errorf("adminRpc ping %s failed : %v", address, call.Error)
				} else {
					if firstPingTime == 0 {
						firstPingTime = reply.TimeUnix
					}
					// if firstPingTime difference : admin is crash. lost all user infomation
					if firstPingTime != reply.TimeUnix {
						// @TODO  client maybe disconnect while doing ReSubAll,
						// so, comet need to send all online users to admin later?
						err := CometServer.ReSubAll()
						if err == nil {
							firstPingTime = reply.TimeUnix
						}
					}
				}
			}

			if r.Client == nil || (call != nil && call.Error == rpc.ErrShutdown) {
				//if isDebug {
				//	log.Debugf("rpc.Dial (%s@%s) failed : %v", network, address, err)
				//}
				if r.Client, err = rpc.Dial(network, address); err == nil {
				} else {
					log.Errorf("adminRpc dial error : %v", err)
				}
			}
		}
		// @TODO : configuration heartbeat between comet and admin
		time.Sleep(5 * time.Second)
	}
}
Beispiel #3
0
func (c *CometRpc) MPush(msg *proto.PushMMsgREQ) {
	var (
		reply = &proto.NoRES{}
		cc    *CometRpcClient
		err   error
	)
	cc, err = c.getCometByServerId(msg.ServerId)
	if err != nil || cc == nil || cc.Client == nil {
		log.Errorf("getCometByServerId(%d) failed : %v", msg.ServerId, err)
		return
	}
	if err = cc.Client.Call(CometServiceMPushMsg, msg, reply); err != nil {
		log.Errorf("c.Call(%s, %v, reply) failed : %v", CometServiceMPushMsg, msg, err)
	}
}
Beispiel #4
0
func (c *CometRpc) processPushMsg(ch chan *proto.MQMsg) {

	for {
		msg := <-ch
		switch msg.OP {
		case define.MQ_MESSAGE:
			if len(msg.ServerId) == 1 {
				pmsg := &proto.PushSMsgREQ{msg.ServerId[0], msg.Msg}
				c.SPush(pmsg)
			}

		case define.MQ_MESSAGE_MULTI:
			if len(msg.ServerId) == 1 {
				pmsg := &proto.PushMMsgREQ{msg.ServerId[0], msg.UserId, msg.Msg}
				c.MPush(pmsg)
			}

		case define.MQ_MESSAGE_BROADCAST:
			pmsg := &proto.PushBroadcastREQ{msg.Msg}
			c.broadcast(pmsg)

		case define.MQ_MESSAGE_BROADCAST_TOPIC:
			pmsg := &proto.PushBroadcastTopicREQ{msg.Topic, msg.ServerId, msg.Msg}
			c.broadcastTopic(pmsg)
		default:
			log.Errorf("unknown operation:%s", msg.OP)
		}
	}
}
Beispiel #5
0
func (server *Server) InitTlsWebsocket(addrs []string, cert, priv string) (err error) {
	var (
		httpServeMux = http.NewServeMux()
	)
	httpServeMux.Handle("/subscribe", websocket.Handler(server.subscribeWS))
	config := &tls.Config{}
	config.Certificates = make([]tls.Certificate, 1)
	if config.Certificates[0], err = tls.LoadX509KeyPair(cert, priv); err != nil {
		return
	}
	for _, bind := range addrs {
		server := &http.Server{Addr: bind, Handler: httpServeMux}
		server.SetKeepAlivesEnabled(true)
		log.Infof("comet server start websocket TLS listen: %s", bind)

		go func() {
			ln, err := net.Listen("tcp", bind)
			if err != nil {
				return
			}

			tlsListener := tls.NewListener(ln, config)
			if err = server.Serve(tlsListener); err != nil {
				log.Errorf("server.Serve(%s) failed : %v", bind, err)
				return
			}
		}()
	}
	return
}
Beispiel #6
0
func reload() {
	err := ReloadConfig()
	if err != nil {
		log.Errorf("ReloadConfig failed : %v", err)
		return
	}
}
Beispiel #7
0
func (r *AdminRpcs) UnSubscribe(a *proto.UnSubREQ) (err error) {
	var (
		uid string
		n   string
	)
	uid = strconv.FormatInt(a.UserId, 10)
	n, err = r.CHash.Get(uid)
	if err != nil {
		return
	}
	c, ok := r.adminRpc[n]
	if !ok {
		err = ErrInternal
		return
	}

	var has bool
	if has, err = c.UnSubscribe(a); err != nil {
		log.Errorf("AdminRpc.UnSubscribe user(%d) to server(%d) error : %v", a.UserId, a.Server, err)
		return
	}
	if !has {
		log.Warnf("AdminRpc.UnSubscribe user: %v not exists", a.UserId)
	}
	return
}
Beispiel #8
0
//  push
// does invalidate msg, if it is invalid, ignore it
func (c *CometRpc) PushMQ(msg []byte) (err error) {
	m := &proto.MQMsg{}
	if err = ffjson.Unmarshal(msg, m); err != nil {
		log.Errorf("json.Unmarshal(%s) error(%s)", msg, err)
		return
	}
	c.push(m)
	return
}
Beispiel #9
0
// broadcast broadcast a message to all
func (c *CometRpc) broadcast(msg *proto.PushBroadcastREQ) {

	for serverId, cc := range c.Clients {
		if cc != nil {
			go c.broadcastComet(cc, msg)
		} else {
			log.Errorf("broadcast to server(%d) failed : rpc.client is nil", serverId)
		}
	}
}
Beispiel #10
0
// broadcast topic
func (c *CometRpc) broadcastTopic(msg *proto.PushBroadcastTopicREQ) {
	var (
		reply = &proto.NoRES{}
		cc    *CometRpcClient
		err   error
	)

	for _, server := range msg.ServerId {
		go func(server int32) {
			cc, err = c.getCometByServerId(server)
			if err != nil || cc == nil || cc.Client == nil {
				log.Errorf("getCometByServerId(%d) failed : %v", msg.ServerId, err)
				return
			}
			if err = cc.Client.Call(CometServiceBroadcastTopic, msg, &reply); err != nil {
				log.Errorf("c.Call(%s, %v, reply) failed : %v", CometServiceBroadcastTopic, msg, err)
			}
		}(server)
	}
}
Beispiel #11
0
// broadcastComet a message to specified comet
func (c *CometRpc) broadcastComet(cc *CometRpcClient, msg *proto.PushBroadcastREQ) (err error) {
	var reply = proto.NoRES{}
	if cc == nil || cc.Client == nil {
		log.Errorln("rpc client is nil")
		return
	}
	if err = cc.Client.Call(CometServiceBroadcast, msg, &reply); err != nil {
		log.Errorf("c.Call(%s, %v, reply) failed : %v", CometServiceBroadcast, msg, err)
	}
	return
}
Beispiel #12
0
func (r *PushRpc) Keepalive(quit chan struct{}, network, address string) {
	//if r.Client == nil {
	//	log.Errorf("keepalive rpc with push : rpc.Client is nil")
	//	//panic(ErrPushRpc)
	//}
	var (
		err   error
		call  *rpc.Call
		done  = make(chan *rpc.Call, 1)
		args  = proto.NoREQ{}
		reply = proto.PingRES{}
		//firstPingTime int64
	)
	for {
		select {
		case <-quit:
			return
		default:
			if r.Client != nil {
				call = <-r.Client.Go(pushRpcPing, &args, &reply, done).Done
				if call.Error != nil {
					log.Errorf("pushRpc ping %s failed : %v", address, call.Error)
				}
			}

			if (call != nil && call.Error == rpc.ErrShutdown) || r.Client == nil {
				//if isDebug {
				//	log.Debugf("rpc.Dial (%s@%s) failed : %v", network, address, err)
				//}
				if r.Client, err = rpc.Dial(network, address); err == nil {
					// @TODO Dial other pushRpc server : but must to avoid avalanche effect
				} else {
					log.Errorf("pushRpc dial error : %v", err)
				}
			}
		}
		// @TODO : configuration heartbeat between comet and push
		time.Sleep(5 * time.Second)
	}
}
Beispiel #13
0
// unsubscribe to admin
func (r *AdminRpc) UnSubscribe(arg *proto.UnSubREQ) (has bool, err error) {
	if r.Client == nil {
		err = ErrAdminRpc
		return
	}
	reply := &proto.UnSubRES{}
	if err = r.Client.Call(adminRpcUnSub, arg, reply); err != nil {
		log.Errorf("adminRpc.Call(%s, %v) failed : %v", adminRpcUnSub, arg, err)
		return
	}
	has = reply.Has
	return
}
Beispiel #14
0
func (r *PushRpc) Init(conf ConfigRpcPush) (err error) {

	network := conf.Network
	addr := conf.Addr

	r.Client, err = rpc.Dial(network, addr)
	if err != nil || r.Client == nil {
		log.Errorf("pushRpc.Dial(%s@%s) failed : %s", network, addr, err)
		//panic(err)
	}
	r.rpcQuit = make(chan struct{}, 1)
	go r.Keepalive(r.rpcQuit, network, addr)
	log.Infof("Init push Rpc (%s@%s) sub successfuls", network, addr)
	return
}
Beispiel #15
0
// keep alive with comet servers
func (c *CometRpc) Keepalive(cometId int32, quit chan struct{}, network, address string) {
	var (
		client *CometRpcClient
		ok     bool
		err    error
		call   *rpc.Call
		done   = make(chan *rpc.Call, 1)
		args   = proto.NoREQ{}
		reply  = proto.NoRES{}
	)
	if client, ok = c.Clients[cometId]; !ok {
		log.Errorf("keepalive : rpc connect to comet(id : %d) failed", cometId)
		panic(ErrConnectComet)
	}

	for {
		select {
		case <-quit:
			return
		default:
			if client.Client != nil {
				call = <-client.Client.Go(CometServicePing, &args, &reply, done).Done
				if call.Error != nil {
					log.Errorf("rpc ping %s error(%v)", address, call.Error)
				}
			}

			if client.Client == nil || (call != nil && call.Error == rpc.ErrShutdown) {
				if client.Client, err = rpc.Dial(network, address); err == nil {
					// @TODO Dial other adminRpc server
				}
			}
		}
		time.Sleep(5 * time.Second)
	}
}
Beispiel #16
0
func rpcListen(network, addr string) {
	l, err := net.Listen(network, addr)
	if err != nil {
		log.Fatalf("Rpc listen(%s, %s) failed(%v)", network, addr, err)
		panic(err)
	}
	log.Infof("Rpc listen(%s@%s) successful", network, addr)
	defer func() {
		log.Infof("net.Listen(%s, %s)  closed", network, addr)
		if err := l.Close(); err != nil {
			log.Errorf("Rpc Close() failed(%v)", err)
		}
	}()

	rpc.Accept(l)
}
Beispiel #17
0
func (r *PushRpcs) Init() (err error) {
	r.Config = Conf.Server.RpcPush
	r.pushRpc = make([]*PushRpc, 0)
	for _, node := range r.Config {
		rpc := NewPushRpc()
		if err = rpc.Init(node); err != nil {
			log.Errorf("init push rpc failed : %v", err)
			panic(err)
		} else {
			r.pushRpc = append(r.pushRpc, rpc)
		}
	}
	// if len(r.pushRpc) == 0 {
	// 	log.Errorln(r.Config)
	// 	return ErrPushRpc
	// }
	return nil
}
Beispiel #18
0
// subscribe to admin
func (r *AdminRpc) Subscribe(arg *proto.SubREQ) (err error) {
	if r.Client == nil {
		err = ErrAdminRpc
		return
	}

	// @TODO memory pool
	reply := &proto.SubRES{}
	if err = r.Client.Call(adminRpcSub, arg, reply); err != nil {
		if isDebug {
			log.Errorf("adminRpc.Call(%s, %v) failed : %v", adminRpcSub, arg, err)
		}
		return
	}
	if !reply.Pass {
		err = ErrAuthFailed
		return
	}

	return
}
Beispiel #19
0
func (server *Server) subscribeWS(conn *websocket.Conn) {
	var (
		err    error
		userid int64
		p      *proto.Proto
		b      *Bucket
		trd    *itime.TimerData
		ch     = NewChannel(server.Config.Bucket.Channel.RingBufferSize, server.Config.Bucket.Channel.PushBufferSize)
		topics []int32
		tr     = server.round.Timer(rand.Int())
		token  string
	)

	//
	if p, err = ch.ProtoRing.Set(); err == nil {
		if userid, topics, token, err = server.wsSubscribe(conn, p); err == nil {
			b = server.Bucket(userid)
			if tch := b.Channel(userid); tch != nil {
				conn.Close()
				return
			}
			err = b.Put(userid, ch)
		} else {
			log.Errorf("wsSubscribe failed user(%d) failed : %v", userid, err)
		}
	}
	if err != nil {
		conn.Close()
		log.Errorf("handshake failed : %v", err)
		return
	}

	//
	trd = tr.Add(time.Duration(server.Config.HeartBeatTimeout), func() {
		conn.Close()
	})

	ch.SetTopics(topics)
	ch.SetToken(token)
	trd.Key = userid
	tr.Set(trd, time.Duration(server.Config.HeartBeatTimeout))

	go server.processWriteWebsocket(userid, conn, ch)
	for {
		if p, err = ch.ProtoRing.Set(); err != nil {
			break
		}
		if err = p.ReadWebsocket(conn); err != nil {
			break
		}

		if p.OpCode == define.OP_HEARTBEAT_REQ {
			// @TODO heartbeat timeout need to close connection
			tr.Set(trd, time.Duration(server.Config.HeartBeatTimeout))
			p.Body = nil
			p.OpCode = define.OP_HEARTBEAT_RES
		}
		ch.ProtoRing.SetAdv()
		ch.Ready()
	}
	conn.Close()
	ch.Close()
	b.DelSafe(userid, ch)

	disArg := &proto.UnSubREQ{userid, server.Config.Id}
	if err = server.UnSubscribe(disArg); err != nil {
		log.Errorf("UnSubscribe user(%d)  failed : %v", userid, err)
	}
	if isDebug {
		//log.Debugf("userid: %d server websocket goroutine exit", userid)
	}
	return
}
Beispiel #20
0
func initWebsocket(userid int64, topics []int32) {
	defer func() {
		if err := recover(); err != nil {
			log.Errorf("%v\n", err)
		}
	}()
	var (
		err error
	)
	origin := "http://" + Conf.Protocol.Addr + "/subscribe"
	url := "ws://" + Conf.Protocol.Addr + "/subscribe"
	conn, err := websocket.Dial(url, "", origin)
	//log.Debugf("dial : %v", url)

	if err != nil {
		log.Errorf("websocket.Dial(\"%s\") error(%v)", Conf.Protocol.Addr, err)
		return
	}

	proto := new(Proto)
	proto.Ver = 1

	proto.Operation = OP_SUB_REQ
	msgId := int64(0)
	proto.MsgId = msgId

	topicstring := ""
	for i, t := range topics {
		if i != 0 {
			topicstring = fmt.Sprintf("%s,%d", topicstring, t)
		} else {
			topicstring = fmt.Sprintf("%d", t)
		}

	}
	bodystr := fmt.Sprintf(`{"userid":%v, "token":"%v", "topics":[%v]}`, userid, userid, topicstring)
	proto.Body = []byte(bodystr)

	//log.Debugf("body : %s\n", string(proto.Body))

	if err = websocketWriteProto(conn, proto); err != nil {
		log.Errorf("websocketWriteProto() error(%v)", err)
		return
	}
	if err = websocketReadProto(conn, proto); err != nil {
		log.Errorf("websocketReadProto() error(%v)", err)
		return
	}
	//log.Debugf("auth ok, proto: %v", proto)
	msgId++
	// writer
	go func() {
		proto1 := new(Proto)
		for {
			// heartbeat
			proto1.Operation = OP_HEARTBEAT_REQ
			proto1.MsgId = msgId
			proto1.Body = nil
			if err = websocketWriteProto(conn, proto1); err != nil {
				log.Errorf("tcpWriteProto() error(%v)", err)
				return
			}
			msgId++
			time.Sleep(100 * time.Second)
		}
	}()
	// reader
	st := time.Now()
	et := time.Now()
	for {
		if err = websocketReadProto(conn, proto); err != nil {
			//if errCount % 100 == 0 {
			log.Errorf("wsReadProto(%d) error(%v)", errCount, err)
			//}
			atomic.AddInt64(&errCount, 1)
			return
		}

		if proto.Operation == OP_HEARTBEAT_RES {
			if err = conn.SetReadDeadline(time.Now().Add(10 * time.Hour)); err != nil {
				log.Errorf("conn.SetReadDeadline() error(%v)", err)
				return
			}
		} else {
			if msgCount%10000 == 0 {
				et = time.Now()
				log.Debugf("-->%d----->%v", msgCount, et.Sub(st))
				st = time.Now()

			}
			atomic.AddInt64(&msgCount, 1)
		}
	}
}