//握手包 func handshake(ga *c.GroupAuth, remoteClient *c.RemotingClient) (bool, error) { for i := 0; i < 3; i++ { p := protocol.MarshalConnMeta(ga.GroupId, ga.SecretKey) rpacket := packet.NewPacket(protocol.CMD_CONN_META, p) resp, err := remoteClient.WriteAndGet(*rpacket, 5*time.Second) if nil != err { //两秒后重试 time.Sleep(2 * time.Second) log.WarnLog("kite_client", "kiteClient|handShake|FAIL|%s|%s\n", ga.GroupId, err) } else { authAck, ok := resp.(*protocol.ConnAuthAck) if !ok { return false, errors.New("Unmatches Handshake Ack Type! ") } else { if authAck.GetStatus() { log.InfoLog("kite_client", "kiteClient|handShake|SUCC|%s|%s\n", ga.GroupId, authAck.GetFeedback()) return true, nil } else { log.WarnLog("kite_client", "kiteClient|handShake|FAIL|%s|%s\n", ga.GroupId, authAck.GetFeedback()) return false, errors.New("Auth FAIL![" + authAck.GetFeedback() + "]") } } } } return false, errors.New("handshake fail! [" + remoteClient.RemoteAddr() + "]") }
//remove segment func (self *MessageStore) remove(s *Segment) { self.Lock() defer self.Unlock() //remove from segments for i, s := range self.segments { if s.sid == s.sid { self.segments = append(self.segments[0:i], self.segments[i+1:]...) break } } //remove from cache for e := self.segmentCache.Front(); nil != e; e = e.Next() { cs := e.Value.(*Segment) if cs.sid == s.sid { self.segmentCache.Remove(e) break } } //close segment s.Close() err := os.Remove(s.path) if nil != err { log.WarnLog("kite_store", "MessageStore|Remove|Segment|FAIL|%s|%s", err, s.path) } err = os.Remove(s.slog.path) if nil != err { log.WarnLog("kite_store", "MessageStore|Remove|SegmentLog|FAIL|%s|%s", err, s.slog.path) } log.InfoLog("kite_store", "MessageStore|Remove|Segment|%s", s.path) }
//订阅关系改变 func (self *BindExchanger) onBindChanged(topic, groupId string, newbinds []*Binding) { if len(groupId) <= 0 { delete(self.exchanger, topic) return } //不是当前服务可以处理的topic则直接丢地啊哦 if sort.SearchStrings(self.topics, topic) == len(self.topics) { log.WarnLog("kite_bind", "BindExchanger|onBindChanged|UnAccept Bindings|%s|%s|%s\n", topic, self.topics, newbinds) return } v, ok := self.exchanger[topic] if !ok { v = make(map[string][]*Binding, 10) self.exchanger[topic] = v } if len(newbinds) > 0 { v[groupId] = newbinds } else { delete(v, groupId) } }
func (self *AccessHandler) Process(ctx *DefaultPipelineContext, event IEvent) error { // log.Debug("accessEvent|Process|%s|%t\n", self.GetName(), event) aevent, ok := self.cast(event) if !ok { return ERROR_INVALID_EVENT_TYPE } //做权限校验............. if false { log.WarnLog("kite_handler", "accessEvent|Process|INVALID AUTH|%s|%s\n", aevent.groupId, aevent.secretKey) } // 权限验证通过 保存到clientmanager self.clientManager.Auth(client.NewGroupAuth(aevent.groupId, aevent.secretKey), aevent.remoteClient) // log.InfoLog("kite_handler", "accessEvent|Process|NEW CONNECTION|AUTH SUCC|%s|%s|%s\n", aevent.groupId, aevent.secretKey, aevent.remoteClient.RemoteAddr()) cmd := protocol.MarshalConnAuthAck(true, "授权成功") //响应包 packet := packet.NewRespPacket(aevent.opaque, protocol.CMD_CONN_AUTH, cmd) //向当前连接写入当前包 remoteEvent := NewRemotingEvent(packet, []string{aevent.remoteClient.RemoteAddr()}) //向后走网络传输 ctx.SendForward(remoteEvent) return nil }
//迁移过期的消息 func (self *KiteMysqlStore) migrateMessage(now int64, hashKey string) { log.InfoLog("kite_store", "KiteMysqlStore|MoveExpired|START|%s|%d", hashKey) //需要将过期的消息迁移到DLQ中 sql := self.sqlwrapper.hashDLQSQL(DLQ_MOVE_QUERY, hashKey) //获取到需要导入的id,然后导入 isql := self.sqlwrapper.hashDLQSQL(DLQ_MOVE_INSERT, hashKey) //删除已导入的数据 dsql := self.sqlwrapper.hashDLQSQL(DLQ_MOVE_DELETE, hashKey) start := 0 limit := 50 for { messageIds := make([]interface{}, 1, 50) err := func() error { rows, err := self.dbshard.FindSlave(hashKey).Query(sql, self.serverName, now, start, limit) if err != nil { log.ErrorLog("kite_store", "KiteMysqlStore|migrateMessage|Query|FAIL|%s|%s|%s", err, hashKey, sql) return err } defer rows.Close() for rows.Next() { var id int var messageId string err = rows.Scan(&id, &messageId) if nil != err { log.ErrorLog("kite_store", "KiteMysqlStore|MoveExpired|Scan|FAIL|%s|%s|%s", err, hashKey, sql) } else { start = id messageIds = append(messageIds, messageId) } } return nil }() //已经搬迁完毕则退出进行下一个 if nil != err || len(messageIds[1:]) <= 0 { log.WarnLog("kite_store", "KiteMysqlStore|MoveExpired|SUCC|%s|%d|%s", hashKey, start, err) break } in := strings.Repeat("?,", len(messageIds[1:])) in = in[:len(in)-1] isqlTmp := strings.Replace(isql, "{ids}", in, 1) _, err = self.dbshard.FindMaster(hashKey).Exec(isqlTmp, messageIds[1:]...) if err != nil { log.ErrorLog("kite_store", "KiteMysqlStore|MoveExpired|Insert|FAIL|%s|%s", err, isqlTmp, messageIds) break } dsqlTmp := strings.Replace(dsql, "{ids}", in, 1) messageIds[0] = self.serverName _, err = self.dbshard.FindMaster(hashKey).Exec(dsqlTmp, messageIds...) if err != nil { log.ErrorLog("kite_store", "KiteMysqlStore|MoveExpired|DELETE|FAIL|%s|%s|%s|%s", err, dsql, dsqlTmp, messageIds) break } } }
func (self *HeartbeatHandler) keepAlive() { for { select { case <-time.After(self.heartbeatPeriod): //心跳检测 func() { id := time.Now().Unix() clients := self.clientMangager.ClientsClone() p := protocol.MarshalHeartbeatPacket(id) for h, c := range clients { i := 0 //关闭的时候发起重连 if c.IsClosed() { i = 3 } else { //如果是空闲的则发起心跳 if c.Idle() { for ; i < 3; i++ { hp := packet.NewPacket(protocol.CMD_HEARTBEAT, p) err := c.Ping(hp, time.Duration(self.heartbeatTimeout)) //如果有错误则需要记录 if nil != err { log.WarnLog("kite_client_handler", "HeartbeatHandler|KeepAlive|FAIL|%s|local:%s|remote:%s|%d\n", err, c.LocalAddr(), h, id) continue } else { log.InfoLog("kite_client_handler", "HeartbeatHandler|KeepAlive|SUCC|local:%s|remote:%s|%d|%d ...\n", c.LocalAddr(), h, id, i) break } } } } if i >= 3 { //说明连接有问题需要重连 c.Shutdown() self.clientMangager.SubmitReconnect(c) log.WarnLog("kite_client_handler", "HeartbeatHandler|SubmitReconnect|%s\n", c.RemoteAddr()) } } }() } } }
//处理push func (self *ApnsHttpServer) handlePush(out http.ResponseWriter, req *http.Request) { resp := &response{} resp.Status = RESP_STATUS_SUCC if req.Method == "GET" { //返回不支持的请求方式 resp.Status = RESP_STATUS_INVALID_PROTO resp.Error = errors.New("Unsupport Get method Invoke!") } else if req.Method == "POST" { //pushType pushType := req.PostFormValue("pt") //先默认采用Enhanced方式 //接卸对应的token和payload token, payload := self.decodePayload(req, resp) trace := req.PostFormValue("trace") expiredSeconds := req.PostFormValue("expiredSeconds") expiredTime := self.expiredTime if len(expiredSeconds) > 0 { t, err := strconv.ParseInt(expiredSeconds, 10, 32) if nil == err { expiredTime = uint32(t) } } //----------------如果依然是成功状态则证明当前可以发送 if RESP_STATUS_SUCC == resp.Status { func() { defer func() { if re := recover(); nil != re { stack := re.(*errors.Error).ErrorStack() log.ErrorLog("push_handler", "ApnsHttpServer|handlePush|SEND|FAIL|%s|%s|%s", stack, payload, trace) resp.Status = RESP_STATUS_ERROR resp.Error = errors.New(fmt.Sprintf("%s", re)) self.write(out, resp) } }() self.innerSend(pushType, token, payload, resp, expiredTime) self.write(out, resp) log.DebugLog("push_handler", "ApnsHttpServer|handlePush|SUCC|%s|%s|%s", resp, payload, trace) }() } else { log.WarnLog("push_handler", "ApnsHttpServer|handlePush|FAIL|%s|%s|%s", resp, payload, trace) self.write(out, resp) } } }
//traverse oplog func (self *SegmentLog) Replay(do func(l *oplog)) { self.Open() offset := int64(0) tmp := make([]byte, 1024) //seek to head self.rf.Seek(0, 0) self.br.Reset(self.rf) for { var length int32 err := binary.Read(self.br, binary.BigEndian, &length) if nil != err { if err == io.EOF { self.br.Reset(self.rf) break } log.WarnLog("kite_store", "SegmentLog|Replay|LEN|%s|Skip...", err) continue } // log.Debug("SegmentLog|Replay|LEN|%d", length) if int(length) > cap(tmp) { grow := make([]byte, int(length)-cap(tmp)) tmp = append(tmp, grow...) } err = binary.Read(self.br, binary.BigEndian, tmp[:int(length)-4]) if nil != err { self.br.Reset(self.rf) log.ErrorLog("kite_store", "SegmentLog|Replay|Data|%s", err) break } var ol oplog r := bytes.NewReader(tmp[:int(length)-4]) deco := gob.NewDecoder(r) err = deco.Decode(&ol) if nil != err { log.ErrorLog("kite_store", "SegmentLog|Replay|unmarshal|oplog|FAIL|%s", err) continue } // log.Debug("SegmentLog|Replay|oplog|%s", ol) do(&ol) //line offset += int64(length) } self.offset = int64(offset) }
func (self *ApnsConnection) sendMessage(msg *entry.Message) error { //将当前的msg强制设置为当前conn的id作为标识 msg.ProcessId = self.connectionId err, packet := msg.Encode() if nil != err { return err } length, sendErr := self.conn.Write(packet) if nil != err || length != len(packet) { log.WarnLog("push_client", "CONNECTION|SEND MESSAGE|FAIL|%s", err) } else { log.DebugLog("push_client", "CONNECTION|SEND MESSAGE|SUCC") } return sendErr }
func (self *BindExchanger) DataChange(path string, binds []*Binding) { //订阅关系变更才处理 if strings.HasPrefix(path, KITEQ_SUB) { split := strings.Split(path, "/") //获取topic topic := split[3] groupId := strings.TrimSuffix(split[4], "-bind") self.lock.Lock() defer self.lock.Unlock() //开始处理变化的订阅关系 self.onBindChanged(topic, groupId, binds) } else { log.WarnLog("kite_bind", "BindExchanger|DataChange|非SUB节点变更|%s\n", path) } }
/* *重连zk */ func (self *ZKManager) reconnect() <-chan bool { ch := make(chan bool, 1) go func() { reconnTimes := int64(0) f := func() error { ss, eventChan, err := zk.Connect(strings.Split(self.zkhosts, ","), 5*time.Second) if nil != err { log.WarnLog("kite_bind", "连接zk失败.....%ds后重连任务重新发起...|", (reconnTimes+1)*5) return err } else { log.InfoLog("kite_bind", "重连ZK任务成功....") //初始化当前的状态 self.session = ss self.eventChan = eventChan ch <- true close(ch) return nil } } //启动重连任务 for !self.isClose { select { case <-time.After(time.Duration(reconnTimes * time.Second.Nanoseconds())): err := f() if nil != err { reconnTimes += 1 } else { //重连成功则推出 break } } } //失败 ch <- false close(ch) }() return ch }
func (self *KiteClientManager) NodeChange(path string, eventType binding.ZkEvent, children []string) { //如果是订阅关系变更则处理 if strings.HasPrefix(path, binding.KITEQ_SERVER) { //获取topic split := strings.Split(path, "/") if len(split) < 4 { //不合法的订阅璐姐 log.WarnLog("kite_client", "KiteClientManager|ChildWatcher|INVALID SERVER PATH |%s|%t\n", path, children) return } //获取topic topic := split[3] //search topic for _, t := range self.topics { if t == topic { self.onQServerChanged(topic, children) break } } } }
func (self *AcceptHandler) Process(ctx *pipe.DefaultPipelineContext, event pipe.IEvent) error { // log.Debug("AcceptHandler|Process|%s|%t\n", self.GetName(), event) ae, ok := self.cast(event) if !ok { return pipe.ERROR_INVALID_EVENT_TYPE } //这里处理一下ae,做一下校验 var msg *store.MessageEntity switch ae.msgType { case protocol.CMD_DELIVER_ACK: //收到投递结果直接attach响应 // log.DebugLog("kite_handler", "AcceptHandler|DELIVER_ACK|%s|%t", ae.opaque, ae.msg) ae.remoteClient.Attach(ae.opaque, ae.msg) return nil case protocol.CMD_HEARTBEAT: hb := ae.msg.(*protocol.HeartBeat) event = pipe.NewHeartbeatEvent(ae.remoteClient, ae.opaque, hb.GetVersion()) ctx.SendForward(event) return nil case protocol.CMD_BYTES_MESSAGE: msg = store.NewMessageEntity(protocol.NewQMessage(ae.msg.(*protocol.BytesMessage))) case protocol.CMD_STRING_MESSAGE: msg = store.NewMessageEntity(protocol.NewQMessage(ae.msg.(*protocol.StringMessage))) default: //这只是一个bug不支持的数据类型能给你 log.WarnLog("kite_handler", "AcceptHandler|Process|%s|%t", INVALID_MSG_TYPE_ERROR, ae.msg) } if nil != msg { msg.PublishTime = time.Now().Unix() msg.KiteServer = self.kiteserver deliver := newPersistentEvent(msg, ae.remoteClient, ae.opaque) ctx.SendForward(deliver) return nil } return INVALID_MSG_TYPE_ERROR }
//处理push func (self *ApnsHttpServer) handlePush(out http.ResponseWriter, req *http.Request) { resp := &response{} resp.Status = RESP_STATUS_SUCC if req.Method == "GET" { //返回不支持的请求方式 resp.Status = RESP_STATUS_INVALID_PROTO resp.Error = errors.New("Unsupport Get method Invoke!") } else if req.Method == "POST" { //pushType pushType := req.PostFormValue("pt") //先默认采用Enhanced方式 //接卸对应的token和payload token, payload := self.decodePayload(req, resp) trace := req.PostFormValue("trace") //----------------如果依然是成功状态则证明当前可以发送 if RESP_STATUS_SUCC == resp.Status { defer func() { if re := recover(); nil != re { log.ErrorLog("push_handler", "ApnsHttpServer|handlePush|SEND|FAIL|%s|%s|%s", re, payload, trace) resp.Status = RESP_STATUS_ERROR resp.Error = errors.New(fmt.Sprintf("%s", re)) self.write(out, resp) } }() self.innerSend(pushType, token, payload, resp) log.InfoLog("push_handler", "ApnsHttpServer|handlePush|SUCC|%s|%s|%s", resp, payload, trace) } else { log.WarnLog("push_handler", "ApnsHttpServer|handlePush|FAIL|%s|%s|%s", resp, payload, trace) } } self.write(out, resp) }
func (self *ApnsConnection) dial() error { config := tls.Config{} config.Certificates = []tls.Certificate{self.cert} config.InsecureSkipVerify = true conn, err := tls.Dial("tcp", self.hostport, &config) if nil != err { //connect fail log.WarnLog("push_client", "CONNECTION|%s|DIAL CONNECT|FAIL|%s|%s", self.name(), self.hostport, err.Error()) return err } // conn.SetDeadline(0 * time.Second) for { state := conn.ConnectionState() if state.HandshakeComplete { log.InfoLog("push_client", "CONNECTION|%s|HANDSHAKE SUCC", self.name()) break } time.Sleep(1 * time.Second) } self.conn = conn return nil }
func (self *ZKManager) Start() { if len(self.zkhosts) <= 0 { log.WarnLog("kite_bind", "使用默认zkhosts!|localhost:2181\n") self.zkhosts = "localhost:2181" } else { log.Info("使用zkhosts:[%s]!\n", self.zkhosts) } ss, eventChan, err := zk.Connect(strings.Split(self.zkhosts, ","), 5*time.Second) if nil != err { panic("连接zk失败..." + err.Error()) return } exist, _, err := ss.Exists(KITEQ) if nil != err { ss.Close() panic("无法创建KITEQ " + err.Error()) } if !exist { resp, err := ss.Create(KITEQ, nil, zk.CreatePersistent, zk.WorldACL(zk.PermAll)) if nil != err { ss.Close() panic("NewZKManager|CREATE ROOT PATH|FAIL|" + KITEQ + "|" + err.Error()) } else { log.InfoLog("kite_bind", "NewZKManager|CREATE ROOT PATH|SUCC|%s", resp) } } self.session = ss self.isClose = false self.eventChan = eventChan go self.listenEvent() }
func (self *ApnsClient) storeInvalidToken(token string) { //这里是里面最后存储不合法的token log.WarnLog("push_client", "APNSCLIENT|UnImplement StoreInvalidToken|%s\n", token) }
//监听数据变更 func (self *ZKManager) listenEvent() { for !self.isClose { //根据zk的文档 Watcher机制是无法保证可靠的,其次需要在每次处理完Watcher后要重新注册Watcher change := <-self.eventChan path := change.Path //开始检查符合的watcher watcher := func() IWatcher { for k, w := range self.wathcers { //以给定的 if strings.Index(path, k) >= 0 { return w } } return nil }() //如果没有wacher那么久忽略 if nil == watcher { log.WarnLog("kite_bind", "ZKManager|listenEvent|NO WATCHER|%s", path) continue } switch change.Type { case zk.EventSession: if change.State == zk.StateExpired { log.WarnLog("kite_bind", "ZKManager|OnSessionExpired!|Reconnect Zk ....") //阻塞等待重连任务成功 succ := <-self.reconnect() if !succ { log.WarnLog("kite_bind", "ZKManager|OnSessionExpired|Reconnect Zk|FAIL| ....") continue } //session失效必须通知所有的watcher func() { for _, w := range self.wathcers { //zk链接开则需要重新链接重新推送 w.OnSessionExpired() } }() } case zk.EventNodeDeleted: self.session.ExistsW(path) watcher.NodeChange(path, ZkEvent(change.Type), []string{}) // log.Info("ZKManager|listenEvent|%s|%s\n", path, change) case zk.EventNodeCreated, zk.EventNodeChildrenChanged: childnodes, _, _, err := self.session.ChildrenW(path) if nil != err { log.ErrorLog("kite_bind", "ZKManager|listenEvent|CD|%s|%s|%t\n", err, path, change.Type) } else { watcher.NodeChange(path, ZkEvent(change.Type), childnodes) // log.Info("ZKManager|listenEvent|%s|%s|%s\n", path, change, childnodes) } case zk.EventNodeDataChanged: split := strings.Split(path, "/") //如果不是bind级别的变更则忽略 if len(split) < 5 || strings.LastIndex(split[4], "-bind") <= 0 { continue } //获取一下数据 binds, err := self.getBindData(path) if nil != err { log.ErrorLog("kite_bind", "ZKManager|listenEvent|Changed|Get DATA|FAIL|%s|%s\n", err, path) //忽略 continue } watcher.DataChange(path, binds) // log.Info("ZKManager|listenEvent|%s|%s|%s\n", path, change, binds) } } }