//握手包
func handshake(ga *c.GroupAuth, remoteClient *c.RemotingClient) (bool, error) {
for i := 0; i < 3; i++ {
p := protocol.MarshalConnMeta(ga.GroupId, ga.SecretKey)
rpacket := packet.NewPacket(protocol.CMD_CONN_META, p)
resp, err := remoteClient.WriteAndGet(*rpacket, 5*time.Second)
if nil != err {
//两秒后重试
time.Sleep(2 * time.Second)
log.WarnLog("kite_client", "kiteClient|handShake|FAIL|%s|%s\n", ga.GroupId, err)
} else {
authAck, ok := resp.(*protocol.ConnAuthAck)
if !ok {
return false, errors.New("Unmatches Handshake Ack Type! ")
} else {
if authAck.GetStatus() {
log.InfoLog("kite_client", "kiteClient|handShake|SUCC|%s|%s\n", ga.GroupId, authAck.GetFeedback())
return true, nil
} else {
log.WarnLog("kite_client", "kiteClient|handShake|FAIL|%s|%s\n", ga.GroupId, authAck.GetFeedback())
return false, errors.New("Auth FAIL![" + authAck.GetFeedback() + "]")
}
}
}
}
return false, errors.New("handshake fail! [" + remoteClient.RemoteAddr() + "]")
}
//remove segment
func (self *MessageStore) remove(s *Segment) {
self.Lock()
defer self.Unlock()
//remove from segments
for i, s := range self.segments {
if s.sid == s.sid {
self.segments = append(self.segments[0:i], self.segments[i+1:]...)
break
}
}
//remove from cache
for e := self.segmentCache.Front(); nil != e; e = e.Next() {
cs := e.Value.(*Segment)
if cs.sid == s.sid {
self.segmentCache.Remove(e)
break
}
}
//close segment
s.Close()
err := os.Remove(s.path)
if nil != err {
log.WarnLog("kite_store", "MessageStore|Remove|Segment|FAIL|%s|%s", err, s.path)
}
err = os.Remove(s.slog.path)
if nil != err {
log.WarnLog("kite_store", "MessageStore|Remove|SegmentLog|FAIL|%s|%s", err, s.slog.path)
}
log.InfoLog("kite_store", "MessageStore|Remove|Segment|%s", s.path)
}
//订阅关系改变
func (self *BindExchanger) onBindChanged(topic, groupId string, newbinds []*Binding) {
if len(groupId) <= 0 {
delete(self.exchanger, topic)
return
}
//不是当前服务可以处理的topic则直接丢地啊哦
if sort.SearchStrings(self.topics, topic) == len(self.topics) {
log.WarnLog("kite_bind", "BindExchanger|onBindChanged|UnAccept Bindings|%s|%s|%s\n", topic, self.topics, newbinds)
return
}
v, ok := self.exchanger[topic]
if !ok {
v = make(map[string][]*Binding, 10)
self.exchanger[topic] = v
}
if len(newbinds) > 0 {
v[groupId] = newbinds
} else {
delete(v, groupId)
}
}
func (self *AccessHandler) Process(ctx *DefaultPipelineContext, event IEvent) error {
// log.Debug("accessEvent|Process|%s|%t\n", self.GetName(), event)
aevent, ok := self.cast(event)
if !ok {
return ERROR_INVALID_EVENT_TYPE
}
//做权限校验.............
if false {
log.WarnLog("kite_handler", "accessEvent|Process|INVALID AUTH|%s|%s\n", aevent.groupId, aevent.secretKey)
}
// 权限验证通过 保存到clientmanager
self.clientManager.Auth(client.NewGroupAuth(aevent.groupId, aevent.secretKey), aevent.remoteClient)
// log.InfoLog("kite_handler", "accessEvent|Process|NEW CONNECTION|AUTH SUCC|%s|%s|%s\n", aevent.groupId, aevent.secretKey, aevent.remoteClient.RemoteAddr())
cmd := protocol.MarshalConnAuthAck(true, "授权成功")
//响应包
packet := packet.NewRespPacket(aevent.opaque, protocol.CMD_CONN_AUTH, cmd)
//向当前连接写入当前包
remoteEvent := NewRemotingEvent(packet, []string{aevent.remoteClient.RemoteAddr()})
//向后走网络传输
ctx.SendForward(remoteEvent)
return nil
}
//迁移过期的消息
func (self *KiteMysqlStore) migrateMessage(now int64, hashKey string) {
log.InfoLog("kite_store", "KiteMysqlStore|MoveExpired|START|%s|%d", hashKey)
//需要将过期的消息迁移到DLQ中
sql := self.sqlwrapper.hashDLQSQL(DLQ_MOVE_QUERY, hashKey)
//获取到需要导入的id，然后导入
isql := self.sqlwrapper.hashDLQSQL(DLQ_MOVE_INSERT, hashKey)
//删除已导入的数据
dsql := self.sqlwrapper.hashDLQSQL(DLQ_MOVE_DELETE, hashKey)
start := 0
limit := 50
for {
messageIds := make([]interface{}, 1, 50)
err := func() error {
rows, err := self.dbshard.FindSlave(hashKey).Query(sql, self.serverName, now, start, limit)
if err != nil {
log.ErrorLog("kite_store", "KiteMysqlStore|migrateMessage|Query|FAIL|%s|%s|%s", err, hashKey, sql)
return err
}
defer rows.Close()
for rows.Next() {
var id int
var messageId string
err = rows.Scan(&id, &messageId)
if nil != err {
log.ErrorLog("kite_store", "KiteMysqlStore|MoveExpired|Scan|FAIL|%s|%s|%s", err, hashKey, sql)
} else {
start = id
messageIds = append(messageIds, messageId)
}
}
return nil
}()
//已经搬迁完毕则退出进行下一个
if nil != err || len(messageIds[1:]) <= 0 {
log.WarnLog("kite_store", "KiteMysqlStore|MoveExpired|SUCC|%s|%d|%s", hashKey, start, err)
break
}
in := strings.Repeat("?,", len(messageIds[1:]))
in = in[:len(in)-1]
isqlTmp := strings.Replace(isql, "{ids}", in, 1)
_, err = self.dbshard.FindMaster(hashKey).Exec(isqlTmp, messageIds[1:]...)
if err != nil {
log.ErrorLog("kite_store", "KiteMysqlStore|MoveExpired|Insert|FAIL|%s|%s", err, isqlTmp, messageIds)
break
}
dsqlTmp := strings.Replace(dsql, "{ids}", in, 1)
messageIds[0] = self.serverName
_, err = self.dbshard.FindMaster(hashKey).Exec(dsqlTmp, messageIds...)
if err != nil {
log.ErrorLog("kite_store", "KiteMysqlStore|MoveExpired|DELETE|FAIL|%s|%s|%s|%s", err, dsql, dsqlTmp, messageIds)
break
}
}
}
func (self *HeartbeatHandler) keepAlive() {
for {
select {
case <-time.After(self.heartbeatPeriod):
//心跳检测
func() {
id := time.Now().Unix()
clients := self.clientMangager.ClientsClone()
p := protocol.MarshalHeartbeatPacket(id)
for h, c := range clients {
i := 0
//关闭的时候发起重连
if c.IsClosed() {
i = 3
} else {
//如果是空闲的则发起心跳
if c.Idle() {
for ; i < 3; i++ {
hp := packet.NewPacket(protocol.CMD_HEARTBEAT, p)
err := c.Ping(hp, time.Duration(self.heartbeatTimeout))
//如果有错误则需要记录
if nil != err {
log.WarnLog("kite_client_handler", "HeartbeatHandler|KeepAlive|FAIL|%s|local:%s|remote:%s|%d\n", err, c.LocalAddr(), h, id)
continue
} else {
log.InfoLog("kite_client_handler", "HeartbeatHandler|KeepAlive|SUCC|local:%s|remote:%s|%d|%d ...\n", c.LocalAddr(), h, id, i)
break
}
}
}
}
if i >= 3 {
//说明连接有问题需要重连
c.Shutdown()
self.clientMangager.SubmitReconnect(c)
log.WarnLog("kite_client_handler", "HeartbeatHandler|SubmitReconnect|%s\n", c.RemoteAddr())
}
}
}()
}
}
}
//处理push
func (self *ApnsHttpServer) handlePush(out http.ResponseWriter, req *http.Request) {
resp := &response{}
resp.Status = RESP_STATUS_SUCC
if req.Method == "GET" {
//返回不支持的请求方式
resp.Status = RESP_STATUS_INVALID_PROTO
resp.Error = errors.New("Unsupport Get method Invoke!")
} else if req.Method == "POST" {
//pushType
pushType := req.PostFormValue("pt") //先默认采用Enhanced方式
//接卸对应的token和payload
token, payload := self.decodePayload(req, resp)
trace := req.PostFormValue("trace")
expiredSeconds := req.PostFormValue("expiredSeconds")
expiredTime := self.expiredTime
if len(expiredSeconds) > 0 {
t, err := strconv.ParseInt(expiredSeconds, 10, 32)
if nil == err {
expiredTime = uint32(t)
}
}
//----------------如果依然是成功状态则证明当前可以发送
if RESP_STATUS_SUCC == resp.Status {
func() {
defer func() {
if re := recover(); nil != re {
stack := re.(*errors.Error).ErrorStack()
log.ErrorLog("push_handler", "ApnsHttpServer|handlePush|SEND|FAIL|%s|%s|%s", stack, payload, trace)
resp.Status = RESP_STATUS_ERROR
resp.Error = errors.New(fmt.Sprintf("%s", re))
self.write(out, resp)
}
}()
self.innerSend(pushType, token, payload, resp, expiredTime)
self.write(out, resp)
log.DebugLog("push_handler", "ApnsHttpServer|handlePush|SUCC|%s|%s|%s", resp, payload, trace)
}()
} else {
log.WarnLog("push_handler", "ApnsHttpServer|handlePush|FAIL|%s|%s|%s", resp, payload, trace)
self.write(out, resp)
}
}
}
//traverse oplog
func (self *SegmentLog) Replay(do func(l *oplog)) {
self.Open()
offset := int64(0)
tmp := make([]byte, 1024)
//seek to head
self.rf.Seek(0, 0)
self.br.Reset(self.rf)
for {
var length int32
err := binary.Read(self.br, binary.BigEndian, &length)
if nil != err {
if err == io.EOF {
self.br.Reset(self.rf)
break
}
log.WarnLog("kite_store", "SegmentLog|Replay|LEN|%s|Skip...", err)
continue
}
// log.Debug("SegmentLog|Replay|LEN|%d", length)
if int(length) > cap(tmp) {
grow := make([]byte, int(length)-cap(tmp))
tmp = append(tmp, grow...)
}
err = binary.Read(self.br, binary.BigEndian, tmp[:int(length)-4])
if nil != err {
self.br.Reset(self.rf)
log.ErrorLog("kite_store", "SegmentLog|Replay|Data|%s", err)
break
}
var ol oplog
r := bytes.NewReader(tmp[:int(length)-4])
deco := gob.NewDecoder(r)
err = deco.Decode(&ol)
if nil != err {
log.ErrorLog("kite_store", "SegmentLog|Replay|unmarshal|oplog|FAIL|%s", err)
continue
}
// log.Debug("SegmentLog|Replay|oplog|%s", ol)
do(&ol)
//line
offset += int64(length)
}
self.offset = int64(offset)
}
func (self *ApnsConnection) sendMessage(msg *entry.Message) error {
//将当前的msg强制设置为当前conn的id作为标识
msg.ProcessId = self.connectionId
err, packet := msg.Encode()
if nil != err {
return err
}
length, sendErr := self.conn.Write(packet)
if nil != err || length != len(packet) {
log.WarnLog("push_client", "CONNECTION|SEND MESSAGE|FAIL|%s", err)
} else {
log.DebugLog("push_client", "CONNECTION|SEND MESSAGE|SUCC")
}
return sendErr
}
func (self *BindExchanger) DataChange(path string, binds []*Binding) {
//订阅关系变更才处理
if strings.HasPrefix(path, KITEQ_SUB) {
split := strings.Split(path, "/")
//获取topic
topic := split[3]
groupId := strings.TrimSuffix(split[4], "-bind")
self.lock.Lock()
defer self.lock.Unlock()
//开始处理变化的订阅关系
self.onBindChanged(topic, groupId, binds)
} else {
log.WarnLog("kite_bind", "BindExchanger|DataChange|非SUB节点变更|%s\n", path)
}
}
/*
*重连zk
*/
func (self *ZKManager) reconnect() <-chan bool {
ch := make(chan bool, 1)
go func() {
reconnTimes := int64(0)
f := func() error {
ss, eventChan, err := zk.Connect(strings.Split(self.zkhosts, ","), 5*time.Second)
if nil != err {
log.WarnLog("kite_bind", "连接zk失败.....%ds后重连任务重新发起...|", (reconnTimes+1)*5)
return err
} else {
log.InfoLog("kite_bind", "重连ZK任务成功....")
//初始化当前的状态
self.session = ss
self.eventChan = eventChan
ch <- true
close(ch)
return nil
}
}
//启动重连任务
for !self.isClose {
select {
case <-time.After(time.Duration(reconnTimes * time.Second.Nanoseconds())):
err := f()
if nil != err {
reconnTimes += 1
} else {
//重连成功则推出
break
}
}
}
//失败
ch <- false
close(ch)
}()
return ch
}
func (self *KiteClientManager) NodeChange(path string, eventType binding.ZkEvent, children []string) {
//如果是订阅关系变更则处理
if strings.HasPrefix(path, binding.KITEQ_SERVER) {
//获取topic
split := strings.Split(path, "/")
if len(split) < 4 {
//不合法的订阅璐姐
log.WarnLog("kite_client", "KiteClientManager|ChildWatcher|INVALID SERVER PATH |%s|%t\n", path, children)
return
}
//获取topic
topic := split[3]
//search topic
for _, t := range self.topics {
if t == topic {
self.onQServerChanged(topic, children)
break
}
}
}
}
func (self *AcceptHandler) Process(ctx *pipe.DefaultPipelineContext, event pipe.IEvent) error {
// log.Debug("AcceptHandler|Process|%s|%t\n", self.GetName(), event)
ae, ok := self.cast(event)
if !ok {
return pipe.ERROR_INVALID_EVENT_TYPE
}
//这里处理一下ae,做一下校验
var msg *store.MessageEntity
switch ae.msgType {
case protocol.CMD_DELIVER_ACK:
//收到投递结果直接attach响应
// log.DebugLog("kite_handler", "AcceptHandler|DELIVER_ACK|%s|%t", ae.opaque, ae.msg)
ae.remoteClient.Attach(ae.opaque, ae.msg)
return nil
case protocol.CMD_HEARTBEAT:
hb := ae.msg.(*protocol.HeartBeat)
event = pipe.NewHeartbeatEvent(ae.remoteClient, ae.opaque, hb.GetVersion())
ctx.SendForward(event)
return nil
case protocol.CMD_BYTES_MESSAGE:
msg = store.NewMessageEntity(protocol.NewQMessage(ae.msg.(*protocol.BytesMessage)))
case protocol.CMD_STRING_MESSAGE:
msg = store.NewMessageEntity(protocol.NewQMessage(ae.msg.(*protocol.StringMessage)))
default:
//这只是一个bug不支持的数据类型能给你
log.WarnLog("kite_handler", "AcceptHandler|Process|%s|%t", INVALID_MSG_TYPE_ERROR, ae.msg)
}
if nil != msg {
msg.PublishTime = time.Now().Unix()
msg.KiteServer = self.kiteserver
deliver := newPersistentEvent(msg, ae.remoteClient, ae.opaque)
ctx.SendForward(deliver)
return nil
}
return INVALID_MSG_TYPE_ERROR
}
//处理push
func (self *ApnsHttpServer) handlePush(out http.ResponseWriter, req *http.Request) {
resp := &response{}
resp.Status = RESP_STATUS_SUCC
if req.Method == "GET" {
//返回不支持的请求方式
resp.Status = RESP_STATUS_INVALID_PROTO
resp.Error = errors.New("Unsupport Get method Invoke!")
} else if req.Method == "POST" {
//pushType
pushType := req.PostFormValue("pt") //先默认采用Enhanced方式
//接卸对应的token和payload
token, payload := self.decodePayload(req, resp)
trace := req.PostFormValue("trace")
//----------------如果依然是成功状态则证明当前可以发送
if RESP_STATUS_SUCC == resp.Status {
defer func() {
if re := recover(); nil != re {
log.ErrorLog("push_handler", "ApnsHttpServer|handlePush|SEND|FAIL|%s|%s|%s", re, payload, trace)
resp.Status = RESP_STATUS_ERROR
resp.Error = errors.New(fmt.Sprintf("%s", re))
self.write(out, resp)
}
}()
self.innerSend(pushType, token, payload, resp)
log.InfoLog("push_handler", "ApnsHttpServer|handlePush|SUCC|%s|%s|%s", resp, payload, trace)
} else {
log.WarnLog("push_handler", "ApnsHttpServer|handlePush|FAIL|%s|%s|%s", resp, payload, trace)
}
}
self.write(out, resp)
}
func (self *ApnsConnection) dial() error {
config := tls.Config{}
config.Certificates = []tls.Certificate{self.cert}
config.InsecureSkipVerify = true
conn, err := tls.Dial("tcp", self.hostport, &config)
if nil != err {
//connect fail
log.WarnLog("push_client", "CONNECTION|%s|DIAL CONNECT|FAIL|%s|%s", self.name(), self.hostport, err.Error())
return err
}
// conn.SetDeadline(0 * time.Second)
for {
state := conn.ConnectionState()
if state.HandshakeComplete {
log.InfoLog("push_client", "CONNECTION|%s|HANDSHAKE SUCC", self.name())
break
}
time.Sleep(1 * time.Second)
}
self.conn = conn
return nil
}
func (self *ZKManager) Start() {
if len(self.zkhosts) <= 0 {
log.WarnLog("kite_bind", "使用默认zkhosts！|localhost:2181\n")
self.zkhosts = "localhost:2181"
} else {
log.Info("使用zkhosts:[%s]！\n", self.zkhosts)
}
ss, eventChan, err := zk.Connect(strings.Split(self.zkhosts, ","), 5*time.Second)
if nil != err {
panic("连接zk失败..." + err.Error())
return
}
exist, _, err := ss.Exists(KITEQ)
if nil != err {
ss.Close()
panic("无法创建KITEQ " + err.Error())
}
if !exist {
resp, err := ss.Create(KITEQ, nil, zk.CreatePersistent, zk.WorldACL(zk.PermAll))
if nil != err {
ss.Close()
panic("NewZKManager|CREATE ROOT PATH|FAIL|" + KITEQ + "|" + err.Error())
} else {
log.InfoLog("kite_bind", "NewZKManager|CREATE ROOT PATH|SUCC|%s", resp)
}
}
self.session = ss
self.isClose = false
self.eventChan = eventChan
go self.listenEvent()
}
func (self *ApnsClient) storeInvalidToken(token string) {
//这里是里面最后存储不合法的token
log.WarnLog("push_client", "APNSCLIENT|UnImplement StoreInvalidToken|%s\n", token)
}
//监听数据变更
func (self *ZKManager) listenEvent() {
for !self.isClose {
//根据zk的文档 Watcher机制是无法保证可靠的，其次需要在每次处理完Watcher后要重新注册Watcher
change := <-self.eventChan
path := change.Path
//开始检查符合的watcher
watcher := func() IWatcher {
for k, w := range self.wathcers {
//以给定的
if strings.Index(path, k) >= 0 {
return w
}
}
return nil
}()
//如果没有wacher那么久忽略
if nil == watcher {
log.WarnLog("kite_bind", "ZKManager|listenEvent|NO WATCHER|%s", path)
continue
}
switch change.Type {
case zk.EventSession:
if change.State == zk.StateExpired {
log.WarnLog("kite_bind", "ZKManager|OnSessionExpired!|Reconnect Zk ....")
//阻塞等待重连任务成功
succ := <-self.reconnect()
if !succ {
log.WarnLog("kite_bind", "ZKManager|OnSessionExpired|Reconnect Zk|FAIL| ....")
continue
}
//session失效必须通知所有的watcher
func() {
for _, w := range self.wathcers {
//zk链接开则需要重新链接重新推送
w.OnSessionExpired()
}
}()
}
case zk.EventNodeDeleted:
self.session.ExistsW(path)
watcher.NodeChange(path, ZkEvent(change.Type), []string{})
// log.Info("ZKManager|listenEvent|%s|%s\n", path, change)
case zk.EventNodeCreated, zk.EventNodeChildrenChanged:
childnodes, _, _, err := self.session.ChildrenW(path)
if nil != err {
log.ErrorLog("kite_bind", "ZKManager|listenEvent|CD|%s|%s|%t\n", err, path, change.Type)
} else {
watcher.NodeChange(path, ZkEvent(change.Type), childnodes)
// log.Info("ZKManager|listenEvent|%s|%s|%s\n", path, change, childnodes)
}
case zk.EventNodeDataChanged:
split := strings.Split(path, "/")
//如果不是bind级别的变更则忽略
if len(split) < 5 || strings.LastIndex(split[4], "-bind") <= 0 {
continue
}
//获取一下数据
binds, err := self.getBindData(path)
if nil != err {
log.ErrorLog("kite_bind", "ZKManager|listenEvent|Changed|Get DATA|FAIL|%s|%s\n", err, path)
//忽略
continue
}
watcher.DataChange(path, binds)
// log.Info("ZKManager|listenEvent|%s|%s|%s\n", path, change, binds)
}
}
}