Example #1
0
func (self *KiteQServer) Start() {

	self.remotingServer = server.NewRemotionServer(self.kc.server, self.kc.rc,
		func(rclient *client.RemotingClient, p *packet.Packet) {
			event := pipe.NewPacketEvent(rclient, p)
			err := self.pipeline.FireWork(event)
			if nil != err {
				log.ErrorLog("kite_server", "RemotingServer|onPacketRecieve|FAIL|%s|%t", err, packet.MarshalPacket(p))
			} else {
				// log.Debug("RemotingServer|onPacketRecieve|SUCC|%s|%t\n", rclient.RemoteAddr(), packet)
			}
		})

	err := self.remotingServer.ListenAndServer()
	if nil != err {
		log.Crashf("KiteQServer|RemotionServer|START|FAIL|%s|%s\n", err, self.kc.server)
	} else {
		log.InfoLog("kite_server", "KiteQServer|RemotionServer|START|SUCC|%s\n", self.kc.server)
	}
	//推送可发送的topic列表并且获取了对应topic下的订阅关系
	succ := self.exchanger.PushQServer(self.kc.server, self.kc.topics)
	if !succ {
		log.Crashf("KiteQServer|PushQServer|FAIL|%s|%s\n", err, self.kc.topics)
	} else {
		log.InfoLog("kite_server", "KiteQServer|PushQServer|SUCC|%s\n", self.kc.topics)
	}

	//开启流量统计
	self.startFlow()

	//开启recover
	self.recoverManager.Start()

}
Example #2
0
func (self *ConnPool) Shutdown() {
	self.mutex.Lock()
	defer self.mutex.Unlock()
	self.running = false

	for i := 0; i < 3; {
		//等待五秒中结束
		time.Sleep(5 * time.Second)
		if self.workPool.Len() <= 0 {
			break
		}

		log.InfoLog("apns_pool", "CONNECTION POOL|CLOSEING|WORK POOL SIZE|:%d", self.workPool.Len())
		i++
	}

	var idleconn *IdleConn
	//关闭掉空闲的client
	for e := self.idlePool.Front(); e != nil; e = e.Next() {
		idleconn = e.Value.(*IdleConn)
		idleconn.conn.Close()
		self.idlePool.Remove(e)
		idleconn = nil
	}

	log.InfoLog("apns_pool", "CONNECTION_POOL|SHUTDOWN")
}
Example #3
0
func (self *StmtPool) Shutdown() {
	self.mutex.Lock()
	defer self.mutex.Unlock()
	self.running = false
	//等待五秒中结束
	time.Sleep(1 * time.Second)
	for i := 0; i < 3; {

		if self.numWork <= 0 {
			break
		}

		log.InfoLog("kite_store", "Statment Pool|CLOSEING|WORK POOL SIZE|:%d\n", self.numWork)
		i++
	}

	var idleStmt *IdleStmt
	//关闭掉空闲的client
	for e := self.idlePool.Front(); e != nil; e = e.Next() {
		idleStmt = e.Value.(*IdleStmt)
		idleStmt.stmt.Close()
		self.idlePool.Remove(e)
		idleStmt = nil
	}

	log.InfoLog("kite_store", "Statment Pool|SHUTDOWN")
}
Example #4
0
//迁移过期的消息
func (self *KiteMysqlStore) migrateMessage(now int64, hashKey string) {

	log.InfoLog("kite_store", "KiteMysqlStore|MoveExpired|START|%s|%d", hashKey)

	//需要将过期的消息迁移到DLQ中
	sql := self.sqlwrapper.hashDLQSQL(DLQ_MOVE_QUERY, hashKey)
	//获取到需要导入的id,然后导入
	isql := self.sqlwrapper.hashDLQSQL(DLQ_MOVE_INSERT, hashKey)
	//删除已导入的数据
	dsql := self.sqlwrapper.hashDLQSQL(DLQ_MOVE_DELETE, hashKey)

	start := 0
	limit := 50
	for {
		rows, err := self.dbshard.FindSlave(hashKey).Query(sql, self.serverName, now, start, limit)
		if err != nil {
			log.ErrorLog("kite_store", "KiteMysqlStore|migrateMessage|Query|FAIL|%s|%s|%s", err, hashKey, sql)
			continue
		}
		defer rows.Close()
		messageIds := make([]interface{}, 1, 50)
		for rows.Next() {
			var id int
			var messageId string
			err = rows.Scan(&id, &messageId)
			if nil != err {
				log.ErrorLog("kite_store", "KiteMysqlStore|MoveExpired|Scan|FAIL|%s|%s|%s", err, hashKey, sql)
			} else {
				start = id
				messageIds = append(messageIds, messageId)
			}
		}

		//已经搬迁完毕则退出进行下一个
		if len(messageIds[1:]) <= 0 {
			log.InfoLog("kite_store", "KiteMysqlStore|MoveExpired|SUCC|%s|%d", hashKey, start)
			break
		}

		in := strings.Repeat("?,", len(messageIds[1:]))
		in = in[:len(in)-1]
		isqlTmp := strings.Replace(isql, "{ids}", in, 1)
		_, err = self.dbshard.FindMaster(hashKey).Exec(isqlTmp, messageIds[1:]...)
		if err != nil {
			log.ErrorLog("kite_store", "KiteMysqlStore|MoveExpired|Insert|FAIL|%s|%s", err, isqlTmp, messageIds)
			continue
		}

		dsqlTmp := strings.Replace(dsql, "{ids}", in, 1)
		messageIds[0] = self.serverName
		_, err = self.dbshard.FindMaster(hashKey).Exec(dsqlTmp, messageIds...)
		if err != nil {
			log.ErrorLog("kite_store", "KiteMysqlStore|MoveExpired|DELETE|FAIL|%s|%s|%s|%s", err, dsql, dsqlTmp, messageIds)
			continue
		}
	}
}
Example #5
0
func (self *KiteQServer) Start() {

	self.remotingServer = server.NewRemotionServer(self.kc.so.bindHost, self.kc.rc,
		func(rclient *client.RemotingClient, p *packet.Packet) {
			event := pipe.NewPacketEvent(rclient, p)
			err := self.pipeline.FireWork(event)
			if nil != err {
				log.ErrorLog("kite_server", "RemotingServer|onPacketRecieve|FAIL|%s|%t", err, packet.MarshalPacket(p))
			} else {
				// log.Debug("RemotingServer|onPacketRecieve|SUCC|%s|%t\n", rclient.RemoteAddr(), packet)
			}
		})

	err := self.remotingServer.ListenAndServer()
	if nil != err {
		log.Crashf("KiteQServer|RemotionServer|START|FAIL|%s|%s\n", err, self.kc.so.bindHost)
	} else {
		log.InfoLog("kite_server", "KiteQServer|RemotionServer|START|SUCC|%s\n", self.kc.so.bindHost)
	}
	//推送可发送的topic列表并且获取了对应topic下的订阅关系
	succ := self.exchanger.PushQServer(self.kc.so.bindHost, self.kc.so.topics)
	if !succ {
		log.Crashf("KiteQServer|PushQServer|FAIL|%s|%s\n", err, self.kc.so.topics)
	} else {
		log.InfoLog("kite_server", "KiteQServer|PushQServer|SUCC|%s\n", self.kc.so.topics)
	}

	//开启流量统计
	self.startFlow()

	//开启recover
	self.recoverManager.Start()

	//启动DLQ的时间
	self.startDLQ()

	//检查配置更新
	if len(self.kc.so.configPath) > 0 {
		self.startCheckConf()
	}

	//启动pprof
	host, _, _ := net.SplitHostPort(self.kc.so.bindHost)
	go func() {
		if self.kc.so.pprofPort > 0 {
			http.HandleFunc("/stat", self.HandleStat)
			http.HandleFunc("/binds", self.HandleBindings)
			log.Error(http.ListenAndServe(host+":"+strconv.Itoa(self.kc.so.pprofPort), nil))
		}
	}()
}
//接受错误的响应并触发重发
func (self *ApnsClient) onErrorResponseRecieve(responseChannel chan *entry.Response) {

	ch := make(chan *entry.Message, 1000)
	//启动重发任务
	go self.resend(ch)

	//开始启动
	for self.running {
		//顺序处理每一个连接的错误数据发送
		resp := <-responseChannel
		//只有 prcessing error 和 shutdown的两种id才会进行重发
		switch resp.Status {

		case entry.RESP_SHUTDOWN, entry.RESP_ERROR, entry.RESP_UNKNOW:

			//只有这三种才重发
			ch := self.storage.Remove(resp.Identifier, 0, func(id uint32, msg *entry.Message) bool {
				expiredTime := int64(entry.UmarshalExpiredTime(msg))

				//过滤掉 不是当前连接ID的消息 或者 当前相同ID的消息 或者 (有过期时间结果已经过期的消息)
				return msg.ProcessId != resp.ProccessId ||
					id == resp.Identifier ||
					(0 != expiredTime && (time.Now().Unix()-expiredTime >= 0))

			})

			for {

				tmp := <-ch
				//如果删除成功并且消息不为空则重发
				if nil != tmp {
					ch <- tmp
				} else {
					break
				}
			}
			log.InfoLog("push_client", "APNSCLIENT|onErrorResponseRecieve|ERROR|%d", resp.Status)

		case entry.RESP_INVALID_TOKEN, entry.RESP_INVALID_TOKEN_SIZE:
			//将错误的token记录在存储中,备后续的过滤使用
			msg := self.storage.Get(resp.Identifier)
			if nil != msg {
				//从msg中拿出token用于记录
				token := entry.UmarshalToken(msg)
				self.storeInvalidToken(token)
				log.InfoLog("push_client", "APNSCLIENT|INVALID TOKEN|%s", resp.Identifier)
			}
		}

	}
}
Example #7
0
func (self *ApnsHttpServer) dial(hp string) {

	log.InfoLog("push_handler", "APNS HTTPSERVER IS STARTING ....")
	http.HandleFunc("/apns/push", self.handlePush)
	http.HandleFunc("/apns/feedback", self.handleFeedBack)

	err := self.httpserver.ListenAndServe()
	if nil != err {
		log.ErrorLog("push_handler", "APNSHTTPSERVER|LISTEN|FAIL|%s", err)
	} else {
		log.InfoLog("push_handler", "APNSHTTPSERVER|LISTEN|SUCC|%s .....", hp)
	}

}
Example #8
0
//启动
func (self *KiteClientManager) Start() {

	self.zkManager = binding.NewZKManager(self.zkAddr)
	//注册kiteqserver的变更
	self.zkManager.RegisteWather(PATH_KITEQ_SERVER, self)

	hostname, _ := os.Hostname()
	//推送本机到
	err := self.zkManager.PublishTopics(self.topics, self.ga.GroupId, hostname)
	if nil != err {
		log.Crashf("KiteClientManager|PublishTopics|FAIL|%s|%s\n", err, self.topics)
	} else {
		log.InfoLog("kite_client", "KiteClientManager|PublishTopics|SUCC|%s\n", self.topics)
	}

outter:
	for _, b := range self.binds {
		for _, t := range self.topics {
			if t == b.Topic {
				continue outter
			}
		}
		self.topics = append(self.topics, b.Topic)
	}

	for _, topic := range self.topics {

		hosts, err := self.zkManager.GetQServerAndWatch(topic)
		if nil != err {
			log.Crashf("KiteClientManager|GetQServerAndWatch|FAIL|%s|%s\n", err, topic)
		} else {
			log.InfoLog("kite_client", "KiteClientManager|GetQServerAndWatch|SUCC|%s|%s\n", topic, hosts)
		}
		self.onQServerChanged(topic, hosts)
	}

	if len(self.kiteClients) <= 0 {
		log.Crashf("KiteClientManager|Start|NO VALID KITESERVER|%s\n", self.topics)
	}

	if len(self.binds) > 0 {
		//订阅关系推送,并拉取QServer
		err = self.zkManager.PublishBindings(self.ga.GroupId, self.binds)
		if nil != err {
			log.Crashf("KiteClientManager|PublishBindings|FAIL|%s|%s\n", err, self.binds)
		}
	}

}
Example #9
0
//开始启动恢复程序
func (self *RecoverManager) Start() {
	for i := 0; i < self.kitestore.RecoverNum(); i++ {
		go self.startRecoverTask(fmt.Sprintf("%x", i))
	}

	log.InfoLog("kite_server", "RecoverManager|Start|SUCC....")
}
Example #10
0
//握手包
func handshake(ga *c.GroupAuth, remoteClient *c.RemotingClient) (bool, error) {

	for i := 0; i < 3; i++ {
		p := protocol.MarshalConnMeta(ga.GroupId, ga.SecretKey)
		rpacket := packet.NewPacket(protocol.CMD_CONN_META, p)
		resp, err := remoteClient.WriteAndGet(*rpacket, 5*time.Second)
		if nil != err {
			//两秒后重试
			time.Sleep(2 * time.Second)
			log.WarnLog("kite_client", "kiteClient|handShake|FAIL|%s|%s\n", ga.GroupId, err)
		} else {
			authAck, ok := resp.(*protocol.ConnAuthAck)
			if !ok {
				return false, errors.New("Unmatches Handshake Ack Type! ")
			} else {
				if authAck.GetStatus() {
					log.InfoLog("kite_client", "kiteClient|handShake|SUCC|%s|%s\n", ga.GroupId, authAck.GetFeedback())
					return true, nil
				} else {
					log.WarnLog("kite_client", "kiteClient|handShake|FAIL|%s|%s\n", ga.GroupId, authAck.GetFeedback())
					return false, errors.New("Auth FAIL![" + authAck.GetFeedback() + "]")
				}
			}
		}
	}

	return false, errors.New("handshake fail! [" + remoteClient.RemoteAddr() + "]")
}
Example #11
0
//remove segment
func (self *MessageStore) remove(s *Segment) {

	self.Lock()
	defer self.Unlock()
	//remove from segments
	for i, s := range self.segments {
		if s.sid == s.sid {
			self.segments = append(self.segments[0:i], self.segments[i+1:]...)
			break
		}
	}
	//remove from cache
	for e := self.segmentCache.Front(); nil != e; e = e.Next() {
		cs := e.Value.(*Segment)
		if cs.sid == s.sid {
			self.segmentCache.Remove(e)
			break
		}
	}

	//close segment
	s.Close()

	err := os.Remove(s.path)
	if nil != err {
		log.WarnLog("kite_store", "MessageStore|Remove|Segment|FAIL|%s|%s", err, s.path)
	}
	err = os.Remove(s.slog.path)
	if nil != err {
		log.WarnLog("kite_store", "MessageStore|Remove|SegmentLog|FAIL|%s|%s", err, s.slog.path)
	}

	log.InfoLog("kite_store", "MessageStore|Remove|Segment|%s", s.path)
}
Example #12
0
//发布订阅关系
func (self *ZKManager) PublishBindings(groupId string, bindings []*Binding) error {

	//按topic分组
	groupBind := make(map[string][]*Binding, 10)
	for _, b := range bindings {
		g, ok := groupBind[b.Topic]
		if !ok {
			g = make([]*Binding, 0, 2)
		}
		b.GroupId = groupId
		g = append(g, b)
		groupBind[b.Topic] = g
	}

	for topic, binds := range groupBind {
		data, err := MarshalBinds(binds)
		if nil != err {
			log.ErrorLog("kite_bind", "ZKManager|PublishBindings|MarshalBind|FAIL|%s|%s|%t\n", err, groupId, binds)
			return err
		}

		createType := zk.CreatePersistent

		path := KITEQ_SUB + "/" + topic
		//注册对应topic的groupId //注册订阅信息
		succpath, err := self.registePath(path, groupId+"-bind", createType, data)
		if nil != err {
			log.ErrorLog("kite_bind", "ZKManager|PublishTopic|Bind|FAIL|%s|%s/%s\n", err, path, binds)
			return err
		} else {
			log.InfoLog("kite_bind", "ZKManager|PublishTopic|Bind|SUCC|%s|%s\n", succpath, binds)
		}
	}
	return nil
}
Example #13
0
//发布topic对应的server
func (self *ZKManager) PublishQServer(hostport string, topics []string) error {

	for _, topic := range topics {
		qpath := KITEQ_SERVER + "/" + topic
		spath := KITEQ_SUB + "/" + topic
		ppath := KITEQ_PUB + "/" + topic

		//创建发送和订阅的根节点
		self.traverseCreatePath(ppath, nil, zk.CreatePersistent)
		// self.session.ExistsW(ppath)
		self.traverseCreatePath(spath, nil, zk.CreatePersistent)
		self.session.ExistsW(spath)

		//先删除当前这个临时节点再注册 避免监听不到临时节点变更的事件
		self.session.Delete(qpath+"/"+hostport, -1)

		//注册当前节点
		path, err := self.registePath(qpath, hostport, zk.CreateEphemeral, nil)
		if nil != err {
			log.ErrorLog("kite_bind", "ZKManager|PublishQServer|FAIL|%s|%s/%s\n", err, qpath, hostport)
			return err
		}
		log.InfoLog("kite_bind", "ZKManager|PublishQServer|SUCC|%s\n", path)
	}

	//注册当前的kiteqserver
	self.session.Delete(KITEQ_ALIVE_SERVERS+"/"+hostport, -1)
	self.registePath(KITEQ_ALIVE_SERVERS, hostport, zk.CreateEphemeral, nil)
	self.registePath(KITEQ_ALL_SERVERS, hostport, zk.CreatePersistent, nil)
	return nil
}
Example #14
0
func NewApnsHttpServer(option Option) *ApnsHttpServer {

	feedbackChan := make(chan *entry.Feedback, 1000)
	var apnsClient *apns.ApnsClient
	if option.startMode == STARTMODE_MOCK {
		//初始化mock apns
		apnsClient = apns.NewMockApnsClient(option.cert,
			option.pushAddr, chan<- *entry.Feedback(feedbackChan), option.feedbackAddr, entry.NewCycleLink(3, option.storageCapacity))
		log.Info("MOCK APNS HTTPSERVER IS STARTING ....")
	} else {
		//初始化apns
		apnsClient = apns.NewDefaultApnsClient(option.cert,
			option.pushAddr, chan<- *entry.Feedback(feedbackChan), option.feedbackAddr, entry.NewCycleLink(3, option.storageCapacity))
		log.InfoLog("push_handler", "ONLINE APNS HTTPSERVER IS STARTING ....")
	}

	server := &ApnsHttpServer{feedbackChan: feedbackChan,
		apnsClient: apnsClient, expiredTime: option.expiredTime}

	//创建http
	server.httpserver = NewMomoHttpServer(option.bindAddr, nil)

	go server.dial(option.bindAddr)

	return server
}
Example #15
0
//关闭掉exchanger
func (self *BindExchanger) Shutdown() {
	//删除掉当前的QServer
	self.zkmanager.UnpushlishQServer(self.kiteqserver, self.topics)
	time.Sleep(10 * time.Second)
	self.zkmanager.Close()
	log.InfoLog("kite_bind", "BindExchanger|Shutdown...")
}
Example #16
0
func (self *KiteQServer) startCheckConf() {
	go func() {
		t := time.NewTicker(1 * time.Minute)
		for !self.stop {
			so := ServerOption{}
			err := loadTomlConf(self.kc.so.configPath, self.kc.so.clusterName,
				self.kc.so.bindHost, self.kc.so.pprofPort, &so)
			if nil != err {
				log.ErrorLog("kite_server", "KiteQServer|startCheckConf|FAIL|%s", err)
			}

			//新增或者减少topics
			if len(so.topics) != len(self.kc.so.topics) {
				//推送可发送的topic列表并且获取了对应topic下的订阅关系
				succ := self.exchanger.PushQServer(self.kc.so.bindHost, so.topics)
				if !succ {
					log.ErrorLog("kite_server", "KiteQServer|startCheckConf|PushQServer|FAIL|%s|%s\n", err, so.topics)
				} else {
					log.InfoLog("kite_server", "KiteQServer|startCheckConf|PushQServer|SUCC|%s\n", so.topics)
				}
				//重置数据
				self.kc.so = so
			}

			<-t.C
		}
		t.Stop()
	}()
}
Example #17
0
func (self *KiteMysqlStore) Start() {

	count := SHARD_SEED
	//创建Hash的channel
	batchDelChan := make([]chan string, 0, count)
	batchUpChan := make([]chan *MessageEntity, 0, count)
	batchComChan := make([]chan string, 0, count)
	for i := 0; i < count; i++ {
		batchUpChan = append(batchUpChan, make(chan *MessageEntity, self.batchUpSize*2))
		batchDelChan = append(batchDelChan, make(chan string, self.batchDelSize*2))
		batchComChan = append(batchComChan, make(chan string, self.batchUpSize*2))
	}

	//批量的channel
	self.batchUpChan = batchUpChan
	self.batchDelChan = batchDelChan
	self.batchComChan = batchComChan

	//创建每种批量的preparedstmt
	stmts := make(map[batchType][][]*StmtPool, 4)
	for k, v := range self.sqlwrapper.batchSQL {
		btype := k
		pool := make([][]*StmtPool, 0, self.dbshard.ShardNum())
		//对每个shard构建stmt的pool
		for i := 0; i < self.dbshard.ShardNum(); i++ {
			innerPool := make([]*StmtPool, 0, self.dbshard.HashNum())
			for j, s := range v {
				psql := s
				db := self.dbshard.FindShardById(i*self.dbshard.HashNum() + j).master
				err, p := NewStmtPool(5, 10, 20, 1*time.Minute, func() (error, *sql.Stmt) {
					stmt, err := db.Prepare(psql)
					if nil != err {
						log.ErrorLog("kite_store", "StmtPool|Create Stmt|FAIL|%s|%s\n", err, psql)
						return err, nil
					}
					return nil, stmt
				})
				if nil != err {
					log.ErrorLog("kite_store", "NewKiteMysql|NewStmtPool|FAIL|%s\n", err)
					panic(err)
				}
				innerPool = append(innerPool, p)
			}
			pool = append(pool, innerPool)
		}
		stmts[btype] = pool
	}

	self.stmtPools = stmts

	for i := 0; i < count; i++ {
		// log.Printf("KiteMysqlStore|start|SQL|%s\n|%s\n", sqlu, sqld)
		self.startBatch(i, self.batchUpChan[i],
			self.batchDelChan[i], self.batchComChan[i])
	}
	log.InfoLog("kite_store", "KiteMysqlStore|Start...")
}
Example #18
0
func (self *KiteClientManager) remointflow() {
	go func() {
		t := time.NewTicker(1 * time.Second)
		for {
			ns := self.rc.FlowStat.Stat()
			log.InfoLog("kite_client", "Remoting read:%d/%d\twrite:%d/%d\tdispatcher_go:%d\tconnetions:%d", ns.ReadBytes, ns.ReadCount,
				ns.WriteBytes, ns.WriteCount, ns.DispatcherGo, self.clientManager.ConnNum())
			<-t.C
		}
	}()
}
Example #19
0
func (self *DeliverResultHandler) Process(ctx *DefaultPipelineContext, event IEvent) error {

	fevent, ok := self.cast(event)
	if !ok {
		return ERROR_INVALID_EVENT_TYPE
	}

	if len(fevent.futures) > 0 {
		tid, ch := self.tw.After(self.deliverTimeout, func() {})
		func() {
			defer self.tw.Remove(tid)
			//等待回调结果
			fevent.wait(ch)
		}()

	}

	//增加投递成功的分组
	if len(fevent.deliverSuccGroups) > 0 {
		fevent.succGroups = append(fevent.succGroups, fevent.deliverSuccGroups...)

	}

	attemptDeliver := nil != fevent.attemptDeliver
	//第一次尝试投递失败了立即通知
	if attemptDeliver {
		fevent.attemptDeliver <- fevent.deliverFailGroups
		close(fevent.attemptDeliver)
	}

	log.InfoLog("kite_handler", "%s|Process|SEND RESULT:\nattemptDeliver:%v\nfly:%v\nmessageId:%s\nTopic:%s\nMessageType:%s\nDeliverCount:%d\n"+
		"DeliverySUCCGROUPS:%s\nSUCCGROUPS:%s\nFAILGROUPS:%s",
		self.GetName(), attemptDeliver, fevent.fly, fevent.deliverEvent.messageId, fevent.deliverEvent.topic, fevent.deliverEvent.messageType,
		fevent.deliverCount, fevent.succGroups, fevent.succGroupFuture, fevent.failGroupFuture)
	//都投递成功
	if len(fevent.deliverFailGroups) <= 0 {
		if !fevent.fly && !attemptDeliver {
			//async batch remove
			self.kitestore.AsyncDelete(fevent.messageId)
		}
	} else {
		//重投策略
		//不是尝试投递也就是第一次投递并且也是满足重投条件
		if !attemptDeliver && self.checkRedelivery(fevent) {
			//去掉当前消息的投递事件
			self.deliveryRegistry.UnRegiste(fevent.messageId)
			//再次发起重投策略
			ctx.SendBackward(fevent.deliverEvent)
		}
	}

	return nil

}
Example #20
0
func (self *KiteQServer) Shutdown() {
	self.stop = true
	//先关闭exchanger让客户端不要再输送数据
	self.exchanger.Shutdown()
	self.recoverManager.Stop()
	self.kitedb.Stop()
	self.clientManager.Shutdown()
	self.remotingServer.Shutdown()
	log.InfoLog("kite_server", "KiteQServer|Shutdown...")

}
Example #21
0
func (self *KiteMysqlStore) Stop() {
	self.stop = true
	for k, v := range self.stmtPools {
		for _, s := range v {
			for _, p := range s {
				p.Shutdown()
			}
		}
		log.InfoLog("kite_store", "KiteMysqlStore|Stop|Stmt|%s", k)
	}
	self.dbshard.Stop()
}
Example #22
0
func (self *KiteQServer) startDLQ() {
	go func() {
		for {
			now := time.Now()
			next := now.Add(time.Hour * 24)
			next = time.Date(next.Year(), next.Month(), next.Day(), self.kc.dlqHour, 0, 0, 0, next.Location())
			t := time.NewTimer(next.Sub(now))
			<-t.C
			func() {
				defer func() {
					if err := recover(); nil != err {
						log.ErrorLog("kite_server", "KiteQServer|startDLQ|FAIL|%s|%s", err, time.Now())
					}
				}()
				//开始做迁移
				self.kitedb.MoveExpired()
			}()
			log.InfoLog("kite_server", "KiteQServer|startDLQ|SUCC|%s", time.Now())
		}
	}()
	log.InfoLog("kite_server", "KiteQServer|startDLQ|SUCC|%s", time.Now())
}
Example #23
0
func (self *KiteFileStore) Start() {
	self.Lock()
	defer self.Unlock()
	if !self.running {
		self.Add(1)
		self.running = true
		//start snapshost
		self.snapshot.Start()
		go self.delSync()
		log.InfoLog("kite_store", "KiteFileStore|Start...\t%s", self.Monitor())
	}

}
Example #24
0
//发布可以使用的topic类型的publisher
func (self *ZKManager) PublishTopics(topics []string, groupId string, hostport string) error {

	for _, topic := range topics {
		pubPath := KITEQ_PUB + "/" + topic + "/" + groupId
		path, err := self.registePath(pubPath, hostport, zk.CreatePersistent, nil)
		if nil != err {
			log.ErrorLog("kite_bind", "ZKManager|PublishTopic|FAIL|%s|%s/%s\n", err, pubPath, hostport)
			return err
		}
		log.InfoLog("kite_bind", "ZKManager|PublishTopic|SUCC|%s\n", path)
	}
	return nil
}
Example #25
0
func (self *Segment) Open(do func(ol *oplog)) error {

	self.Lock()
	defer self.Unlock()
	if atomic.CompareAndSwapInt32(&self.isOpen, 0, 1) {

		// log.Info("Segment|Open|BEGIN|%s|%s", self.path, self.name)
		var rf *os.File
		var wf *os.File
		_, err := os.Stat(self.path)
		//file exist
		if os.IsNotExist(err) {
			_, err := os.Create(self.path)
			if nil != err {
				log.ErrorLog("kite_store", "Segment|Create|FAIL|%s|%s", err, self.path)
				return err
			}
		}

		//file not exist create file
		wf, err = os.OpenFile(self.path, os.O_RDWR|os.O_APPEND, os.ModePerm)
		if nil != err {
			log.ErrorLog("kite_store", "Segment|Open|FAIL|%s|%s", err, self.name)
			return err
		}

		rf, err = os.OpenFile(self.path, os.O_RDWR, os.ModePerm)
		if nil != err {
			log.ErrorLog("kite_store", "Segment|Open|FAIL|%s|%s", err, self.name)
			return err
		}

		self.rf = rf
		self.wf = wf
		//buffer
		self.br = bufio.NewReader(rf)
		//load
		self.loadCheck()
		// //seek
		// self.wf.Seek(self.offset, 0)
		self.bw = bufio.NewWriter(wf)
		//op segment log
		self.slog.Open()
		//recover segment
		self.recover(do)
		total, n, d, e := self.stat()
		log.InfoLog("kite_store", "Segment|Open|SUCC|%s|total:%d,n:%d,d:%d,e:%d", self.name, total, n, d, e)
		return nil
	}
	return nil
}
Example #26
0
func (self *KiteFileStore) Stop() {

	self.Lock()
	defer self.Unlock()
	if self.running {
		self.running = false
		close(self.delChan)
		//wait delete finish
		self.Wait()
		self.snapshot.Destory()
		log.InfoLog("kite_store", "KiteFileStore|Stop...")
	}

}
Example #27
0
//去除掉当前的KiteQServer
func (self *ZKManager) UnpushlishQServer(hostport string, topics []string) {
	for _, topic := range topics {

		qpath := KITEQ_SERVER + "/" + topic + "/" + hostport

		//删除当前该Topic下的本机
		err := self.session.Delete(qpath, -1)
		if nil != err {
			log.ErrorLog("kite_bind", "ZKManager|UnpushlishQServer|FAIL|%s|%s\n", err, qpath)
		} else {
			log.InfoLog("kite_bind", "ZKManager|UnpushlishQServer|SUCC|%s\n", qpath)
		}
	}
	//注册当前的kiteqserver
	self.session.Delete(KITEQ_ALIVE_SERVERS+"/"+hostport, -1)
}
Example #28
0
func (self *ApnsConnection) waitRepsonse() {
	//这里需要优化是否同步读取结果
	buff := make([]byte, entry.ERROR_RESPONSE, entry.ERROR_RESPONSE)
	//同步读取当前conn的结果
	length, err := self.conn.Read(buff[:entry.ERROR_RESPONSE])
	if nil != err || length != len(buff) {
		log.InfoLog("push_client", "CONNECTION|%s|READ RESPONSE|FAIL|%s|%d", self.conn.RemoteAddr().String(), err, buff)
	} else {
		response := &entry.Response{}
		response.Unmarshal(buff)
		self.responseChan <- response
	}

	//已经读取到了错误信息直接关闭
	self.Close()

}
Example #29
0
//监听topics的对应的订阅关系的变更
func (self *BindExchanger) subscribeBinds(topics []string) bool {
	self.lock.Lock()
	defer self.lock.Unlock()
	for _, topic := range topics {
		binds, err := self.zkmanager.GetBindAndWatch(topic)
		if nil != err {
			log.ErrorLog("kite_bind", "BindExchanger|SubscribeBinds|FAIL|%s|%s\n", err, topic)
			return false
		} else {
			for groupId, bs := range binds {
				self.onBindChanged(topic, groupId, bs)
				log.InfoLog("kite_bind", "BindExchanger|SubscribeBinds|SUCC|%s|%s\n", topic, binds)
			}
		}
	}

	return true
}
Example #30
0
func (self *HeartbeatHandler) keepAlive() {

	for {
		select {
		case <-time.After(self.heartbeatPeriod):
			//心跳检测
			func() {
				id := time.Now().Unix()
				clients := self.clientMangager.ClientsClone()
				p := protocol.MarshalHeartbeatPacket(id)
				for h, c := range clients {
					i := 0
					//关闭的时候发起重连
					if c.IsClosed() {
						i = 3
					} else {
						//如果是空闲的则发起心跳
						if c.Idle() {
							for ; i < 3; i++ {
								hp := packet.NewPacket(protocol.CMD_HEARTBEAT, p)
								err := c.Ping(hp, time.Duration(self.heartbeatTimeout))
								//如果有错误则需要记录
								if nil != err {
									log.WarnLog("kite_client_handler", "HeartbeatHandler|KeepAlive|FAIL|%s|local:%s|remote:%s|%d\n", err, c.LocalAddr(), h, id)
									continue
								} else {
									log.InfoLog("kite_client_handler", "HeartbeatHandler|KeepAlive|SUCC|local:%s|remote:%s|%d|%d ...\n", c.LocalAddr(), h, id, i)
									break
								}
							}
						}
					}
					if i >= 3 {
						//说明连接有问题需要重连
						c.Shutdown()
						self.clientMangager.SubmitReconnect(c)
						log.WarnLog("kite_client_handler", "HeartbeatHandler|SubmitReconnect|%s\n", c.RemoteAddr())
					}
				}
			}()
		}
	}

}