func (self *KiteQServer) Start() {
self.remotingServer = server.NewRemotionServer(self.kc.server, self.kc.rc,
func(rclient *client.RemotingClient, p *packet.Packet) {
event := pipe.NewPacketEvent(rclient, p)
err := self.pipeline.FireWork(event)
if nil != err {
log.Error("RemotingServer|onPacketRecieve|FAIL|%s|%t\n", err, p)
} else {
// log.Debug("RemotingServer|onPacketRecieve|SUCC|%s|%t\n", rclient.RemoteAddr(), packet)
}
})
err := self.remotingServer.ListenAndServer()
if nil != err {
log.Crashf("KiteQServer|RemotionServer|START|FAIL|%s|%s\n", err, self.kc.server)
} else {
log.Info("KiteQServer|RemotionServer|START|SUCC|%s\n", self.kc.server)
}
//推送可发送的topic列表并且获取了对应topic下的订阅关系
succ := self.exchanger.PushQServer(self.kc.server, self.kc.topics)
if !succ {
log.Crashf("KiteQServer|PushQServer|FAIL|%s|%s\n", err, self.kc.topics)
} else {
log.Info("KiteQServer|PushQServer|SUCC|%s\n", self.kc.topics)
}
//开启流量统计
self.kc.flowstat.Start()
//开启recover
self.recoverManager.Start()
}
func (self *KiteQServer) Start() {
self.remotingServer = server.NewRemotionServer(self.kc.so.bindHost, self.kc.rc,
func(rclient *client.RemotingClient, p *packet.Packet) {
event := pipe.NewPacketEvent(rclient, p)
err := self.pipeline.FireWork(event)
if nil != err {
log.ErrorLog("kite_server", "RemotingServer|onPacketRecieve|FAIL|%s|%t", err, packet.MarshalPacket(p))
} else {
// log.Debug("RemotingServer|onPacketRecieve|SUCC|%s|%t\n", rclient.RemoteAddr(), packet)
}
})
err := self.remotingServer.ListenAndServer()
if nil != err {
log.Crashf("KiteQServer|RemotionServer|START|FAIL|%s|%s\n", err, self.kc.so.bindHost)
} else {
log.InfoLog("kite_server", "KiteQServer|RemotionServer|START|SUCC|%s\n", self.kc.so.bindHost)
}
//推送可发送的topic列表并且获取了对应topic下的订阅关系
succ := self.exchanger.PushQServer(self.kc.so.bindHost, self.kc.so.topics)
if !succ {
log.Crashf("KiteQServer|PushQServer|FAIL|%s|%s\n", err, self.kc.so.topics)
} else {
log.InfoLog("kite_server", "KiteQServer|PushQServer|SUCC|%s\n", self.kc.so.topics)
}
//开启流量统计
self.startFlow()
//开启recover
self.recoverManager.Start()
//启动DLQ的时间
self.startDLQ()
//检查配置更新
if len(self.kc.so.configPath) > 0 {
self.startCheckConf()
}
//启动pprof
host, _, _ := net.SplitHostPort(self.kc.so.bindHost)
go func() {
if self.kc.so.pprofPort > 0 {
http.HandleFunc("/stat", self.HandleStat)
http.HandleFunc("/binds", self.HandleBindings)
log.Error(http.ListenAndServe(host+":"+strconv.Itoa(self.kc.so.pprofPort), nil))
}
}()
}
//启动
func (self *KiteClientManager) Start() {
self.zkManager = binding.NewZKManager(self.zkAddr)
//注册kiteqserver的变更
self.zkManager.RegisteWather(PATH_KITEQ_SERVER, self)
hostname, _ := os.Hostname()
//推送本机到
err := self.zkManager.PublishTopics(self.topics, self.ga.GroupId, hostname)
if nil != err {
log.Crashf("KiteClientManager|PublishTopics|FAIL|%s|%s\n", err, self.topics)
} else {
log.Info("KiteClientManager|PublishTopics|SUCC|%s\n", self.topics)
}
outter:
for _, b := range self.binds {
for _, t := range self.topics {
if t == b.Topic {
continue outter
}
}
self.topics = append(self.topics, b.Topic)
}
for _, topic := range self.topics {
hosts, err := self.zkManager.GetQServerAndWatch(topic)
if nil != err {
log.Crashf("KiteClientManager|GetQServerAndWatch|FAIL|%s|%s\n", err, topic)
} else {
log.Info("KiteClientManager|GetQServerAndWatch|SUCC|%s|%s\n", topic, hosts)
}
self.onQServerChanged(topic, hosts)
}
if len(self.kiteClients) <= 0 {
log.Crashf("KiteClientManager|Start|NO VALID KITESERVER|%s\n", self.topics)
}
if len(self.binds) > 0 {
//订阅关系推送，并拉取QServer
err = self.zkManager.PublishBindings(self.ga.GroupId, self.binds)
if nil != err {
log.Crashf("KiteClientManager|PublishBindings|FAIL|%s|%s\n", err, self.binds)
}
}
}
func parseDB(kc KiteQConfig) store.IKiteStore {
db := kc.db
var kitedb store.IKiteStore
if strings.HasPrefix(db, "mock://") {
kitedb = &store.MockKiteStore{}
} else if strings.HasPrefix(db, "memory://") {
url := strings.TrimPrefix(db, "memory://")
split := strings.Split(url, "&")
params := make(map[string]string, len(split))
for _, v := range split {
p := strings.SplitN(v, "=", 2)
if len(p) >= 2 {
params[p[0]] = p[1]
}
}
initval := 10 * 10000
initcap, ok := params["initcap"]
if ok {
v, e := strconv.ParseInt(initcap, 10, 32)
if nil != e {
log.Crashf("NewKiteQServer|INVALID|INIT CAP|%s\n", db)
}
initval = int(v)
}
max := 50 * 10000
maxcap, ok := params["maxcap"]
if ok {
v, e := strconv.ParseInt(maxcap, 10, 32)
if nil != e {
log.Crashf("NewKiteQServer|INVALID|MAX CAP|%s\n", db)
}
max = int(v)
}
kitedb = sm.NewKiteMemoryStore(initval, max)
} else if strings.HasPrefix(db, "mysql://") {
url := strings.TrimPrefix(db, "mysql://")
mp := strings.Split(url, "?")
params := make(map[string]string, 5)
if len(mp) > 1 {
split := strings.Split(mp[1], "&")
for _, v := range split {
p := strings.SplitN(v, "=", 2)
params[p[0]] = p[1]
}
}
bus := 100
u, ok := params["batchUpdateSize"]
if ok {
v, e := strconv.ParseInt(u, 10, 32)
if nil != e {
log.Crashf("NewKiteQServer|INVALID|batchUpdateSize|%s\n", db)
}
bus = int(v)
}
bds := 100
d, ok := params["batchDelSize"]
if ok {
v, e := strconv.ParseInt(d, 10, 32)
if nil != e {
log.Crashf("NewKiteQServer|INVALID|batchDelSize|%s\n", db)
}
bds = int(v)
}
flushPeriod := 1 * time.Second
fp, ok := params["flushPeriod"]
if ok {
v, e := strconv.ParseInt(fp, 10, 32)
if nil != e {
log.Crashf("NewKiteQServer|INVALID|batchDelSize|%s\n", db)
}
flushPeriod = time.Duration(v * int64(flushPeriod))
}
maxConn := 500
mc, ok := params["maxConn"]
if ok {
v, e := strconv.ParseInt(mc, 10, 32)
if nil != e {
log.Crashf("NewKiteQServer|INVALID|batchDelSize|%s\n", db)
}
maxConn = int(v)
}
//解析Mysql的host
master := mp[0]
slave := ""
mysqlHosts := strings.Split(mp[0], ",")
if len(mysqlHosts) > 1 {
master = mysqlHosts[0]
slave = mysqlHosts[1]
}
//解析用户名密码：
username, _ := params["username"]
password, pok := params["password"]
if !pok {
password = ""
}
//shard的数量
shardnum := 8
sd, sdok := params["shardnum"]
if sdok {
v, e := strconv.ParseInt(sd, 10, 32)
if nil != e {
log.Crashf("NewKiteQServer|INVALID|ShardNum|%s\n", db)
}
shardnum = int(v)
}
options := smq.MysqlOptions{
Addr: master,
SlaveAddr: slave,
ShardNum: shardnum,
DB: params["db"],
Username: username,
Password: password,
BatchUpSize: bus,
BatchDelSize: bds,
FlushPeriod: flushPeriod,
MaxIdleConn: maxConn / 2,
MaxOpenConn: maxConn}
kitedb = smq.NewKiteMysql(options)
} else if strings.HasPrefix(db, "file://") {
url := strings.TrimPrefix(db, "file://")
mp := strings.Split(url, "?")
params := make(map[string]string, 5)
if len(mp) > 1 {
split := strings.Split(mp[1], "&")
for _, v := range split {
p := strings.SplitN(v, "=", 2)
params[p[0]] = p[1]
}
}
if len(mp[0]) <= 0 {
log.Crashf("NewKiteQServer|INVALID|FilePath|%s\n", db)
}
//最大消息容量
maxcap := 100
d, ok := params["cap"]
if ok {
v, e := strconv.ParseInt(d, 10, 32)
if nil != e {
log.Crashf("NewKiteQServer|INVALID|cap|%s\n", db)
}
maxcap = int(v)
}
//检查文件过期时间
checkPeriod := 1 * time.Second
fp, ok := params["checkPeriod"]
if ok {
v, e := strconv.ParseInt(fp, 10, 32)
if nil != e {
log.Crashf("NewKiteQServer|INVALID|checkPeriod|%s\n", db)
}
checkPeriod = time.Duration(v * int64(checkPeriod))
}
kitedb = smf.NewKiteFileStore(mp[0], maxcap, checkPeriod)
log.Debug("NewKiteQServer|FILESTORE|%s|%d|%d", mp[0], maxcap, int(checkPeriod.Seconds()))
} else {
log.Crashf("NewKiteQServer|UNSUPPORT DB PROTOCOL|%s\n", db)
}
kc.flowstat.Kitestore = kitedb
return kitedb
}