示例#1
0
func (self *KiteQServer) Start() {

	self.remotingServer = server.NewRemotionServer(self.kc.server, self.kc.rc,
		func(rclient *client.RemotingClient, p *packet.Packet) {
			event := pipe.NewPacketEvent(rclient, p)
			err := self.pipeline.FireWork(event)
			if nil != err {
				log.Error("RemotingServer|onPacketRecieve|FAIL|%s|%t\n", err, p)
			} else {
				// log.Debug("RemotingServer|onPacketRecieve|SUCC|%s|%t\n", rclient.RemoteAddr(), packet)
			}
		})

	err := self.remotingServer.ListenAndServer()
	if nil != err {
		log.Crashf("KiteQServer|RemotionServer|START|FAIL|%s|%s\n", err, self.kc.server)
	} else {
		log.Info("KiteQServer|RemotionServer|START|SUCC|%s\n", self.kc.server)
	}
	//推送可发送的topic列表并且获取了对应topic下的订阅关系
	succ := self.exchanger.PushQServer(self.kc.server, self.kc.topics)
	if !succ {
		log.Crashf("KiteQServer|PushQServer|FAIL|%s|%s\n", err, self.kc.topics)
	} else {
		log.Info("KiteQServer|PushQServer|SUCC|%s\n", self.kc.topics)
	}

	//开启流量统计
	self.kc.flowstat.Start()

	//开启recover
	self.recoverManager.Start()

}
示例#2
0
func (self *KiteQServer) Start() {

	self.remotingServer = server.NewRemotionServer(self.kc.so.bindHost, self.kc.rc,
		func(rclient *client.RemotingClient, p *packet.Packet) {
			event := pipe.NewPacketEvent(rclient, p)
			err := self.pipeline.FireWork(event)
			if nil != err {
				log.ErrorLog("kite_server", "RemotingServer|onPacketRecieve|FAIL|%s|%t", err, packet.MarshalPacket(p))
			} else {
				// log.Debug("RemotingServer|onPacketRecieve|SUCC|%s|%t\n", rclient.RemoteAddr(), packet)
			}
		})

	err := self.remotingServer.ListenAndServer()
	if nil != err {
		log.Crashf("KiteQServer|RemotionServer|START|FAIL|%s|%s\n", err, self.kc.so.bindHost)
	} else {
		log.InfoLog("kite_server", "KiteQServer|RemotionServer|START|SUCC|%s\n", self.kc.so.bindHost)
	}
	//推送可发送的topic列表并且获取了对应topic下的订阅关系
	succ := self.exchanger.PushQServer(self.kc.so.bindHost, self.kc.so.topics)
	if !succ {
		log.Crashf("KiteQServer|PushQServer|FAIL|%s|%s\n", err, self.kc.so.topics)
	} else {
		log.InfoLog("kite_server", "KiteQServer|PushQServer|SUCC|%s\n", self.kc.so.topics)
	}

	//开启流量统计
	self.startFlow()

	//开启recover
	self.recoverManager.Start()

	//启动DLQ的时间
	self.startDLQ()

	//检查配置更新
	if len(self.kc.so.configPath) > 0 {
		self.startCheckConf()
	}

	//启动pprof
	host, _, _ := net.SplitHostPort(self.kc.so.bindHost)
	go func() {
		if self.kc.so.pprofPort > 0 {
			http.HandleFunc("/stat", self.HandleStat)
			http.HandleFunc("/binds", self.HandleBindings)
			log.Error(http.ListenAndServe(host+":"+strconv.Itoa(self.kc.so.pprofPort), nil))
		}
	}()
}
示例#3
0
//启动
func (self *KiteClientManager) Start() {

	self.zkManager = binding.NewZKManager(self.zkAddr)
	//注册kiteqserver的变更
	self.zkManager.RegisteWather(PATH_KITEQ_SERVER, self)

	hostname, _ := os.Hostname()
	//推送本机到
	err := self.zkManager.PublishTopics(self.topics, self.ga.GroupId, hostname)
	if nil != err {
		log.Crashf("KiteClientManager|PublishTopics|FAIL|%s|%s\n", err, self.topics)
	} else {
		log.Info("KiteClientManager|PublishTopics|SUCC|%s\n", self.topics)
	}

outter:
	for _, b := range self.binds {
		for _, t := range self.topics {
			if t == b.Topic {
				continue outter
			}
		}
		self.topics = append(self.topics, b.Topic)
	}

	for _, topic := range self.topics {

		hosts, err := self.zkManager.GetQServerAndWatch(topic)
		if nil != err {
			log.Crashf("KiteClientManager|GetQServerAndWatch|FAIL|%s|%s\n", err, topic)
		} else {
			log.Info("KiteClientManager|GetQServerAndWatch|SUCC|%s|%s\n", topic, hosts)
		}
		self.onQServerChanged(topic, hosts)
	}

	if len(self.kiteClients) <= 0 {
		log.Crashf("KiteClientManager|Start|NO VALID KITESERVER|%s\n", self.topics)
	}

	if len(self.binds) > 0 {
		//订阅关系推送,并拉取QServer
		err = self.zkManager.PublishBindings(self.ga.GroupId, self.binds)
		if nil != err {
			log.Crashf("KiteClientManager|PublishBindings|FAIL|%s|%s\n", err, self.binds)
		}
	}

}
示例#4
0
func parseDB(kc KiteQConfig) store.IKiteStore {
	db := kc.db

	var kitedb store.IKiteStore
	if strings.HasPrefix(db, "mock://") {
		kitedb = &store.MockKiteStore{}
	} else if strings.HasPrefix(db, "memory://") {
		url := strings.TrimPrefix(db, "memory://")
		split := strings.Split(url, "&")
		params := make(map[string]string, len(split))
		for _, v := range split {
			p := strings.SplitN(v, "=", 2)
			if len(p) >= 2 {
				params[p[0]] = p[1]
			}
		}

		initval := 10 * 10000
		initcap, ok := params["initcap"]
		if ok {
			v, e := strconv.ParseInt(initcap, 10, 32)
			if nil != e {
				log.Crashf("NewKiteQServer|INVALID|INIT CAP|%s\n", db)
			}
			initval = int(v)
		}
		max := 50 * 10000
		maxcap, ok := params["maxcap"]
		if ok {
			v, e := strconv.ParseInt(maxcap, 10, 32)
			if nil != e {
				log.Crashf("NewKiteQServer|INVALID|MAX CAP|%s\n", db)
			}
			max = int(v)
		}
		kitedb = sm.NewKiteMemoryStore(initval, max)
	} else if strings.HasPrefix(db, "mysql://") {
		url := strings.TrimPrefix(db, "mysql://")
		mp := strings.Split(url, "?")
		params := make(map[string]string, 5)
		if len(mp) > 1 {
			split := strings.Split(mp[1], "&")
			for _, v := range split {
				p := strings.SplitN(v, "=", 2)
				params[p[0]] = p[1]
			}
		}

		bus := 100
		u, ok := params["batchUpdateSize"]
		if ok {
			v, e := strconv.ParseInt(u, 10, 32)
			if nil != e {
				log.Crashf("NewKiteQServer|INVALID|batchUpdateSize|%s\n", db)
			}
			bus = int(v)
		}

		bds := 100
		d, ok := params["batchDelSize"]
		if ok {
			v, e := strconv.ParseInt(d, 10, 32)
			if nil != e {
				log.Crashf("NewKiteQServer|INVALID|batchDelSize|%s\n", db)
			}
			bds = int(v)
		}

		flushPeriod := 1 * time.Second
		fp, ok := params["flushPeriod"]
		if ok {
			v, e := strconv.ParseInt(fp, 10, 32)
			if nil != e {
				log.Crashf("NewKiteQServer|INVALID|batchDelSize|%s\n", db)
			}
			flushPeriod = time.Duration(v * int64(flushPeriod))
		}

		maxConn := 500
		mc, ok := params["maxConn"]
		if ok {
			v, e := strconv.ParseInt(mc, 10, 32)
			if nil != e {
				log.Crashf("NewKiteQServer|INVALID|batchDelSize|%s\n", db)
			}
			maxConn = int(v)
		}

		//解析Mysql的host
		master := mp[0]
		slave := ""
		mysqlHosts := strings.Split(mp[0], ",")
		if len(mysqlHosts) > 1 {
			master = mysqlHosts[0]
			slave = mysqlHosts[1]
		}

		//解析用户名密码:
		username, _ := params["username"]
		password, pok := params["password"]
		if !pok {
			password = ""
		}

		//shard的数量
		shardnum := 8
		sd, sdok := params["shardnum"]
		if sdok {
			v, e := strconv.ParseInt(sd, 10, 32)
			if nil != e {
				log.Crashf("NewKiteQServer|INVALID|ShardNum|%s\n", db)
			}
			shardnum = int(v)
		}

		options := smq.MysqlOptions{
			Addr:         master,
			SlaveAddr:    slave,
			ShardNum:     shardnum,
			DB:           params["db"],
			Username:     username,
			Password:     password,
			BatchUpSize:  bus,
			BatchDelSize: bds,
			FlushPeriod:  flushPeriod,
			MaxIdleConn:  maxConn / 2,
			MaxOpenConn:  maxConn}
		kitedb = smq.NewKiteMysql(options)
	} else if strings.HasPrefix(db, "file://") {
		url := strings.TrimPrefix(db, "file://")
		mp := strings.Split(url, "?")
		params := make(map[string]string, 5)
		if len(mp) > 1 {
			split := strings.Split(mp[1], "&")
			for _, v := range split {
				p := strings.SplitN(v, "=", 2)
				params[p[0]] = p[1]
			}
		}
		if len(mp[0]) <= 0 {
			log.Crashf("NewKiteQServer|INVALID|FilePath|%s\n", db)
		}

		//最大消息容量
		maxcap := 100
		d, ok := params["cap"]
		if ok {
			v, e := strconv.ParseInt(d, 10, 32)
			if nil != e {
				log.Crashf("NewKiteQServer|INVALID|cap|%s\n", db)
			}
			maxcap = int(v)
		}

		//检查文件过期时间
		checkPeriod := 1 * time.Second
		fp, ok := params["checkPeriod"]
		if ok {
			v, e := strconv.ParseInt(fp, 10, 32)
			if nil != e {
				log.Crashf("NewKiteQServer|INVALID|checkPeriod|%s\n", db)
			}
			checkPeriod = time.Duration(v * int64(checkPeriod))
		}

		kitedb = smf.NewKiteFileStore(mp[0], maxcap, checkPeriod)
		log.Debug("NewKiteQServer|FILESTORE|%s|%d|%d", mp[0], maxcap, int(checkPeriod.Seconds()))
	} else {
		log.Crashf("NewKiteQServer|UNSUPPORT DB PROTOCOL|%s\n", db)
	}
	kc.flowstat.Kitestore = kitedb
	return kitedb
}