func (self *SourceManager) initSourceServer(business string, flumenodes []config.HostPort) *SourceServer { //首先判断当前是否该sink支持该种business _, ok := self.watcherPool[business] if !ok { log.Printf("unsupport business[%s],HostPorts:[%s]\n", business, flumenodes) return nil } if len(flumenodes) == 0 { log.Println("no valid flume agent node for [" + business + "]") return nil } //新增的消费类型 //使用的pool pools := list.New() for _, hp := range flumenodes { poollink, ok := self.hp2flumeClientPool[hp] if !ok { err, tmppool := pool.NewFlumePoolLink(hp) if nil != err { log.Println("SOURCE_MANGER|INIT FLUMEPOOLLINE|FAIL|%s", err) continue } poollink = tmppool self.hp2flumeClientPool[hp] = poollink } defer func() { if nil == poollink { return } if err := recover(); nil != err { log.Printf("SOURCE_MANGER|CREATE FLUMECLIENT|FAIL|[%s]\n", hp) poollink = nil } }() if nil == poollink { continue } poollink.AttachBusiness(business) pools.PushFront(poollink) } //创建一个sourceserver sourceserver := newSourceServer(business, pools) self.sourceServers[business] = sourceserver return sourceserver }
func Test_SourceServer(t *testing.T) { hp := config.HostPort{Host: "localhost", Port: 44444} err, poollink := pool.NewFlumePoolLink(hp) if nil != err { t.Fail() } list := list.New() list.PushFront(poollink) sourceserver := newSourceServer("location", list) go func() { sourceserver.start() }() for i := 0; i < 100; i++ { _, message := decodeCommand([]byte(LOG)) sourceserver.buffChannel <- message } time.Sleep(10 * time.Second) sourceserver.stop() }
func (self *FlumeWatcher) ChildWatcher(business string, childNode []config.HostPort) { //当前业务下的flume节点发生了变更会全量推送一次新的节点 if len(childNode) <= 0 { self.BusinessWatcher(business, config.Deleted) return } self.sourcemanger.mutex.Lock() defer self.sourcemanger.mutex.Unlock() val, ok := self.sourcemanger.sourceServers[business] if ok { //判断该业务下已经被停掉的节点 // for _, link := range val.flumeClientPool { for e := val.flumeClientPool.Back(); nil != e; e = e.Prev() { link := e.Value.(*pool.FlumePoolLink) hp := link.FlumePool.GetHostPort() contain := false for _, chp := range childNode { //如果当前节点没有变更,即存在在childnode中不变更 if hp == chp { contain = true } } //如果当前的node没有在childnode中则删除该pooL if !contain { size := val.flumeClientPool.Len() link.DetachBusiness(business) val.flumeClientPool.Remove(e) self.clearPool(business, link) //从Business的clientpool中移除该client self.sourcemanger.watcherLog.Printf("WATCHER|BUSINESS:%s|REMOVE FLUME:%s|SIZE:[%d,%d]\n", business, hp, size, val.flumeClientPool.Len()) } } //已经存在那么就检查节点变更 for _, hp := range childNode { //先创建该业务节点: fpool, ok := self.sourcemanger.hp2flumeClientPool[hp] //如果存在Pool直接使用 if ok { //检查该业务已有是否已经该flumepool //如果不包含则创建该池子并加入该业务对应的flumeclientpoollink中 if !fpool.IsAttached(business) { val.flumeClientPool.PushFront(fpool) fpool.AttachBusiness(business) self.sourcemanger.watcherLog.Printf("WATCHER|BUSINESS:[%s]|ADD POOL|[%s]\n", business, hp) } //如果已经包含了,则啥事都不干 } else { //如果不存在该flumepool,直接创建并且添加到该pool种 err, poollink := pool.NewFlumePoolLink(hp) if nil == err || nil != poollink { self.sourcemanger.hp2flumeClientPool[hp] = poollink val.flumeClientPool.PushFront(poollink) poollink.AttachBusiness(business) } else if nil != err { self.sourcemanger.watcherLog.Printf("WATCHER|BUSINESS:[%s]|ADD POOL|FAIL|[%s]|%s\n", business, hp, err.Error()) } } } } else { sourceserver := self.sourcemanger.initSourceServer(business, childNode) sourceserver.start() } }