func (self *SourceManager) initSourceServer(business string, flumenodes []config.HostPort) *SourceServer {
//首先判断当前是否该sink支持该种business
_, ok := self.watcherPool[business]
if !ok {
log.Printf("unsupport business[%s],HostPorts:[%s]\n", business, flumenodes)
return nil
}
if len(flumenodes) == 0 {
log.Println("no valid flume agent node for [" + business + "]")
return nil
}
//新增的消费类型
//使用的pool
pools := list.New()
for _, hp := range flumenodes {
poollink, ok := self.hp2flumeClientPool[hp]
if !ok {
err, tmppool := pool.NewFlumePoolLink(hp)
if nil != err {
log.Println("SOURCE_MANGER|INIT FLUMEPOOLLINE|FAIL|%s", err)
continue
}
poollink = tmppool
self.hp2flumeClientPool[hp] = poollink
}
defer func() {
if nil == poollink {
return
}
if err := recover(); nil != err {
log.Printf("SOURCE_MANGER|CREATE FLUMECLIENT|FAIL|[%s]\n", hp)
poollink = nil
}
}()
if nil == poollink {
continue
}
poollink.AttachBusiness(business)
pools.PushFront(poollink)
}
//创建一个sourceserver
sourceserver := newSourceServer(business, pools)
self.sourceServers[business] = sourceserver
return sourceserver
}
func Test_SourceServer(t *testing.T) {
hp := config.HostPort{Host: "localhost", Port: 44444}
err, poollink := pool.NewFlumePoolLink(hp)
if nil != err {
t.Fail()
}
list := list.New()
list.PushFront(poollink)
sourceserver := newSourceServer("location", list)
go func() { sourceserver.start() }()
for i := 0; i < 100; i++ {
_, message := decodeCommand([]byte(LOG))
sourceserver.buffChannel <- message
}
time.Sleep(10 * time.Second)
sourceserver.stop()
}
func (self *FlumeWatcher) ChildWatcher(business string, childNode []config.HostPort) {
//当前业务下的flume节点发生了变更会全量推送一次新的节点
if len(childNode) <= 0 {
self.BusinessWatcher(business, config.Deleted)
return
}
self.sourcemanger.mutex.Lock()
defer self.sourcemanger.mutex.Unlock()
val, ok := self.sourcemanger.sourceServers[business]
if ok {
//判断该业务下已经被停掉的节点
// for _, link := range val.flumeClientPool {
for e := val.flumeClientPool.Back(); nil != e; e = e.Prev() {
link := e.Value.(*pool.FlumePoolLink)
hp := link.FlumePool.GetHostPort()
contain := false
for _, chp := range childNode {
//如果当前节点没有变更，即存在在childnode中不变更
if hp == chp {
contain = true
}
}
//如果当前的node没有在childnode中则删除该pooL
if !contain {
size := val.flumeClientPool.Len()
link.DetachBusiness(business)
val.flumeClientPool.Remove(e)
self.clearPool(business, link)
//从Business的clientpool中移除该client
self.sourcemanger.watcherLog.Printf("WATCHER|BUSINESS:%s|REMOVE FLUME:%s|SIZE:[%d,%d]\n",
business, hp, size, val.flumeClientPool.Len())
}
}
//已经存在那么就检查节点变更
for _, hp := range childNode {
//先创建该业务节点：
fpool, ok := self.sourcemanger.hp2flumeClientPool[hp]
//如果存在Pool直接使用
if ok {
//检查该业务已有是否已经该flumepool
//如果不包含则创建该池子并加入该业务对应的flumeclientpoollink中
if !fpool.IsAttached(business) {
val.flumeClientPool.PushFront(fpool)
fpool.AttachBusiness(business)
self.sourcemanger.watcherLog.Printf("WATCHER|BUSINESS:[%s]|ADD POOL|[%s]\n", business, hp)
}
//如果已经包含了，则啥事都不干
} else {
//如果不存在该flumepool，直接创建并且添加到该pool种
err, poollink := pool.NewFlumePoolLink(hp)
if nil == err || nil != poollink {
self.sourcemanger.hp2flumeClientPool[hp] = poollink
val.flumeClientPool.PushFront(poollink)
poollink.AttachBusiness(business)
} else if nil != err {
self.sourcemanger.watcherLog.Printf("WATCHER|BUSINESS:[%s]|ADD POOL|FAIL|[%s]|%s\n", business, hp, err.Error())
}
}
}
} else {
sourceserver := self.sourcemanger.initSourceServer(business, childNode)
sourceserver.start()
}
}