func (self *SourceServer) innerSend(events []*flume.ThriftFlumeEvent) {
for i := 0; i < 3; i++ {
pool := self.getFlumeClientPool()
flumeclient, err := pool.Get(5 * time.Second)
if nil != err || nil == flumeclient {
log.Printf("LOG_SOURCE|GET FLUMECLIENT|FAIL|%s|%s|TRY:%d\n", self.business, err, i)
continue
}
err = flumeclient.AppendBatch(events)
defer func() {
if err := recover(); nil != err {
//回收这个坏的连接
pool.ReleaseBroken(flumeclient)
} else {
pool.Release(flumeclient)
}
}()
if nil != err {
atomic.AddInt64(&self.monitorCount.currFailValue, int64(1*self.batchSize))
log.Printf("LOG_SOURCE|SEND FLUME|FAIL|%s|%s|TRY:%d\n", self.business, err.Error(), i)
} else {
atomic.AddInt64(&self.monitorCount.currSuccValue, int64(1*self.batchSize))
if rand.Int()%10000 == 0 {
log.Printf("trace|send 2 flume succ|%s|%d\n", flumeclient.HostPort(), len(events))
}
break
}
}
}
func (self *SourceManager) startWorker() {
for k, v := range self.redisPool {
log.Println("LOG_SOURCE|REDIS|[" + k + "]|START")
for _, pool := range v {
go func(queuename string, pool *redis.Pool) {
//批量收集数据
conn := pool.Get()
defer pool.Release(conn)
for self.isRunning {
reply, err := conn.Do("LPOP", queuename)
if nil != err || nil == reply {
if nil != err {
log.Printf("LPOP|FAIL|%T", err)
conn.Close()
conn = pool.Get()
} else {
time.Sleep(100 * time.Millisecond)
}
continue
}
resp := reply.([]byte)
businessName, event := decodeCommand(resp)
if nil == event {
continue
}
//提交到对应business的channel中
sourceServer, ok := self.sourceServers[businessName]
//如果存在则转发出去
if ok {
sourceServer.buffChannel <- event
} else {
log.Printf("LOG_SOURCE_MANGER|NO MATCHES SOURCE_SERVER|%s\n", businessName)
}
}
}(k, pool)
}
}
}
