func collectDataOnce() int {
keys := config.Keys()
keysLen := len(keys)
// 并发+同步控制
cfg := g.Config().Collector
concurrent := int(cfg.Concurrent)
if concurrent < 1 || concurrent > 50 {
concurrent = 10
}
sema := tsema.NewSemaphore(concurrent)
batch := int(cfg.Batch)
if batch < 100 || batch > 1000 {
batch = 200 //batch不能太小, 否则channel将会很大
}
batchCnt := (keysLen + batch - 1) / batch
rch := make(chan int, batchCnt+1)
i := 0
for i < keysLen {
leftLen := keysLen - i
fetchSize := batch // 每次处理batch个配置
if leftLen < fetchSize {
fetchSize = leftLen
}
fetchKeys := keys[i : i+fetchSize]
// 并发collect数据
sema.Acquire()
go func(keys []string, keySize int) {
defer sema.Release()
size, _ := fetchItemsAndStore(keys, keySize)
rch <- size
}(fetchKeys, fetchSize)
i += fetchSize
}
collectCnt := 0
for i := 0; i < batchCnt; i++ {
select {
case cnt := <-rch:
collectCnt += cnt
}
}
return collectCnt
}
// Do Judge
func judge() {
now := time.Now().Unix()
keys := config.Keys()
for _, key := range keys {
ndcfg, found := config.GetNdConfig(key)
if !found { //策略不存在,不处理
continue
}
step := ndcfg.Step
mock := ndcfg.Mock
item, found := collector.GetFirstItem(key)
if !found { //没有数据,未开始采集,不处理
continue
}
lastTs := now - getTimeout(step)
if item.FStatus != "OK" || item.FTs < lastTs { //数据采集失败,不处理
continue
}
if fCompare(mock, item.Value) == 0 { //采集到的数据为mock数据,则认为上报超时了
if LastTs(key)+step <= now {
TurnNodata(key, now)
genMock(genTs(now, step), key, ndcfg)
}
continue
}
if item.Ts < lastTs { //数据过期, 则认为上报超时
if LastTs(key)+step <= now {
TurnNodata(key, now)
genMock(genTs(now, step), key, ndcfg)
}
continue
}
TurnOk(key, now)
}
}