Esempio n. 1
0
func collectDataOnce() int {
	keys := config.Keys()
	keysLen := len(keys)

	// 并发+同步控制
	cfg := g.Config().Collector
	concurrent := int(cfg.Concurrent)
	if concurrent < 1 || concurrent > 50 {
		concurrent = 10
	}
	sema := tsema.NewSemaphore(concurrent)

	batch := int(cfg.Batch)
	if batch < 100 || batch > 1000 {
		batch = 200 //batch不能太小, 否则channel将会很大
	}

	batchCnt := (keysLen + batch - 1) / batch
	rch := make(chan int, batchCnt+1)

	i := 0
	for i < keysLen {
		leftLen := keysLen - i
		fetchSize := batch // 每次处理batch个配置
		if leftLen < fetchSize {
			fetchSize = leftLen
		}
		fetchKeys := keys[i : i+fetchSize]

		// 并发collect数据
		sema.Acquire()
		go func(keys []string, keySize int) {
			defer sema.Release()
			size, _ := fetchItemsAndStore(keys, keySize)
			rch <- size
		}(fetchKeys, fetchSize)

		i += fetchSize
	}

	collectCnt := 0
	for i := 0; i < batchCnt; i++ {
		select {
		case cnt := <-rch:
			collectCnt += cnt
		}
	}

	return collectCnt
}
Esempio n. 2
0
// Do Judge
func judge() {
	now := time.Now().Unix()
	keys := config.Keys()
	for _, key := range keys {
		ndcfg, found := config.GetNdConfig(key)
		if !found { //策略不存在,不处理
			continue
		}
		step := ndcfg.Step
		mock := ndcfg.Mock

		item, found := collector.GetFirstItem(key)
		if !found { //没有数据,未开始采集,不处理
			continue
		}

		lastTs := now - getTimeout(step)
		if item.FStatus != "OK" || item.FTs < lastTs { //数据采集失败,不处理
			continue
		}

		if fCompare(mock, item.Value) == 0 { //采集到的数据为mock数据,则认为上报超时了
			if LastTs(key)+step <= now {
				TurnNodata(key, now)
				genMock(genTs(now, step), key, ndcfg)
			}
			continue
		}

		if item.Ts < lastTs { //数据过期, 则认为上报超时
			if LastTs(key)+step <= now {
				TurnNodata(key, now)
				genMock(genTs(now, step), key, ndcfg)
			}
			continue
		}

		TurnOk(key, now)
	}
}