// TODO 添加对发送任务的控制,比如stop等
func startSendTasks() {
cfg := g.Config()
// init semaphore
judgeConcurrent := cfg.Judge.MaxIdle / 2
graphConcurrent := cfg.Graph.MaxIdle / 2
if judgeConcurrent < 1 {
judgeConcurrent = 1
}
if graphConcurrent < 1 {
graphConcurrent = 1
}
semaSendToJudge = nsema.NewSemaphore(judgeConcurrent)
semaSendToGraph = nsema.NewSemaphore(graphConcurrent)
semaSendToGraphMigrating = nsema.NewSemaphore(graphConcurrent)
// init send go-routines
for node, _ := range cfg.Judge.Cluster {
queue := JudgeQueues[node]
go forward2JudgeTask(queue, node, judgeConcurrent)
}
for node, _ := range cfg.Graph.Cluster {
queue := GraphQueues[node]
go forward2GraphTask(queue, node, graphConcurrent)
}
if cfg.Graph.Migrating {
for node, _ := range cfg.Graph.ClusterMigrating {
queue := GraphMigratingQueues[node]
go forward2GraphMigratingTask(queue, node, graphConcurrent)
}
}
}
func SWifMetricToTransfer() {
log.Println("start SWifMetricToTransfer")
sema := nsema.NewSemaphore(10)
for {
items := IfstatsQueue.PopBackBy(5000)
count := len(items)
if count == 0 {
time.Sleep(DefaultSendTaskSleepInterval)
continue
}
mvsSend := make([]*model.MetricValue, count)
for i := 0; i < count; i++ {
mvsSend[i] = items[i].(*model.MetricValue)
}
// 同步Call + 有限并发 进行发送
sema.Acquire()
go func(mvsend []*model.MetricValue) {
defer sema.Release()
g.SendToTransfer(mvsend)
}(mvsSend)
}
}
func collectDataOnce() int {
keys := config.Keys()
keysLen := len(keys)
// 并发+同步控制
cfg := g.Config().Collector
concurrent := int(cfg.Concurrent)
if concurrent < 1 || concurrent > 50 {
concurrent = 10
}
sema := tsema.NewSemaphore(concurrent)
batch := int(cfg.Batch)
if batch < 100 || batch > 1000 {
batch = 200 //batch不能太小, 否则channel将会很大
}
batchCnt := (keysLen + batch - 1) / batch
rch := make(chan int, batchCnt+1)
i := 0
for i < keysLen {
leftLen := keysLen - i
fetchSize := batch // 每次处理batch个配置
if leftLen < fetchSize {
fetchSize = leftLen
}
fetchKeys := keys[i : i+fetchSize]
// 并发collect数据
sema.Acquire()
go func(keys []string, keySize int) {
defer sema.Release()
size, _ := fetchItemsAndStore(keys, keySize)
rch <- size
}(fetchKeys, fetchSize)
i += fetchSize
}
collectCnt := 0
for i := 0; i < batchCnt; i++ {
select {
case cnt := <-rch:
collectCnt += cnt
}
}
return collectCnt
}
// Judge定时任务, 将 Judge发送缓存中的数据 通过rpc连接池 发送到Judge
func forward2JudgeTask(Q *list.SafeLinkedListLimited, node string, concurrent int) {
batch := g.Config().Judge.Batch // 一次发送,最多batch条数据
addr := g.Config().Judge.Cluster[node]
sema := nsema.NewSemaphore(concurrent)
for {
items := Q.PopBack(batch)
count := len(items)
if count == 0 {
time.Sleep(DefaultSendTaskSleepInterval)
continue
}
judgeItems := make([]*cmodel.JudgeItem, count)
for i := 0; i < count; i++ {
judgeItems[i] = items[i].(*cmodel.JudgeItem)
}
// 同步Call + 有限并发 进行发送
sema.Acquire()
go func(addr string, judgeItems []*cmodel.JudgeItem, count int) {
defer sema.Release()
resp := &cmodel.SimpleRpcResponse{}
var err error
sendOk := false
for i := 0; i < 3; i++ { //最多重试3次
err = JudgeConnPools.Call(addr, "Judge.Send", judgeItems, resp)
if err == nil {
sendOk = true
break
}
time.Sleep(time.Millisecond * 10)
}
if !sendOk {
log.Printf("send judge %s fail: %v", addr, err)
// statistics
proc.SendToJudgeFailCnt.IncrBy(int64(count))
} else {
// statistics
proc.SendToJudgeCnt.IncrBy(int64(count))
}
}(addr, judgeItems, count)
}
}
// Graph定时任务, 进行数据迁移时的 数据冗余发送
func forward2GraphMigratingTask(Q *list.SafeLinkedListLimited, node string, concurrent int) {
batch := g.Config().Graph.Batch // 一次发送,最多batch条数据
addr := g.Config().Graph.ClusterMigrating[node]
sema := nsema.NewSemaphore(concurrent)
for {
items := Q.PopBack(batch)
count := len(items)
if count == 0 {
time.Sleep(DefaultSendTaskSleepInterval)
continue
}
graphItems := make([]*cmodel.GraphItem, count)
for i := 0; i < count; i++ {
graphItems[i] = items[i].(*cmodel.GraphItem)
}
sema.Acquire()
go func(addr string, graphItems []*cmodel.GraphItem, count int) {
defer sema.Release()
resp := &cmodel.SimpleRpcResponse{}
var err error
sendOk := false
for i := 0; i < 3; i++ { //最多重试3次
err = GraphMigratingConnPools.Call(addr, "Graph.Send", graphItems, resp)
if err == nil {
sendOk = true
break
}
time.Sleep(time.Millisecond * 10) //发送失败了,睡10ms
}
if !sendOk {
log.Printf("send to graph migrating %s fail: %v", addr, err)
// statistics
proc.SendToGraphMigratingFailCnt.IncrBy(int64(count))
} else {
// statistics
proc.SendToGraphMigratingCnt.IncrBy(int64(count))
}
}(addr, graphItems, count)
}
}
// Tsdb定时任务, 将数据通过api发送到tsdb
func forward2TsdbTask(concurrent int) {
batch := g.Config().Tsdb.Batch // 一次发送,最多batch条数据
retry := g.Config().Tsdb.MaxRetry
sema := nsema.NewSemaphore(concurrent)
for {
items := TsdbQueue.PopBackBy(batch)
if len(items) == 0 {
time.Sleep(DefaultSendTaskSleepInterval)
continue
}
// 同步Call + 有限并发 进行发送
sema.Acquire()
go func(itemList []interface{}) {
defer sema.Release()
var tsdbBuffer bytes.Buffer
for i := 0; i < len(itemList); i++ {
tsdbItem := itemList[i].(*cmodel.TsdbItem)
tsdbBuffer.WriteString(tsdbItem.TsdbString())
tsdbBuffer.WriteString("\n")
}
var err error
for i := 0; i < retry; i++ {
err = TsdbConnPoolHelper.Send(tsdbBuffer.Bytes())
if err == nil {
proc.SendToTsdbCnt.IncrBy(int64(len(itemList)))
break
}
time.Sleep(100 * time.Millisecond)
}
if err != nil {
proc.SendToTsdbFailCnt.IncrBy(int64(len(itemList)))
log.Println(err)
return
}
}(items)
}
}
// Tsdb定时任务, 将 Tsdb发送缓存中的数据 通过api连接池 发送到Tsdb
// 单个Cluster配置多个influxdb地址，修改为并发发送。
func forward2InfluxdbTask(Q *list.SafeListLimited, node string, concurrent int) {
cfg := g.Config()
batch := cfg.Influxdb.Batch // 一次发送,最多batch条数据
sema := nsema.NewSemaphore(concurrent * len(cfg.Influxdb.Cluster2[node].Addrs))
retry := cfg.Influxdb.MaxRetry
for {
items := Q.PopBackBy(batch)
count := len(items)
if count == 0 {
time.Sleep(DefaultSendTaskSleepInterval)
continue
}
pts := make([]*client.Point, count)
for i := 0; i < count; i++ {
pts[i] = items[i].(*client.Point)
}
for _, addr := range cfg.Influxdb.Cluster2[node].Addrs {
sema.Acquire()
go coreSend2Influxdb(addr, sema, retry, pts)
}
}
}
"net/http"
"time"
cmodel "github.com/open-falcon/common/model"
tsema "github.com/toolkits/concurrent/semaphore"
"github.com/toolkits/container/nmap"
thttpclient "github.com/toolkits/http/httpclient"
ttime "github.com/toolkits/time"
"github.com/open-falcon/nodata/config"
"github.com/open-falcon/nodata/g"
)
var (
MockMap = nmap.NewSafeMap()
sema = tsema.NewSemaphore(1)
)
func Start() {
if !g.Config().Sender.Enabled {
log.Println("sender.Start warning, not enabled")
return
}
startGaussCron()
log.Println("sender.Start ok")
}
func AddMock(key string, endpoint string, metric string, tags string, ts int64, dstype string, step int64, value interface{}) {
item := &cmodel.JsonMetaData{metric, endpoint, ts, step, value, dstype, tags}
MockMap.Put(key, item)
}
cutils "github.com/open-falcon/common/utils"
"github.com/open-falcon/graph/g"
proc "github.com/open-falcon/graph/proc"
nsema "github.com/toolkits/concurrent/semaphore"
ntime "github.com/toolkits/time"
"log"
"time"
)
const (
DefaultUpdateStepInSec = 2 * 24 * 3600 //更新步长,一定不能大于删除步长. 两天内的数据,都可以用来建立索引
ConcurrentOfUpdateIndexAll = 1
)
var (
semaIndexUpdateAllTask = nsema.NewSemaphore(ConcurrentOfUpdateIndexAll) //全量同步任务 并发控制器
semaIndexUpdateAll = nsema.NewSemaphore(4) // 索引全量更新时的mysql操作并发控制
)
// 更新一条监控数据对应的索引. 用于手动添加索引,一般情况下不会使用
func UpdateIndexOne(endpoint string, metric string, tags map[string]string, dstype string, step int) error {
log.Println("1")
itemDemo := &cmodel.GraphItem{
Endpoint: endpoint,
Metric: metric,
Tags: tags,
DsType: dstype,
Step: step,
}
md5 := itemDemo.Checksum()
uuid := itemDemo.UUID()
nsema "github.com/toolkits/concurrent/semaphore"
ntime "github.com/toolkits/time"
cmodel "github.com/open-falcon/common/model"
cutils "github.com/open-falcon/common/utils"
"github.com/open-falcon/graph/g"
proc "github.com/open-falcon/graph/proc"
)
const (
IndexUpdateIncrTaskSleepInterval = time.Duration(1) * time.Second // 增量更新间隔时间, 默认30s
)
var (
semaUpdateIndexIncr = nsema.NewSemaphore(2) // 索引增量更新时操作mysql的并发控制
)
// 启动索引的 异步、增量更新 任务
func StartIndexUpdateIncrTask() {
for {
time.Sleep(IndexUpdateIncrTaskSleepInterval)
startTs := time.Now().Unix()
cnt := updateIndexIncr()
endTs := time.Now().Unix()
// statistics
proc.IndexUpdateIncrCnt.SetCnt(int64(cnt))
proc.IndexUpdateIncr.Incr()
proc.IndexUpdateIncr.PutOther("lastStartTs", ntime.FormatTs(startTs))
proc.IndexUpdateIncr.PutOther("lastTimeConsumingInSec", endTs-startTs)
}
import (
Mdb "github.com/open-falcon/common/db"
"github.com/open-falcon/task/proc"
TSemaphore "github.com/toolkits/concurrent/semaphore"
cron "github.com/toolkits/cron"
"log"
"time"
)
const (
indexDeleteCronSpec = "0 0 2 ? * 6" // 每周6晚上22:00执行一次
deteleStepInSec = 7 * 24 * 3600 // 索引的最大生存周期, sec
)
var (
semaIndexDelete = TSemaphore.NewSemaphore(1)
indexDeleteCron = cron.New()
)
// 启动 索引全量更新 定时任务
func StartIndexDeleteTask() {
indexDeleteCron.AddFunc(indexDeleteCronSpec, func() {
DeleteIndex()
})
indexDeleteCron.Start()
}
func StopIndexDeleteTask() {
indexDeleteCron.Stop()
}
nsema "github.com/toolkits/concurrent/semaphore"
nmap "github.com/toolkits/container/nmap"
ncron "github.com/toolkits/cron"
nhttpclient "github.com/toolkits/http/httpclient"
ntime "github.com/toolkits/time"
"io/ioutil"
"log"
"net/http"
"strings"
"sync"
"time"
)
var (
monitorCron = ncron.New()
sema = nsema.NewSemaphore(1)
statusCache = nmap.NewSafeMap()
alarmCache = nmap.NewSafeMap()
cronSpec = "30 * * * * ?"
)
func Start() {
if !g.Config().Monitor.Enabled {
log.Println("monitor.Start, not enable")
return
}
//
monitorCron.AddFunc(cronSpec, func() {
monitor()
})
monitorCron.Start()
func forward2TransferTask(Q *nlist.SafeListLimited, concurrent int32) {
cfg := g.Config()
batch := int(cfg.Transfer.Batch)
maxConns := int64(cfg.Transfer.MaxConns)
retry := int(cfg.Transfer.Retry)
if retry < 1 {
retry = 1
}
sema := nsema.NewSemaphore(int(concurrent))
transNum := len(TransferHostnames)
for {
items := Q.PopBackBy(batch)
count := len(items)
if count == 0 {
time.Sleep(time.Millisecond * 50)
continue
}
transItems := make([]*cmodel.MetricValue, count)
for i := 0; i < count; i++ {
transItems[i] = convert(items[i].(*cmodel.MetaData))
}
sema.Acquire()
go func(transItems []*cmodel.MetricValue, count int) {
defer sema.Release()
var err error
// 随机遍历transfer列表，直到数据发送成功 或者 遍历完;随机遍历，可以缓解慢transfer
resp := &g.TransferResp{}
sendOk := false
for j := 0; j < retry && !sendOk; j++ {
rint := rand.Int()
for i := 0; i < transNum && !sendOk; i++ {
idx := (i + rint) % transNum
host := TransferHostnames[idx]
addr := TransferMap[host]
// 过滤掉建连缓慢的host, 否则会严重影响发送速率
cc := pfc.GetCounterCount(host)
if cc >= maxConns {
continue
}
pfc.Counter(host, 1)
err = SenderConnPools.Call(addr, "Transfer.Update", transItems, resp)
pfc.Counter(host, -1)
if err == nil {
sendOk = true
// statistics
TransferSendCnt[host].IncrBy(int64(count))
} else {
// statistics
TransferSendFailCnt[host].IncrBy(int64(count))
}
}
}
// statistics
if !sendOk {
if cfg.Debug {
log.Printf("send to transfer fail, connpool:%v", SenderConnPools.Proc())
}
pfc.Meter("SendFail", int64(count))
} else {
pfc.Meter("Send", int64(count))
}
}(transItems, count)
}
}
"github.com/open-falcon/task/g"
"github.com/open-falcon/task/proc"
sema "github.com/toolkits/concurrent/semaphore"
cron "github.com/toolkits/cron"
nhttpclient "github.com/toolkits/http/httpclient"
"io/ioutil"
"log"
"net/http"
"strings"
"time"
)
var (
collectorCron = cron.New()
collectorCronSpec = "0 * * * * ?"
collectorSema = sema.NewSemaphore(1)
srcUrlFmt = "http://%s/statistics/all"
destUrl = "http://127.0.0.1:1988/v1/push"
)
func Start() {
if !g.Config().Collector.Enabled {
log.Println("collector.Start, not enable")
return
}
// init url
if g.Config().Collector.DestUrl != "" {
destUrl = g.Config().Collector.DestUrl
}
if g.Config().Collector.SrcUrlFmt != "" {