Beispiel #1
0
func SendMockOnce() int {
	if !sema.TryAcquire() {
		return -1
	}
	defer sema.Release()

	// not enabled
	if !g.Config().Sender.Enabled {
		return 0
	}

	start := time.Now().Unix()
	cnt, _ := sendMock()
	end := time.Now().Unix()
	if g.Config().Debug {
		log.Printf("sender cron, cnt %d, time %ds, start %s", cnt, end-start, ttime.FormatTs(start))
	}

	// statistics
	g.SenderCronCnt.Incr()
	g.SenderLastTs.SetCnt(end - start)
	g.SenderCnt.IncrBy(int64(cnt))

	return cnt
}
Beispiel #2
0
// internal
func makeDbConn() (conn *sql.DB, err error) {
	conn, err = sql.Open("mysql", g.Config().Config.Dsn)
	if err != nil {
		return nil, err
	}

	conn.SetMaxIdleConns(int(g.Config().Config.MaxIdle))
	err = conn.Ping()

	return conn, err
}
Beispiel #3
0
func getThreshold() int32 {
	lock.RLock()
	defer lock.RUnlock()

	cfg := g.Config().Sender.Block
	if avg < 0 { // gauss not inited
		return cfg.Threshold
	}

	// 3-sigma
	dt_gauss_3sigma_min := cfg.Gauss3SigmaMin
	dt_gauss_3sigma_max := cfg.Gauss3SigmaMax
	threeSigma := 3 * dev
	if threeSigma < dt_gauss_3sigma_min {
		threeSigma = dt_gauss_3sigma_min
	} else if threeSigma > dt_gauss_3sigma_max {
		threeSigma = dt_gauss_3sigma_max
	}

	// threshold
	gaussThreshold := int32(math.Ceil(avg + threeSigma))
	if gaussThreshold < cfg.Threshold {
		gaussThreshold = cfg.Threshold
	}

	return gaussThreshold
}
Beispiel #4
0
func configCommonRoutes() {
	http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("ok\n"))
	})

	http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(fmt.Sprintf("%s\n", g.VERSION)))
	})

	http.HandleFunc("/workdir", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(fmt.Sprintf("%s\n", file.SelfDir())))
	})

	http.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) {
		RenderDataJson(w, g.Config())
	})

	http.HandleFunc("/config/reload", func(w http.ResponseWriter, r *http.Request) {
		if strings.HasPrefix(r.RemoteAddr, "127.0.0.1") {
			g.ParseConfig(g.ConfigFile)
			RenderDataJson(w, "ok")
		} else {
			RenderDataJson(w, "no privilege")
		}
	})
}
Beispiel #5
0
func Start() {
	if !g.Config().Sender.Enabled {
		log.Println("sender.Start warning, not enabled")
		return
	}
	startGaussCron()
	log.Println("sender.Start ok")
}
Beispiel #6
0
func Start() {
	if !g.Config().Collector.Enabled {
		log.Println("collector.Start warning, not enabled")
		return
	}

	StartCollectorCron()
	log.Println("collector.Start ok")
}
Beispiel #7
0
func startGaussCron() {
	if !g.Config().Sender.Block.EnableGauss {
		log.Println("sender.StartGaussCron warning, not enabled")
		return
	}

	// start gauss cron
	gaussCron.AddFuncCC("40 */20 * * * ?", func() {
		start := time.Now().Unix()
		cnt := calcGaussOnce()
		end := time.Now().Unix()
		if g.Config().Debug {
			log.Printf("gause cron, cnt %d, time %ds, start %s\n", cnt, end-start, ttime.FormatTs(start))
		}
	}, 1)
	gaussCron.Start()
	log.Println("sender.StartGaussCron ok")
}
Beispiel #8
0
func Start() {
	if !g.Config().Config.Enabled {
		log.Println("config.Start warning, not enabled")
		return
	}

	service.InitDB()
	StartNdConfigCron()
	log.Println("config.Start ok")
}
Beispiel #9
0
func startHttpServer() {
	if !g.Config().Http.Enabled {
		return
	}

	addr := g.Config().Http.Listen
	if addr == "" {
		return
	}

	configRoutes()

	s := &http.Server{
		Addr:           addr,
		MaxHeaderBytes: 1 << 30,
	}

	log.Println("http.startHttpServer ok, listening", addr)
	log.Fatalln(s.ListenAndServe())
}
Beispiel #10
0
func collectDataOnce() int {
	keys := config.Keys()
	keysLen := len(keys)

	// 并发+同步控制
	cfg := g.Config().Collector
	concurrent := int(cfg.Concurrent)
	if concurrent < 1 || concurrent > 50 {
		concurrent = 10
	}
	sema := tsema.NewSemaphore(concurrent)

	batch := int(cfg.Batch)
	if batch < 100 || batch > 1000 {
		batch = 200 //batch不能太小, 否则channel将会很大
	}

	batchCnt := (keysLen + batch - 1) / batch
	rch := make(chan int, batchCnt+1)

	i := 0
	for i < keysLen {
		leftLen := keysLen - i
		fetchSize := batch // 每次处理batch个配置
		if leftLen < fetchSize {
			fetchSize = leftLen
		}
		fetchKeys := keys[i : i+fetchSize]

		// 并发collect数据
		sema.Acquire()
		go func(keys []string, keySize int) {
			defer sema.Release()
			size, _ := fetchItemsAndStore(keys, keySize)
			rch <- size
		}(fetchKeys, fetchSize)

		i += fetchSize
	}

	collectCnt := 0
	for i := 0; i < batchCnt; i++ {
		select {
		case cnt := <-rch:
			collectCnt += cnt
		}
	}

	return collectCnt
}
Beispiel #11
0
func calcGaussOnce() int {
	values := fetchRawItems()
	size := len(values)
	if size < 100 {
		return size
	}

	// gauss
	myavg, mydev := gaussDistribution(values)

	// filter
	nvals := make([]float64, 0)
	filter := mydev
	if filter < g.Config().Sender.Block.GaussFilter { //防止过度的过滤
		filter = g.Config().Sender.Block.GaussFilter
	}
	for _, val := range values {
		if (val-myavg) > filter || val-myavg < (-filter) {
			continue
		}
		nvals = append(nvals, val)
	}
	if len(nvals) < 100 {
		return size
	}

	// gauss
	myavg, mydev = gaussDistribution(nvals)

	lock.Lock()
	defer lock.Unlock()
	avg = myavg
	dev = mydev
	log.Printf("gause status, avg %f, dev %f\n", avg, dev)

	return size
}
Beispiel #12
0
func StartNdConfigCron() {
	ndconfigCron.AddFuncCC(ndconfigCronSpec, func() {
		start := time.Now().Unix()
		cnt, _ := syncNdConfig()
		end := time.Now().Unix()
		if g.Config().Debug {
			log.Printf("config cron, cnt %d, time %ds, start %s\n", cnt, end-start, ttime.FormatTs(start))
		}

		// statistics
		g.ConfigCronCnt.Incr()
		g.ConfigLastTs.SetCnt(end - start)
		g.ConfigLastCnt.SetCnt(int64(cnt))
	}, 1)
	ndconfigCron.Start()
}
Beispiel #13
0
func StartCollectorCron() {
	collectorCron.AddFuncCC("*/20 * * * * ?", func() {
		start := time.Now().Unix()
		cnt := collectDataOnce()
		end := time.Now().Unix()
		if g.Config().Debug {
			log.Printf("collect cron, cnt %d, time %ds, start %s\n", cnt, end-start, ttime.FormatTs(start))
		}

		// statistics
		g.CollectorCronCnt.Incr()
		g.CollectorLastTs.SetCnt(end - start)
		g.CollectorLastCnt.SetCnt(int64(cnt))
		g.CollectorCnt.IncrBy(int64(cnt))
	}, 1)
	collectorCron.Start()
}
Beispiel #14
0
func StartJudgeCron() {
	judgeCron.AddFuncCC(judgeCronSpec, func() {
		start := time.Now().Unix()
		judge()
		end := time.Now().Unix()
		if g.Config().Debug {
			log.Printf("judge cron, time %ds, start %s\n", end-start, ttime.FormatTs(start))
		}

		// statistics
		g.JudgeCronCnt.Incr()
		g.JudgeLastTs.SetCnt(end - start)

		// trigger sender
		sender.SendMockOnceAsync()
	}, 1)
	judgeCron.Start()
}
Beispiel #15
0
func sendItemsToTransfer(items []*cmodel.JsonMetaData, size int, httpcliname string,
	connT, reqT time.Duration) (cnt int, errt error) {
	if size < 1 {
		return
	}

	cfg := g.Config()
	transUlr := fmt.Sprintf("http://%s/api/push", cfg.Sender.TransferAddr)
	hcli := thttpclient.GetHttpClient(httpcliname, connT, reqT)

	// form request args
	itemsBody, err := json.Marshal(items)
	if err != nil {
		log.Println(transUlr+", format body error,", err)
		errt = err
		return
	}

	// fetch items
	req, err := http.NewRequest("POST", transUlr, bytes.NewBuffer(itemsBody))
	req.Header.Set("Content-Type", "application/json; charset=UTF-8")
	req.Header.Set("Connection", "close")
	postResp, err := hcli.Do(req)
	if err != nil {
		log.Println(transUlr+", post to dest error,", err)
		errt = err
		return
	}
	defer postResp.Body.Close()

	if postResp.StatusCode/100 != 2 {
		log.Println(transUlr+", post to dest, bad response,", postResp.Body)
		errt = fmt.Errorf("request failed, %s", postResp.Body)
		return
	}

	return size, nil
}
Beispiel #16
0
func sendMock() (cnt int, errt error) {
	// check nodata flood
	cfgsize := config.Size()
	ndsize := MockMap.Size()
	if cfgsize < 1 {
		return
	}

	rate := int32(100 * ndsize / cfgsize)
	threshold := getThreshold()
	if g.Config().Debug {
		log.Printf("nodata threshold: %d", threshold)
	}
	// statistics
	g.FloodRate.SetCnt(int64(rate))
	g.Threshold.SetCnt(int64(threshold))

	if g.Config().Sender.Block.Enabled {
		if rate > threshold || g.Config().Sender.Block.SetBlock { // nodata flooding, blocking
			log.Printf("nodata blocking: flood rate %d, threshold %d", int(rate), int(threshold))
			// statistics
			g.Blocking.SetCnt(1)
			// clear send buffer
			MockMap.Clear()
			return 0, nil
		}
	}
	// statistics
	g.Blocking.SetCnt(0)

	cfg := g.Config().Sender
	batch := int(cfg.Batch)
	connTimeout := cfg.ConnectTimeout
	requTimeout := cfg.RequestTimeout

	// send mock to transfer
	mocks := MockMap.Slice()
	MockMap.Clear()
	mockSize := len(mocks)
	i := 0
	for i < mockSize {
		leftLen := mockSize - i
		sendSize := batch
		if leftLen < sendSize {
			sendSize = leftLen
		}
		fetchMocks := mocks[i : i+sendSize]
		i += sendSize

		items := make([]*cmodel.JsonMetaData, 0)
		for _, val := range fetchMocks {
			if val == nil {
				continue
			}
			items = append(items, val.(*cmodel.JsonMetaData))
		}
		cntonce, err := sendItemsToTransfer(items, len(items), "nodata.mock",
			time.Millisecond*time.Duration(connTimeout),
			time.Millisecond*time.Duration(requTimeout))
		if err == nil {
			if g.Config().Debug {
				log.Println("send items:", items)
			}
			cnt += cntonce
		}
	}

	return cnt, nil
}
Beispiel #17
0
func fetchItemsAndStore(fetchKeys []string, fetchSize int) (size int, errt error) {
	if fetchSize < 1 {
		return
	}

	cfg := g.Config()
	queryUlr := fmt.Sprintf("http://%s/graph/last", cfg.Query.QueryAddr)
	hcli := thttpclient.GetHttpClient("nodata.collector",
		time.Millisecond*time.Duration(cfg.Query.ConnectTimeout),
		time.Millisecond*time.Duration(cfg.Query.RequestTimeout))

	// form request args
	args := make([]*cmodel.GraphLastParam, 0)
	for _, key := range fetchKeys {
		ndcfg, found := config.GetNdConfig(key)
		if !found {
			continue
		}

		endpoint := ndcfg.Endpoint
		counter := cutils.Counter(ndcfg.Metric, ndcfg.Tags)
		arg := &cmodel.GraphLastParam{endpoint, counter}
		args = append(args, arg)
	}
	if len(args) < 1 {
		return
	}
	argsBody, err := json.Marshal(args)
	if err != nil {
		log.Println(queryUlr+", format body error,", err)
		errt = err
		return
	}

	// fetch items
	req, err := http.NewRequest("POST", queryUlr, bytes.NewBuffer(argsBody))
	req.Header.Set("Content-Type", "application/json; charset=UTF-8")
	req.Header.Set("Connection", "close")
	postResp, err := hcli.Do(req)
	if err != nil {
		log.Println(queryUlr+", post to dest error,", err)
		errt = err
		return
	}
	defer postResp.Body.Close()

	if postResp.StatusCode/100 != 2 {
		log.Println(queryUlr+", post to dest, bad response,", postResp.Body)
		errt = fmt.Errorf("request failed, %s", postResp.Body)
		return
	}

	body, err := ioutil.ReadAll(postResp.Body)
	if err != nil {
		log.Println(queryUlr+", read response error,", err)
		errt = err
		return
	}

	resp := []*cmodel.GraphLastResp{}
	err = json.Unmarshal(body, &resp)
	if err != nil {
		log.Println(queryUlr+", unmarshal error,", err)
		errt = err
		return
	}

	// store items
	fts := time.Now().Unix()
	for _, glr := range resp {
		//log.Printf("collect:%v\n", glr)
		if glr == nil || glr.Value == nil {
			continue
		}
		AddItem(cutils.PK2(glr.Endpoint, glr.Counter), NewDataItem(glr.Value.Timestamp, float64(glr.Value.Value), "OK", fts))
	}

	return len(resp), nil
}
Beispiel #18
0
func fetchRawItems() (values []float64) {
	cfg := g.Config()
	queryUlr := fmt.Sprintf("http://%s/graph/history", cfg.Query.QueryAddr)
	hcli := thttpclient.GetHttpClient("nodata.gauss",
		time.Millisecond*time.Duration(cfg.Query.ConnectTimeout),
		time.Millisecond*time.Duration(cfg.Query.RequestTimeout))

	// form request args
	nowTs := time.Now().Unix()
	endTs := nowTs - nowTs%1200
	startTs := endTs - 24*3600*5 //用5天的数据,做高斯拟合
	hostname, _ := os.Hostname()
	if len(cfg.Sender.Block.Hostname) > 0 {
		hostname = cfg.Sender.Block.Hostname
	}
	fcounter := "FloodRate/module=nodata,pdl=falcon,port=6090,type=statistics"
	if len(cfg.Sender.Block.FloodCounter) > 0 {
		fcounter = cfg.Sender.Block.FloodCounter
	}
	endpointCounters := make([]cmodel.GraphInfoParam, 0)
	endpointCounters = append(endpointCounters, cmodel.GraphInfoParam{Endpoint: hostname, Counter: fcounter})
	args := GraphHistoryParam{Start: startTs, End: endTs, CF: "AVERAGE", EndpointCounters: endpointCounters}

	argsBody, err := json.Marshal(args)
	if err != nil {
		log.Println(queryUlr+", format body error,", err)
		return
	}

	// fetch items
	req, err := http.NewRequest("POST", queryUlr, bytes.NewBuffer(argsBody))
	req.Header.Set("Content-Type", "application/json; charset=UTF-8")
	req.Header.Set("Connection", "close")
	postResp, err := hcli.Do(req)
	if err != nil {
		log.Println(queryUlr+", post to dest error,", err)
		return
	}
	defer postResp.Body.Close()

	if postResp.StatusCode/100 != 2 {
		log.Println(queryUlr+", post to dest, bad response,", postResp.Body)
		return
	}

	body, err := ioutil.ReadAll(postResp.Body)
	if err != nil {
		log.Println(queryUlr+", read response error,", err)
		return
	}

	resp := make([]*cmodel.GraphQueryResponse, 0)
	err = json.Unmarshal(body, &resp)
	if err != nil {
		log.Println(queryUlr+", unmarshal error,", err)
		return
	}
	if len(resp) != 1 || resp[0] == nil {
		return
	}

	// store items
	values = make([]float64, 0)
	for _, glr := range resp[0].Values {
		if glr == nil || math.IsNaN(float64(glr.Value)) {
			continue
		}
		values = append(values, float64(glr.Value))
	}

	return values
}