Пример #1
0
// alarm judge
func alarmJudge() {
	interval := time.Duration(10) * time.Second
	for {
		time.Sleep(interval)
		var content bytes.Buffer

		keys := alarmCache.Keys()
		if len(keys) == 0 {
			continue
		}
		for _, key := range keys {
			aitem, found := alarmCache.GetAndRemove(key)
			if !found {
				continue
			}
			content.WriteString(aitem.(*Alarm).String() + "\n")
		}

		if content.Len() > 5 {
			mailContent := formAlarmMailContent(g.Config().Monitor.MailTos, "Self-Monitor.Alarm",
				content.String(), "Falcon")
			err := sendMail(g.Config().Monitor.MailUrl, mailContent)
			if err != nil {
				log.Println("alarm send mail error, mail:", mailContent, "", err)
			} else {
				// statistics
				proc.MonitorAlarmMailCnt.Incr()
			}
		}
	}
}
Пример #2
0
func GetDbConn() (conn *sql.DB, err error) {
	conn, err = sql.Open("mysql", g.Config().Index.Dsn)
	if err != nil {
		return nil, err
	}

	conn.SetMaxIdleConns(g.Config().Index.MaxIdle)

	err = conn.Ping()
	if err != nil {
		conn.Close()
	}

	return conn, err
}
Пример #3
0
func configCommonRoutes() {
	http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("ok\n"))
	})

	http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(fmt.Sprintf("%s\n", g.VERSION)))
	})

	http.HandleFunc("/workdir", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(fmt.Sprintf("%s\n", file.SelfDir())))
	})

	http.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) {
		RenderDataJson(w, g.Config())
	})

	http.HandleFunc("/config/reload", func(w http.ResponseWriter, r *http.Request) {
		if strings.HasPrefix(r.RemoteAddr, "127.0.0.1") {
			g.ParseConfig(g.ConfigFile)
			RenderDataJson(w, "ok")
		} else {
			RenderDataJson(w, "no privilege")
		}
	})
}
Пример #4
0
func _monitor() {
	client := nhttpclient.GetHttpClient("monitor.get", 5*time.Second, 10*time.Second)
	for _, host := range g.Config().Monitor.Cluster {
		hostInfo := strings.Split(host, ",") // "module,hostname:port/health"
		if len(hostInfo) != 2 {
			continue
		}
		//hostType := hostInfo[0]
		hostUrl := hostInfo[1]
		if !strings.Contains(hostUrl, "http://") {
			hostUrl = "http://" + hostUrl
		}

		req, _ := http.NewRequest("GET", hostUrl, nil)
		req.Header.Set("Connection", "close")
		getResp, err := client.Do(req)
		if err != nil {
			log.Printf(host+", monitor error,", err)
			onMonitorErr(host)
			continue
		}
		defer getResp.Body.Close()

		body, err := ioutil.ReadAll(getResp.Body)                        // body=['o','k',...]
		if !(err == nil && len(body) >= 2 && string(body[:2]) == "ok") { // err
			log.Println(host, ", error,", err)
			onMonitorErr(host)
		} else { // get "ok"
			onMonitorOk(host)
		}
	}
}
Пример #5
0
// 启动 索引全量更新 定时任务
func StartIndexUpdateAllTask() {
	for graphAddr, cronSpec := range g.Config().Index.Cluster {
		ga := graphAddr
		indexUpdateAllCron.AddFuncCC(cronSpec, func() { UpdateIndexOfOneGraph(ga, "cron") }, 1)
	}

	indexUpdateAllCron.Start()
}
Пример #6
0
func Start() {
	if !g.Config().Collector.Enable {
		log.Println("collector.Start warning, not enable")
		return
	}

	// init url
	if g.Config().Collector.DestUrl != "" {
		destUrl = g.Config().Collector.DestUrl
	}
	if g.Config().Collector.SrcUrlFmt != "" {
		srcUrlFmt = g.Config().Collector.SrcUrlFmt
	}
	// start
	go startCollectorCron()
	log.Println("collector.Start, ok")
}
Пример #7
0
// 初始化索引功能模块
func Start() {
	if g.Config().Index.Enabled {
		StartDB()
		StartIndexDeleteTask()
		StartIndexUpdateAllTask()
		log.Println("index:Start, ok")
	} else {
		log.Println("index:Start, index not enabled")
	}
}
Пример #8
0
func Start() {
	if !g.Config().Monitor.Enabled {
		log.Println("monitor.Start, not enable")
		return
	}
	//
	monitorCron.AddFunc(cronSpec, func() {
		monitor()
	})
	monitorCron.Start()
	go alarmJudge()
	log.Println("monitor.Start, ok")
}
Пример #9
0
func startHttpServer() {
	if !g.Config().Http.Enable {
		return
	}

	addr := g.Config().Http.Listen
	if addr == "" {
		return
	}

	// init url mapping
	configCommonRoutes()
	configProcHttpRoutes()
	configIndexHttpRoutes()

	s := &http.Server{
		Addr:           addr,
		MaxHeaderBytes: 1 << 30,
	}

	log.Println("http:startHttpServer, ok, listening ", addr)
	log.Fatalln(s.ListenAndServe())
}
Пример #10
0
// 初始化索引功能模块
func Start() {
	cfg := g.Config()
	if !cfg.Index.Enable {
		log.Println("index.Start warning, not enable")
		return
	}

	InitDB()
	if cfg.Index.AutoDelete {
		StartIndexDeleteTask()
		log.Println("index.Start warning, index cleaner enable")
	}
	StartIndexUpdateAllTask()
	log.Println("index.Start ok")
}
Пример #11
0
func updateAllIndex() {
	client := nhttpclient.GetHttpClient("index.updateall", 5*time.Second, 10*time.Second)
	for _, hostNamePort := range g.Config().Index.Cluster {
		if hostNamePort == "" {
			continue
		}

		destUrl := fmt.Sprintf(destUrlFmt, hostNamePort)
		req, _ := http.NewRequest("GET", destUrl, nil)
		req.Header.Set("Connection", "close")
		getResp, err := client.Do(req)
		if err != nil {
			log.Printf(hostNamePort+", index update all error,", err)
			continue
		}
		defer getResp.Body.Close()

		body, err := ioutil.ReadAll(getResp.Body)
		if err != nil {
			log.Println(hostNamePort+", index update all error,", err)
			continue
		}

		var data Dto
		err = json.Unmarshal(body, &data)
		if err != nil {
			log.Println(hostNamePort+", index update all error,", err)
			continue
		}

		if data.Data != "ok" {
			log.Println(hostNamePort+", index update all error, bad result,", data.Data)
			continue
		}
	}
}
Пример #12
0
// 手动触发全量更新
func UpdateAllIndex() {
	for graphAddr, _ := range g.Config().Index.Cluster {
		UpdateIndexOfOneGraph(graphAddr, "manual")
	}
}
Пример #13
0
func _collect() {
	clientGet := nhttpclient.GetHttpClient("collector.get", 10*time.Second, 20*time.Second)
	tags := "type=statistics,pdl=falcon"
	for _, host := range g.Config().Collector.Cluster {
		ts := time.Now().Unix()
		jsonList := make([]*cmodel.JsonMetaData, 0)

		// get statistics by http-get
		hostInfo := strings.Split(host, ",") // "module,hostname:port"
		if len(hostInfo) != 2 {
			continue
		}
		hostModule := hostInfo[0]
		hostNamePort := hostInfo[1]

		hostNamePortList := strings.Split(hostNamePort, ":")
		if len(hostNamePortList) != 2 {
			continue
		}
		hostName := hostNamePortList[0]
		hostPort := hostNamePortList[1]

		myTags := tags + ",module=" + hostModule + ",port=" + hostPort
		srcUrl := fmt.Sprintf(srcUrlFmt, hostNamePort)
		reqGet, _ := http.NewRequest("GET", srcUrl, nil)
		reqGet.Header.Set("Connection", "close")
		getResp, err := clientGet.Do(reqGet)
		if err != nil {
			log.Printf(hostNamePort+", get statistics error,", err)
			continue
		}
		defer getResp.Body.Close()

		body, err := ioutil.ReadAll(getResp.Body)
		if err != nil {
			log.Println(hostNamePort+", get statistics error,", err)
			continue
		}

		var data Dto
		err = json.Unmarshal(body, &data)
		if err != nil {
			log.Println(hostNamePort+", get statistics error,", err)
			continue
		}

		for _, item := range data.Data {
			if item["Name"] == nil {
				continue
			}
			itemName := item["Name"].(string)

			if item["Cnt"] != nil {
				var jmdCnt cmodel.JsonMetaData
				jmdCnt.Endpoint = hostName
				jmdCnt.Metric = itemName
				jmdCnt.Timestamp = ts
				jmdCnt.Step = 60
				jmdCnt.Value = int64(item["Cnt"].(float64))
				jmdCnt.CounterType = "GAUGE"
				jmdCnt.Tags = myTags
				jsonList = append(jsonList, &jmdCnt)
			}

			if item["Qps"] != nil {
				var jmdQps cmodel.JsonMetaData
				jmdQps.Endpoint = hostName
				jmdQps.Metric = itemName + ".Qps"
				jmdQps.Timestamp = ts
				jmdQps.Step = 60
				jmdQps.Value = int64(item["Qps"].(float64))
				jmdQps.CounterType = "GAUGE"
				jmdQps.Tags = myTags
				jsonList = append(jsonList, &jmdQps)
			}
		}

		// format result
		err = sendToTransfer(jsonList, destUrl)
		if err != nil {
			log.Println(hostNamePort, "send to transfer error,", err.Error())
		}
	}

	// collector.alive
	_collectorAlive()
}
Пример #14
0
func _collect() {
	clientGet := nhttpclient.GetHttpClient("collector.get", 10*time.Second, 20*time.Second)
	clientPost := nhttpclient.GetHttpClient("collector.post", 5*time.Second, 10*time.Second)

	tags := "type=statistics,pdl=falcon"

	for _, host := range g.Config().Collector.Cluster {
		ts := time.Now().Unix()
		jsonList := make([]model.MetricValue, 0)

		// get statistics by http-get
		hostInfo := strings.Split(host, ",") // "module,hostname:port"
		if len(hostInfo) != 2 {
			continue
		}
		hostModule := hostInfo[0]
		hostNamePort := hostInfo[1]

		hostNamePortList := strings.Split(hostNamePort, ":")
		if len(hostNamePortList) != 2 {
			continue
		}
		hostName := hostNamePortList[0]
		hostPort := hostNamePortList[1]

		myTags := tags + ",module=" + hostModule + ",port=" + hostPort
		srcUrl := fmt.Sprintf(srcUrlFmt, hostNamePort)
		reqGet, _ := http.NewRequest("GET", srcUrl, nil)
		reqGet.Header.Set("Connection", "close")
		getResp, err := clientGet.Do(reqGet)
		if err != nil {
			log.Printf(hostNamePort+", get statistics error,", err)
			continue
		}
		defer getResp.Body.Close()

		body, err := ioutil.ReadAll(getResp.Body)
		if err != nil {
			log.Println(hostNamePort+", get statistics error,", err)
			continue
		}

		var data Dto
		err = json.Unmarshal(body, &data)
		if err != nil {
			log.Println(hostNamePort+", get statistics error,", err)
			continue
		}

		for _, item := range data.Data {
			if item["Name"] == nil {
				continue
			}
			itemName := item["Name"].(string)

			if item["Cnt"] != nil {
				var jmdCnt model.MetricValue
				jmdCnt.Endpoint = hostName
				jmdCnt.Metric = itemName
				jmdCnt.Timestamp = ts
				jmdCnt.Step = 60
				jmdCnt.Value = int64(item["Cnt"].(float64))
				jmdCnt.Type = "GAUGE"
				jmdCnt.Tags = myTags
				jsonList = append(jsonList, jmdCnt)
			}

			if item["Qps"] != nil {
				var jmdQps model.MetricValue
				jmdQps.Endpoint = hostName
				jmdQps.Metric = itemName + ".Qps"
				jmdQps.Timestamp = ts
				jmdQps.Step = 60
				jmdQps.Value = int64(item["Qps"].(float64))
				jmdQps.Type = "GAUGE"
				jmdQps.Tags = myTags
				jsonList = append(jsonList, jmdQps)
			}
		}

		if len(jsonList) < 1 { //没取到数据
			log.Println("get null from ", hostNamePort)
			continue
		}

		// format result
		jsonBody, err := json.Marshal(jsonList)
		if err != nil {
			log.Println(hostNamePort+", format body error,", err)
			continue
		}

		// send by http-post
		req, err := http.NewRequest("POST", destUrl, bytes.NewBuffer(jsonBody))
		req.Header.Set("Content-Type", "application/json; charset=UTF-8")
		req.Header.Set("Connection", "close")
		postResp, err := clientPost.Do(req)
		if err != nil {
			log.Println(hostNamePort+", post to dest error,", err)
			continue
		}
		defer postResp.Body.Close()

		if postResp.StatusCode/100 != 2 {
			log.Println(hostNamePort+", post to dest, bad response,", postResp.StatusCode)
		}
	}
}