// alarm judge
func alarmJudge() {
interval := time.Duration(10) * time.Second
for {
time.Sleep(interval)
var content bytes.Buffer
keys := alarmCache.Keys()
if len(keys) == 0 {
continue
}
for _, key := range keys {
aitem, found := alarmCache.GetAndRemove(key)
if !found {
continue
}
content.WriteString(aitem.(*Alarm).String() + "\n")
}
if content.Len() > 5 {
mailContent := formAlarmMailContent(g.Config().Monitor.MailTos, "Self-Monitor.Alarm",
content.String(), "Falcon")
err := sendMail(g.Config().Monitor.MailUrl, mailContent)
if err != nil {
log.Println("alarm send mail error, mail:", mailContent, "", err)
} else {
// statistics
proc.MonitorAlarmMailCnt.Incr()
}
}
}
}
func GetDbConn() (conn *sql.DB, err error) {
conn, err = sql.Open("mysql", g.Config().Index.Dsn)
if err != nil {
return nil, err
}
conn.SetMaxIdleConns(g.Config().Index.MaxIdle)
err = conn.Ping()
if err != nil {
conn.Close()
}
return conn, err
}
func configCommonRoutes() {
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("ok\n"))
})
http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(fmt.Sprintf("%s\n", g.VERSION)))
})
http.HandleFunc("/workdir", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(fmt.Sprintf("%s\n", file.SelfDir())))
})
http.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) {
RenderDataJson(w, g.Config())
})
http.HandleFunc("/config/reload", func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.RemoteAddr, "127.0.0.1") {
g.ParseConfig(g.ConfigFile)
RenderDataJson(w, "ok")
} else {
RenderDataJson(w, "no privilege")
}
})
}
func _monitor() {
client := nhttpclient.GetHttpClient("monitor.get", 5*time.Second, 10*time.Second)
for _, host := range g.Config().Monitor.Cluster {
hostInfo := strings.Split(host, ",") // "module,hostname:port/health"
if len(hostInfo) != 2 {
continue
}
//hostType := hostInfo[0]
hostUrl := hostInfo[1]
if !strings.Contains(hostUrl, "http://") {
hostUrl = "http://" + hostUrl
}
req, _ := http.NewRequest("GET", hostUrl, nil)
req.Header.Set("Connection", "close")
getResp, err := client.Do(req)
if err != nil {
log.Printf(host+", monitor error,", err)
onMonitorErr(host)
continue
}
defer getResp.Body.Close()
body, err := ioutil.ReadAll(getResp.Body) // body=['o','k',...]
if !(err == nil && len(body) >= 2 && string(body[:2]) == "ok") { // err
log.Println(host, ", error,", err)
onMonitorErr(host)
} else { // get "ok"
onMonitorOk(host)
}
}
}
// 启动 索引全量更新 定时任务
func StartIndexUpdateAllTask() {
for graphAddr, cronSpec := range g.Config().Index.Cluster {
ga := graphAddr
indexUpdateAllCron.AddFuncCC(cronSpec, func() { UpdateIndexOfOneGraph(ga, "cron") }, 1)
}
indexUpdateAllCron.Start()
}
func Start() {
if !g.Config().Collector.Enable {
log.Println("collector.Start warning, not enable")
return
}
// init url
if g.Config().Collector.DestUrl != "" {
destUrl = g.Config().Collector.DestUrl
}
if g.Config().Collector.SrcUrlFmt != "" {
srcUrlFmt = g.Config().Collector.SrcUrlFmt
}
// start
go startCollectorCron()
log.Println("collector.Start, ok")
}
// 初始化索引功能模块
func Start() {
if g.Config().Index.Enabled {
StartDB()
StartIndexDeleteTask()
StartIndexUpdateAllTask()
log.Println("index:Start, ok")
} else {
log.Println("index:Start, index not enabled")
}
}
func Start() {
if !g.Config().Monitor.Enabled {
log.Println("monitor.Start, not enable")
return
}
//
monitorCron.AddFunc(cronSpec, func() {
monitor()
})
monitorCron.Start()
go alarmJudge()
log.Println("monitor.Start, ok")
}
func startHttpServer() {
if !g.Config().Http.Enable {
return
}
addr := g.Config().Http.Listen
if addr == "" {
return
}
// init url mapping
configCommonRoutes()
configProcHttpRoutes()
configIndexHttpRoutes()
s := &http.Server{
Addr: addr,
MaxHeaderBytes: 1 << 30,
}
log.Println("http:startHttpServer, ok, listening ", addr)
log.Fatalln(s.ListenAndServe())
}
// 初始化索引功能模块
func Start() {
cfg := g.Config()
if !cfg.Index.Enable {
log.Println("index.Start warning, not enable")
return
}
InitDB()
if cfg.Index.AutoDelete {
StartIndexDeleteTask()
log.Println("index.Start warning, index cleaner enable")
}
StartIndexUpdateAllTask()
log.Println("index.Start ok")
}
func updateAllIndex() {
client := nhttpclient.GetHttpClient("index.updateall", 5*time.Second, 10*time.Second)
for _, hostNamePort := range g.Config().Index.Cluster {
if hostNamePort == "" {
continue
}
destUrl := fmt.Sprintf(destUrlFmt, hostNamePort)
req, _ := http.NewRequest("GET", destUrl, nil)
req.Header.Set("Connection", "close")
getResp, err := client.Do(req)
if err != nil {
log.Printf(hostNamePort+", index update all error,", err)
continue
}
defer getResp.Body.Close()
body, err := ioutil.ReadAll(getResp.Body)
if err != nil {
log.Println(hostNamePort+", index update all error,", err)
continue
}
var data Dto
err = json.Unmarshal(body, &data)
if err != nil {
log.Println(hostNamePort+", index update all error,", err)
continue
}
if data.Data != "ok" {
log.Println(hostNamePort+", index update all error, bad result,", data.Data)
continue
}
}
}
// 手动触发全量更新
func UpdateAllIndex() {
for graphAddr, _ := range g.Config().Index.Cluster {
UpdateIndexOfOneGraph(graphAddr, "manual")
}
}
func _collect() {
clientGet := nhttpclient.GetHttpClient("collector.get", 10*time.Second, 20*time.Second)
tags := "type=statistics,pdl=falcon"
for _, host := range g.Config().Collector.Cluster {
ts := time.Now().Unix()
jsonList := make([]*cmodel.JsonMetaData, 0)
// get statistics by http-get
hostInfo := strings.Split(host, ",") // "module,hostname:port"
if len(hostInfo) != 2 {
continue
}
hostModule := hostInfo[0]
hostNamePort := hostInfo[1]
hostNamePortList := strings.Split(hostNamePort, ":")
if len(hostNamePortList) != 2 {
continue
}
hostName := hostNamePortList[0]
hostPort := hostNamePortList[1]
myTags := tags + ",module=" + hostModule + ",port=" + hostPort
srcUrl := fmt.Sprintf(srcUrlFmt, hostNamePort)
reqGet, _ := http.NewRequest("GET", srcUrl, nil)
reqGet.Header.Set("Connection", "close")
getResp, err := clientGet.Do(reqGet)
if err != nil {
log.Printf(hostNamePort+", get statistics error,", err)
continue
}
defer getResp.Body.Close()
body, err := ioutil.ReadAll(getResp.Body)
if err != nil {
log.Println(hostNamePort+", get statistics error,", err)
continue
}
var data Dto
err = json.Unmarshal(body, &data)
if err != nil {
log.Println(hostNamePort+", get statistics error,", err)
continue
}
for _, item := range data.Data {
if item["Name"] == nil {
continue
}
itemName := item["Name"].(string)
if item["Cnt"] != nil {
var jmdCnt cmodel.JsonMetaData
jmdCnt.Endpoint = hostName
jmdCnt.Metric = itemName
jmdCnt.Timestamp = ts
jmdCnt.Step = 60
jmdCnt.Value = int64(item["Cnt"].(float64))
jmdCnt.CounterType = "GAUGE"
jmdCnt.Tags = myTags
jsonList = append(jsonList, &jmdCnt)
}
if item["Qps"] != nil {
var jmdQps cmodel.JsonMetaData
jmdQps.Endpoint = hostName
jmdQps.Metric = itemName + ".Qps"
jmdQps.Timestamp = ts
jmdQps.Step = 60
jmdQps.Value = int64(item["Qps"].(float64))
jmdQps.CounterType = "GAUGE"
jmdQps.Tags = myTags
jsonList = append(jsonList, &jmdQps)
}
}
// format result
err = sendToTransfer(jsonList, destUrl)
if err != nil {
log.Println(hostNamePort, "send to transfer error,", err.Error())
}
}
// collector.alive
_collectorAlive()
}
func _collect() {
clientGet := nhttpclient.GetHttpClient("collector.get", 10*time.Second, 20*time.Second)
clientPost := nhttpclient.GetHttpClient("collector.post", 5*time.Second, 10*time.Second)
tags := "type=statistics,pdl=falcon"
for _, host := range g.Config().Collector.Cluster {
ts := time.Now().Unix()
jsonList := make([]model.MetricValue, 0)
// get statistics by http-get
hostInfo := strings.Split(host, ",") // "module,hostname:port"
if len(hostInfo) != 2 {
continue
}
hostModule := hostInfo[0]
hostNamePort := hostInfo[1]
hostNamePortList := strings.Split(hostNamePort, ":")
if len(hostNamePortList) != 2 {
continue
}
hostName := hostNamePortList[0]
hostPort := hostNamePortList[1]
myTags := tags + ",module=" + hostModule + ",port=" + hostPort
srcUrl := fmt.Sprintf(srcUrlFmt, hostNamePort)
reqGet, _ := http.NewRequest("GET", srcUrl, nil)
reqGet.Header.Set("Connection", "close")
getResp, err := clientGet.Do(reqGet)
if err != nil {
log.Printf(hostNamePort+", get statistics error,", err)
continue
}
defer getResp.Body.Close()
body, err := ioutil.ReadAll(getResp.Body)
if err != nil {
log.Println(hostNamePort+", get statistics error,", err)
continue
}
var data Dto
err = json.Unmarshal(body, &data)
if err != nil {
log.Println(hostNamePort+", get statistics error,", err)
continue
}
for _, item := range data.Data {
if item["Name"] == nil {
continue
}
itemName := item["Name"].(string)
if item["Cnt"] != nil {
var jmdCnt model.MetricValue
jmdCnt.Endpoint = hostName
jmdCnt.Metric = itemName
jmdCnt.Timestamp = ts
jmdCnt.Step = 60
jmdCnt.Value = int64(item["Cnt"].(float64))
jmdCnt.Type = "GAUGE"
jmdCnt.Tags = myTags
jsonList = append(jsonList, jmdCnt)
}
if item["Qps"] != nil {
var jmdQps model.MetricValue
jmdQps.Endpoint = hostName
jmdQps.Metric = itemName + ".Qps"
jmdQps.Timestamp = ts
jmdQps.Step = 60
jmdQps.Value = int64(item["Qps"].(float64))
jmdQps.Type = "GAUGE"
jmdQps.Tags = myTags
jsonList = append(jsonList, jmdQps)
}
}
if len(jsonList) < 1 { //没取到数据
log.Println("get null from ", hostNamePort)
continue
}
// format result
jsonBody, err := json.Marshal(jsonList)
if err != nil {
log.Println(hostNamePort+", format body error,", err)
continue
}
// send by http-post
req, err := http.NewRequest("POST", destUrl, bytes.NewBuffer(jsonBody))
req.Header.Set("Content-Type", "application/json; charset=UTF-8")
req.Header.Set("Connection", "close")
postResp, err := clientPost.Do(req)
if err != nil {
log.Println(hostNamePort+", post to dest error,", err)
continue
}
defer postResp.Body.Close()
if postResp.StatusCode/100 != 2 {
log.Println(hostNamePort+", post to dest, bad response,", postResp.StatusCode)
}
}
}