// alarm judge func alarmJudge() { interval := time.Duration(10) * time.Second for { time.Sleep(interval) var content bytes.Buffer keys := alarmCache.Keys() if len(keys) == 0 { continue } for _, key := range keys { aitem, found := alarmCache.GetAndRemove(key) if !found { continue } content.WriteString(aitem.(*Alarm).String() + "\n") } if content.Len() > 5 { mailContent := formAlarmMailContent(g.Config().Monitor.MailTos, "Self-Monitor.Alarm", content.String(), "Falcon") err := sendMail(g.Config().Monitor.MailUrl, mailContent) if err != nil { log.Println("alarm send mail error, mail:", mailContent, "", err) } else { // statistics proc.MonitorAlarmMailCnt.Incr() } } } }
func GetDbConn() (conn *sql.DB, err error) { conn, err = sql.Open("mysql", g.Config().Index.Dsn) if err != nil { return nil, err } conn.SetMaxIdleConns(g.Config().Index.MaxIdle) err = conn.Ping() if err != nil { conn.Close() } return conn, err }
func configCommonRoutes() { http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ok\n")) }) http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(fmt.Sprintf("%s\n", g.VERSION))) }) http.HandleFunc("/workdir", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(fmt.Sprintf("%s\n", file.SelfDir()))) }) http.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) { RenderDataJson(w, g.Config()) }) http.HandleFunc("/config/reload", func(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(r.RemoteAddr, "127.0.0.1") { g.ParseConfig(g.ConfigFile) RenderDataJson(w, "ok") } else { RenderDataJson(w, "no privilege") } }) }
func _monitor() { client := nhttpclient.GetHttpClient("monitor.get", 5*time.Second, 10*time.Second) for _, host := range g.Config().Monitor.Cluster { hostInfo := strings.Split(host, ",") // "module,hostname:port/health" if len(hostInfo) != 2 { continue } //hostType := hostInfo[0] hostUrl := hostInfo[1] if !strings.Contains(hostUrl, "http://") { hostUrl = "http://" + hostUrl } req, _ := http.NewRequest("GET", hostUrl, nil) req.Header.Set("Connection", "close") getResp, err := client.Do(req) if err != nil { log.Printf(host+", monitor error,", err) onMonitorErr(host) continue } defer getResp.Body.Close() body, err := ioutil.ReadAll(getResp.Body) // body=['o','k',...] if !(err == nil && len(body) >= 2 && string(body[:2]) == "ok") { // err log.Println(host, ", error,", err) onMonitorErr(host) } else { // get "ok" onMonitorOk(host) } } }
// 启动 索引全量更新 定时任务 func StartIndexUpdateAllTask() { for graphAddr, cronSpec := range g.Config().Index.Cluster { ga := graphAddr indexUpdateAllCron.AddFuncCC(cronSpec, func() { UpdateIndexOfOneGraph(ga, "cron") }, 1) } indexUpdateAllCron.Start() }
func Start() { if !g.Config().Collector.Enable { log.Println("collector.Start warning, not enable") return } // init url if g.Config().Collector.DestUrl != "" { destUrl = g.Config().Collector.DestUrl } if g.Config().Collector.SrcUrlFmt != "" { srcUrlFmt = g.Config().Collector.SrcUrlFmt } // start go startCollectorCron() log.Println("collector.Start, ok") }
// 初始化索引功能模块 func Start() { if g.Config().Index.Enabled { StartDB() StartIndexDeleteTask() StartIndexUpdateAllTask() log.Println("index:Start, ok") } else { log.Println("index:Start, index not enabled") } }
func Start() { if !g.Config().Monitor.Enabled { log.Println("monitor.Start, not enable") return } // monitorCron.AddFunc(cronSpec, func() { monitor() }) monitorCron.Start() go alarmJudge() log.Println("monitor.Start, ok") }
func startHttpServer() { if !g.Config().Http.Enable { return } addr := g.Config().Http.Listen if addr == "" { return } // init url mapping configCommonRoutes() configProcHttpRoutes() configIndexHttpRoutes() s := &http.Server{ Addr: addr, MaxHeaderBytes: 1 << 30, } log.Println("http:startHttpServer, ok, listening ", addr) log.Fatalln(s.ListenAndServe()) }
// 初始化索引功能模块 func Start() { cfg := g.Config() if !cfg.Index.Enable { log.Println("index.Start warning, not enable") return } InitDB() if cfg.Index.AutoDelete { StartIndexDeleteTask() log.Println("index.Start warning, index cleaner enable") } StartIndexUpdateAllTask() log.Println("index.Start ok") }
func updateAllIndex() { client := nhttpclient.GetHttpClient("index.updateall", 5*time.Second, 10*time.Second) for _, hostNamePort := range g.Config().Index.Cluster { if hostNamePort == "" { continue } destUrl := fmt.Sprintf(destUrlFmt, hostNamePort) req, _ := http.NewRequest("GET", destUrl, nil) req.Header.Set("Connection", "close") getResp, err := client.Do(req) if err != nil { log.Printf(hostNamePort+", index update all error,", err) continue } defer getResp.Body.Close() body, err := ioutil.ReadAll(getResp.Body) if err != nil { log.Println(hostNamePort+", index update all error,", err) continue } var data Dto err = json.Unmarshal(body, &data) if err != nil { log.Println(hostNamePort+", index update all error,", err) continue } if data.Data != "ok" { log.Println(hostNamePort+", index update all error, bad result,", data.Data) continue } } }
// 手动触发全量更新 func UpdateAllIndex() { for graphAddr, _ := range g.Config().Index.Cluster { UpdateIndexOfOneGraph(graphAddr, "manual") } }
func _collect() { clientGet := nhttpclient.GetHttpClient("collector.get", 10*time.Second, 20*time.Second) tags := "type=statistics,pdl=falcon" for _, host := range g.Config().Collector.Cluster { ts := time.Now().Unix() jsonList := make([]*cmodel.JsonMetaData, 0) // get statistics by http-get hostInfo := strings.Split(host, ",") // "module,hostname:port" if len(hostInfo) != 2 { continue } hostModule := hostInfo[0] hostNamePort := hostInfo[1] hostNamePortList := strings.Split(hostNamePort, ":") if len(hostNamePortList) != 2 { continue } hostName := hostNamePortList[0] hostPort := hostNamePortList[1] myTags := tags + ",module=" + hostModule + ",port=" + hostPort srcUrl := fmt.Sprintf(srcUrlFmt, hostNamePort) reqGet, _ := http.NewRequest("GET", srcUrl, nil) reqGet.Header.Set("Connection", "close") getResp, err := clientGet.Do(reqGet) if err != nil { log.Printf(hostNamePort+", get statistics error,", err) continue } defer getResp.Body.Close() body, err := ioutil.ReadAll(getResp.Body) if err != nil { log.Println(hostNamePort+", get statistics error,", err) continue } var data Dto err = json.Unmarshal(body, &data) if err != nil { log.Println(hostNamePort+", get statistics error,", err) continue } for _, item := range data.Data { if item["Name"] == nil { continue } itemName := item["Name"].(string) if item["Cnt"] != nil { var jmdCnt cmodel.JsonMetaData jmdCnt.Endpoint = hostName jmdCnt.Metric = itemName jmdCnt.Timestamp = ts jmdCnt.Step = 60 jmdCnt.Value = int64(item["Cnt"].(float64)) jmdCnt.CounterType = "GAUGE" jmdCnt.Tags = myTags jsonList = append(jsonList, &jmdCnt) } if item["Qps"] != nil { var jmdQps cmodel.JsonMetaData jmdQps.Endpoint = hostName jmdQps.Metric = itemName + ".Qps" jmdQps.Timestamp = ts jmdQps.Step = 60 jmdQps.Value = int64(item["Qps"].(float64)) jmdQps.CounterType = "GAUGE" jmdQps.Tags = myTags jsonList = append(jsonList, &jmdQps) } } // format result err = sendToTransfer(jsonList, destUrl) if err != nil { log.Println(hostNamePort, "send to transfer error,", err.Error()) } } // collector.alive _collectorAlive() }
func _collect() { clientGet := nhttpclient.GetHttpClient("collector.get", 10*time.Second, 20*time.Second) clientPost := nhttpclient.GetHttpClient("collector.post", 5*time.Second, 10*time.Second) tags := "type=statistics,pdl=falcon" for _, host := range g.Config().Collector.Cluster { ts := time.Now().Unix() jsonList := make([]model.MetricValue, 0) // get statistics by http-get hostInfo := strings.Split(host, ",") // "module,hostname:port" if len(hostInfo) != 2 { continue } hostModule := hostInfo[0] hostNamePort := hostInfo[1] hostNamePortList := strings.Split(hostNamePort, ":") if len(hostNamePortList) != 2 { continue } hostName := hostNamePortList[0] hostPort := hostNamePortList[1] myTags := tags + ",module=" + hostModule + ",port=" + hostPort srcUrl := fmt.Sprintf(srcUrlFmt, hostNamePort) reqGet, _ := http.NewRequest("GET", srcUrl, nil) reqGet.Header.Set("Connection", "close") getResp, err := clientGet.Do(reqGet) if err != nil { log.Printf(hostNamePort+", get statistics error,", err) continue } defer getResp.Body.Close() body, err := ioutil.ReadAll(getResp.Body) if err != nil { log.Println(hostNamePort+", get statistics error,", err) continue } var data Dto err = json.Unmarshal(body, &data) if err != nil { log.Println(hostNamePort+", get statistics error,", err) continue } for _, item := range data.Data { if item["Name"] == nil { continue } itemName := item["Name"].(string) if item["Cnt"] != nil { var jmdCnt model.MetricValue jmdCnt.Endpoint = hostName jmdCnt.Metric = itemName jmdCnt.Timestamp = ts jmdCnt.Step = 60 jmdCnt.Value = int64(item["Cnt"].(float64)) jmdCnt.Type = "GAUGE" jmdCnt.Tags = myTags jsonList = append(jsonList, jmdCnt) } if item["Qps"] != nil { var jmdQps model.MetricValue jmdQps.Endpoint = hostName jmdQps.Metric = itemName + ".Qps" jmdQps.Timestamp = ts jmdQps.Step = 60 jmdQps.Value = int64(item["Qps"].(float64)) jmdQps.Type = "GAUGE" jmdQps.Tags = myTags jsonList = append(jsonList, jmdQps) } } if len(jsonList) < 1 { //没取到数据 log.Println("get null from ", hostNamePort) continue } // format result jsonBody, err := json.Marshal(jsonList) if err != nil { log.Println(hostNamePort+", format body error,", err) continue } // send by http-post req, err := http.NewRequest("POST", destUrl, bytes.NewBuffer(jsonBody)) req.Header.Set("Content-Type", "application/json; charset=UTF-8") req.Header.Set("Connection", "close") postResp, err := clientPost.Do(req) if err != nil { log.Println(hostNamePort+", post to dest error,", err) continue } defer postResp.Body.Close() if postResp.StatusCode/100 != 2 { log.Println(hostNamePort+", post to dest, bad response,", postResp.StatusCode) } } }