func sendToTransfer(items []*cmodel.JsonMetaData, destUrl string) error {
if len(items) < 1 {
return nil
}
// format result
jsonBody, err := json.Marshal(items)
if err != nil {
return fmt.Errorf("json.Marshal failed with %v", err)
}
// send by http-post
req, err := http.NewRequest("POST", destUrl, bytes.NewBuffer(jsonBody))
req.Header.Set("Content-Type", "application/json; charset=UTF-8")
req.Header.Set("Connection", "close")
clientPost := nhttpclient.GetHttpClient("collector.post", 5*time.Second, 10*time.Second)
postResp, err := clientPost.Do(req)
if err != nil {
return fmt.Errorf("post to %s, resquest failed with %v", destUrl, err)
}
defer postResp.Body.Close()
if postResp.StatusCode/100 != 2 {
return fmt.Errorf("post to %s, got bad response %d", destUrl, postResp.StatusCode)
}
return nil
}
func _monitor() {
client := nhttpclient.GetHttpClient("monitor.get", 5*time.Second, 10*time.Second)
for _, host := range g.Config().Monitor.Cluster {
hostInfo := strings.Split(host, ",") // "module,hostname:port/health"
if len(hostInfo) != 2 {
continue
}
//hostType := hostInfo[0]
hostUrl := hostInfo[1]
if !strings.Contains(hostUrl, "http://") {
hostUrl = "http://" + hostUrl
}
req, _ := http.NewRequest("GET", hostUrl, nil)
req.Header.Set("Connection", "close")
getResp, err := client.Do(req)
if err != nil {
log.Printf(host+", monitor error,", err)
onMonitorErr(host)
continue
}
defer getResp.Body.Close()
body, err := ioutil.ReadAll(getResp.Body) // body=['o','k',...]
if !(err == nil && len(body) >= 2 && string(body[:2]) == "ok") { // err
log.Println(host, ", error,", err)
onMonitorErr(host)
} else { // get "ok"
onMonitorOk(host)
}
}
}
func sendMail(mailUrl string, content string) error {
client := nhttpclient.GetHttpClient("monitor.mail", 5*time.Second, 10*time.Second)
// send by http-post
req, err := http.NewRequest("POST", mailUrl, bytes.NewBufferString(content))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
req.Header.Set("Connection", "close")
postResp, err := client.Do(req)
if err != nil {
return err
}
defer postResp.Body.Close()
if postResp.StatusCode/100 != 2 {
return fmt.Errorf("Http-Post Error, Code %d", postResp.StatusCode)
}
return nil
}
func sendItemsToTransfer(items []*cmodel.JsonMetaData, size int, httpcliname string,
connT, reqT time.Duration) (cnt int, errt error) {
if size < 1 {
return
}
cfg := g.Config()
transUlr := fmt.Sprintf("http://%s/api/push", cfg.Sender.TransferAddr)
hcli := thttpclient.GetHttpClient(httpcliname, connT, reqT)
// form request args
itemsBody, err := json.Marshal(items)
if err != nil {
log.Println(transUlr+", format body error,", err)
errt = err
return
}
// fetch items
req, err := http.NewRequest("POST", transUlr, bytes.NewBuffer(itemsBody))
req.Header.Set("Content-Type", "application/json; charset=UTF-8")
req.Header.Set("Connection", "close")
postResp, err := hcli.Do(req)
if err != nil {
log.Println(transUlr+", post to dest error,", err)
errt = err
return
}
defer postResp.Body.Close()
if postResp.StatusCode/100 != 2 {
log.Println(transUlr+", post to dest, bad response,", postResp.Body)
errt = fmt.Errorf("request failed, %s", postResp.Body)
return
}
return size, nil
}
func updateIndexOfOneGraph(hostNamePort string) error {
if hostNamePort == "" {
return fmt.Errorf("index update error, bad host")
}
client := nhttpclient.GetHttpClient("index.update."+hostNamePort, 5*time.Second, 10*time.Second)
destUrl := fmt.Sprintf(destUrlFmt, hostNamePort)
req, _ := http.NewRequest("GET", destUrl, nil)
req.Header.Set("Connection", "close")
getResp, err := client.Do(req)
if err != nil {
log.Printf(hostNamePort+", index update error,", err)
return err
}
defer getResp.Body.Close()
body, err := ioutil.ReadAll(getResp.Body)
if err != nil {
log.Println(hostNamePort+", index update error,", err)
return err
}
var data Dto
err = json.Unmarshal(body, &data)
if err != nil {
log.Println(hostNamePort+", index update error,", err)
return err
}
if data.Data != "ok" {
log.Println(hostNamePort+", index update error, bad result,", data.Data)
return err
}
return nil
}
func updateAllIndex() {
client := nhttpclient.GetHttpClient("index.updateall", 5*time.Second, 10*time.Second)
for _, hostNamePort := range g.Config().Index.Cluster {
if hostNamePort == "" {
continue
}
destUrl := fmt.Sprintf(destUrlFmt, hostNamePort)
req, _ := http.NewRequest("GET", destUrl, nil)
req.Header.Set("Connection", "close")
getResp, err := client.Do(req)
if err != nil {
log.Printf(hostNamePort+", index update all error,", err)
continue
}
defer getResp.Body.Close()
body, err := ioutil.ReadAll(getResp.Body)
if err != nil {
log.Println(hostNamePort+", index update all error,", err)
continue
}
var data Dto
err = json.Unmarshal(body, &data)
if err != nil {
log.Println(hostNamePort+", index update all error,", err)
continue
}
if data.Data != "ok" {
log.Println(hostNamePort+", index update all error, bad result,", data.Data)
continue
}
}
}
func fetchRawItems() (values []float64) {
cfg := g.Config()
queryUlr := fmt.Sprintf("http://%s/graph/history", cfg.Query.QueryAddr)
hcli := thttpclient.GetHttpClient("nodata.gauss",
time.Millisecond*time.Duration(cfg.Query.ConnectTimeout),
time.Millisecond*time.Duration(cfg.Query.RequestTimeout))
// form request args
nowTs := time.Now().Unix()
endTs := nowTs - nowTs%1200
startTs := endTs - 24*3600*5 //用5天的数据,做高斯拟合
hostname, _ := os.Hostname()
if len(cfg.Sender.Block.Hostname) > 0 {
hostname = cfg.Sender.Block.Hostname
}
fcounter := "FloodRate/module=nodata,pdl=falcon,port=6090,type=statistics"
if len(cfg.Sender.Block.FloodCounter) > 0 {
fcounter = cfg.Sender.Block.FloodCounter
}
endpointCounters := make([]cmodel.GraphInfoParam, 0)
endpointCounters = append(endpointCounters, cmodel.GraphInfoParam{Endpoint: hostname, Counter: fcounter})
args := GraphHistoryParam{Start: startTs, End: endTs, CF: "AVERAGE", EndpointCounters: endpointCounters}
argsBody, err := json.Marshal(args)
if err != nil {
log.Println(queryUlr+", format body error,", err)
return
}
// fetch items
req, err := http.NewRequest("POST", queryUlr, bytes.NewBuffer(argsBody))
req.Header.Set("Content-Type", "application/json; charset=UTF-8")
req.Header.Set("Connection", "close")
postResp, err := hcli.Do(req)
if err != nil {
log.Println(queryUlr+", post to dest error,", err)
return
}
defer postResp.Body.Close()
if postResp.StatusCode/100 != 2 {
log.Println(queryUlr+", post to dest, bad response,", postResp.Body)
return
}
body, err := ioutil.ReadAll(postResp.Body)
if err != nil {
log.Println(queryUlr+", read response error,", err)
return
}
resp := make([]*cmodel.GraphQueryResponse, 0)
err = json.Unmarshal(body, &resp)
if err != nil {
log.Println(queryUlr+", unmarshal error,", err)
return
}
if len(resp) != 1 || resp[0] == nil {
return
}
// store items
values = make([]float64, 0)
for _, glr := range resp[0].Values {
if glr == nil || math.IsNaN(float64(glr.Value)) {
continue
}
values = append(values, float64(glr.Value))
}
return values
}
func _collect() {
clientGet := nhttpclient.GetHttpClient("collector.get", 10*time.Second, 20*time.Second)
tags := "type=statistics,pdl=falcon"
for _, host := range g.Config().Collector.Cluster {
ts := time.Now().Unix()
jsonList := make([]*cmodel.JsonMetaData, 0)
// get statistics by http-get
hostInfo := strings.Split(host, ",") // "module,hostname:port"
if len(hostInfo) != 2 {
continue
}
hostModule := hostInfo[0]
hostNamePort := hostInfo[1]
hostNamePortList := strings.Split(hostNamePort, ":")
if len(hostNamePortList) != 2 {
continue
}
hostName := hostNamePortList[0]
hostPort := hostNamePortList[1]
myTags := tags + ",module=" + hostModule + ",port=" + hostPort
srcUrl := fmt.Sprintf(srcUrlFmt, hostNamePort)
reqGet, _ := http.NewRequest("GET", srcUrl, nil)
reqGet.Header.Set("Connection", "close")
getResp, err := clientGet.Do(reqGet)
if err != nil {
log.Printf(hostNamePort+", get statistics error,", err)
continue
}
defer getResp.Body.Close()
body, err := ioutil.ReadAll(getResp.Body)
if err != nil {
log.Println(hostNamePort+", get statistics error,", err)
continue
}
var data Dto
err = json.Unmarshal(body, &data)
if err != nil {
log.Println(hostNamePort+", get statistics error,", err)
continue
}
for _, item := range data.Data {
if item["Name"] == nil {
continue
}
itemName := item["Name"].(string)
if item["Cnt"] != nil {
var jmdCnt cmodel.JsonMetaData
jmdCnt.Endpoint = hostName
jmdCnt.Metric = itemName
jmdCnt.Timestamp = ts
jmdCnt.Step = 60
jmdCnt.Value = int64(item["Cnt"].(float64))
jmdCnt.CounterType = "GAUGE"
jmdCnt.Tags = myTags
jsonList = append(jsonList, &jmdCnt)
}
if item["Qps"] != nil {
var jmdQps cmodel.JsonMetaData
jmdQps.Endpoint = hostName
jmdQps.Metric = itemName + ".Qps"
jmdQps.Timestamp = ts
jmdQps.Step = 60
jmdQps.Value = int64(item["Qps"].(float64))
jmdQps.CounterType = "GAUGE"
jmdQps.Tags = myTags
jsonList = append(jsonList, &jmdQps)
}
}
// format result
err = sendToTransfer(jsonList, destUrl)
if err != nil {
log.Println(hostNamePort, "send to transfer error,", err.Error())
}
}
// collector.alive
_collectorAlive()
}
func _collect() {
clientGet := nhttpclient.GetHttpClient("collector.get", 10*time.Second, 20*time.Second)
clientPost := nhttpclient.GetHttpClient("collector.post", 5*time.Second, 10*time.Second)
tags := "type=statistics,pdl=falcon"
for _, host := range g.Config().Collector.Cluster {
ts := time.Now().Unix()
jsonList := make([]model.MetricValue, 0)
// get statistics by http-get
hostInfo := strings.Split(host, ",") // "module,hostname:port"
if len(hostInfo) != 2 {
continue
}
hostModule := hostInfo[0]
hostNamePort := hostInfo[1]
hostNamePortList := strings.Split(hostNamePort, ":")
if len(hostNamePortList) != 2 {
continue
}
hostName := hostNamePortList[0]
hostPort := hostNamePortList[1]
myTags := tags + ",module=" + hostModule + ",port=" + hostPort
srcUrl := fmt.Sprintf(srcUrlFmt, hostNamePort)
reqGet, _ := http.NewRequest("GET", srcUrl, nil)
reqGet.Header.Set("Connection", "close")
getResp, err := clientGet.Do(reqGet)
if err != nil {
log.Printf(hostNamePort+", get statistics error,", err)
continue
}
defer getResp.Body.Close()
body, err := ioutil.ReadAll(getResp.Body)
if err != nil {
log.Println(hostNamePort+", get statistics error,", err)
continue
}
var data Dto
err = json.Unmarshal(body, &data)
if err != nil {
log.Println(hostNamePort+", get statistics error,", err)
continue
}
for _, item := range data.Data {
if item["Name"] == nil {
continue
}
itemName := item["Name"].(string)
if item["Cnt"] != nil {
var jmdCnt model.MetricValue
jmdCnt.Endpoint = hostName
jmdCnt.Metric = itemName
jmdCnt.Timestamp = ts
jmdCnt.Step = 60
jmdCnt.Value = int64(item["Cnt"].(float64))
jmdCnt.Type = "GAUGE"
jmdCnt.Tags = myTags
jsonList = append(jsonList, jmdCnt)
}
if item["Qps"] != nil {
var jmdQps model.MetricValue
jmdQps.Endpoint = hostName
jmdQps.Metric = itemName + ".Qps"
jmdQps.Timestamp = ts
jmdQps.Step = 60
jmdQps.Value = int64(item["Qps"].(float64))
jmdQps.Type = "GAUGE"
jmdQps.Tags = myTags
jsonList = append(jsonList, jmdQps)
}
}
if len(jsonList) < 1 { //没取到数据
log.Println("get null from ", hostNamePort)
continue
}
// format result
jsonBody, err := json.Marshal(jsonList)
if err != nil {
log.Println(hostNamePort+", format body error,", err)
continue
}
// send by http-post
req, err := http.NewRequest("POST", destUrl, bytes.NewBuffer(jsonBody))
req.Header.Set("Content-Type", "application/json; charset=UTF-8")
req.Header.Set("Connection", "close")
postResp, err := clientPost.Do(req)
if err != nil {
log.Println(hostNamePort+", post to dest error,", err)
continue
}
defer postResp.Body.Close()
if postResp.StatusCode/100 != 2 {
log.Println(hostNamePort+", post to dest, bad response,", postResp.StatusCode)
}
}
}
func fetchItemsAndStore(fetchKeys []string, fetchSize int) (size int, errt error) {
if fetchSize < 1 {
return
}
cfg := g.Config()
queryUlr := fmt.Sprintf("http://%s/graph/last", cfg.Query.QueryAddr)
hcli := thttpclient.GetHttpClient("nodata.collector",
time.Millisecond*time.Duration(cfg.Query.ConnectTimeout),
time.Millisecond*time.Duration(cfg.Query.RequestTimeout))
// form request args
args := make([]*cmodel.GraphLastParam, 0)
for _, key := range fetchKeys {
ndcfg, found := config.GetNdConfig(key)
if !found {
continue
}
endpoint := ndcfg.Endpoint
counter := cutils.Counter(ndcfg.Metric, ndcfg.Tags)
arg := &cmodel.GraphLastParam{endpoint, counter}
args = append(args, arg)
}
if len(args) < 1 {
return
}
argsBody, err := json.Marshal(args)
if err != nil {
log.Println(queryUlr+", format body error,", err)
errt = err
return
}
// fetch items
req, err := http.NewRequest("POST", queryUlr, bytes.NewBuffer(argsBody))
req.Header.Set("Content-Type", "application/json; charset=UTF-8")
req.Header.Set("Connection", "close")
postResp, err := hcli.Do(req)
if err != nil {
log.Println(queryUlr+", post to dest error,", err)
errt = err
return
}
defer postResp.Body.Close()
if postResp.StatusCode/100 != 2 {
log.Println(queryUlr+", post to dest, bad response,", postResp.Body)
errt = fmt.Errorf("request failed, %s", postResp.Body)
return
}
body, err := ioutil.ReadAll(postResp.Body)
if err != nil {
log.Println(queryUlr+", read response error,", err)
errt = err
return
}
resp := []*cmodel.GraphLastResp{}
err = json.Unmarshal(body, &resp)
if err != nil {
log.Println(queryUlr+", unmarshal error,", err)
errt = err
return
}
// store items
fts := time.Now().Unix()
for _, glr := range resp {
//log.Printf("collect:%v\n", glr)
if glr == nil || glr.Value == nil {
continue
}
AddItem(cutils.PK2(glr.Endpoint, glr.Counter), NewDataItem(glr.Value.Timestamp, float64(glr.Value.Value), "OK", fts))
}
return len(resp), nil
}