func Start(httpMsg chan<- string) { defer func() { httpMsg <- "http" }() if !g.Config().Http.Enabled { log.Println("http.Start warning, not enable") return } // config http routes configCommonRoutes() configProcHttpRoutes() configGraphRoutes() configAPIRoutes() configAlertRoutes() configGrafanaRoutes() configZabbixRoutes() configNqmRoutes() configNQMRoutes() // start mysql database InitDatabase() // start http server addr := g.Config().Http.Listen s := &http.Server{ Addr: addr, MaxHeaderBytes: 1 << 30, } log.Println("http.Start ok, listening on", addr) log.Fatalln(s.ListenAndServe()) }
func getHosts(reqHost string, hostKeyword string) []interface{} { if len(hostKeyword) == 0 { hostKeyword = ".+" } rand.Seed(time.Now().UTC().UnixNano()) random64 := rand.Float64() _r := strconv.FormatFloat(random64, 'f', -1, 32) maxQuery := g.Config().Api.Max url := fmt.Sprintf("/api/endpoints?q=%s&tags&limit=%d&_r=%s®ex_query=1", hostKeyword, maxQuery, _r) if strings.Index(g.Config().Api.Query, reqHost) >= 0 { url = "http://localhost:9966" + url } else { url = g.Config().Api.Query + url } nodes := doHTTPQuery(url) result := []interface{}{} chart := map[string]interface{}{ "text": "chart", "expandable": true, } result = append(result, chart) for _, host := range nodes["data"].([]interface{}) { item := map[string]interface{}{ "text": host, "expandable": true, } result = append(result, item) } return result }
func Start() { if !g.Config().Http.Enable { log.Println("http.Start warning, not enable") return } // config http routes configCommonRoutes() configProcHttpRoutes() configGraphRoutes() configApiRoutes() configGrafanaRoutes() configZabbixRoutes() // start mysql database InitDatabase() // start http server addr := g.Config().Http.Listen s := &http.Server{ Addr: addr, MaxHeaderBytes: 1 << 30, } log.Println("http.Start ok, listening on", addr) log.Fatalln(s.ListenAndServe()) }
/** * @function name: func getHosts(rw http.ResponseWriter, req *http.Request, hostKeyword string) * @description: This function returns available hosts for Grafana query editor. * @related issues: OWL-221, OWL-151 * @param: rw http.ResponseWriter * @param: req *http.Request * @param: hostKeyword string * @return: void * @author: Don Hsieh * @since: 11/17/2015 * @last modified: 12/18/2015 * @called by: func setQueryEditor(rw http.ResponseWriter, req *http.Request, hostKeyword string) */ func getHosts(rw http.ResponseWriter, req *http.Request, hostKeyword string) { if len(hostKeyword) == 0 { hostKeyword = ".+" } rand.Seed(time.Now().UTC().UnixNano()) random64 := rand.Float64() _r := strconv.FormatFloat(random64, 'f', -1, 32) maxQuery := strconv.Itoa(g.Config().Api.Max) url := "/api/endpoints" + "?q=" + hostKeyword + "&tags&limit=" + maxQuery + "&_r=" + _r + "®ex_query=1" log.Println("req.Host =", req.Host) log.Println("g.Config().Api.Query =", g.Config().Api.Query) log.Println("maxQuery = ", maxQuery) if strings.Index(g.Config().Api.Query, req.Host) >= 0 { url = "http://localhost:9966" + url } else { url = g.Config().Api.Query + url } log.Println("url =", url) reqGet, err := http.NewRequest("GET", url, nil) if err != nil { log.Println("Error =", err.Error()) } client := &http.Client{} resp, err := client.Do(reqGet) if err != nil { log.Println("Error =", err.Error()) } defer resp.Body.Close() result := []interface{}{} chart := map[string]interface{}{ "text": "chart", "expandable": true, } result = append(result, chart) if resp.Status == "200 OK" { body, _ := ioutil.ReadAll(resp.Body) var nodes = make(map[string]interface{}) if err := json.Unmarshal(body, &nodes); err != nil { log.Println(err.Error()) } for _, host := range nodes["data"].([]interface{}) { item := map[string]interface{}{ "text": host, "expandable": true, } result = append(result, item) } RenderJson(rw, result) } else { RenderJson(rw, result) } }
func getLocation(pop_id int) map[string]string { location := map[string]string{ "area": "", "province": "", "city": "", } fcname := g.Config().Api.Name fctoken := getFctoken() url := g.Config().Api.Geo args := map[string]interface{}{ "fcname": fcname, "fctoken": fctoken, "pop_id": pop_id, } bs, err := json.Marshal(args) if err != nil { log.Println("Error =", err.Error()) } reqPost, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(bs))) if err != nil { log.Println("Error =", err.Error()) } reqPost.Header.Set("Content-Type", "application/json") client := &http.Client{} resp, err := client.Do(reqPost) if err != nil { log.Println("Error =", err.Error()) } defer resp.Body.Close() if resp.Status == "200 OK" { body, _ := ioutil.ReadAll(resp.Body) nodes := map[string]interface{}{} if err := json.Unmarshal(body, &nodes); err != nil { log.Println(err.Error()) } status := int(nodes["status"].(float64)) if status == 1 { result := nodes["result"] location["area"] = result.(map[string]interface{})["area"].(string) location["province"] = result.(map[string]interface{})["province"].(string) location["city"] = result.(map[string]interface{})["city"].(string) } } return location }
func queryAgentAlive(queries []*cmodel.GraphLastParam, reqHost string, result map[string]interface{}) []cmodel.GraphLastResp { data := []cmodel.GraphLastResp{} if len(queries) > 0 { if strings.Index(g.Config().Api.Query, reqHost) >= 0 { proc.LastRequestCnt.Incr() for _, param := range queries { if param == nil { continue } last, err := graph.Last(*param) if err != nil { log.Printf("graph.last fail, resp: %v, err: %v", last, err) } if last == nil { continue } data = append(data, *last) } proc.LastRequestItemCnt.IncrBy(int64(len(data))) } else { s, err := json.Marshal(queries) if err != nil { setError(err.Error(), result) } url := g.Config().Api.Query + "/graph/last" reqPost, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(s))) if err != nil { setError(err.Error(), result) } reqPost.Header.Set("Content-Type", "application/json") client := &http.Client{} resp, err := client.Do(reqPost) if err != nil { setError(err.Error(), result) } defer resp.Body.Close() body, _ := ioutil.ReadAll(resp.Body) err = json.Unmarshal(body, &data) if err != nil { setError(err.Error(), result) } } } return data }
func Init() { conf := g.Config() db, err = gorm.Open("mysql", conf.GraphDB.Addr) if err != nil { log.Println(err.Error()) } }
func main() { cfg := flag.String("c", "cfg.json", "specify config file") version := flag.Bool("v", false, "show version") versionGit := flag.Bool("vg", false, "show version and git commit log") flag.Parse() if *version { fmt.Println(g.VERSION) os.Exit(0) } if *versionGit { fmt.Println(g.VERSION, g.COMMIT) os.Exit(0) } // config g.ParseConfig(*cfg) gconf := g.Config() // proc proc.Start() // graph graph.Start() grpcMsg := make(chan string) if gconf.Grpc.Enabled { // grpc go grpc.Start(grpcMsg) } ginMsg := make(chan string) if gconf.GinHttp.Enabled { //lambdaSetup database.Init() conf.ReadConf("./conf/lambdaSetup.json") go ginHttp.StartWeb(ginMsg) } httpMsg := make(chan string) if gconf.Http.Enabled { // http go http.Start(httpMsg) } select { case <-grpcMsg: log.Printf("%v is crashed", grpcMsg) case <-ginMsg: log.Printf("%v is crashed", ginMsg) case <-httpMsg: log.Printf("%v is crashed", ginMsg) } }
// internal functions func initConnPools() { cfg := g.Config() // TODO 为了得到Slice,这里做的太复杂了 graphInstances := nset.NewSafeSet() for _, address := range cfg.Graph.Cluster { graphInstances.Add(address) } GraphConnPools = spool.CreateSafeRpcConnPools(cfg.Graph.MaxConns, cfg.Graph.MaxIdle, cfg.Graph.ConnTimeout, cfg.Graph.CallTimeout, graphInstances.ToSlice()) }
func Info(para cmodel.GraphInfoParam) (resp *cmodel.GraphFullyInfo, err error) { endpoint, counter := para.Endpoint, para.Counter pool, addr, err := selectPool(endpoint, counter) if err != nil { return nil, err } conn, err := pool.Fetch() if err != nil { return nil, err } rpcConn := conn.(spool.RpcClient) if rpcConn.Closed() { pool.ForceClose(conn) return nil, errors.New("conn closed") } type ChResult struct { Err error Resp *cmodel.GraphInfoResp } ch := make(chan *ChResult, 1) go func() { resp := &cmodel.GraphInfoResp{} err := rpcConn.Call("Graph.Info", para, resp) ch <- &ChResult{Err: err, Resp: resp} }() select { case <-time.After(time.Duration(g.Config().Graph.CallTimeout) * time.Millisecond): pool.ForceClose(conn) return nil, fmt.Errorf("%s, call timeout. proc: %s", addr, pool.Proc()) case r := <-ch: if r.Err != nil { pool.ForceClose(conn) return nil, fmt.Errorf("%s, call failed, err %v. proc: %s", addr, r.Err, pool.Proc()) } else { pool.Release(conn) fullyInfo := cmodel.GraphFullyInfo{ Endpoint: endpoint, Counter: counter, ConsolFun: r.Resp.ConsolFun, Step: r.Resp.Step, Filename: r.Resp.Filename, Addr: addr, } return &fullyInfo, nil } } }
func getPlatformJSON(nodes map[string]interface{}, result map[string]interface{}) { fcname := g.Config().Api.Name fctoken := getFctoken() url := g.Config().Api.Map + "/fcname/" + fcname + "/fctoken/" + fctoken url += "/show_active/yes/hostname/yes/pop_id/yes/ip/yes.json" req, err := http.NewRequest("GET", url, nil) if err != nil { setError(err.Error(), result) } client := &http.Client{} resp, err := client.Do(req) if err != nil { setError(err.Error(), result) } defer resp.Body.Close() body, _ := ioutil.ReadAll(resp.Body) if err := json.Unmarshal(body, &nodes); err != nil { setError(err.Error(), result) } }
/** * @function name: func getFctoken() fctoken string * @description: This function returns fctoken for API request. * @related issues: OWL-159 * @param: void * @return: fctoken string * @author: Don Hsieh * @since: 11/24/2015 * @last modified: 11/24/2015 * @called by: func apiAlert(rw http.ResponseWriter, req *http.Request) * in query/http/zabbix.go * func getMapValues(chartType string) map[string]interface{} * in query/http/grafana.go */ func getFctoken() string { hasher := md5.New() io.WriteString(hasher, g.Config().Api.Token) s := hex.EncodeToString(hasher.Sum(nil)) t := time.Now() now := t.Format("20060102") s = now + s hasher = md5.New() io.WriteString(hasher, s) fctoken := hex.EncodeToString(hasher.Sum(nil)) return fctoken }
func InitDatabase() { // set default database config := g.Config() orm.RegisterDataBase("default", "mysql", config.Db.Addr, config.Db.Idle, config.Db.Max) // register model orm.RegisterModel(new(Host), new(Grp), new(Grp_host), new(Grp_tpl), new(Tpl)) strConn := strings.Replace(config.Db.Addr, "graph", "grafana", 1) orm.RegisterDataBase("grafana", "mysql", strConn, config.Db.Idle, config.Db.Max) orm.RegisterModel(new(Province), new(City), new(Idc)) if config.Debug == true { orm.Debug = true } }
func configCommonRoutes() { http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ok\n")) }) http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(fmt.Sprintf("%s\n", g.VERSION))) }) http.HandleFunc("/workdir", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(fmt.Sprintf("%s\n", file.SelfDir()))) }) http.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) { RenderDataJson(w, g.Config()) }) }
// Used to output JSON message even if the execution is panic func outputJsonForPanic(ctx *context.Context) { r := recover() if r == nil { return } if g.Config().Debug { debug.PrintStack() } log.Printf("Error on HTTP Request[%v/%v]. Error: %v", ctx.Input.Method(), ctx.Input.URI(), r) ctx.Output.SetStatus(http.StatusBadRequest) ctx.Output.JSON(&jsonDslError{ Code: -1, Message: fmt.Sprintf("%v", r), }, jsonIndent, jsonCoding) }
func Start(grpcMsg chan<- string) { defer func() { grpcMsg <- "grpc" }() port := fmt.Sprintf(":%v", g.Config().Grpc.Port) log.Printf("start grpc in port %v ..", port) //queryrrd(1452806153, 1452827753, "AVERAGE", "docker-agent", "cpu.idle") lis, err := net.Listen("tcp", port) if err != nil { log.Fatalf("failed to listen: %v", err) } s := grpc.NewServer() pb.RegisterOwlQueryServer(s, &server{}) s.Serve(lis) }
func LastRaw(para cmodel.GraphLastParam) (r *cmodel.GraphLastResp, err error) { endpoint, counter := para.Endpoint, para.Counter pool, addr, err := selectPool(endpoint, counter) if err != nil { return nil, err } conn, err := pool.Fetch() if err != nil { return nil, err } rpcConn := conn.(spool.RpcClient) if rpcConn.Closed() { pool.ForceClose(conn) return nil, errors.New("conn closed") } type ChResult struct { Err error Resp *cmodel.GraphLastResp } ch := make(chan *ChResult, 1) go func() { resp := &cmodel.GraphLastResp{} err := rpcConn.Call("Graph.LastRaw", para, resp) ch <- &ChResult{Err: err, Resp: resp} }() select { case <-time.After(time.Duration(g.Config().Graph.CallTimeout) * time.Millisecond): pool.ForceClose(conn) return nil, fmt.Errorf("%s, call timeout. proc: %s", addr, pool.Proc()) case r := <-ch: if r.Err != nil { pool.ForceClose(conn) return r.Resp, fmt.Errorf("%s, call failed, err %v. proc: %s", addr, r.Err, pool.Proc()) } else { pool.Release(conn) return r.Resp, nil } } }
func getAgentAliveData(hostnames []string, versions map[string]string, result map[string]interface{}) []cmodel.GraphLastResp { var queries []cmodel.GraphLastParam o := orm.NewOrm() var hosts []*Host _, err := o.Raw("SELECT hostname, agent_version FROM falcon_portal.host ORDER BY hostname ASC").QueryRows(&hosts) if err != nil { setError(err.Error(), result) } else { for _, host := range hosts { var query cmodel.GraphLastParam if !strings.Contains(host.Hostname, ".") && strings.Contains(host.Hostname, "-") { hostnames = append(hostnames, host.Hostname) versions[host.Hostname] = host.Agent_version query.Endpoint = host.Hostname query.Counter = "agent.alive" queries = append(queries, query) } } } s, err := json.Marshal(queries) if err != nil { setError(err.Error(), result) } url := g.Config().Api.Query + "/graph/last" reqPost, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(s))) if err != nil { setError(err.Error(), result) } reqPost.Header.Set("Content-Type", "application/json") client := &http.Client{} resp, err := client.Do(reqPost) if err != nil { setError(err.Error(), result) } defer resp.Body.Close() body, _ := ioutil.ReadAll(resp.Body) data := []cmodel.GraphLastResp{} err = json.Unmarshal(body, &data) if err != nil { setError(err.Error(), result) } return data }
func selectPool(endpoint, counter string) (rpool *spool.ConnPool, raddr string, rerr error) { pkey := cutils.PK2(endpoint, counter) node, err := GraphNodeRing.GetNode(pkey) if err != nil { return nil, "", err } addr, found := g.Config().Graph.Cluster[node] if !found { return nil, "", errors.New("node not found") } pool, found := GraphConnPools.Get(addr) if !found { return nil, addr, errors.New("addr not found") } return pool, addr, nil }
func EndpointQuery() (endpointList []string) { database.Init() db := database.DBConn() gconf := g.Config() var enps []Endpoint var sqlStr string if gconf.GraphDB.Limit == -1 { sqlStr = "SELECT * from graph.endpoint" } else { sqlStr = fmt.Sprintf("SELECT * from graph.endpoint limit %v", gconf.GraphDB.Limit) } db.Raw(sqlStr).Scan(&enps) if len(enps) != 0 { for _, host := range enps { endpointList = append(endpointList, host.Endpoint) } } return }
func StartWeb(ginMsg chan<- string) { defer func() { ginMsg <- "gin" }() handler := gin.Default() handler.Use(CORSMiddleware()) compute := handler.Group("/func") conf := g.Config() compute.GET("/compute", computeFunc.Compute) compute.GET("/funcations", computeFunc.GetAvaibleFun) compute.GET("/smapledata", computeFunc.GetTestData) openfalcon := handler.Group("/owl") openfalcon.GET("/endpoints", openFalcon.GetEndpoints) openfalcon.GET("/queryrrd", openFalcon.QueryData) log.Println("open gin port on:", conf.GinHttp.Listen) handler.Run(conf.GinHttp.Listen) }
func InitDatabase() { config := g.Config() // set default database // orm.RegisterDataBase("default", "mysql", config.Db.Addr, config.Db.Idle, config.Db.Max) // register model orm.RegisterModel(new(Host), new(Grp), new(Grp_host), new(Grp_tpl), new(Plugin_dir), new(Tpl)) // set grafana database strConn := strings.Replace(config.Db.Addr, "falcon_portal", "grafana", 1) orm.RegisterDataBase("grafana", "mysql", strConn, config.Db.Idle, config.Db.Max) orm.RegisterModel(new(Province), new(City), new(Idc)) orm.RegisterDataBase("gz_nqm", "mysql", config.Nqm.Addr, config.Nqm.Idle, config.Nqm.Max) orm.RegisterModel(new(Nqm_node)) if config.Debug == true { orm.Debug = true } }
func DoPost() { conf := g.Config() url := fmt.Sprintf("%s%s", conf.Http.Listen, "/graph/history") // host := []string{"endpointA", "endpointB"} host := gethost() dd := generatePostData(host, "cpu.idle", 1464761471, 1464847858) data, _ := json.Marshal(&dd) req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(data))) if err != nil { log.Println(err.Error()) } req.Header.Set("Content-Type", "application/json; charset=UTF-8") client := &http.Client{} resp, err := client.Do(req) if err != nil { panic(err) } defer resp.Body.Close() body, _ := ioutil.ReadAll(resp.Body) fmt.Println("response Body:", string(body)) }
/** * @function name: func dashboardEndpoints(rw http.ResponseWriter, req *http.Request) * @description: This function handles /api/endpoints API request. * @related issues: OWL-159, OWL-171 * @param: rw http.ResponseWriter * @param: req *http.Request * @return: void * @author: Don Hsieh * @since: 11/12/2015 * @last modified: 11/24/2015 * @called by: func configApiRoutes() */ func dashboardEndpoints(rw http.ResponseWriter, req *http.Request) { url := g.Config().Api.Dashboard + req.URL.RequestURI() getRequest(rw, url) }
func initIcmp() { config := g.Config() rpcServiceCaller = jsonrpc.NewService(config.NqmLog.JsonrpcUrl) }
/** * @function name: func queryHistory(rw http.ResponseWriter, req *http.Request) * @description: This function handles /graph/history API request. * @related issues: OWL-171 * @param: rw http.ResponseWriter * @param: req *http.Request * @return: void * @author: Don Hsieh * @since: 11/12/2015 * @last modified: 11/13/2015 * @called by: func configApiRoutes() */ func queryHistory(rw http.ResponseWriter, req *http.Request) { url := g.Config().Api.Query + "/graph/history" postByJson(rw, req, url) }
/** * @function name: func queryInfo(rw http.ResponseWriter, req *http.Request) * @description: This function handles /graph/info API request. * @related issues: OWL-171 * @param: rw http.ResponseWriter * @param: req *http.Request * @return: void * @author: Don Hsieh * @since: 11/12/2015 * @last modified: 11/13/2015 * @called by: func configApiRoutes() */ func queryInfo(rw http.ResponseWriter, req *http.Request) { url := g.Config().Api.Query + "/graph/info" postByJson(rw, req, url) }
/** * @function name: func dashboardChart(rw http.ResponseWriter, req *http.Request) * @description: This function handles /api/chart API request. * @related issues: OWL-171 * @param: rw http.ResponseWriter * @param: req *http.Request * @return: void * @author: Don Hsieh * @since: 11/13/2015 * @last modified: 11/13/2015 * @called by: func configApiRoutes() */ func dashboardChart(rw http.ResponseWriter, req *http.Request) { url := g.Config().Api.Dashboard + "/chart" postByForm(rw, req, url) }
/** * @function name: func dashboardCounters(rw http.ResponseWriter, req *http.Request) * @description: This function handles /api/counters API request. * @related issues: OWL-171 * @param: rw http.ResponseWriter * @param: req *http.Request * @return: void * @author: Don Hsieh * @since: 11/13/2015 * @last modified: 11/13/2015 * @called by: func configApiRoutes() */ func dashboardCounters(rw http.ResponseWriter, req *http.Request) { url := g.Config().Api.Dashboard + "/api/counters" postByForm(rw, req, url) }
func QueryOne(para cmodel.GraphQueryParam) (resp *cmodel.GraphQueryResponse, err error) { start, end := para.Start, para.End endpoint, counter := para.Endpoint, para.Counter pool, addr, err := selectPool(endpoint, counter) if err != nil { return nil, err } conn, err := pool.Fetch() if err != nil { return nil, err } rpcConn := conn.(spool.RpcClient) if rpcConn.Closed() { pool.ForceClose(conn) return nil, errors.New("conn closed") } type ChResult struct { Err error Resp *cmodel.GraphQueryResponse } ch := make(chan *ChResult, 1) go func() { resp := &cmodel.GraphQueryResponse{} err := rpcConn.Call("Graph.Query", para, resp) ch <- &ChResult{Err: err, Resp: resp} }() select { case <-time.After(time.Duration(g.Config().Graph.CallTimeout) * time.Millisecond): pool.ForceClose(conn) return nil, fmt.Errorf("%s, call timeout. proc: %s", addr, pool.Proc()) case r := <-ch: if r.Err != nil { pool.ForceClose(conn) return r.Resp, fmt.Errorf("%s, call failed, err %v. proc: %s", addr, r.Err, pool.Proc()) } else { pool.Release(conn) if len(r.Resp.Values) < 1 { r.Resp.Values = []*cmodel.RRDData{} return r.Resp, nil } // TODO query不该做这些事情, 说明graph没做好 fixed := []*cmodel.RRDData{} for _, v := range r.Resp.Values { if v == nil || !(v.Timestamp >= start && v.Timestamp <= end) { continue } //FIXME: 查询数据的时候,把所有的负值都过滤掉,因为transfer之前在设置最小值的时候为U if (r.Resp.DsType == "DERIVE" || r.Resp.DsType == "COUNTER") && v.Value < 0 { fixed = append(fixed, &cmodel.RRDData{Timestamp: v.Timestamp, Value: cmodel.JsonFloat(math.NaN())}) } else { fixed = append(fixed, v) } } r.Resp.Values = fixed } return r.Resp, nil } }