func ReloadBackends() {
cfg := g.Config().Graph
for {
time.Sleep(time.Duration(cfg.ReloadInterval) * time.Second)
err := InitBackends()
if err != nil {
logger.Error("reload backends fail: %v", err)
}
}
}
// internal functions
func initConnPools() {
cfg := g.Config()
// TODO 为了得到Slice,这里做的太复杂了
graphInstances := nset.NewSafeSet()
for _, address := range cfg.Graph.Cluster {
graphInstances.Add(address)
}
GraphConnPools = spool.CreateSafeRpcConnPools(cfg.Graph.MaxConns, cfg.Graph.MaxIdle,
cfg.Graph.ConnTimeout, cfg.Graph.CallTimeout, graphInstances.ToSlice())
}
func Start() {
if !g.Config().Http.Enabled {
log.Println("http.Start warning, not enabled")
return
}
// config http routes
configCommonRoutes()
configProcHttpRoutes()
configGraphRoutes()
// start http server
addr := g.Config().Http.Listen
s := &http.Server{
Addr: addr,
MaxHeaderBytes: 1 << 30,
}
log.Println("http.Start ok, listening on", addr)
log.Fatalln(s.ListenAndServe())
}
func Start() {
addr := g.Config().Http.Listen
if addr == "" {
return
}
s := &http.Server{
Addr: addr,
MaxHeaderBytes: 1 << 30,
}
log.Println("listening", addr)
log.Fatalln(s.ListenAndServe())
}
func Info(para cmodel.GraphInfoParam) (resp *cmodel.GraphFullyInfo, err error) {
endpoint, counter := para.Endpoint, para.Counter
pool, addr, err := selectPool(endpoint, counter)
if err != nil {
return nil, err
}
conn, err := pool.Fetch()
if err != nil {
return nil, err
}
rpcConn := conn.(spool.RpcClient)
if rpcConn.Closed() {
pool.ForceClose(conn)
return nil, errors.New("conn closed")
}
type ChResult struct {
Err error
Resp *cmodel.GraphInfoResp
}
ch := make(chan *ChResult, 1)
go func() {
resp := &cmodel.GraphInfoResp{}
err := rpcConn.Call("Graph.Info", para, resp)
ch <- &ChResult{Err: err, Resp: resp}
}()
select {
case <-time.After(time.Duration(g.Config().Graph.CallTimeout) * time.Millisecond):
pool.ForceClose(conn)
return nil, fmt.Errorf("%s, call timeout. proc: %s", addr, pool.Proc())
case r := <-ch:
if r.Err != nil {
pool.ForceClose(conn)
return nil, fmt.Errorf("%s, call failed, err %v. proc: %s", addr, r.Err, pool.Proc())
} else {
pool.Release(conn)
fullyInfo := cmodel.GraphFullyInfo{
Endpoint: endpoint,
Counter: counter,
ConsolFun: r.Resp.ConsolFun,
Step: r.Resp.Step,
Filename: r.Resp.Filename,
Addr: addr,
}
return &fullyInfo, nil
}
}
}
func Last(endpoint, counter string) (r *model.GraphLastResp, err error) {
pool, err := selectPool(endpoint, counter)
if err != nil {
return nil, err
}
conn, err := pool.Get()
if err != nil {
return nil, err
}
rpc_conn := conn.(RpcConn)
if rpc_conn.cli == nil {
pool.CloseClean(conn)
return nil, errors.New("nil rpc conn")
}
type ChResult struct {
Err error
Resp *model.GraphLastResp
}
ch := make(chan *ChResult, 1)
go func() {
param := model.GraphLastParam{
Endpoint: endpoint,
Counter: counter,
}
resp := &model.GraphLastResp{}
err := rpc_conn.cli.Call("Graph.Last", param, resp)
r := &ChResult{
Err: err,
Resp: resp,
}
ch <- r
}()
cfg := g.Config().Graph
select {
case r := <-ch:
if r.Err != nil {
pool.CloseClean(conn)
return nil, r.Err
} else {
pool.Release(conn)
return r.Resp, nil
}
case <-time.After(time.Duration(cfg.Timeout) * time.Millisecond):
pool.Release(conn)
return nil, errors.New("i/o timeout")
}
}
func configCommonRoutes() {
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("ok\n"))
})
http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(fmt.Sprintf("%s\n", g.VERSION)))
})
http.HandleFunc("/workdir", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(fmt.Sprintf("%s\n", file.SelfDir())))
})
http.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) {
RenderDataJson(w, g.Config())
})
}
func LastRaw(para cmodel.GraphLastParam) (r *cmodel.GraphLastResp, err error) {
endpoint, counter := para.Endpoint, para.Counter
pool, addr, err := selectPool(endpoint, counter)
if err != nil {
return nil, err
}
conn, err := pool.Fetch()
if err != nil {
return nil, err
}
rpcConn := conn.(spool.RpcClient)
if rpcConn.Closed() {
pool.ForceClose(conn)
return nil, errors.New("conn closed")
}
type ChResult struct {
Err error
Resp *cmodel.GraphLastResp
}
ch := make(chan *ChResult, 1)
go func() {
resp := &cmodel.GraphLastResp{}
err := rpcConn.Call("Graph.LastRaw", para, resp)
ch <- &ChResult{Err: err, Resp: resp}
}()
select {
case <-time.After(time.Duration(g.Config().Graph.CallTimeout) * time.Millisecond):
pool.ForceClose(conn)
return nil, fmt.Errorf("%s, call timeout. proc: %s", addr, pool.Proc())
case r := <-ch:
if r.Err != nil {
pool.ForceClose(conn)
return r.Resp, fmt.Errorf("%s, call failed, err %v. proc: %s", addr, r.Err, pool.Proc())
} else {
pool.Release(conn)
return r.Resp, nil
}
}
}
func InitBackends() error {
var err error
cfg := g.Config().Graph
file_path := cfg.Backends
err = backend.LoadAddrs(file_path)
if err != nil {
return err
}
backend.InitRing(cfg.Replicas)
err = initConnPools()
if err != nil {
return err
}
return nil
}
func selectPool(endpoint, counter string) (rpool *spool.ConnPool, raddr string, rerr error) {
pkey := cutils.PK2(endpoint, counter)
node, err := GraphNodeRing.GetNode(pkey)
if err != nil {
return nil, "", err
}
addr, found := g.Config().Graph.Cluster[node]
if !found {
return nil, "", errors.New("node not found")
}
pool, found := GraphConnPools.Get(addr)
if !found {
return nil, addr, errors.New("addr not found")
}
return pool, addr, nil
}
func initConnPools() error {
cfg := g.Config()
if cfg.LogLevel == "trace" || cfg.LogLevel == "debug" {
conn_pool.EnableSlowLog(true, cfg.SlowLog)
}
var (
tmp_addrs map[string][]string
tmp_pools map[string]*conn_pool.ConnPool
)
backend.RLock()
tmp_addrs = backend.Addrs
tmp_pools = backend.Pools
backend.RUnlock()
c := cfg.Graph
for name, addr_list := range tmp_addrs {
for _, addr := range addr_list {
if _, ok := tmp_pools[addr]; !ok {
pool := conn_pool.NewConnPool(addr, c.MaxConns, c.MaxIdle)
pool.New = func() (io.Closer, error) {
_, err := net.ResolveTCPAddr("tcp", pool.Name)
if err != nil {
return nil, err
}
conn, err := net.DialTimeout("tcp", pool.Name, time.Duration(c.Timeout)*time.Millisecond)
if err != nil {
return nil, err
}
return RpcConn{rpc.NewClient(conn)}, nil
}
pool.Ping = func(conn io.Closer) error {
rpc_conn := conn.(RpcConn)
if rpc_conn.cli == nil {
return errors.New("nil conn")
}
resp := &model.SimpleRpcResponse{}
err := rpc_conn.cli.Call("Graph.Ping", model.NullRpcRequest{}, resp)
logger.Trace("Graph.Ping resp: %v", resp)
return err
}
tmp_pools[addr] = pool
logger.Info("create the pool: %s %s", name, addr)
} else {
logger.Trace("keep the pool: %s %s", name, addr)
}
}
}
backend.Lock()
defer backend.Unlock()
backend.Pools = tmp_pools
return nil
}
func QueryOne(para cmodel.GraphQueryParam) (resp *cmodel.GraphQueryResponse, err error) {
start, end := para.Start, para.End
endpoint, counter := para.Endpoint, para.Counter
pool, addr, err := selectPool(endpoint, counter)
if err != nil {
return nil, err
}
conn, err := pool.Fetch()
if err != nil {
return nil, err
}
rpcConn := conn.(spool.RpcClient)
if rpcConn.Closed() {
pool.ForceClose(conn)
return nil, errors.New("conn closed")
}
type ChResult struct {
Err error
Resp *cmodel.GraphQueryResponse
}
ch := make(chan *ChResult, 1)
go func() {
resp := &cmodel.GraphQueryResponse{}
err := rpcConn.Call("Graph.Query", para, resp)
ch <- &ChResult{Err: err, Resp: resp}
}()
select {
case <-time.After(time.Duration(g.Config().Graph.CallTimeout) * time.Millisecond):
pool.ForceClose(conn)
return nil, fmt.Errorf("%s, call timeout. proc: %s", addr, pool.Proc())
case r := <-ch:
if r.Err != nil {
pool.ForceClose(conn)
return r.Resp, fmt.Errorf("%s, call failed, err %v. proc: %s", addr, r.Err, pool.Proc())
} else {
pool.Release(conn)
if len(r.Resp.Values) < 1 {
return r.Resp, nil
}
// TODO query不该做这些事情, 说明graph没做好
fixed := []*cmodel.RRDData{}
for _, v := range r.Resp.Values {
if v == nil || !(v.Timestamp >= start && v.Timestamp <= end) {
continue
}
//FIXME: 查询数据的时候，把所有的负值都过滤掉，因为transfer之前在设置最小值的时候为U
if (r.Resp.DsType == "DERIVE" || r.Resp.DsType == "COUNTER") && v.Value < 0 {
fixed = append(fixed, &cmodel.RRDData{Timestamp: v.Timestamp, Value: cmodel.JsonFloat(math.NaN())})
} else {
fixed = append(fixed, v)
}
}
r.Resp.Values = fixed
}
return r.Resp, nil
}
}
func initNodeRings() {
cfg := g.Config()
GraphNodeRing = rings.NewConsistentHashNodesRing(cfg.Graph.Replicas, cutils.KeysOfMap(cfg.Graph.Cluster))
}
func Info(endpoint, counter string) (r *model.GraphFullyInfo, err error) {
pool, err := selectPool(endpoint, counter)
if err != nil {
return nil, err
}
conn, err := pool.Get()
if err != nil {
return nil, err
}
rpc_conn := conn.(RpcConn)
if rpc_conn.cli == nil {
pool.CloseClean(conn)
return nil, errors.New("nil rpc conn")
}
type ChResult struct {
Err error
Resp *model.GraphInfoResp
}
ch := make(chan *ChResult, 1)
go func() {
param := model.GraphInfoParam{
Endpoint: endpoint,
Counter: counter,
}
resp := &model.GraphInfoResp{}
err := rpc_conn.cli.Call("Graph.Info", param, resp)
r := &ChResult{
Err: err,
Resp: resp,
}
ch <- r
}()
cfg := g.Config().Graph
select {
case r := <-ch:
if r.Err != nil {
pool.CloseClean(conn)
return nil, r.Err
} else {
pool.Release(conn)
logger.Trace("graph.info resp: %v, addr: %v", r.Resp, pool.Name)
fullyInfo := model.GraphFullyInfo{
Endpoint: endpoint,
Counter: counter,
ConsolFun: r.Resp.ConsolFun,
Step: r.Resp.Step,
Filename: r.Resp.Filename,
Addr: pool.Name,
}
return &fullyInfo, nil
}
case <-time.After(time.Duration(cfg.Timeout) * time.Millisecond):
pool.Release(conn)
logger.Trace("graph.info timeout err: i/o timeout, addr: %v", pool.Name)
return nil, errors.New("i/o timeout")
}
}
func QueryOne(start, end int64, cf, endpoint, counter string) (r *model.GraphQueryResponse, err error) {
pool, err := selectPool(endpoint, counter)
if err != nil {
return nil, err
}
conn, err := pool.Get()
if err != nil {
return nil, err
}
rpc_conn := conn.(RpcConn)
if rpc_conn.cli == nil {
pool.CloseClean(conn)
return nil, errors.New("nil rpc conn")
}
type ChResult struct {
Err error
Resp *model.GraphQueryResponse
}
ch := make(chan *ChResult, 1)
go func() {
param := model.GraphQueryParam{
Start: start,
End: end,
ConsolFun: cf,
Endpoint: endpoint,
Counter: counter,
}
resp := &model.GraphQueryResponse{}
err := rpc_conn.cli.Call("Graph.Query", param, resp)
r := &ChResult{
Err: err,
Resp: resp,
}
ch <- r
}()
cfg := g.Config().Graph
select {
case r := <-ch:
if r.Err != nil {
pool.CloseClean(conn)
return nil, r.Err
} else {
pool.Release(conn)
logger.Trace("graph: query graph resp: %v, addr: %v", r.Resp, pool.Name)
fixedResp := &model.GraphQueryResponse{
Endpoint: r.Resp.Endpoint,
Counter: r.Resp.Counter,
DsType: r.Resp.DsType,
Step: r.Resp.Step,
}
size := len(r.Resp.Values)
//NOTICE:最后一个点是坏点，过滤点，可能是rrdtool的bug
if size < 1 {
return fixedResp, nil
} else {
dsType := r.Resp.DsType
fixedValues := []*model.RRDData{}
for _, v := range r.Resp.Values[0:size] {
if v == nil {
continue
}
if v.Timestamp < start || v.Timestamp > end {
continue
}
//FIXME: 查询数据的时候，把所有的负值都过滤掉，因为transfer之前在设置最小值的时候为U
if (dsType == "DERIVE" || dsType == "COUNTER") && v.Value < 0 {
fixedValues = append(fixedValues, &model.RRDData{
Timestamp: v.Timestamp,
Value: model.JsonFloat(math.NaN()),
})
} else {
fixedValues = append(fixedValues, v)
}
}
fixedResp.Values = fixedValues
return fixedResp, nil
}
}
case <-time.After(time.Duration(cfg.Timeout) * time.Millisecond):
pool.Release(conn)
logger.Trace("query graph timeout err: i/o timeout, addr: %v", pool.Name)
return nil, errors.New("i/o timeout")
}
}