Example #1
0
func ReloadBackends() {
	cfg := g.Config().Graph
	for {
		time.Sleep(time.Duration(cfg.ReloadInterval) * time.Second)
		err := InitBackends()
		if err != nil {
			logger.Error("reload backends fail: %v", err)
		}
	}
}
Example #2
0
// internal functions
func initConnPools() {
	cfg := g.Config()

	// TODO 为了得到Slice,这里做的太复杂了
	graphInstances := nset.NewSafeSet()
	for _, address := range cfg.Graph.Cluster {
		graphInstances.Add(address)
	}
	GraphConnPools = spool.CreateSafeRpcConnPools(cfg.Graph.MaxConns, cfg.Graph.MaxIdle,
		cfg.Graph.ConnTimeout, cfg.Graph.CallTimeout, graphInstances.ToSlice())
}
Example #3
0
func Start() {
	if !g.Config().Http.Enabled {
		log.Println("http.Start warning, not enabled")
		return
	}

	// config http routes
	configCommonRoutes()
	configProcHttpRoutes()
	configGraphRoutes()

	// start http server
	addr := g.Config().Http.Listen
	s := &http.Server{
		Addr:           addr,
		MaxHeaderBytes: 1 << 30,
	}

	log.Println("http.Start ok, listening on", addr)
	log.Fatalln(s.ListenAndServe())
}
Example #4
0
func Start() {
	addr := g.Config().Http.Listen
	if addr == "" {
		return
	}
	s := &http.Server{
		Addr:           addr,
		MaxHeaderBytes: 1 << 30,
	}
	log.Println("listening", addr)
	log.Fatalln(s.ListenAndServe())
}
Example #5
0
func Info(para cmodel.GraphInfoParam) (resp *cmodel.GraphFullyInfo, err error) {
	endpoint, counter := para.Endpoint, para.Counter

	pool, addr, err := selectPool(endpoint, counter)
	if err != nil {
		return nil, err
	}

	conn, err := pool.Fetch()
	if err != nil {
		return nil, err
	}

	rpcConn := conn.(spool.RpcClient)
	if rpcConn.Closed() {
		pool.ForceClose(conn)
		return nil, errors.New("conn closed")
	}

	type ChResult struct {
		Err  error
		Resp *cmodel.GraphInfoResp
	}
	ch := make(chan *ChResult, 1)
	go func() {
		resp := &cmodel.GraphInfoResp{}
		err := rpcConn.Call("Graph.Info", para, resp)
		ch <- &ChResult{Err: err, Resp: resp}
	}()

	select {
	case <-time.After(time.Duration(g.Config().Graph.CallTimeout) * time.Millisecond):
		pool.ForceClose(conn)
		return nil, fmt.Errorf("%s, call timeout. proc: %s", addr, pool.Proc())
	case r := <-ch:
		if r.Err != nil {
			pool.ForceClose(conn)
			return nil, fmt.Errorf("%s, call failed, err %v. proc: %s", addr, r.Err, pool.Proc())
		} else {
			pool.Release(conn)
			fullyInfo := cmodel.GraphFullyInfo{
				Endpoint:  endpoint,
				Counter:   counter,
				ConsolFun: r.Resp.ConsolFun,
				Step:      r.Resp.Step,
				Filename:  r.Resp.Filename,
				Addr:      addr,
			}
			return &fullyInfo, nil
		}
	}
}
Example #6
0
func Last(endpoint, counter string) (r *model.GraphLastResp, err error) {
	pool, err := selectPool(endpoint, counter)
	if err != nil {
		return nil, err
	}

	conn, err := pool.Get()
	if err != nil {
		return nil, err
	}

	rpc_conn := conn.(RpcConn)
	if rpc_conn.cli == nil {
		pool.CloseClean(conn)
		return nil, errors.New("nil rpc conn")
	}

	type ChResult struct {
		Err  error
		Resp *model.GraphLastResp
	}
	ch := make(chan *ChResult, 1)
	go func() {
		param := model.GraphLastParam{
			Endpoint: endpoint,
			Counter:  counter,
		}
		resp := &model.GraphLastResp{}
		err := rpc_conn.cli.Call("Graph.Last", param, resp)
		r := &ChResult{
			Err:  err,
			Resp: resp,
		}
		ch <- r
	}()

	cfg := g.Config().Graph
	select {
	case r := <-ch:
		if r.Err != nil {
			pool.CloseClean(conn)
			return nil, r.Err
		} else {
			pool.Release(conn)
			return r.Resp, nil
		}
	case <-time.After(time.Duration(cfg.Timeout) * time.Millisecond):
		pool.Release(conn)
		return nil, errors.New("i/o timeout")
	}
}
Example #7
0
func configCommonRoutes() {
	http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("ok\n"))
	})

	http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(fmt.Sprintf("%s\n", g.VERSION)))
	})

	http.HandleFunc("/workdir", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(fmt.Sprintf("%s\n", file.SelfDir())))
	})

	http.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) {
		RenderDataJson(w, g.Config())
	})

}
Example #8
0
func LastRaw(para cmodel.GraphLastParam) (r *cmodel.GraphLastResp, err error) {
	endpoint, counter := para.Endpoint, para.Counter

	pool, addr, err := selectPool(endpoint, counter)
	if err != nil {
		return nil, err
	}

	conn, err := pool.Fetch()
	if err != nil {
		return nil, err
	}

	rpcConn := conn.(spool.RpcClient)
	if rpcConn.Closed() {
		pool.ForceClose(conn)
		return nil, errors.New("conn closed")
	}

	type ChResult struct {
		Err  error
		Resp *cmodel.GraphLastResp
	}
	ch := make(chan *ChResult, 1)
	go func() {
		resp := &cmodel.GraphLastResp{}
		err := rpcConn.Call("Graph.LastRaw", para, resp)
		ch <- &ChResult{Err: err, Resp: resp}
	}()

	select {
	case <-time.After(time.Duration(g.Config().Graph.CallTimeout) * time.Millisecond):
		pool.ForceClose(conn)
		return nil, fmt.Errorf("%s, call timeout. proc: %s", addr, pool.Proc())
	case r := <-ch:
		if r.Err != nil {
			pool.ForceClose(conn)
			return r.Resp, fmt.Errorf("%s, call failed, err %v. proc: %s", addr, r.Err, pool.Proc())
		} else {
			pool.Release(conn)
			return r.Resp, nil
		}
	}
}
Example #9
0
func InitBackends() error {
	var err error
	cfg := g.Config().Graph

	file_path := cfg.Backends
	err = backend.LoadAddrs(file_path)
	if err != nil {
		return err
	}

	backend.InitRing(cfg.Replicas)

	err = initConnPools()
	if err != nil {
		return err
	}

	return nil
}
Example #10
0
func selectPool(endpoint, counter string) (rpool *spool.ConnPool, raddr string, rerr error) {
	pkey := cutils.PK2(endpoint, counter)
	node, err := GraphNodeRing.GetNode(pkey)
	if err != nil {
		return nil, "", err
	}

	addr, found := g.Config().Graph.Cluster[node]
	if !found {
		return nil, "", errors.New("node not found")
	}

	pool, found := GraphConnPools.Get(addr)
	if !found {
		return nil, addr, errors.New("addr not found")
	}

	return pool, addr, nil
}
Example #11
0
func initConnPools() error {
	cfg := g.Config()
	if cfg.LogLevel == "trace" || cfg.LogLevel == "debug" {
		conn_pool.EnableSlowLog(true, cfg.SlowLog)
	}

	var (
		tmp_addrs map[string][]string
		tmp_pools map[string]*conn_pool.ConnPool
	)

	backend.RLock()
	tmp_addrs = backend.Addrs
	tmp_pools = backend.Pools
	backend.RUnlock()

	c := cfg.Graph
	for name, addr_list := range tmp_addrs {
		for _, addr := range addr_list {
			if _, ok := tmp_pools[addr]; !ok {
				pool := conn_pool.NewConnPool(addr, c.MaxConns, c.MaxIdle)

				pool.New = func() (io.Closer, error) {
					_, err := net.ResolveTCPAddr("tcp", pool.Name)
					if err != nil {
						return nil, err
					}
					conn, err := net.DialTimeout("tcp", pool.Name, time.Duration(c.Timeout)*time.Millisecond)
					if err != nil {
						return nil, err
					}

					return RpcConn{rpc.NewClient(conn)}, nil
				}

				pool.Ping = func(conn io.Closer) error {
					rpc_conn := conn.(RpcConn)
					if rpc_conn.cli == nil {
						return errors.New("nil conn")
					}

					resp := &model.SimpleRpcResponse{}
					err := rpc_conn.cli.Call("Graph.Ping", model.NullRpcRequest{}, resp)
					logger.Trace("Graph.Ping resp: %v", resp)

					return err
				}

				tmp_pools[addr] = pool
				logger.Info("create the pool: %s %s", name, addr)
			} else {
				logger.Trace("keep the pool: %s %s", name, addr)
			}
		}
	}

	backend.Lock()
	defer backend.Unlock()
	backend.Pools = tmp_pools

	return nil
}
Example #12
0
func QueryOne(para cmodel.GraphQueryParam) (resp *cmodel.GraphQueryResponse, err error) {
	start, end := para.Start, para.End
	endpoint, counter := para.Endpoint, para.Counter

	pool, addr, err := selectPool(endpoint, counter)
	if err != nil {
		return nil, err
	}

	conn, err := pool.Fetch()
	if err != nil {
		return nil, err
	}

	rpcConn := conn.(spool.RpcClient)
	if rpcConn.Closed() {
		pool.ForceClose(conn)
		return nil, errors.New("conn closed")
	}

	type ChResult struct {
		Err  error
		Resp *cmodel.GraphQueryResponse
	}

	ch := make(chan *ChResult, 1)
	go func() {
		resp := &cmodel.GraphQueryResponse{}
		err := rpcConn.Call("Graph.Query", para, resp)
		ch <- &ChResult{Err: err, Resp: resp}
	}()

	select {
	case <-time.After(time.Duration(g.Config().Graph.CallTimeout) * time.Millisecond):
		pool.ForceClose(conn)
		return nil, fmt.Errorf("%s, call timeout. proc: %s", addr, pool.Proc())
	case r := <-ch:
		if r.Err != nil {
			pool.ForceClose(conn)
			return r.Resp, fmt.Errorf("%s, call failed, err %v. proc: %s", addr, r.Err, pool.Proc())
		} else {
			pool.Release(conn)

			if len(r.Resp.Values) < 1 {
				return r.Resp, nil
			}

			// TODO query不该做这些事情, 说明graph没做好
			fixed := []*cmodel.RRDData{}
			for _, v := range r.Resp.Values {
				if v == nil || !(v.Timestamp >= start && v.Timestamp <= end) {
					continue
				}
				//FIXME: 查询数据的时候,把所有的负值都过滤掉,因为transfer之前在设置最小值的时候为U
				if (r.Resp.DsType == "DERIVE" || r.Resp.DsType == "COUNTER") && v.Value < 0 {
					fixed = append(fixed, &cmodel.RRDData{Timestamp: v.Timestamp, Value: cmodel.JsonFloat(math.NaN())})
				} else {
					fixed = append(fixed, v)
				}
			}
			r.Resp.Values = fixed
		}
		return r.Resp, nil
	}
}
Example #13
0
func initNodeRings() {
	cfg := g.Config()
	GraphNodeRing = rings.NewConsistentHashNodesRing(cfg.Graph.Replicas, cutils.KeysOfMap(cfg.Graph.Cluster))
}
Example #14
0
func Info(endpoint, counter string) (r *model.GraphFullyInfo, err error) {
	pool, err := selectPool(endpoint, counter)
	if err != nil {
		return nil, err
	}

	conn, err := pool.Get()
	if err != nil {
		return nil, err
	}

	rpc_conn := conn.(RpcConn)
	if rpc_conn.cli == nil {
		pool.CloseClean(conn)
		return nil, errors.New("nil rpc conn")
	}

	type ChResult struct {
		Err  error
		Resp *model.GraphInfoResp
	}
	ch := make(chan *ChResult, 1)
	go func() {
		param := model.GraphInfoParam{
			Endpoint: endpoint,
			Counter:  counter,
		}
		resp := &model.GraphInfoResp{}
		err := rpc_conn.cli.Call("Graph.Info", param, resp)
		r := &ChResult{
			Err:  err,
			Resp: resp,
		}
		ch <- r
	}()

	cfg := g.Config().Graph
	select {
	case r := <-ch:
		if r.Err != nil {
			pool.CloseClean(conn)
			return nil, r.Err
		} else {
			pool.Release(conn)
			logger.Trace("graph.info resp: %v, addr: %v", r.Resp, pool.Name)
			fullyInfo := model.GraphFullyInfo{
				Endpoint:  endpoint,
				Counter:   counter,
				ConsolFun: r.Resp.ConsolFun,
				Step:      r.Resp.Step,
				Filename:  r.Resp.Filename,
				Addr:      pool.Name,
			}
			return &fullyInfo, nil
		}

	case <-time.After(time.Duration(cfg.Timeout) * time.Millisecond):
		pool.Release(conn)
		logger.Trace("graph.info timeout err: i/o timeout, addr: %v", pool.Name)
		return nil, errors.New("i/o timeout")
	}
}
Example #15
0
func QueryOne(start, end int64, cf, endpoint, counter string) (r *model.GraphQueryResponse, err error) {
	pool, err := selectPool(endpoint, counter)
	if err != nil {
		return nil, err
	}

	conn, err := pool.Get()
	if err != nil {
		return nil, err
	}

	rpc_conn := conn.(RpcConn)
	if rpc_conn.cli == nil {
		pool.CloseClean(conn)
		return nil, errors.New("nil rpc conn")
	}

	type ChResult struct {
		Err  error
		Resp *model.GraphQueryResponse
	}
	ch := make(chan *ChResult, 1)
	go func() {
		param := model.GraphQueryParam{
			Start:     start,
			End:       end,
			ConsolFun: cf,
			Endpoint:  endpoint,
			Counter:   counter,
		}
		resp := &model.GraphQueryResponse{}
		err := rpc_conn.cli.Call("Graph.Query", param, resp)
		r := &ChResult{
			Err:  err,
			Resp: resp,
		}
		ch <- r
	}()

	cfg := g.Config().Graph
	select {
	case r := <-ch:
		if r.Err != nil {
			pool.CloseClean(conn)
			return nil, r.Err
		} else {
			pool.Release(conn)
			logger.Trace("graph: query graph resp: %v, addr: %v", r.Resp, pool.Name)

			fixedResp := &model.GraphQueryResponse{
				Endpoint: r.Resp.Endpoint,
				Counter:  r.Resp.Counter,
				DsType:   r.Resp.DsType,
				Step:     r.Resp.Step,
			}
			size := len(r.Resp.Values)

			//NOTICE:最后一个点是坏点,过滤点,可能是rrdtool的bug
			if size < 1 {
				return fixedResp, nil
			} else {
				dsType := r.Resp.DsType
				fixedValues := []*model.RRDData{}
				for _, v := range r.Resp.Values[0:size] {
					if v == nil {
						continue
					}
					if v.Timestamp < start || v.Timestamp > end {
						continue
					}
					//FIXME: 查询数据的时候,把所有的负值都过滤掉,因为transfer之前在设置最小值的时候为U
					if (dsType == "DERIVE" || dsType == "COUNTER") && v.Value < 0 {
						fixedValues = append(fixedValues, &model.RRDData{
							Timestamp: v.Timestamp,
							Value:     model.JsonFloat(math.NaN()),
						})
					} else {
						fixedValues = append(fixedValues, v)
					}
				}
				fixedResp.Values = fixedValues
				return fixedResp, nil
			}
		}

	case <-time.After(time.Duration(cfg.Timeout) * time.Millisecond):
		pool.Release(conn)
		logger.Trace("query graph timeout err: i/o timeout, addr: %v", pool.Name)
		return nil, errors.New("i/o timeout")
	}

}