func Start() {
if !g.Config().Http.Enabled {
return
}
addr := g.Config().Http.Listen
if addr == "" {
return
}
s := &http.Server{
Addr: addr,
MaxHeaderBytes: 1 << 30,
}
log.Println("http listening", addr)
ln, err := net.Listen("tcp", addr)
if err != nil {
log.Fatalln(err)
return
}
l := ln.(*net.TCPListener)
go s.Serve(TcpKeepAliveListener{l})
select {
case <-Close_chan:
log.Println("http recv sigout and exit...")
l.Close()
Close_done_chan <- 1
return
}
}
func Start() {
if !g.Config().Rpc.Enabled {
return
}
addr := g.Config().Rpc.Listen
tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
if err != nil {
log.Fatalf("net.ResolveTCPAddr fail: %s", err)
}
listener, err := net.ListenTCP("tcp", tcpAddr)
if err != nil {
log.Fatalf("listen %s fail: %s", addr, err)
} else {
log.Println("rpc listening", addr)
}
rpc.Register(new(Graph))
go func() {
var tempDelay time.Duration // how long to sleep on accept failure
for {
conn, err := listener.Accept()
if err != nil {
if tempDelay == 0 {
tempDelay = 5 * time.Millisecond
} else {
tempDelay *= 2
}
if max := 1 * time.Second; tempDelay > max {
tempDelay = max
}
time.Sleep(tempDelay)
continue
}
tempDelay = 0
go func() {
e := connects.insert(conn)
defer connects.remove(e)
rpc.ServeConn(conn)
}()
}
}()
select {
case <-Close_chan:
log.Println("api recv sigout and exit...")
listener.Close()
Close_done_chan <- 1
connects.Lock()
for e := connects.list.Front(); e != nil; e = e.Next() {
e.Value.(net.Conn).Close()
}
connects.Unlock()
return
}
}
func FlushRRD(idx int) {
var debug_checksum string
var debug bool
storageDir := g.Config().RRD.Storage
if g.Config().Debug {
debug = true
debug_checksum = g.Config().DebugChecksum
} else {
debug = false
}
keys := store.GraphItems.KeysByIndex(idx)
if len(keys) == 0 {
return
}
for _, checksum := range keys {
items := store.GraphItems.PopAll(checksum)
size := len(items)
if size == 0 {
continue
}
first := items[0]
filename := fmt.Sprintf("%s/%s/%s_%s_%d.rrd", storageDir, checksum[0:2], checksum, first.DsType, first.Step)
if debug && debug_checksum == checksum {
for _, item := range items {
log.Printf(
"2-flush:%d:%s:%lf",
item.Timestamp,
time.Unix(item.Timestamp, 0).Format("2006-01-02 15:04:05"),
item.Value,
)
}
}
err := Flush(filename, items)
if err != nil && debug && debug_checksum == checksum {
log.Println("flush fail:", err, "filename:", filename)
}
Counter += 1
}
if debug {
log.Println("flushrrd counter:", Counter)
}
}
func FlushRRD(idx int) {
storageDir := g.Config().RRD.Storage
keys := store.GraphItems.KeysByIndex(idx)
if len(keys) == 0 {
return
}
for _, ckey := range keys {
// get md5, dstype, step
checksum, dsType, step, err := g.SplitRrdCacheKey(ckey)
if err != nil {
continue
}
items := store.GraphItems.PopAll(ckey)
size := len(items)
if size == 0 {
continue
}
filename := g.RrdFileName(storageDir, checksum, dsType, step)
Flush(filename, items)
Counter += 1
}
}
func configCommonRoutes() {
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("ok"))
})
http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(g.VERSION))
})
http.HandleFunc("/workdir", func(w http.ResponseWriter, r *http.Request) {
RenderDataJson(w, file.SelfDir())
})
http.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) {
RenderDataJson(w, g.Config())
})
http.HandleFunc("/config/reload", func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.RemoteAddr, "127.0.0.1") {
g.ParseConfig(g.ConfigFile)
RenderDataJson(w, "ok")
} else {
RenderDataJson(w, "no privilege")
}
})
}
func main() {
cfg := flag.String("c", "cfg.json", "specify config file")
version := flag.Bool("v", false, "show version")
versionGit := flag.Bool("vg", false, "show version and git commit log")
flag.Parse()
if *version {
fmt.Println(g.VERSION)
os.Exit(0)
}
if *versionGit {
fmt.Println(g.VERSION, g.COMMIT)
os.Exit(0)
}
// global config
g.ParseConfig(*cfg)
// init db
g.InitDB()
// rrdtool before api for disable loopback connection
rrdtool.Start()
// start api
go api.Start()
// start indexing
index.Start()
// start http server
go http.Start()
start_signal(os.Getpid(), g.Config())
}
// index收到一条新上报的监控数据,尝试用于增量更新索引
func ReceiveItem(item *cmodel.GraphItem, md5 string) {
if item == nil {
return
}
uuid := item.UUID()
// 已上报过的数据
if indexedItemCache.ContainsKey(md5) {
old := indexedItemCache.Get(md5).(*IndexCacheItem)
if uuid == old.UUID { // dsType+step没有发生变化,只更新缓存 TODO 存在线程安全的问题
old.Item = item
} else { // dsType+step变化了,当成一个新的增量来处理(甚至,不用rrd文件来过滤)
//indexedItemCache.Remove(md5)
unIndexedItemCache.Put(md5, NewIndexCacheItem(uuid, item))
}
return
}
// 是否有rrdtool文件存在,如果有 认为已建立索引
// 针对 索引缓存重建场景 做的优化, 结合索引全量更新 来保证一致性
rrdFileName := g.RrdFileName(g.Config().RRD.Storage, md5, item.DsType, item.Step)
if g.IsRrdFileExist(rrdFileName) {
indexedItemCache.Put(md5, NewIndexCacheItem(uuid, item))
return
}
// 缓存未命中, 放入增量更新队列
unIndexedItemCache.Put(md5, NewIndexCacheItem(uuid, item))
}
func Start() {
// check data dir
dataDir := g.Config().RRD.Storage
if err := file.EnsureDirRW(dataDir); err != nil {
log.Fatalln("rrdtool.Start error, bad data dir", dataDir+",", err)
}
log.Println("rrdtool.Start, ok")
}
func fetch_rrd(client **rpc.Client, key string, addr string) error {
var (
err error
flag uint32
md5 string
dsType string
filename string
step, i int
rrdfile g.File
)
cfg := g.Config()
if flag, err = store.GraphItems.GetFlag(key); err != nil {
return err
}
store.GraphItems.SetFlag(key, flag|g.GRAPH_F_FETCHING)
md5, dsType, step, _ = g.SplitRrdCacheKey(key)
filename = g.RrdFileName(cfg.RRD.Storage, md5, dsType, step)
for i = 0; i < 3; i++ {
err = rpc_call(*client, "Graph.GetRrd", key, &rrdfile,
time.Duration(cfg.CallTimeout)*time.Millisecond)
if err == nil {
done := make(chan error, 1)
io_task_chan <- &io_task_t{
method: IO_TASK_M_WRITE,
args: &g.File{
Filename: filename,
Body: rrdfile.Body[:],
},
done: done,
}
if err = <-done; err != nil {
goto out
} else {
flag &= ^g.GRAPH_F_MISS
goto out
}
} else {
log.Println(err)
}
if err == rpc.ErrShutdown {
reconnection(client, addr)
}
}
out:
flag &= ^g.GRAPH_F_FETCHING
store.GraphItems.SetFlag(key, flag)
return err
}
func CommitByKey(key string) {
md5, dsType, step, err := g.SplitRrdCacheKey(key)
if err != nil {
return
}
filename := g.RrdFileName(g.Config().RRD.Storage, md5, dsType, step)
items := store.GraphItems.PopAll(key)
if len(items) == 0 {
return
}
FlushFile(filename, items)
}
func (this *Graph) GetRrd(key string, rrdfile *g.File) (err error) {
if md5, dsType, step, err := g.SplitRrdCacheKey(key); err != nil {
return err
} else {
rrdfile.Filename = g.RrdFileName(g.Config().RRD.Storage, md5, dsType, step)
}
items := store.GraphItems.PopAll(key)
if len(items) > 0 {
rrdtool.FlushFile(rrdfile.Filename, items)
}
rrdfile.Body, err = rrdtool.ReadFile(rrdfile.Filename)
return
}
func Start() {
cfg := g.Config()
var err error
// check data dir
if err = file.EnsureDirRW(cfg.RRD.Storage); err != nil {
log.Fatalln("rrdtool.Start error, bad data dir "+cfg.RRD.Storage+",", err)
}
migrate_start(cfg)
// sync disk
go syncDisk()
go ioWorker()
log.Println("rrdtool.Start ok")
}
func (this *Graph) Info(param cmodel.GraphInfoParam, resp *cmodel.GraphInfoResp) error {
// statistics
proc.GraphInfoCnt.Incr()
dsType, step, exists := index.GetTypeAndStep(param.Endpoint, param.Counter)
if !exists {
return nil
}
md5 := cutils.Md5(param.Endpoint + "/" + param.Counter)
filename := fmt.Sprintf("%s/%s/%s_%s_%d.rrd", g.Config().RRD.Storage, md5[0:2], md5, dsType, step)
resp.ConsolFun = dsType
resp.Step = step
resp.Filename = filename
return nil
}
func send_data(client **rpc.Client, key string, addr string) error {
var (
err error
flag uint32
resp *cmodel.SimpleRpcResponse
i int
)
//remote
if flag, err = store.GraphItems.GetFlag(key); err != nil {
return err
}
cfg := g.Config()
store.GraphItems.SetFlag(key, flag|g.GRAPH_F_SENDING)
items := store.GraphItems.PopAll(key)
items_size := len(items)
if items_size == 0 {
goto out
}
resp = &cmodel.SimpleRpcResponse{}
for i = 0; i < 3; i++ {
err = rpc_call(*client, "Graph.Send", items, resp,
time.Duration(cfg.CallTimeout)*time.Millisecond)
if err == nil {
goto out
}
if err == rpc.ErrShutdown {
reconnection(client, addr)
}
}
// err
store.GraphItems.PushAll(key, items)
//flag |= g.GRAPH_F_ERR
out:
flag &= ^g.GRAPH_F_SENDING
store.GraphItems.SetFlag(key, flag)
return err
}
func query_data(client **rpc.Client, addr string,
args interface{}, resp interface{}) error {
var (
err error
i int
)
for i = 0; i < 3; i++ {
err = rpc_call(*client, "Graph.Query", args, resp,
time.Duration(g.Config().CallTimeout)*time.Millisecond)
if err == nil {
break
}
if err == rpc.ErrShutdown {
reconnection(client, addr)
}
}
return err
}
func main() {
cfg := flag.String("c", "cfg.json", "configuration file")
version := flag.Bool("v", false, "show version")
versionGit := flag.Bool("vg", false, "show version")
flag.Parse()
if *version {
fmt.Println(g.VERSION)
os.Exit(0)
}
if *versionGit {
fmt.Println(g.VERSION, g.COMMIT)
os.Exit(0)
}
// global config
g.ParseConfig(*cfg)
// init db
g.InitDB()
// start rrdtool
rrdtool.Start()
go api.Start()
// 刷硬盘
go cron.SyncDisk()
// 索引更新2.0
index.Start()
// http
go http.Start()
start_signal(os.Getpid(), *g.Config())
}
func handleItems(items []*cmodel.GraphItem) {
if items == nil {
return
}
count := len(items)
if count == 0 {
return
}
cfg := g.Config()
for i := 0; i < count; i++ {
if items[i] == nil {
continue
}
dsType := items[i].DsType
step := items[i].Step
checksum := items[i].Checksum()
key := g.FormRrdCacheKey(checksum, dsType, step)
//statistics
proc.GraphRpcRecvCnt.Incr()
// To Graph
first := store.GraphItems.First(key)
if first != nil && items[i].Timestamp <= first.Timestamp {
continue
}
store.GraphItems.PushFront(key, items[i], checksum, cfg)
// To Index
index.ReceiveItem(items[i], checksum)
// To History
store.AddItem(checksum, items[i])
}
}
func FlushRRD(idx int, force bool) {
begin := time.Now()
atomic.StoreInt32(&flushrrd_timeout, 0)
keys := store.GraphItems.KeysByIndex(idx)
if len(keys) == 0 {
return
}
for _, key := range keys {
flag, _ := store.GraphItems.GetFlag(key)
//write err data to local filename
if force == false && g.Config().Migrate.Enabled && flag&g.GRAPH_F_MISS != 0 {
if time.Since(begin) > time.Millisecond*g.FLUSH_DISK_STEP {
atomic.StoreInt32(&flushrrd_timeout, 1)
}
PullByKey(key)
} else {
CommitByKey(key)
}
}
}
func (this *Graph) Query(param cmodel.GraphQueryParam, resp *cmodel.GraphQueryResponse) error {
// statistics
proc.GraphQueryCnt.Incr()
resp.Values = []*cmodel.RRDData{}
dsType, step, exists := index.GetTypeAndStep(param.Endpoint, param.Counter)
if !exists {
return nil
}
md5 := cutils.Md5(param.Endpoint + "/" + param.Counter)
filename := fmt.Sprintf("%s/%s/%s_%s_%d.rrd", g.Config().RRD.Storage, md5[0:2], md5, dsType, step)
datas, err := rrdtool.Fetch(filename, param.ConsolFun, param.Start, param.End, step)
if err != nil {
if store.GraphItems.LenOf(md5) <= 2 {
return nil
}
// TODO not atomic, fix me
items := store.GraphItems.PopAll(md5)
size := len(items)
if size > 2 {
filename := fmt.Sprintf("%s/%s/%s_%s_%d.rrd", g.Config().RRD.Storage, md5[0:2],
md5, items[0].DsType, items[0].Step)
err := rrdtool.Flush(filename, items)
if err != nil && g.Config().Debug && g.Config().DebugChecksum == md5 {
log.Println("flush fail:", err, "filename:", filename)
}
} else {
return nil
}
}
items := store.GraphItems.FetchAll(md5)
// merge
items_size := len(items)
datas_size := len(datas)
if items_size > 1 && datas_size > 2 &&
int(datas[1].Timestamp-datas[0].Timestamp) == step &&
items[items_size-1].Timestamp > datas[0].Timestamp {
var val cmodel.JsonFloat
cache_size := int(items[items_size-1].Timestamp-items[0].Timestamp)/step + 1
cache := make([]*cmodel.RRDData, cache_size, cache_size)
//fix items
items_idx := 0
ts := items[0].Timestamp
if dsType == g.DERIVE || dsType == g.COUNTER {
for i := 0; i < cache_size; i++ {
if items_idx < items_size-1 &&
ts == items[items_idx].Timestamp &&
ts != items[items_idx+1].Timestamp {
val = cmodel.JsonFloat(items[items_idx+1].Value-items[items_idx].Value) /
cmodel.JsonFloat(items[items_idx+1].Timestamp-items[items_idx].Timestamp)
if val < 0 {
val = cmodel.JsonFloat(math.NaN())
}
items_idx++
} else {
// miss
val = cmodel.JsonFloat(math.NaN())
}
cache[i] = &cmodel.RRDData{
Timestamp: ts,
Value: val,
}
ts = ts + int64(step)
}
} else if dsType == g.GAUGE {
for i := 0; i < cache_size; i++ {
if items_idx < items_size && ts == items[items_idx].Timestamp {
val = cmodel.JsonFloat(items[items_idx].Value)
items_idx++
} else {
// miss
val = cmodel.JsonFloat(math.NaN())
}
cache[i] = &cmodel.RRDData{
Timestamp: ts,
Value: val,
}
ts = ts + int64(step)
}
} else {
log.Println("not support dstype")
return nil
}
size := int(items[items_size-1].Timestamp-datas[0].Timestamp)/step + 1
ret := make([]*cmodel.RRDData, size, size)
cache_idx := 0
ts = datas[0].Timestamp
if g.Config().Debug && g.Config().DebugChecksum == md5 {
log.Println("param.start", param.Start, "param.End:", param.End,
"items:", items, "datas:", datas)
}
for i := 0; i < size; i++ {
if g.Config().Debug && g.Config().DebugChecksum == md5 {
log.Println("i", i, "size:", size, "items_idx:", items_idx, "ts:", ts)
}
if i < datas_size {
if ts == cache[cache_idx].Timestamp {
if math.IsNaN(float64(cache[cache_idx].Value)) {
val = datas[i].Value
} else {
val = cache[cache_idx].Value
}
cache_idx++
} else {
val = datas[i].Value
}
} else {
if cache_idx < cache_size && ts == cache[cache_idx].Timestamp {
val = cache[cache_idx].Value
cache_idx++
} else {
//miss
val = cmodel.JsonFloat(math.NaN())
}
}
ret[i] = &cmodel.RRDData{
Timestamp: ts,
Value: val,
}
ts = ts + int64(step)
}
resp.Values = ret
} else {
resp.Values = datas
}
resp.Endpoint = param.Endpoint
resp.Counter = param.Counter
resp.DsType = dsType
resp.Step = step
// statistics
proc.GraphQueryItemCnt.IncrBy(int64(len(resp.Values)))
return nil
}
func (this *Graph) Query(param cmodel.GraphQueryParam, resp *cmodel.GraphQueryResponse) error {
// statistics
proc.GraphQueryCnt.Incr()
// form empty response
resp.Values = []*cmodel.RRDData{}
resp.Endpoint = param.Endpoint
resp.Counter = param.Counter
dsType, step, exists := index.GetTypeAndStep(param.Endpoint, param.Counter) // complete dsType and step
if !exists {
return nil
}
resp.DsType = dsType
resp.Step = step
start_ts := param.Start - param.Start%int64(step)
end_ts := param.End - param.End%int64(step) + int64(step)
if end_ts-start_ts-int64(step) < 1 {
return nil
}
md5 := cutils.Md5(param.Endpoint + "/" + param.Counter)
ckey := g.FormRrdCacheKey(md5, dsType, step)
filename := g.RrdFileName(g.Config().RRD.Storage, md5, dsType, step)
// read data from rrd file
datas, _ := rrdtool.Fetch(filename, param.ConsolFun, start_ts, end_ts, step)
datas_size := len(datas)
// read cached items
items := store.GraphItems.FetchAll(ckey)
items_size := len(items)
nowTs := time.Now().Unix()
lastUpTs := nowTs - nowTs%int64(step)
rra1StartTs := lastUpTs - int64(rrdtool.RRA1PointCnt*step)
// consolidated, do not merge
if start_ts < rra1StartTs {
resp.Values = datas
goto _RETURN_OK
}
// no cached items, do not merge
if items_size < 1 {
resp.Values = datas
goto _RETURN_OK
}
// merge
{
// fmt cached items
var val cmodel.JsonFloat
cache := make([]*cmodel.RRDData, 0)
ts := items[0].Timestamp
itemEndTs := items[items_size-1].Timestamp
itemIdx := 0
if dsType == g.DERIVE || dsType == g.COUNTER {
for ts < itemEndTs {
if itemIdx < items_size-1 && ts == items[itemIdx].Timestamp &&
ts == items[itemIdx+1].Timestamp-int64(step) {
val = cmodel.JsonFloat(items[itemIdx+1].Value-items[itemIdx].Value) / cmodel.JsonFloat(step)
if val < 0 {
val = cmodel.JsonFloat(math.NaN())
}
itemIdx++
} else {
// missing
val = cmodel.JsonFloat(math.NaN())
}
if ts >= start_ts && ts <= end_ts {
cache = append(cache, &cmodel.RRDData{Timestamp: ts, Value: val})
}
ts = ts + int64(step)
}
} else if dsType == g.GAUGE {
for ts <= itemEndTs {
if itemIdx < items_size && ts == items[itemIdx].Timestamp {
val = cmodel.JsonFloat(items[itemIdx].Value)
itemIdx++
} else {
// missing
val = cmodel.JsonFloat(math.NaN())
}
if ts >= start_ts && ts <= end_ts {
cache = append(cache, &cmodel.RRDData{Timestamp: ts, Value: val})
}
ts = ts + int64(step)
}
}
cache_size := len(cache)
// do merging
merged := make([]*cmodel.RRDData, 0)
if datas_size > 0 {
for _, val := range datas {
if val.Timestamp >= start_ts && val.Timestamp <= end_ts {
merged = append(merged, val) //rrdtool返回的数据,时间戳是连续的、不会有跳点的情况
}
}
}
if cache_size > 0 {
rrdDataSize := len(merged)
lastTs := cache[0].Timestamp
// find junction
rrdDataIdx := 0
for rrdDataIdx = rrdDataSize - 1; rrdDataIdx >= 0; rrdDataIdx-- {
if merged[rrdDataIdx].Timestamp < cache[0].Timestamp {
lastTs = merged[rrdDataIdx].Timestamp
break
}
}
// fix missing
for ts := lastTs + int64(step); ts < cache[0].Timestamp; ts += int64(step) {
merged = append(merged, &cmodel.RRDData{Timestamp: ts, Value: cmodel.JsonFloat(math.NaN())})
}
// merge cached items to result
rrdDataIdx += 1
for cacheIdx := 0; cacheIdx < cache_size; cacheIdx++ {
if rrdDataIdx < rrdDataSize {
if !math.IsNaN(float64(cache[cacheIdx].Value)) {
merged[rrdDataIdx] = cache[cacheIdx]
}
} else {
merged = append(merged, cache[cacheIdx])
}
rrdDataIdx++
}
}
mergedSize := len(merged)
// fmt result
ret_size := int((end_ts - start_ts) / int64(step))
ret := make([]*cmodel.RRDData, ret_size, ret_size)
mergedIdx := 0
ts = start_ts
for i := 0; i < ret_size; i++ {
if mergedIdx < mergedSize && ts == merged[mergedIdx].Timestamp {
ret[i] = merged[mergedIdx]
mergedIdx++
} else {
ret[i] = &cmodel.RRDData{Timestamp: ts, Value: cmodel.JsonFloat(math.NaN())}
}
ts += int64(step)
}
resp.Values = ret
}
_RETURN_OK:
// statistics
proc.GraphQueryItemCnt.IncrBy(int64(len(resp.Values)))
return nil
}