// 更新一条监控数据对应的索引. 用于手动添加索引,一般情况下不会使用
func UpdateIndexOne(endpoint string, metric string, tags map[string]string, dstype string, step int) error {
log.Println("1")
itemDemo := &cmodel.GraphItem{
Endpoint: endpoint,
Metric: metric,
Tags: tags,
DsType: dstype,
Step: step,
}
md5 := itemDemo.Checksum()
uuid := itemDemo.UUID()
cached := indexedItemCache.Get(md5)
if cached == nil {
return fmt.Errorf("not found")
}
icitem := cached.(*IndexCacheItem)
if icitem.UUID != uuid {
return fmt.Errorf("bad type or step")
}
gitem := icitem.Item
log.Println("2")
dbConn, err := g.GetDbConn("UpdateIndexAllTask")
if err != nil {
log.Println("[ERROR] make dbConn fail", err)
return err
}
log.Println("3")
return updateIndexFromOneItem(gitem, dbConn)
}
// 进行一次增量更新
func updateIndexIncr() int {
ret := 0
if unIndexedItemCache == nil || unIndexedItemCache.Size() <= 0 {
return ret
}
dbConn, err := g.GetDbConn("UpdateIndexIncrTask")
if err != nil {
log.Println("[ERROR] get dbConn fail", err)
return ret
}
keys := unIndexedItemCache.Keys()
for _, key := range keys {
icitem := unIndexedItemCache.Get(key)
unIndexedItemCache.Remove(key)
if icitem != nil {
// 并发更新mysql
semaUpdateIndexIncr.Acquire()
go func(key string, icitem *IndexCacheItem, dbConn *sql.DB) {
defer semaUpdateIndexIncr.Release()
err := maybeUpdateIndexFromOneItem(icitem.Item, dbConn)
if err != nil {
proc.IndexUpdateIncrErrorCnt.Incr()
} else {
indexedItemCache.Put(key, icitem)
}
}(key, icitem.(*IndexCacheItem), dbConn)
ret++
}
}
return ret
}
func updateIndexAll(updateStepInSec int64) int {
var ret int = 0
if indexedItemCache == nil || indexedItemCache.Size() <= 0 {
return ret
}
dbConn, err := g.GetDbConn("UpdateIndexAllTask")
if err != nil {
log.Println("[ERROR] make dbConn fail", err)
return ret
}
// lastTs for update index
ts := time.Now().Unix()
lastTs := ts - updateStepInSec
keys := indexedItemCache.Keys()
for _, key := range keys {
icitem := indexedItemCache.Get(key)
if icitem == nil {
continue
}
gitem := icitem.(*IndexCacheItem).Item
if gitem.Timestamp < lastTs { //缓存中的数据太旧了,不能用于索引的全量更新
indexedItemCache.Remove(key) //在这里做个删除,有点恶心
continue
}
// 并发写mysql
semaIndexUpdateAll.Acquire()
go func(gitem *cmodel.GraphItem, dbConn *sql.DB) {
defer semaIndexUpdateAll.Release()
err := updateIndexFromOneItem(gitem, dbConn)
if err != nil {
proc.IndexUpdateAllErrorCnt.Incr()
}
}(gitem, dbConn)
ret++
}
return ret
}