func (r *Raper) createAndUploadData(domain string, pathList *domainPathList, dataInfo *specifiledDataInfo) {
hostName, _ := os.Hostname()
var conftags []string
for {
select {
case <-time.After(time.Duration(dataInfo.sTime-utils.CurrentMilliSecond()+int64(config.UploadFrequency)*1000) * time.Millisecond):
dataInfo.mutex.Lock()
tsdbMap := dataInfo.data
dataInfo.data = make(map[string]*TsdbPutInfo)
dataInfo.sTime, _ = utils.CurrentStamp(config.UploadFrequency)
dataInfo.mutex.Unlock()
pathList.rmux.Lock()
countMap := pathList.data
pathList.data = make(map[string]int)
pathList.rmux.Unlock()
finalTsdbMap := make(domainMetricTsdbMap)
paths := utils.UTF8Filter(AggregationPath(countMap))
r.mutexBindPath.RLock()
domainBinds := r.domainBindPath[domain]
r.mutexBindPath.RUnlock()
paths = utils.AppendListToList(domainBinds, paths)
conftags = r.getDomainTags(domain)
var total float64
for key, tsdb := range tsdbMap {
oldPath := tsdb.Tags["path"]
tsdb.Tags["path"], _ = utils.FindConfigPath(oldPath, paths)
_, newKey := utils.TagsToKey(conftags, tsdb.Tags)
parts := strings.Split(key, "|")
lastPart := parts[len(parts)-1]
finalKey := newKey + "|" + lastPart
if lastPart == config.UrlCodeMetric {
total += tsdb.Value
}
if _, ok := finalTsdbMap[finalKey]; ok {
finalTsdbMap[finalKey].Value += tsdbMap[key].Value
} else {
finalTsdbMap[finalKey] = tsdbMap[key]
}
}
logger.Printf("domain: %s, total: %d\n", domain, int64(total))
go func() {
for _, v := range finalTsdbMap {
var data string
intPart, frac := math.Modf(v.Value)
if frac == 0 {
data = fmt.Sprintf("put %s %d %d consumer=%s", v.Metric, v.TimeStamp, int64(intPart), hostName)
} else {
data = fmt.Sprintf("put %s %d %.2f consumer=%s", v.Metric, v.TimeStamp, v.Value, hostName)
}
for key, value := range v.Tags {
data += fmt.Sprintf(" %s=%s", key, value)
}
// put test.url.code 1440492540000 2000 domain=api.wandoujia.com path=/sre-test code=200
data += "\n"
r.tsdb.MessageChan <- data
}
}()
case <-r.Dying():
return
}
}
}
func (r *Raper) distDomainMertic() {
r.domainsChanMap = make(map[string]chan domainMetricTsdbMap)
for {
select {
case AD := <-r.aggreMessageChan:
domain, data := AD.domain, AD.data
dataInfo := make(domainMetricTsdbMap)
r.mutexDomains.Lock()
if !utils.StrInStrings(domain, r.domains) {
r.domains = append(r.domains, domain)
}
r.mutexDomains.Unlock()
key, tags := "", make(map[string]string)
configTags := r.getDomainTags(domain)
tags, key = utils.TagsToKey(configTags, data)
length, err := utils.ParseFloat64(data["length"])
if err != nil {
logger.Printf("parse length error: %+v\n", data)
break
}
requestTime, err := utils.ParseFloat64(data["reqtime"])
if err != nil {
logger.Printf("request time parse error: %+v", data)
requestTime = 0
}
upstreamTime, err := utils.ParseFloat64(data["upstream_resptime"])
if err != nil {
upstreamTime = requestTime
}
/*
code 408 means server wait for client request timeout, so ignore here.
*/
if c := data["code"]; c == "408" || c == "499" && requestTime < config.Code499Timeout {
break
}
for _, v := range config.TotalUrlMetric {
k := key + "|" + v
if _, ok := dataInfo[k]; !ok {
dataInfo[k] = &TsdbPutInfo{
Metric: v,
Tags: tags,
}
}
}
dataInfo[key+"|"+config.UrlQpsMetric].Value += 1 / float64(config.UploadFrequency)
dataInfo[key+"|"+config.UrlCodeMetric].Value += 1
dataInfo[key+"|"+config.UrlUpstreamMetric].Value += upstreamTime * 1000
dataInfo[key+"|"+config.UrlTimeMetric].Value += requestTime * 1000
dataInfo[key+"|"+config.UrlTrafficMetric].Value += length / float64(config.UploadFrequency)
var channel chan domainMetricTsdbMap
r.mutexDomainMap.RLock()
_, ok := r.domainsChanMap[domain]
r.mutexDomainMap.RUnlock()
if !ok {
r.mutexDomainMap.Lock()
_, ok := r.domainsChanMap[domain]
if !ok {
r.domainsChanMap[domain] = make(chan domainMetricTsdbMap)
go r.updateDomainMetric(domain, r.domainsChanMap[domain])
}
channel = r.domainsChanMap[domain]
r.mutexDomainMap.Unlock()
} else {
r.mutexDomainMap.RLock()
channel = r.domainsChanMap[domain]
r.mutexDomainMap.RUnlock()
}
channel <- dataInfo
case <-r.Dying():
return
}
}
}