Ejemplo n.º 1
0
func (r *Raper) createAndUploadData(domain string, pathList *domainPathList, dataInfo *specifiledDataInfo) {
	hostName, _ := os.Hostname()
	var conftags []string
	for {
		select {
		case <-time.After(time.Duration(dataInfo.sTime-utils.CurrentMilliSecond()+int64(config.UploadFrequency)*1000) * time.Millisecond):
			dataInfo.mutex.Lock()
			tsdbMap := dataInfo.data
			dataInfo.data = make(map[string]*TsdbPutInfo)
			dataInfo.sTime, _ = utils.CurrentStamp(config.UploadFrequency)
			dataInfo.mutex.Unlock()

			pathList.rmux.Lock()
			countMap := pathList.data
			pathList.data = make(map[string]int)
			pathList.rmux.Unlock()

			finalTsdbMap := make(domainMetricTsdbMap)
			paths := utils.UTF8Filter(AggregationPath(countMap))
			r.mutexBindPath.RLock()
			domainBinds := r.domainBindPath[domain]
			r.mutexBindPath.RUnlock()
			paths = utils.AppendListToList(domainBinds, paths)
			conftags = r.getDomainTags(domain)

			var total float64
			for key, tsdb := range tsdbMap {

				oldPath := tsdb.Tags["path"]

				tsdb.Tags["path"], _ = utils.FindConfigPath(oldPath, paths)
				_, newKey := utils.TagsToKey(conftags, tsdb.Tags)
				parts := strings.Split(key, "|")

				lastPart := parts[len(parts)-1]
				finalKey := newKey + "|" + lastPart

				if lastPart == config.UrlCodeMetric {
					total += tsdb.Value
				}

				if _, ok := finalTsdbMap[finalKey]; ok {
					finalTsdbMap[finalKey].Value += tsdbMap[key].Value
				} else {
					finalTsdbMap[finalKey] = tsdbMap[key]
				}
			}
			logger.Printf("domain: %s, total: %d\n", domain, int64(total))
			go func() {
				for _, v := range finalTsdbMap {
					var data string
					intPart, frac := math.Modf(v.Value)
					if frac == 0 {
						data = fmt.Sprintf("put %s %d %d consumer=%s", v.Metric, v.TimeStamp, int64(intPart), hostName)
					} else {
						data = fmt.Sprintf("put %s %d %.2f consumer=%s", v.Metric, v.TimeStamp, v.Value, hostName)
					}
					for key, value := range v.Tags {
						data += fmt.Sprintf(" %s=%s", key, value)
					}
					// put test.url.code 1440492540000 2000 domain=api.wandoujia.com path=/sre-test code=200
					data += "\n"
					r.tsdb.MessageChan <- data
				}
			}()
		case <-r.Dying():
			return
		}
	}
}
Ejemplo n.º 2
0
func (r *Raper) distDomainMertic() {

	r.domainsChanMap = make(map[string]chan domainMetricTsdbMap)
	for {
		select {
		case AD := <-r.aggreMessageChan:
			domain, data := AD.domain, AD.data
			dataInfo := make(domainMetricTsdbMap)
			r.mutexDomains.Lock()
			if !utils.StrInStrings(domain, r.domains) {
				r.domains = append(r.domains, domain)
			}
			r.mutexDomains.Unlock()

			key, tags := "", make(map[string]string)

			configTags := r.getDomainTags(domain)

			tags, key = utils.TagsToKey(configTags, data)

			length, err := utils.ParseFloat64(data["length"])
			if err != nil {
				logger.Printf("parse length error: %+v\n", data)
				break
			}

			requestTime, err := utils.ParseFloat64(data["reqtime"])
			if err != nil {
				logger.Printf("request time parse error: %+v", data)
				requestTime = 0
			}

			upstreamTime, err := utils.ParseFloat64(data["upstream_resptime"])
			if err != nil {
				upstreamTime = requestTime
			}

			/*
				code 408 means server wait for client request timeout, so ignore here.
			*/
			if c := data["code"]; c == "408" || c == "499" && requestTime < config.Code499Timeout {
				break
			}

			for _, v := range config.TotalUrlMetric {
				k := key + "|" + v
				if _, ok := dataInfo[k]; !ok {
					dataInfo[k] = &TsdbPutInfo{
						Metric: v,
						Tags:   tags,
					}
				}
			}
			dataInfo[key+"|"+config.UrlQpsMetric].Value += 1 / float64(config.UploadFrequency)
			dataInfo[key+"|"+config.UrlCodeMetric].Value += 1
			dataInfo[key+"|"+config.UrlUpstreamMetric].Value += upstreamTime * 1000
			dataInfo[key+"|"+config.UrlTimeMetric].Value += requestTime * 1000
			dataInfo[key+"|"+config.UrlTrafficMetric].Value += length / float64(config.UploadFrequency)

			var channel chan domainMetricTsdbMap
			r.mutexDomainMap.RLock()
			_, ok := r.domainsChanMap[domain]
			r.mutexDomainMap.RUnlock()
			if !ok {
				r.mutexDomainMap.Lock()
				_, ok := r.domainsChanMap[domain]
				if !ok {
					r.domainsChanMap[domain] = make(chan domainMetricTsdbMap)
					go r.updateDomainMetric(domain, r.domainsChanMap[domain])
				}
				channel = r.domainsChanMap[domain]
				r.mutexDomainMap.Unlock()

			} else {
				r.mutexDomainMap.RLock()
				channel = r.domainsChanMap[domain]
				r.mutexDomainMap.RUnlock()
			}

			channel <- dataInfo
		case <-r.Dying():
			return
		}
	}

}