Example #1
0
func AggregationPath(pathCount map[string]int) []string {
	for {
		paths := utils.MapKeyToList(pathCount)
		sort.Sort(sorted.ByCharCount(paths, "/"))
		aggregated := 0
		for i := range paths {
			path := paths[len(paths)-1-i] // reverse loop
			if path == "/" {
				continue
			}

			oldPath := path
			countNum := pathCount[path]
			lastAggreNum := aggregated

			if strings.Index(path, "http://") == 0 {
				path = utils.CutHttpHead(path)
				aggregated += 1

			}

			parts := strings.Split(path, "/")
			depth := len(parts) - 1
			lastLen := len(parts[depth])

			if depth > config.PathMaxDepth {
				path = strings.Join(parts[:config.PathMaxDepth+1], "/")
				aggregated += 1
				depth = config.PathMaxDepth
			}

			if countNum < config.PathAggreNumber || lastLen > config.PathLastlength || len(path) > config.PathAggrelength {
				if depth == 1 {
					path = "/"
				} else {
					path = strings.Join(parts[:len(parts)-1], "/")
				}
				aggregated += 1
			}

			if aggregated > lastAggreNum {
				if _, ok := pathCount[path]; ok {
					pathCount[path] += countNum

				} else {
					pathCount[path] = countNum
				}
				delete(pathCount, oldPath)
			}
		}
		if aggregated == 0 {
			break
		}
	}
	final := utils.MapKeyToList(pathCount)
	sStruct := sorted.ByCharCount(final, "/")
	sort.Sort(sStruct)
	return final
}
Example #2
0
func (r *Raper) saveBindData(c DomainBindInfo) {

	var newBindData map[string][]string = make(map[string][]string)

	for _, v1 := range c.Data {
		for _, v2 := range v1 {
			domain := v2.Domain
			path := v2.Path
			if ok := utils.StrInStrings(path, newBindData[domain]); !ok {
				newBindData[domain] = append(newBindData[domain], path)
			}

		}
	}

	r.mutexBindPath.Lock()
	if ok := utils.MapStrStringsCmp(r.domainBindPath, newBindData); !ok {
		logger.Println("save new domain bind data")

		r.domainBindPath = newBindData

	}
	r.mutexBindPath.Unlock()

	var domains []string

	for k := range r.domainBindPath {
		domains = append(domains, k)
	}
	for _, d := range domains {
		r.mutexBindPath.RLock()
		a := make([]string, len(r.domainBindPath[d]))
		copy(a, r.domainBindPath[d])
		r.mutexBindPath.RUnlock()
		bcc := sorted.ByCharCount(a, "/")
		sort.Sort(bcc)
		r.mutexBindPath.Lock()
		r.domainBindPath[d] = bcc.List
		r.mutexBindPath.Unlock()
	}

}