예제 #1
0
파일: sitedb.go 프로젝트: vkuznet/sitestat
// helper function to get CMS site names for given siteName
func siteNames(site string) []string {
	var out []string
	api := "site-names"
	furl := fmt.Sprintf("%s/%s", sitedbUrl(), api)
	if utils.VERBOSE > 1 {
		fmt.Println("furl", furl)
	}
	response := utils.FetchResponse(furl, "")
	if response.Error == nil {
		records := loadSiteDBData(furl, response.Data)
		for _, r := range records {
			siteName := r["alias"].(string)
			siteType := r["type"].(string)
			if siteType == "phedex" && strings.HasPrefix(siteName, site) {
				if strings.HasPrefix(siteName, "T1_") {
					if strings.HasSuffix(siteName, "_Disk") {
						out = append(out, siteName)
					}
				} else {
					out = append(out, siteName)
				}
			}
		}
	}
	return utils.List2Set(out)
}
예제 #2
0
파일: cms.go 프로젝트: vkuznet/sitestat
// helper function to collect popularity results and merge them into bins of given metric
// with the help of updateDict function.
// return rdict which is a dictionary of bins and corresponding dataset names
func popdb2Bins(metric string, bins []int, records []Record, siteName string, tstamps []string) BinRecord {
	var popdbNames []string
	rdict := make(BinRecord)
	for _, bin := range bins {
		rdict[bin] = []string{} // init all bins
	}
	recType := "dataset"            // type of record we'll process
	for idx, rec := range records { // loop over popularity records
		mval := int(rec[metric].(float64))
		name := rec["name"].(string)
		if idx == 0 && strings.Contains(name, "#") {
			recType = "block"
		}
		popdbNames = append(popdbNames, name)
		updateDict(bins, rdict, mval, name)
	}
	// loop over site content and collect zero bin for given metric
	siteNames := siteContent(siteName, tstamps[0], recType)
	var zeroMetricNames []string
	for _, name := range siteNames {
		if !utils.InList(name, popdbNames) {
			zeroMetricNames = append(zeroMetricNames, name)
		}
	}
	rdict[0] = zeroMetricNames

	// fetch old datasets, those who are in zero bin but their creation time
	// is older then interval we're intersting.
	thr := float64(utils.UnixTime(tstamps[0]))
	olds := oldDatasets(rdict[0].([]string), thr)
	rdict[-1] = olds
	newd := utils.Substruct(rdict[0].([]string), rdict[-1].([]string))
	if utils.VERBOSE > 0 {
		fmt.Println("Bin-zero division, bin0-old", len(olds), "bin0-new", len(newd))
	}
	rdict[0] = newd

	// make sure that we have unique list of datasets in every bin
	allbins := []int{-1, 0}
	for _, bin := range bins {
		allbins = append(allbins, bin)
	}
	for _, bin := range allbins {
		arr := rdict[bin].([]string)
		rdict[bin] = utils.List2Set(arr)
		val := rdict[bin]
		fmt.Println(siteName, "bin ", bin, " contains ", len(val.([]string)))
	}
	return rdict
}