// helper function to get CMS site names for given siteName
func siteNames(site string) []string {
var out []string
api := "site-names"
furl := fmt.Sprintf("%s/%s", sitedbUrl(), api)
if utils.VERBOSE > 1 {
fmt.Println("furl", furl)
}
response := utils.FetchResponse(furl, "")
if response.Error == nil {
records := loadSiteDBData(furl, response.Data)
for _, r := range records {
siteName := r["alias"].(string)
siteType := r["type"].(string)
if siteType == "phedex" && strings.HasPrefix(siteName, site) {
if strings.HasPrefix(siteName, "T1_") {
if strings.HasSuffix(siteName, "_Disk") {
out = append(out, siteName)
}
} else {
out = append(out, siteName)
}
}
}
}
return utils.List2Set(out)
}
// helper function to collect popularity results and merge them into bins of given metric
// with the help of updateDict function.
// return rdict which is a dictionary of bins and corresponding dataset names
func popdb2Bins(metric string, bins []int, records []Record, siteName string, tstamps []string) BinRecord {
var popdbNames []string
rdict := make(BinRecord)
for _, bin := range bins {
rdict[bin] = []string{} // init all bins
}
recType := "dataset" // type of record we'll process
for idx, rec := range records { // loop over popularity records
mval := int(rec[metric].(float64))
name := rec["name"].(string)
if idx == 0 && strings.Contains(name, "#") {
recType = "block"
}
popdbNames = append(popdbNames, name)
updateDict(bins, rdict, mval, name)
}
// loop over site content and collect zero bin for given metric
siteNames := siteContent(siteName, tstamps[0], recType)
var zeroMetricNames []string
for _, name := range siteNames {
if !utils.InList(name, popdbNames) {
zeroMetricNames = append(zeroMetricNames, name)
}
}
rdict[0] = zeroMetricNames
// fetch old datasets, those who are in zero bin but their creation time
// is older then interval we're intersting.
thr := float64(utils.UnixTime(tstamps[0]))
olds := oldDatasets(rdict[0].([]string), thr)
rdict[-1] = olds
newd := utils.Substruct(rdict[0].([]string), rdict[-1].([]string))
if utils.VERBOSE > 0 {
fmt.Println("Bin-zero division, bin0-old", len(olds), "bin0-new", len(newd))
}
rdict[0] = newd
// make sure that we have unique list of datasets in every bin
allbins := []int{-1, 0}
for _, bin := range bins {
allbins = append(allbins, bin)
}
for _, bin := range allbins {
arr := rdict[bin].([]string)
rdict[bin] = utils.List2Set(arr)
val := rdict[bin]
fmt.Println(siteName, "bin ", bin, " contains ", len(val.([]string)))
}
return rdict
}