// helper function to get site content. Return either list of blocks or datasets on site.
func siteContent(siteName, tstamp, recType string) []string {
api := "blockreplicasummary"
if strings.HasPrefix(siteName, "T1_") && !strings.HasSuffix(siteName, "_Disk") {
siteName += "_Disk"
}
furl := fmt.Sprintf("%s/%s?node=%s&create_since=%d", phedexUrl(), api, siteName, utils.UnixTime(tstamp))
if utils.VERBOSE > 1 {
fmt.Println("furl", furl)
}
response := utils.FetchResponse(furl, "")
// use a map to collect dataset names as keys
ddict := make(Record)
if response.Error == nil {
records := loadPhedexData(furl, response.Data)
for _, rec := range records {
val := rec["phedex"].(map[string]interface{})
blocks := val["block"].([]interface{})
for _, item := range blocks {
brec := item.(map[string]interface{})
blk := brec["name"].(string)
if recType == "block" {
ddict[blk] = struct{}{}
} else { // look-up dataset name
dataset := strings.Split(blk, "#")[0]
if datasetNameOk(dataset) {
ddict[dataset] = struct{}{}
}
}
}
}
// return map keys, they're unique already
return utils.MapKeys(ddict)
}
return []string{}
}
// helper function to find all dataset at a given tier-site
func datasetsDictAtSite(siteName, tstamp string) Record {
rdict := make(Record)
api := "blockreplicas"
furl := fmt.Sprintf("%s/%s?node=%s&create_since=%d", phedexUrl(), api, siteName, utils.UnixTime(tstamp))
if utils.VERBOSE > 1 {
fmt.Println("furl", furl)
}
if strings.HasPrefix(siteName, "T1_") && !strings.HasSuffix(siteName, "_Disk") {
siteName += "_Disk"
}
response := utils.FetchResponse(furl, "")
if response.Error == nil {
records := loadPhedexData(furl, response.Data)
for _, rec := range records {
val := rec["phedex"].(map[string]interface{})
blocks := val["block"].([]interface{})
for _, item := range blocks {
brec := item.(map[string]interface{})
dataset := strings.Split(brec["name"].(string), "#")[0]
bytes := brec["bytes"].(float64)
val, ok := rdict[dataset]
if ok {
rdict[dataset] = bytes + val.(float64)
} else {
rdict[dataset] = bytes
}
}
}
}
return rdict
}
// helper function to find all dataset at a given tier-site
func datasetInfoAtSite(dataset, siteName, tstamp string, ch chan Record) {
if !datasetNameOk(dataset) {
ch <- Record{"dataset": dataset, "size": 0.0, "tier": "unknown"}
return
}
api := "blockreplicas"
furl := fmt.Sprintf("%s/%s?dataset=%s&node=%s&create_since=%d", phedexUrl(), api, dataset, siteName, utils.UnixTime(tstamp))
if utils.VERBOSE > 1 {
fmt.Println("furl", furl)
}
// if strings.HasPrefix(siteName, "T1_") && !strings.HasSuffix(siteName, "_Disk") {
// siteName += "_Disk"
// }
response := utils.FetchResponse(furl, "")
size := 0.
if response.Error == nil {
records := loadPhedexData(furl, response.Data)
for _, rec := range records {
val := rec["phedex"].(map[string]interface{})
blocks := val["block"].([]interface{})
for _, item := range blocks {
brec := item.(map[string]interface{})
bytes := brec["bytes"].(float64)
size += bytes
}
}
}
ch <- Record{"dataset": dataset, "size": size, "tier": utils.DataTier(dataset)}
}
// helper function to get CMS site names for given siteName
func siteNames(site string) []string {
var out []string
api := "site-names"
furl := fmt.Sprintf("%s/%s", sitedbUrl(), api)
if utils.VERBOSE > 1 {
fmt.Println("furl", furl)
}
response := utils.FetchResponse(furl, "")
if response.Error == nil {
records := loadSiteDBData(furl, response.Data)
for _, r := range records {
siteName := r["alias"].(string)
siteType := r["type"].(string)
if siteType == "phedex" && strings.HasPrefix(siteName, site) {
if strings.HasPrefix(siteName, "T1_") {
if strings.HasSuffix(siteName, "_Disk") {
out = append(out, siteName)
}
} else {
out = append(out, siteName)
}
}
}
}
return utils.List2Set(out)
}
// helper function to collect dataset usage from popularity DB
func datasetStats(siteName string, tstamps []string, tier string) []Record {
var out []Record
api := "DSStatInTimeWindow"
tstart := popDBtstamp(tstamps[0])
tstop := popDBtstamp(tstamps[len(tstamps)-1])
siteName = strings.Replace(siteName, "_Disk", "", 1)
furl := fmt.Sprintf("%s/%s/?sitename=%s&tstart=%s&tstop=%s", popdbUrl(), api, siteName, tstart, tstop)
if utils.VERBOSE > 1 {
fmt.Println("furl", furl)
}
response := utils.FetchResponse(furl, "")
if response.Error == nil {
records := loadPopDBData(furl, response.Data, tier)
return records
}
return out
}