// helper function to load PopDB data stream
// if data tier is given we only filter datasets with given tier
func loadPopDBData(furl string, data []byte, tier string) []Record {
var out []Record
var rec Record
err := json.Unmarshal(data, &rec)
if err != nil {
if utils.VERBOSE > 0 {
msg := fmt.Sprintf("PopDB unable to unmarshal the data, furl=%s, data=%s, error=%v", furl, string(data), err)
fmt.Println(msg)
}
return out
}
values := rec["DATA"].([]interface{})
for _, item := range values {
rec := item.(map[string]interface{})
dataset := rec["COLLNAME"].(string)
rec["name"] = dataset
dataTier := utils.DataTier(dataset)
keep := true
if tier != "" {
if dataTier != tier {
keep = false
}
}
if keep {
out = append(out, rec)
}
}
return out
}
// helper function to find all dataset at a given tier-site
func datasetInfoAtSite(dataset, siteName, tstamp string, ch chan Record) {
if !datasetNameOk(dataset) {
ch <- Record{"dataset": dataset, "size": 0.0, "tier": "unknown"}
return
}
api := "blockreplicas"
furl := fmt.Sprintf("%s/%s?dataset=%s&node=%s&create_since=%d", phedexUrl(), api, dataset, siteName, utils.UnixTime(tstamp))
if utils.VERBOSE > 1 {
fmt.Println("furl", furl)
}
// if strings.HasPrefix(siteName, "T1_") && !strings.HasSuffix(siteName, "_Disk") {
// siteName += "_Disk"
// }
response := utils.FetchResponse(furl, "")
size := 0.
if response.Error == nil {
records := loadPhedexData(furl, response.Data)
for _, rec := range records {
val := rec["phedex"].(map[string]interface{})
blocks := val["block"].([]interface{})
for _, item := range blocks {
brec := item.(map[string]interface{})
bytes := brec["bytes"].(float64)
size += bytes
}
}
}
ch <- Record{"dataset": dataset, "size": size, "tier": utils.DataTier(dataset)}
}
// helper function to load PopDB data stream
// if data tier is given we only filter datasets with given tier
func loadVictorDBData(furl string, data []byte, site, tier string) []Record {
var out []Record
var rec Record
err := json.Unmarshal(data, &rec)
if err != nil {
if utils.VERBOSE > 0 {
msg := fmt.Sprintf("PopDB unable to unmarshal the data, furl=%s, data=%s, error=%v", furl, string(data), err)
fmt.Println(msg)
}
return out
}
val := rec[site]
if val == nil {
return out
}
values := val.(map[string]interface{})
for blk, idict := range values {
rec := idict.(map[string]interface{})
rec["name"] = blk
dataset := strings.Split(blk, "#")[0]
dataTier := utils.DataTier(dataset)
keep := true
if tier != "" {
if dataTier != tier {
keep = false
}
}
if keep {
out = append(out, rec)
}
}
return out
}
func datasetInfoPBR(dataset string, ch chan Record) {
rec := make(Record)
rec["name"] = dataset
rec["size"] = PBRMAP[dataset]
// rec["size"] = sizeFromPBR(dataset, PHGROUP)
rec["tier"] = utils.DataTier(dataset)
ch <- rec
}