// helper function to get CMS site names for given siteName
func siteNames(site string) []string {
var out []string
api := "site-names"
furl := fmt.Sprintf("%s/%s", sitedbUrl(), api)
if utils.VERBOSE > 1 {
fmt.Println("furl", furl)
}
response := utils.FetchResponse(furl, "")
if response.Error == nil {
records := loadSiteDBData(furl, response.Data)
for _, r := range records {
siteName := r["alias"].(string)
siteType := r["type"].(string)
if siteType == "phedex" && strings.HasPrefix(siteName, site) {
if strings.HasPrefix(site, "T1_") {
if strings.HasSuffix(siteName, "_Disk") {
out = append(out, siteName)
}
} else {
out = append(out, siteName)
}
}
}
}
return utils.List2Set(out)
}
// helper function to find all dataset at a given tier-site
func datasetSites(dataset string, ch chan Record) {
api := "blockreplicas"
furl := fmt.Sprintf("%s/%s?dataset=%s", phedexUrl(), api, dataset)
if utils.VERBOSE > 1 {
fmt.Println("furl", furl)
}
response := utils.FetchResponse(furl, "")
sdict := make(Record)
var se string
if response.Error == nil {
records := loadPhedexData(furl, response.Data)
for _, rec := range records {
if rec["phedex"] != nil {
val := rec["phedex"].(map[string]interface{})
blocks := val["block"].([]interface{})
for _, item := range blocks {
brec := item.(map[string]interface{})
replicas := brec["replica"].([]interface{})
for _, r := range replicas {
rep := r.(map[string]interface{})
site := rep["node"].(string)
v := rep["se"]
if v == nil {
se = ""
} else {
se = v.(string)
}
sdict[site] = se
}
}
}
}
}
var s0, s1, s2, s3, su int
for site, _ := range sdict {
if strings.HasPrefix(site, "T0_") {
s0 += 1
} else if strings.HasPrefix(site, "T1_") {
s1 += 1
} else if strings.HasPrefix(site, "T2_") {
s2 += 1
} else if strings.HasPrefix(site, "T3_") {
s3 += 1
} else {
su += 1
}
}
ch <- Record{"s_0": s0, "s_1": s1, "s_2": s2, "s_3": s3, "s_u": su}
}
// helper function to collect dataset usage from popularity DB
func datasetStats(start, stop string) []Record {
var out []Record
api := "DSStatInTimeWindow"
tstart := popDBtstamp(start)
tstop := popDBtstamp(stop)
furl := fmt.Sprintf("%s/%s/?tstart=%s&tstop=%s", popdbUrl(), api, tstart, tstop)
if utils.VERBOSE > 1 {
fmt.Println("furl", furl)
}
response := utils.FetchResponse(furl, "")
if response.Error == nil {
records := loadPopDBData(furl, response.Data)
return records
}
return out
}
// helper function to get DBS datasets for given time interval
func datasetsInTimeWindow(start, stop string) []string {
var out []string
api := "datasets"
mint := utils.UnixTime(start)
maxt := utils.UnixTime(stop)
for dbsinst, _ := range dbsInstances() {
furl := fmt.Sprintf("%s/%s/?min_cdate=%d&max_cdate=%d&dataset_access_type=VALID", dbsUrl(dbsinst), api, mint, maxt)
response := utils.FetchResponse(furl, "")
if response.Error == nil {
records := loadDBSData(furl, response.Data)
for _, rec := range records {
dataset := rec["dataset"].(string)
out = append(out, dataset)
}
}
}
return out
}
// helper function to get all datasets from DBS instance
func dbsDatasets(dbsinst string) []string {
var datasets []string
api := "datasets"
for dbsinst, _ := range dbsInstances() {
furl := fmt.Sprintf("%s/%s", dbsUrl(dbsinst), api)
response := utils.FetchResponse(furl, "")
if response.Error == nil {
records := loadDBSData(furl, response.Data)
if utils.VERBOSE > 1 {
fmt.Println("furl", furl, records)
}
for _, rec := range records {
datasets = append(datasets, rec["dataset"].(string))
}
}
}
return datasets
}
// Unmarshal McM data stream and return DAS records based on api
func mcmProduces(dataset string, ch chan Record) {
api := "produces"
furl := fmt.Sprintf("%s/%s/%s", mcmUrl(), api, dataset)
if utils.VERBOSE > 1 {
fmt.Println("furl", furl)
}
response := utils.FetchResponse(furl, "")
if response.Error == nil {
records := loadMcMData(furl, response.Data)
for _, r := range records {
if r == nil {
continue
}
v := r["results"]
if v == nil {
continue
}
res := v.(map[string]interface{})
rec := make(Record)
rec["energy"] = utils.GetFloatValue(res["energy"])
s := res["sequences"]
if s != nil {
rec["nseq"] = len(s.([]interface{}))
} else {
rec["nseq"] = 0
}
rec["flown_with"] = utils.GetValue(res["flown_with"])
rec["mcmtype"] = utils.GetValue(res["type"])
rec["pwg"] = utils.GetValue(res["pwg"])
rec["idataset"] = utils.GetValue(res["input_dataset"])
rec["pdataset"] = utils.GetValue(res["pileup_dataset_name"])
rec["campain"] = utils.GetValue(res["member_of_campaign"])
rec["mcmevts"] = utils.GetFloatValue(res["nevents"])
rec["mcmpid"] = utils.GetValue(res["prepid"])
ch <- rec
return
}
}
rec := make(Record)
ch <- rec // we must send at least empty record
}
// helper function to get release information about dataset
func dbsInfo(dataset, api, details string) ([]Record, string) {
var out []Record
for dbsinst, _ := range dbsInstances() {
furl := fmt.Sprintf("%s/%s/?dataset=%s", dbsUrl(dbsinst), api, dataset)
if len(details) > 0 {
furl += "&detail=True"
}
response := utils.FetchResponse(furl, "")
if response.Error == nil {
records := loadDBSData(furl, response.Data)
if utils.VERBOSE > 1 {
fmt.Println("furl", furl, records)
}
if len(records) != 0 { // we got records from specific dbs instance
return records, dbsinst
}
} else {
if utils.VERBOSE > 1 {
fmt.Println("DBS error with", furl, response)
}
}
}
return out, ""
}