// helper function to get CMS site names for given siteName func siteNames(site string) []string { var out []string api := "site-names" furl := fmt.Sprintf("%s/%s", sitedbUrl(), api) if utils.VERBOSE > 1 { fmt.Println("furl", furl) } response := utils.FetchResponse(furl, "") if response.Error == nil { records := loadSiteDBData(furl, response.Data) for _, r := range records { siteName := r["alias"].(string) siteType := r["type"].(string) if siteType == "phedex" && strings.HasPrefix(siteName, site) { if strings.HasPrefix(site, "T1_") { if strings.HasSuffix(siteName, "_Disk") { out = append(out, siteName) } } else { out = append(out, siteName) } } } } return utils.List2Set(out) }
// helper function to find all dataset at a given tier-site func datasetSites(dataset string, ch chan Record) { api := "blockreplicas" furl := fmt.Sprintf("%s/%s?dataset=%s", phedexUrl(), api, dataset) if utils.VERBOSE > 1 { fmt.Println("furl", furl) } response := utils.FetchResponse(furl, "") sdict := make(Record) var se string if response.Error == nil { records := loadPhedexData(furl, response.Data) for _, rec := range records { if rec["phedex"] != nil { val := rec["phedex"].(map[string]interface{}) blocks := val["block"].([]interface{}) for _, item := range blocks { brec := item.(map[string]interface{}) replicas := brec["replica"].([]interface{}) for _, r := range replicas { rep := r.(map[string]interface{}) site := rep["node"].(string) v := rep["se"] if v == nil { se = "" } else { se = v.(string) } sdict[site] = se } } } } } var s0, s1, s2, s3, su int for site, _ := range sdict { if strings.HasPrefix(site, "T0_") { s0 += 1 } else if strings.HasPrefix(site, "T1_") { s1 += 1 } else if strings.HasPrefix(site, "T2_") { s2 += 1 } else if strings.HasPrefix(site, "T3_") { s3 += 1 } else { su += 1 } } ch <- Record{"s_0": s0, "s_1": s1, "s_2": s2, "s_3": s3, "s_u": su} }
// helper function to collect dataset usage from popularity DB func datasetStats(start, stop string) []Record { var out []Record api := "DSStatInTimeWindow" tstart := popDBtstamp(start) tstop := popDBtstamp(stop) furl := fmt.Sprintf("%s/%s/?tstart=%s&tstop=%s", popdbUrl(), api, tstart, tstop) if utils.VERBOSE > 1 { fmt.Println("furl", furl) } response := utils.FetchResponse(furl, "") if response.Error == nil { records := loadPopDBData(furl, response.Data) return records } return out }
// helper function to get DBS datasets for given time interval func datasetsInTimeWindow(start, stop string) []string { var out []string api := "datasets" mint := utils.UnixTime(start) maxt := utils.UnixTime(stop) for dbsinst, _ := range dbsInstances() { furl := fmt.Sprintf("%s/%s/?min_cdate=%d&max_cdate=%d&dataset_access_type=VALID", dbsUrl(dbsinst), api, mint, maxt) response := utils.FetchResponse(furl, "") if response.Error == nil { records := loadDBSData(furl, response.Data) for _, rec := range records { dataset := rec["dataset"].(string) out = append(out, dataset) } } } return out }
// helper function to get all datasets from DBS instance func dbsDatasets(dbsinst string) []string { var datasets []string api := "datasets" for dbsinst, _ := range dbsInstances() { furl := fmt.Sprintf("%s/%s", dbsUrl(dbsinst), api) response := utils.FetchResponse(furl, "") if response.Error == nil { records := loadDBSData(furl, response.Data) if utils.VERBOSE > 1 { fmt.Println("furl", furl, records) } for _, rec := range records { datasets = append(datasets, rec["dataset"].(string)) } } } return datasets }
// Unmarshal McM data stream and return DAS records based on api func mcmProduces(dataset string, ch chan Record) { api := "produces" furl := fmt.Sprintf("%s/%s/%s", mcmUrl(), api, dataset) if utils.VERBOSE > 1 { fmt.Println("furl", furl) } response := utils.FetchResponse(furl, "") if response.Error == nil { records := loadMcMData(furl, response.Data) for _, r := range records { if r == nil { continue } v := r["results"] if v == nil { continue } res := v.(map[string]interface{}) rec := make(Record) rec["energy"] = utils.GetFloatValue(res["energy"]) s := res["sequences"] if s != nil { rec["nseq"] = len(s.([]interface{})) } else { rec["nseq"] = 0 } rec["flown_with"] = utils.GetValue(res["flown_with"]) rec["mcmtype"] = utils.GetValue(res["type"]) rec["pwg"] = utils.GetValue(res["pwg"]) rec["idataset"] = utils.GetValue(res["input_dataset"]) rec["pdataset"] = utils.GetValue(res["pileup_dataset_name"]) rec["campain"] = utils.GetValue(res["member_of_campaign"]) rec["mcmevts"] = utils.GetFloatValue(res["nevents"]) rec["mcmpid"] = utils.GetValue(res["prepid"]) ch <- rec return } } rec := make(Record) ch <- rec // we must send at least empty record }
// helper function to get release information about dataset func dbsInfo(dataset, api, details string) ([]Record, string) { var out []Record for dbsinst, _ := range dbsInstances() { furl := fmt.Sprintf("%s/%s/?dataset=%s", dbsUrl(dbsinst), api, dataset) if len(details) > 0 { furl += "&detail=True" } response := utils.FetchResponse(furl, "") if response.Error == nil { records := loadDBSData(furl, response.Data) if utils.VERBOSE > 1 { fmt.Println("furl", furl, records) } if len(records) != 0 { // we got records from specific dbs instance return records, dbsinst } } else { if utils.VERBOSE > 1 { fmt.Println("DBS error with", furl, response) } } } return out, "" }