Пример #1
0
// helper function to get CMS site names for given siteName
func siteNames(site string) []string {
	var out []string
	api := "site-names"
	furl := fmt.Sprintf("%s/%s", sitedbUrl(), api)
	if utils.VERBOSE > 1 {
		fmt.Println("furl", furl)
	}
	response := utils.FetchResponse(furl, "")
	if response.Error == nil {
		records := loadSiteDBData(furl, response.Data)
		for _, r := range records {
			siteName := r["alias"].(string)
			siteType := r["type"].(string)
			if siteType == "phedex" && strings.HasPrefix(siteName, site) {
				if strings.HasPrefix(site, "T1_") {
					if strings.HasSuffix(siteName, "_Disk") {
						out = append(out, siteName)
					}
				} else {
					out = append(out, siteName)
				}
			}
		}
	}
	return utils.List2Set(out)
}
Пример #2
0
// helper function to find all dataset at a given tier-site
func datasetSites(dataset string, ch chan Record) {
	api := "blockreplicas"
	furl := fmt.Sprintf("%s/%s?dataset=%s", phedexUrl(), api, dataset)
	if utils.VERBOSE > 1 {
		fmt.Println("furl", furl)
	}
	response := utils.FetchResponse(furl, "")
	sdict := make(Record)
	var se string
	if response.Error == nil {
		records := loadPhedexData(furl, response.Data)
		for _, rec := range records {
			if rec["phedex"] != nil {
				val := rec["phedex"].(map[string]interface{})
				blocks := val["block"].([]interface{})
				for _, item := range blocks {
					brec := item.(map[string]interface{})
					replicas := brec["replica"].([]interface{})
					for _, r := range replicas {
						rep := r.(map[string]interface{})
						site := rep["node"].(string)
						v := rep["se"]
						if v == nil {
							se = ""
						} else {
							se = v.(string)
						}
						sdict[site] = se
					}
				}
			}
		}
	}
	var s0, s1, s2, s3, su int
	for site, _ := range sdict {
		if strings.HasPrefix(site, "T0_") {
			s0 += 1
		} else if strings.HasPrefix(site, "T1_") {
			s1 += 1
		} else if strings.HasPrefix(site, "T2_") {
			s2 += 1
		} else if strings.HasPrefix(site, "T3_") {
			s3 += 1
		} else {
			su += 1
		}
	}
	ch <- Record{"s_0": s0, "s_1": s1, "s_2": s2, "s_3": s3, "s_u": su}
}
Пример #3
0
// helper function to collect dataset usage from popularity DB
func datasetStats(start, stop string) []Record {
	var out []Record
	api := "DSStatInTimeWindow"
	tstart := popDBtstamp(start)
	tstop := popDBtstamp(stop)
	furl := fmt.Sprintf("%s/%s/?tstart=%s&tstop=%s", popdbUrl(), api, tstart, tstop)
	if utils.VERBOSE > 1 {
		fmt.Println("furl", furl)
	}
	response := utils.FetchResponse(furl, "")
	if response.Error == nil {
		records := loadPopDBData(furl, response.Data)
		return records
	}
	return out
}
Пример #4
0
// helper function to get DBS datasets for given time interval
func datasetsInTimeWindow(start, stop string) []string {
	var out []string
	api := "datasets"
	mint := utils.UnixTime(start)
	maxt := utils.UnixTime(stop)
	for dbsinst, _ := range dbsInstances() {
		furl := fmt.Sprintf("%s/%s/?min_cdate=%d&max_cdate=%d&dataset_access_type=VALID", dbsUrl(dbsinst), api, mint, maxt)
		response := utils.FetchResponse(furl, "")
		if response.Error == nil {
			records := loadDBSData(furl, response.Data)
			for _, rec := range records {
				dataset := rec["dataset"].(string)
				out = append(out, dataset)
			}
		}
	}
	return out
}
Пример #5
0
// helper function to get all datasets from DBS instance
func dbsDatasets(dbsinst string) []string {
	var datasets []string
	api := "datasets"
	for dbsinst, _ := range dbsInstances() {
		furl := fmt.Sprintf("%s/%s", dbsUrl(dbsinst), api)
		response := utils.FetchResponse(furl, "")
		if response.Error == nil {
			records := loadDBSData(furl, response.Data)
			if utils.VERBOSE > 1 {
				fmt.Println("furl", furl, records)
			}
			for _, rec := range records {
				datasets = append(datasets, rec["dataset"].(string))
			}
		}
	}
	return datasets
}
Пример #6
0
// Unmarshal McM data stream and return DAS records based on api
func mcmProduces(dataset string, ch chan Record) {
	api := "produces"
	furl := fmt.Sprintf("%s/%s/%s", mcmUrl(), api, dataset)
	if utils.VERBOSE > 1 {
		fmt.Println("furl", furl)
	}
	response := utils.FetchResponse(furl, "")
	if response.Error == nil {
		records := loadMcMData(furl, response.Data)
		for _, r := range records {
			if r == nil {
				continue
			}
			v := r["results"]
			if v == nil {
				continue
			}
			res := v.(map[string]interface{})
			rec := make(Record)
			rec["energy"] = utils.GetFloatValue(res["energy"])
			s := res["sequences"]
			if s != nil {
				rec["nseq"] = len(s.([]interface{}))
			} else {
				rec["nseq"] = 0
			}
			rec["flown_with"] = utils.GetValue(res["flown_with"])
			rec["mcmtype"] = utils.GetValue(res["type"])
			rec["pwg"] = utils.GetValue(res["pwg"])
			rec["idataset"] = utils.GetValue(res["input_dataset"])
			rec["pdataset"] = utils.GetValue(res["pileup_dataset_name"])
			rec["campain"] = utils.GetValue(res["member_of_campaign"])
			rec["mcmevts"] = utils.GetFloatValue(res["nevents"])
			rec["mcmpid"] = utils.GetValue(res["prepid"])
			ch <- rec
			return
		}
	}
	rec := make(Record)
	ch <- rec // we must send at least empty record
}
Пример #7
0
// helper function to get release information about dataset
func dbsInfo(dataset, api, details string) ([]Record, string) {
	var out []Record
	for dbsinst, _ := range dbsInstances() {
		furl := fmt.Sprintf("%s/%s/?dataset=%s", dbsUrl(dbsinst), api, dataset)
		if len(details) > 0 {
			furl += "&detail=True"
		}
		response := utils.FetchResponse(furl, "")
		if response.Error == nil {
			records := loadDBSData(furl, response.Data)
			if utils.VERBOSE > 1 {
				fmt.Println("furl", furl, records)
			}
			if len(records) != 0 { // we got records from specific dbs instance
				return records, dbsinst
			}
		} else {
			if utils.VERBOSE > 1 {
				fmt.Println("DBS error with", furl, response)
			}
		}
	}
	return out, ""
}