Beispiel #1
0
// helper function to load PopDB data stream
// if data tier is given we only filter datasets with given tier
func loadPopDBData(furl string, data []byte, tier string) []Record {
	var out []Record
	var rec Record
	err := json.Unmarshal(data, &rec)
	if err != nil {
		if utils.VERBOSE > 0 {
			msg := fmt.Sprintf("PopDB unable to unmarshal the data, furl=%s, data=%s, error=%v", furl, string(data), err)
			fmt.Println(msg)
		}
		return out
	}
	values := rec["DATA"].([]interface{})
	for _, item := range values {
		rec := item.(map[string]interface{})
		dataset := rec["COLLNAME"].(string)
		rec["name"] = dataset
		dataTier := utils.DataTier(dataset)
		keep := true
		if tier != "" {
			if dataTier != tier {
				keep = false
			}
		}
		if keep {
			out = append(out, rec)
		}
	}
	return out
}
Beispiel #2
0
// helper function to find all dataset at a given tier-site
func datasetInfoAtSite(dataset, siteName, tstamp string, ch chan Record) {
	if !datasetNameOk(dataset) {
		ch <- Record{"dataset": dataset, "size": 0.0, "tier": "unknown"}
		return
	}
	api := "blockreplicas"
	furl := fmt.Sprintf("%s/%s?dataset=%s&node=%s&create_since=%d", phedexUrl(), api, dataset, siteName, utils.UnixTime(tstamp))
	if utils.VERBOSE > 1 {
		fmt.Println("furl", furl)
	}
	//     if strings.HasPrefix(siteName, "T1_") && !strings.HasSuffix(siteName, "_Disk") {
	//         siteName += "_Disk"
	//     }
	response := utils.FetchResponse(furl, "")
	size := 0.
	if response.Error == nil {
		records := loadPhedexData(furl, response.Data)
		for _, rec := range records {
			val := rec["phedex"].(map[string]interface{})
			blocks := val["block"].([]interface{})
			for _, item := range blocks {
				brec := item.(map[string]interface{})
				bytes := brec["bytes"].(float64)
				size += bytes
			}
		}
	}
	ch <- Record{"dataset": dataset, "size": size, "tier": utils.DataTier(dataset)}
}
Beispiel #3
0
// helper function to load PopDB data stream
// if data tier is given we only filter datasets with given tier
func loadVictorDBData(furl string, data []byte, site, tier string) []Record {
	var out []Record
	var rec Record
	err := json.Unmarshal(data, &rec)
	if err != nil {
		if utils.VERBOSE > 0 {
			msg := fmt.Sprintf("PopDB unable to unmarshal the data, furl=%s, data=%s, error=%v", furl, string(data), err)
			fmt.Println(msg)
		}
		return out
	}
	val := rec[site]
	if val == nil {
		return out
	}
	values := val.(map[string]interface{})
	for blk, idict := range values {
		rec := idict.(map[string]interface{})
		rec["name"] = blk
		dataset := strings.Split(blk, "#")[0]
		dataTier := utils.DataTier(dataset)
		keep := true
		if tier != "" {
			if dataTier != tier {
				keep = false
			}
		}
		if keep {
			out = append(out, rec)
		}
	}
	return out
}
Beispiel #4
0
func datasetInfoPBR(dataset string, ch chan Record) {
	rec := make(Record)
	rec["name"] = dataset
	rec["size"] = PBRMAP[dataset]
	//     rec["size"] = sizeFromPBR(dataset, PHGROUP)
	rec["tier"] = utils.DataTier(dataset)
	ch <- rec
}