Exemple #1
0
// helper function to find all dataset at a given tier-site
func datasetsDictAtSite(siteName, tstamp string) Record {
	rdict := make(Record)
	api := "blockreplicas"
	furl := fmt.Sprintf("%s/%s?node=%s&create_since=%d", phedexUrl(), api, siteName, utils.UnixTime(tstamp))
	if utils.VERBOSE > 1 {
		fmt.Println("furl", furl)
	}
	if strings.HasPrefix(siteName, "T1_") && !strings.HasSuffix(siteName, "_Disk") {
		siteName += "_Disk"
	}
	response := utils.FetchResponse(furl, "")
	if response.Error == nil {
		records := loadPhedexData(furl, response.Data)
		for _, rec := range records {
			val := rec["phedex"].(map[string]interface{})
			blocks := val["block"].([]interface{})
			for _, item := range blocks {
				brec := item.(map[string]interface{})
				dataset := strings.Split(brec["name"].(string), "#")[0]
				bytes := brec["bytes"].(float64)
				val, ok := rdict[dataset]
				if ok {
					rdict[dataset] = bytes + val.(float64)
				} else {
					rdict[dataset] = bytes
				}
			}
		}
	}
	return rdict
}
Exemple #2
0
// helper function to get site content. Return either list of blocks or datasets on site.
func siteContent(siteName, tstamp, recType string) []string {
	api := "blockreplicasummary"
	if strings.HasPrefix(siteName, "T1_") && !strings.HasSuffix(siteName, "_Disk") {
		siteName += "_Disk"
	}
	furl := fmt.Sprintf("%s/%s?node=%s&create_since=%d", phedexUrl(), api, siteName, utils.UnixTime(tstamp))
	if utils.VERBOSE > 1 {
		fmt.Println("furl", furl)
	}
	response := utils.FetchResponse(furl, "")
	// use a map to collect dataset names as keys
	ddict := make(Record)
	if response.Error == nil {
		records := loadPhedexData(furl, response.Data)
		for _, rec := range records {
			val := rec["phedex"].(map[string]interface{})
			blocks := val["block"].([]interface{})
			for _, item := range blocks {
				brec := item.(map[string]interface{})
				blk := brec["name"].(string)
				if recType == "block" {
					ddict[blk] = struct{}{}
				} else { // look-up dataset name
					dataset := strings.Split(blk, "#")[0]
					if datasetNameOk(dataset) {
						ddict[dataset] = struct{}{}
					}
				}
			}
		}
		// return map keys, they're unique already
		return utils.MapKeys(ddict)
	}
	return []string{}
}
Exemple #3
0
// helper function to collect popularity results and merge them into bins of given metric
// with the help of updateDict function.
// return rdict which is a dictionary of bins and corresponding dataset names
func popdb2Bins(metric string, bins []int, records []Record, siteName string, tstamps []string) BinRecord {
	var popdbNames []string
	rdict := make(BinRecord)
	for _, bin := range bins {
		rdict[bin] = []string{} // init all bins
	}
	recType := "dataset"            // type of record we'll process
	for idx, rec := range records { // loop over popularity records
		mval := int(rec[metric].(float64))
		name := rec["name"].(string)
		if idx == 0 && strings.Contains(name, "#") {
			recType = "block"
		}
		popdbNames = append(popdbNames, name)
		updateDict(bins, rdict, mval, name)
	}
	// loop over site content and collect zero bin for given metric
	siteNames := siteContent(siteName, tstamps[0], recType)
	var zeroMetricNames []string
	for _, name := range siteNames {
		if !utils.InList(name, popdbNames) {
			zeroMetricNames = append(zeroMetricNames, name)
		}
	}
	rdict[0] = zeroMetricNames

	// fetch old datasets, those who are in zero bin but their creation time
	// is older then interval we're intersting.
	thr := float64(utils.UnixTime(tstamps[0]))
	olds := oldDatasets(rdict[0].([]string), thr)
	rdict[-1] = olds
	newd := utils.Substruct(rdict[0].([]string), rdict[-1].([]string))
	if utils.VERBOSE > 0 {
		fmt.Println("Bin-zero division, bin0-old", len(olds), "bin0-new", len(newd))
	}
	rdict[0] = newd

	// make sure that we have unique list of datasets in every bin
	allbins := []int{-1, 0}
	for _, bin := range bins {
		allbins = append(allbins, bin)
	}
	for _, bin := range allbins {
		arr := rdict[bin].([]string)
		rdict[bin] = utils.List2Set(arr)
		val := rdict[bin]
		fmt.Println(siteName, "bin ", bin, " contains ", len(val.([]string)))
	}
	return rdict
}
Exemple #4
0
// helper function to find all dataset at a given tier-site
func datasetInfoAtSite(dataset, siteName, tstamp string, ch chan Record) {
	if !datasetNameOk(dataset) {
		ch <- Record{"dataset": dataset, "size": 0.0, "tier": "unknown"}
		return
	}
	api := "blockreplicas"
	furl := fmt.Sprintf("%s/%s?dataset=%s&node=%s&create_since=%d", phedexUrl(), api, dataset, siteName, utils.UnixTime(tstamp))
	if utils.VERBOSE > 1 {
		fmt.Println("furl", furl)
	}
	//     if strings.HasPrefix(siteName, "T1_") && !strings.HasSuffix(siteName, "_Disk") {
	//         siteName += "_Disk"
	//     }
	response := utils.FetchResponse(furl, "")
	size := 0.
	if response.Error == nil {
		records := loadPhedexData(furl, response.Data)
		for _, rec := range records {
			val := rec["phedex"].(map[string]interface{})
			blocks := val["block"].([]interface{})
			for _, item := range blocks {
				brec := item.(map[string]interface{})
				bytes := brec["bytes"].(float64)
				size += bytes
			}
		}
	}
	ch <- Record{"dataset": dataset, "size": size, "tier": utils.DataTier(dataset)}
}