Exemplo n.º 1
0
// helper function to get site content. Return either list of blocks or datasets on site.
func siteContent(siteName, tstamp, recType string) []string {
	api := "blockreplicasummary"
	if strings.HasPrefix(siteName, "T1_") && !strings.HasSuffix(siteName, "_Disk") {
		siteName += "_Disk"
	}
	furl := fmt.Sprintf("%s/%s?node=%s&create_since=%d", phedexUrl(), api, siteName, utils.UnixTime(tstamp))
	if utils.VERBOSE > 1 {
		fmt.Println("furl", furl)
	}
	response := utils.FetchResponse(furl, "")
	// use a map to collect dataset names as keys
	ddict := make(Record)
	if response.Error == nil {
		records := loadPhedexData(furl, response.Data)
		for _, rec := range records {
			val := rec["phedex"].(map[string]interface{})
			blocks := val["block"].([]interface{})
			for _, item := range blocks {
				brec := item.(map[string]interface{})
				blk := brec["name"].(string)
				if recType == "block" {
					ddict[blk] = struct{}{}
				} else { // look-up dataset name
					dataset := strings.Split(blk, "#")[0]
					if datasetNameOk(dataset) {
						ddict[dataset] = struct{}{}
					}
				}
			}
		}
		// return map keys, they're unique already
		return utils.MapKeys(ddict)
	}
	return []string{}
}
Exemplo n.º 2
0
// helper function to find all dataset at a given tier-site
func datasetsDictAtSite(siteName, tstamp string) Record {
	rdict := make(Record)
	api := "blockreplicas"
	furl := fmt.Sprintf("%s/%s?node=%s&create_since=%d", phedexUrl(), api, siteName, utils.UnixTime(tstamp))
	if utils.VERBOSE > 1 {
		fmt.Println("furl", furl)
	}
	if strings.HasPrefix(siteName, "T1_") && !strings.HasSuffix(siteName, "_Disk") {
		siteName += "_Disk"
	}
	response := utils.FetchResponse(furl, "")
	if response.Error == nil {
		records := loadPhedexData(furl, response.Data)
		for _, rec := range records {
			val := rec["phedex"].(map[string]interface{})
			blocks := val["block"].([]interface{})
			for _, item := range blocks {
				brec := item.(map[string]interface{})
				dataset := strings.Split(brec["name"].(string), "#")[0]
				bytes := brec["bytes"].(float64)
				val, ok := rdict[dataset]
				if ok {
					rdict[dataset] = bytes + val.(float64)
				} else {
					rdict[dataset] = bytes
				}
			}
		}
	}
	return rdict
}
Exemplo n.º 3
0
// helper function to find all dataset at a given tier-site
func datasetInfoAtSite(dataset, siteName, tstamp string, ch chan Record) {
	if !datasetNameOk(dataset) {
		ch <- Record{"dataset": dataset, "size": 0.0, "tier": "unknown"}
		return
	}
	api := "blockreplicas"
	furl := fmt.Sprintf("%s/%s?dataset=%s&node=%s&create_since=%d", phedexUrl(), api, dataset, siteName, utils.UnixTime(tstamp))
	if utils.VERBOSE > 1 {
		fmt.Println("furl", furl)
	}
	//     if strings.HasPrefix(siteName, "T1_") && !strings.HasSuffix(siteName, "_Disk") {
	//         siteName += "_Disk"
	//     }
	response := utils.FetchResponse(furl, "")
	size := 0.
	if response.Error == nil {
		records := loadPhedexData(furl, response.Data)
		for _, rec := range records {
			val := rec["phedex"].(map[string]interface{})
			blocks := val["block"].([]interface{})
			for _, item := range blocks {
				brec := item.(map[string]interface{})
				bytes := brec["bytes"].(float64)
				size += bytes
			}
		}
	}
	ch <- Record{"dataset": dataset, "size": size, "tier": utils.DataTier(dataset)}
}
Exemplo n.º 4
0
// helper function to get CMS site names for given siteName
func siteNames(site string) []string {
	var out []string
	api := "site-names"
	furl := fmt.Sprintf("%s/%s", sitedbUrl(), api)
	if utils.VERBOSE > 1 {
		fmt.Println("furl", furl)
	}
	response := utils.FetchResponse(furl, "")
	if response.Error == nil {
		records := loadSiteDBData(furl, response.Data)
		for _, r := range records {
			siteName := r["alias"].(string)
			siteType := r["type"].(string)
			if siteType == "phedex" && strings.HasPrefix(siteName, site) {
				if strings.HasPrefix(siteName, "T1_") {
					if strings.HasSuffix(siteName, "_Disk") {
						out = append(out, siteName)
					}
				} else {
					out = append(out, siteName)
				}
			}
		}
	}
	return utils.List2Set(out)
}
Exemplo n.º 5
0
// helper function to collect dataset usage from popularity DB
func datasetStats(siteName string, tstamps []string, tier string) []Record {
	var out []Record
	api := "DSStatInTimeWindow"
	tstart := popDBtstamp(tstamps[0])
	tstop := popDBtstamp(tstamps[len(tstamps)-1])
	siteName = strings.Replace(siteName, "_Disk", "", 1)
	furl := fmt.Sprintf("%s/%s/?sitename=%s&tstart=%s&tstop=%s", popdbUrl(), api, siteName, tstart, tstop)
	if utils.VERBOSE > 1 {
		fmt.Println("furl", furl)
	}
	response := utils.FetchResponse(furl, "")
	if response.Error == nil {
		records := loadPopDBData(furl, response.Data, tier)
		return records
	}
	return out
}