// helper function to get site content. Return either list of blocks or datasets on site. func siteContent(siteName, tstamp, recType string) []string { api := "blockreplicasummary" if strings.HasPrefix(siteName, "T1_") && !strings.HasSuffix(siteName, "_Disk") { siteName += "_Disk" } furl := fmt.Sprintf("%s/%s?node=%s&create_since=%d", phedexUrl(), api, siteName, utils.UnixTime(tstamp)) if utils.VERBOSE > 1 { fmt.Println("furl", furl) } response := utils.FetchResponse(furl, "") // use a map to collect dataset names as keys ddict := make(Record) if response.Error == nil { records := loadPhedexData(furl, response.Data) for _, rec := range records { val := rec["phedex"].(map[string]interface{}) blocks := val["block"].([]interface{}) for _, item := range blocks { brec := item.(map[string]interface{}) blk := brec["name"].(string) if recType == "block" { ddict[blk] = struct{}{} } else { // look-up dataset name dataset := strings.Split(blk, "#")[0] if datasetNameOk(dataset) { ddict[dataset] = struct{}{} } } } } // return map keys, they're unique already return utils.MapKeys(ddict) } return []string{} }
// helper function to find all dataset at a given tier-site func datasetsDictAtSite(siteName, tstamp string) Record { rdict := make(Record) api := "blockreplicas" furl := fmt.Sprintf("%s/%s?node=%s&create_since=%d", phedexUrl(), api, siteName, utils.UnixTime(tstamp)) if utils.VERBOSE > 1 { fmt.Println("furl", furl) } if strings.HasPrefix(siteName, "T1_") && !strings.HasSuffix(siteName, "_Disk") { siteName += "_Disk" } response := utils.FetchResponse(furl, "") if response.Error == nil { records := loadPhedexData(furl, response.Data) for _, rec := range records { val := rec["phedex"].(map[string]interface{}) blocks := val["block"].([]interface{}) for _, item := range blocks { brec := item.(map[string]interface{}) dataset := strings.Split(brec["name"].(string), "#")[0] bytes := brec["bytes"].(float64) val, ok := rdict[dataset] if ok { rdict[dataset] = bytes + val.(float64) } else { rdict[dataset] = bytes } } } } return rdict }
// helper function to find all dataset at a given tier-site func datasetInfoAtSite(dataset, siteName, tstamp string, ch chan Record) { if !datasetNameOk(dataset) { ch <- Record{"dataset": dataset, "size": 0.0, "tier": "unknown"} return } api := "blockreplicas" furl := fmt.Sprintf("%s/%s?dataset=%s&node=%s&create_since=%d", phedexUrl(), api, dataset, siteName, utils.UnixTime(tstamp)) if utils.VERBOSE > 1 { fmt.Println("furl", furl) } // if strings.HasPrefix(siteName, "T1_") && !strings.HasSuffix(siteName, "_Disk") { // siteName += "_Disk" // } response := utils.FetchResponse(furl, "") size := 0. if response.Error == nil { records := loadPhedexData(furl, response.Data) for _, rec := range records { val := rec["phedex"].(map[string]interface{}) blocks := val["block"].([]interface{}) for _, item := range blocks { brec := item.(map[string]interface{}) bytes := brec["bytes"].(float64) size += bytes } } } ch <- Record{"dataset": dataset, "size": size, "tier": utils.DataTier(dataset)} }
// helper function to get CMS site names for given siteName func siteNames(site string) []string { var out []string api := "site-names" furl := fmt.Sprintf("%s/%s", sitedbUrl(), api) if utils.VERBOSE > 1 { fmt.Println("furl", furl) } response := utils.FetchResponse(furl, "") if response.Error == nil { records := loadSiteDBData(furl, response.Data) for _, r := range records { siteName := r["alias"].(string) siteType := r["type"].(string) if siteType == "phedex" && strings.HasPrefix(siteName, site) { if strings.HasPrefix(siteName, "T1_") { if strings.HasSuffix(siteName, "_Disk") { out = append(out, siteName) } } else { out = append(out, siteName) } } } } return utils.List2Set(out) }
// helper function to collect dataset usage from popularity DB func datasetStats(siteName string, tstamps []string, tier string) []Record { var out []Record api := "DSStatInTimeWindow" tstart := popDBtstamp(tstamps[0]) tstop := popDBtstamp(tstamps[len(tstamps)-1]) siteName = strings.Replace(siteName, "_Disk", "", 1) furl := fmt.Sprintf("%s/%s/?sitename=%s&tstart=%s&tstop=%s", popdbUrl(), api, siteName, tstart, tstop) if utils.VERBOSE > 1 { fmt.Println("furl", furl) } response := utils.FetchResponse(furl, "") if response.Error == nil { records := loadPopDBData(furl, response.Data, tier) return records } return out }