// function to generate datamap
func GenerateMap(datamap, fout string) {
startTime := time.Now()
if fout == "" {
fout = fmt.Sprintf("%s.txt", datamap)
}
utils.TestEnv()
var names []string
var records []Record
for dbsinst, dbsId := range dbsInstances() {
if datamap == "dataset" || datamap == "datasets" {
names = dbsDatasets(dbsinst)
} else if datamap == "tier" || datamap == "tiers" {
names = dbsTiers(dbsinst)
} else {
fmt.Printf("Unsupported map name '%s'\n", datamap)
os.Exit(-1)
}
for _, n := range names {
rec := make(Record)
rec[datamap] = n
rec["dbsinst"] = dbsId
rec["hash"] = utils.Hash1(n)
records = append(records, rec)
}
}
// process extra dataset
writeRecords(records, fout)
if utils.PROFILE {
fmt.Printf("Processed %d urls\n", utils.UrlCounter)
fmt.Printf("Elapsed time %s\n", time.Since(startTime))
}
if utils.VERBOSE > 0 {
fmt.Println("Job finished", time.Now())
}
}
// get dataset information from various CMS data-services
func datasetInfo(prec Record, start, stop string, och chan Record) {
dataset := prec["dataset"].(string)
ch := make(chan Record)
counter := 0 // count how may go routines we send
// DBS calls
go datasetReleases(dataset, ch)
counter += 1
go datasetSummary(dataset, ch)
counter += 1
go datasetParent(dataset, ch)
counter += 1
go datasetDetails(dataset, ch)
counter += 1
// McM calls
go mcmProduces(dataset, ch)
counter += 1
// Phedex calls
// go datasetSites(dataset, ch)
// counter += 1
// collect results
var records []Record
for {
select {
case r := <-ch:
records = append(records, r)
default:
time.Sleep(time.Duration(10) * time.Millisecond) // wait for response
}
if len(records) == counter {
break
}
}
close(ch) // close channel when we're done with collect phase
rec := make(Record)
for _, srec := range records { // loop over all received service records
for k, v := range srec { // loop over service record key-value pairs
switch v.(type) {
case string:
rec[k] = utils.Hash1(v.(string))
default:
rec[k] = v
}
}
}
// popdb metrics
rec["naccess"] = prec["naccess"]
rec["totcpu"] = prec["totcpu"]
rec["nusers"] = prec["nusers"]
rec["rnaccess"] = prec["rnaccess"]
rec["rtotcpu"] = prec["rtotcpu"]
rec["rnusers"] = prec["rnusers"]
v := rec["dataset"]
if v == nil || v.(uint64) == 0 {
if utils.VERBOSE > 0 {
fmt.Println("miss dataset", dataset, rec)
}
rec["dataset"] = dataset
}
if utils.VERBOSE > 1 {
fmt.Println("rec", rec)
}
och <- rec
}
// helper function to get data tier from dataset/block name
func datasetParts(dataset string) (uint64, uint64, uint64) {
dparts := strings.Split(dataset, "/")
return utils.Hash1(dparts[1]), utils.Hash1(dparts[2]), utils.Hash1(dparts[3])
}