func FatalWithMessage(arvLogger *logger.Logger, message string) {
if arvLogger != nil {
arvLogger.FinalUpdate(func(p map[string]interface{}, e map[string]interface{}) {
p["FATAL"] = message
runInfo := logger.GetOrCreateMap(p, "run_info")
runInfo["finished_at"] = time.Now()
})
}
log.Fatalf(message)
}
func singlerun() error {
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
log.Fatalf("Error setting up arvados client %s", err.Error())
}
if is_admin, err := util.UserIsAdmin(arv); err != nil {
log.Fatalf("Error querying current arvados user %s", err.Error())
} else if !is_admin {
log.Fatalf("Current user is not an admin. Datamanager can only be run by admins.")
}
var arvLogger *logger.Logger
if logEventTypePrefix != "" {
arvLogger = logger.NewLogger(logger.LoggerParams{
Client: arv,
EventTypePrefix: logEventTypePrefix,
WriteInterval: time.Second * time.Duration(logFrequencySeconds)})
}
loggerutil.LogRunInfo(arvLogger)
if arvLogger != nil {
arvLogger.AddWriteHook(loggerutil.LogMemoryAlloc)
}
var (
dataFetcher summary.DataFetcher
readCollections collection.ReadCollections
keepServerInfo keep.ReadServers
)
if summary.ShouldReadData() {
dataFetcher = summary.ReadData
} else {
dataFetcher = BuildDataFetcher(arv)
}
dataFetcher(arvLogger, &readCollections, &keepServerInfo)
summary.MaybeWriteData(arvLogger, readCollections, keepServerInfo)
buckets := summary.BucketReplication(readCollections, keepServerInfo)
bucketCounts := buckets.Counts()
replicationSummary := buckets.SummarizeBuckets(readCollections)
replicationCounts := replicationSummary.ComputeCounts()
log.Printf("Blocks In Collections: %d, "+
"\nBlocks In Keep: %d.",
len(readCollections.BlockToDesiredReplication),
len(keepServerInfo.BlockToServers))
log.Println(replicationCounts.PrettyPrint())
log.Printf("Blocks Histogram:")
for _, rlbss := range bucketCounts {
log.Printf("%+v: %10d",
rlbss.Levels,
rlbss.Count)
}
kc, err := keepclient.MakeKeepClient(&arv)
if err != nil {
loggerutil.FatalWithMessage(arvLogger,
fmt.Sprintf("Error setting up keep client %s", err.Error()))
}
// Log that we're finished. We force the recording, since go will
// not wait for the write timer before exiting.
if arvLogger != nil {
defer arvLogger.FinalUpdate(func(p map[string]interface{}, e map[string]interface{}) {
summaryInfo := logger.GetOrCreateMap(p, "summary_info")
summaryInfo["block_replication_counts"] = bucketCounts
summaryInfo["replication_summary"] = replicationCounts
p["summary_info"] = summaryInfo
p["run_info"].(map[string]interface{})["finished_at"] = time.Now()
})
}
pullServers := summary.ComputePullServers(kc,
&keepServerInfo,
readCollections.BlockToDesiredReplication,
replicationSummary.UnderReplicatedBlocks)
pullLists := summary.BuildPullLists(pullServers)
trashLists, trashErr := summary.BuildTrashLists(kc,
&keepServerInfo,
replicationSummary.KeepBlocksNotInCollections)
summary.WritePullLists(arvLogger, pullLists)
if trashErr != nil {
return err
} else {
keep.SendTrashLists(keep.GetDataManagerToken(arvLogger), kc, trashLists)
}
return nil
}