func main() {
defer common.LogPanic()
// Setup DB flags.
dbConf := db.DBConfigFromFlags()
common.InitWithMetricsCB("ingest", func() string {
common.DecodeTomlFile(*configFilename, &config)
return config.Common.GraphiteServer
})
// Initialize the database. We might not need the oauth dialog if it fails.
if !config.Common.Local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
if err := dbConf.InitDB(); err != nil {
glog.Fatal(err)
}
// Get a backoff transport.
transport := util.NewBackOffTransport()
// Determine the oauth scopes we are going to use. Storage is used by all
// ingesters.
scopes := []string{storage.CloudPlatformScope}
for _, ingesterConfig := range config.Ingesters {
if ingesterConfig.ConstructorName == gconfig.CONSTRUCTOR_ANDROID_GOLD {
scopes = append(scopes, androidbuildinternal.AndroidbuildInternalScope)
}
}
// Initialize the oauth client that is used to access all scopes.
var client *http.Client
var err error
if config.Common.Local {
if config.Common.DoOAuth {
client, err = auth.InstalledAppClient(config.Common.OAuthCacheFile,
config.Common.OAuthClientSecretFile,
transport, scopes...)
if err != nil {
glog.Fatalf("Failed to auth: %s", err)
}
} else {
client = nil
// Add back service account access here when it's fixed.
}
} else {
// Assume we are on a GCE instance.
client = auth.GCEServiceAccountClient(transport)
}
// Initialize the ingester and gold ingester.
ingester.Init(client)
if _, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD]; ok {
if err := goldingester.Init(client, filepath.Join(config.Ingesters["gold"].StatusDir, "android-build-info")); err != nil {
glog.Fatalf("Unable to initialize GoldIngester: %s", err)
}
}
// TODO(stephana): Cleanup the way trybots are instantiated so that each trybot has it's own
// database connection and they are all instantiated the same way instead of the
// one-off approaches.
if ingesterConf, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD_TRYBOT]; ok {
dbConf := &database.DatabaseConfig{
Host: ingesterConf.DBHost,
Port: ingesterConf.DBPort,
User: database.USER_RW,
Name: ingesterConf.DBName,
MigrationSteps: golddb.MigrationSteps(),
}
if !config.Common.Local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
vdb, err := dbConf.NewVersionedDB()
if err != nil {
glog.Fatalf("Unable to open db connection: %s", err)
}
goldtrybot.Init(vdb)
}
git, err := gitinfo.NewGitInfo(config.Common.GitRepoDir, true, false)
if err != nil {
glog.Fatalf("Failed loading Git info: %s\n", err)
}
for dataset, ingesterConfig := range config.Ingesters {
// Get duration equivalent to the number of days.
minDuration := 24 * time.Hour * time.Duration(ingesterConfig.MinDays)
constructorName := ingesterConfig.ConstructorName
if constructorName == "" {
constructorName = dataset
}
constructor := ingester.Constructor(constructorName)
resultIngester := constructor()
glog.Infof("Process name: %s", dataset)
startProcess := NewIngestionProcess(git,
config.Common.TileDir,
dataset,
resultIngester,
ingesterConfig.ExtraParams,
ingesterConfig.RunEvery.Duration,
ingesterConfig.NCommits,
minDuration,
ingesterConfig.StatusDir,
ingesterConfig.MetricName)
startProcess()
}
select {}
}
func main() {
defer common.LogPanic()
// Setup DB flags.
dbConf := idb.DBConfigFromFlags()
common.InitWithMetrics("skiaperf", graphiteServer)
Init()
if !*local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
if err := dbConf.InitDB(); err != nil {
glog.Fatal(err)
}
stats.Start(masterTileBuilder, git)
alerting.Start(masterTileBuilder)
var redirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port)
if !*local {
redirectURL = "https://perf.skia.org/oauth2callback/"
}
if err := login.InitFromMetadataOrJSON(redirectURL, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST); err != nil {
glog.Fatalf("Failed to initialize the login system: %s", err)
}
// Resources are served directly.
router := mux.NewRouter()
router.PathPrefix("/res/").HandlerFunc(makeResourceHandler())
router.HandleFunc("/", templateHandler("index.html"))
router.HandleFunc("/frame/", templateHandler("frame.html"))
router.HandleFunc("/shortcuts/", shortcutHandler)
router.PathPrefix("/tiles/").HandlerFunc(tileHandler)
router.PathPrefix("/single/").HandlerFunc(singleHandler)
router.PathPrefix("/query/").HandlerFunc(queryHandler)
router.HandleFunc("/commits/", commitsHandler)
router.HandleFunc("/_/commits/", commitsJSONHandler)
router.HandleFunc("/shortcommits/", shortCommitsHandler)
router.HandleFunc("/trybots/", trybotHandler)
router.HandleFunc("/clusters/", templateHandler("clusters.html"))
router.HandleFunc("/clustering/", clusteringHandler)
router.PathPrefix("/cl/").HandlerFunc(clHandler)
router.PathPrefix("/activitylog/").HandlerFunc(activityHandler)
router.HandleFunc("/alerts/", templateHandler("alerting.html"))
router.HandleFunc("/alerting/", alertingHandler)
router.HandleFunc("/alert_reset/", alertResetHandler)
router.HandleFunc("/annotate/", annotate.Handler)
router.HandleFunc("/compare/", templateHandler("compare.html"))
router.HandleFunc("/kernel/", templateHandler("kernel.html"))
router.HandleFunc("/_/kernel/", kernelJSONHandler)
router.HandleFunc("/calc/", calcHandler)
router.HandleFunc("/help/", helpHandler)
router.HandleFunc("/oauth2callback/", login.OAuth2CallbackHandler)
router.HandleFunc("/logout/", login.LogoutHandler)
router.HandleFunc("/loginstatus/", login.StatusHandler)
http.Handle("/", util.LoggingGzipRequestResponse(router))
glog.Infoln("Ready to serve.")
glog.Fatal(http.ListenAndServe(*port, nil))
}
func main() {
// Setup DB flags.
dbConf := db.DBConfigFromFlags()
common.InitWithMetrics("skiaperf", graphiteServer)
Init()
if !*local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
if err := dbConf.InitDB(); err != nil {
glog.Fatal(err)
}
stats.Start(nanoTileStore, git)
alerting.Start(nanoTileStore, *apikey)
// By default use a set of credentials setup for localhost access.
var cookieSalt = "notverysecret"
var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com"
var clientSecret = "cw0IosPu4yjaG2KWmppj2guj"
var redirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port)
if !*local {
cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT))
clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID))
clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET))
redirectURL = "https://perf.skia.org/oauth2callback/"
}
login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, *local)
glog.Infoln("Begin loading data.")
// Resources are served directly.
router := mux.NewRouter()
router.PathPrefix("/res/").HandlerFunc(makeResourceHandler())
router.HandleFunc("/", mainHandler)
router.HandleFunc("/shortcuts/", shortcutHandler)
router.PathPrefix("/tiles/").HandlerFunc(tileHandler)
router.PathPrefix("/single/").HandlerFunc(singleHandler)
router.PathPrefix("/query/").HandlerFunc(queryHandler)
router.HandleFunc("/commits/", commitsHandler)
router.HandleFunc("/shortcommits/", shortCommitsHandler)
router.HandleFunc("/trybots/", trybotHandler)
router.HandleFunc("/clusters/", clustersHandler)
router.HandleFunc("/clustering/", clusteringHandler)
router.PathPrefix("/cl/").HandlerFunc(clHandler)
router.PathPrefix("/activitylog/").HandlerFunc(activityHandler)
router.HandleFunc("/alerts/", alertsHandler)
router.HandleFunc("/alerting/", alertingHandler)
router.HandleFunc("/alert_reset/", alertResetHandler)
router.HandleFunc("/annotate/", annotate.Handler)
router.HandleFunc("/compare/", compareHandler)
router.HandleFunc("/calc/", calcHandler)
router.HandleFunc("/help/", helpHandler)
router.HandleFunc("/oauth2callback/", login.OAuth2CallbackHandler)
router.HandleFunc("/logout/", login.LogoutHandler)
router.HandleFunc("/loginstatus/", login.StatusHandler)
http.Handle("/", util.LoggingGzipRequestResponse(router))
glog.Infoln("Ready to serve.")
glog.Fatal(http.ListenAndServe(*port, nil))
}