func main() {
defer common.LogPanic()
// Global init.
common.InitWithMetrics(APP_NAME, graphiteServer)
// Parse the time period.
period, err := human.ParseDuration(*timePeriod)
if err != nil {
glog.Fatal(err)
}
// Initialize the buildbot database.
db, err := buildbot.NewRemoteDB(*buildbotDbHost)
if err != nil {
glog.Fatal(err)
}
// Initialize the BuildBucket client.
c, err := auth.NewClient(*local, path.Join(*workdir, "oauth_token_cache"), buildbucket.DEFAULT_SCOPES...)
if err != nil {
glog.Fatal(err)
}
bb := buildbucket.NewClient(c)
// Build the queue.
repos := gitinfo.NewRepoMap(*workdir)
for _, r := range REPOS {
if _, err := repos.Repo(r); err != nil {
glog.Fatal(err)
}
}
q, err := build_queue.NewBuildQueue(period, repos, *scoreThreshold, *scoreDecay24Hr, BOT_BLACKLIST, db)
if err != nil {
glog.Fatal(err)
}
// Start scheduling builds in a loop.
liveness := metrics.NewLiveness(APP_NAME)
if err := scheduleBuilds(q, bb); err != nil {
glog.Errorf("Failed to schedule builds: %v", err)
}
for _ = range time.Tick(time.Minute) {
liveness.Update()
if err := scheduleBuilds(q, bb); err != nil {
glog.Errorf("Failed to schedule builds: %v", err)
}
}
}
func main() {
defer common.LogPanic()
// Setup flags.
influxdb.SetupFlags()
common.InitWithMetrics("status", graphiteServer)
v, err := skiaversion.GetVersion()
if err != nil {
glog.Fatal(err)
}
glog.Infof("Version %s, built at %s", v.Commit, v.Date)
Init()
if *testing {
*useMetadata = false
}
serverURL := "https://" + *host
if *testing {
serverURL = "http://" + *host + *port
}
// Create buildbot remote DB.
db, err = buildbot.NewRemoteDB(*buildbotDbHost)
if err != nil {
glog.Fatal(err)
}
// Setup InfluxDB client.
dbClient, err = influxdb.NewClientFromFlagsAndMetadata(*testing)
if err != nil {
glog.Fatal(err)
}
// By default use a set of credentials setup for localhost access.
var cookieSalt = "notverysecret"
var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com"
var clientSecret = "cw0IosPu4yjaG2KWmppj2guj"
var redirectURL = serverURL + "/oauth2callback/"
if *useMetadata {
cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT))
clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID))
clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET))
}
login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, false)
// Check out source code.
skiaRepo, err := gitinfo.CloneOrUpdate("https://skia.googlesource.com/skia.git", path.Join(*workdir, "skia"), true)
if err != nil {
glog.Fatalf("Failed to check out Skia: %v", err)
}
infraRepoPath := path.Join(*workdir, "infra")
infraRepo, err := gitinfo.CloneOrUpdate("https://skia.googlesource.com/buildbot.git", infraRepoPath, true)
if err != nil {
glog.Fatalf("Failed to checkout Infra: %v", err)
}
glog.Info("CloneOrUpdate complete")
// Create the commit caches.
commitCaches = map[string]*commit_cache.CommitCache{}
skiaCache, err := commit_cache.New(skiaRepo, path.Join(*workdir, "commit_cache.gob"), DEFAULT_COMMITS_TO_LOAD, db)
if err != nil {
glog.Fatalf("Failed to create commit cache: %v", err)
}
commitCaches[SKIA_REPO] = skiaCache
infraCache, err := commit_cache.New(infraRepo, path.Join(*workdir, "commit_cache_infra.gob"), DEFAULT_COMMITS_TO_LOAD, db)
if err != nil {
glog.Fatalf("Failed to create commit cache: %v", err)
}
commitCaches[INFRA_REPO] = infraCache
glog.Info("commit_cache complete")
// Load Perf and Gold data in a loop.
perfStatus, err = dbClient.Int64PollingStatus("select value from \"alerting.new.value\" where app='skiaperf' and host='skia-perf' order by time desc limit 1", time.Minute)
if err != nil {
glog.Fatalf("Failed to create polling Perf status: %v", err)
}
goldGMStatus, err = dbClient.Int64PollingStatus(fmt.Sprintf(GOLD_STATUS_QUERY_TMPL, "gm"), time.Minute)
if err != nil {
glog.Fatalf("Failed to create polling Gold status: %v", err)
}
goldImageStatus, err = dbClient.Int64PollingStatus(fmt.Sprintf(GOLD_STATUS_QUERY_TMPL, "image"), time.Minute)
if err != nil {
glog.Fatalf("Failed to create polling Gold status: %v", err)
}
// Load slave_hosts_cfg and device cfgs in a loop.
slaveHosts, err = buildbot.SlaveHostsCfgPoller(infraRepoPath)
if err != nil {
glog.Fatal(err)
}
androidDevices, err = device_cfg.AndroidDeviceCfgPoller(*workdir)
if err != nil {
glog.Fatal(err)
}
sshDevices, err = device_cfg.SSHDeviceCfgPoller(*workdir)
if err != nil {
glog.Fatal(err)
}
runServer(serverURL)
}
func main() {
defer common.LogPanic()
alertDBConf := alerting.DBConfigFromFlags()
influxdb.SetupFlags()
common.InitWithMetrics("alertserver", graphiteServer)
v, err := skiaversion.GetVersion()
if err != nil {
glog.Fatal(err)
}
glog.Infof("Version %s, built at %s", v.Commit, v.Date)
Init()
if *validateAndExit {
if _, err := rules.MakeRules(*alertsFile, nil, time.Second, nil, *testing); err != nil {
glog.Fatalf("Failed to set up rules: %v", err)
}
return
}
parsedPollInterval, err := time.ParseDuration(*alertPollInterval)
if err != nil {
glog.Fatalf("Failed to parse -alertPollInterval: %s", *alertPollInterval)
}
if *testing {
*useMetadata = false
}
dbClient, err := influxdb.NewClientFromFlagsAndMetadata(*testing)
if err != nil {
glog.Fatalf("Failed to initialize InfluxDB client: %s", err)
}
serverURL := "https://" + *host
if *testing {
serverURL = "http://" + *host + *port
}
usr, err := user.Current()
if err != nil {
glog.Fatal(err)
}
tokenFile, err := filepath.Abs(usr.HomeDir + "/" + GMAIL_TOKEN_CACHE_FILE)
if err != nil {
glog.Fatal(err)
}
// By default use a set of credentials setup for localhost access.
var cookieSalt = "notverysecret"
var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com"
var clientSecret = "cw0IosPu4yjaG2KWmppj2guj"
var redirectURL = serverURL + "/oauth2callback/"
var emailClientId = *emailClientIdFlag
var emailClientSecret = *emailClientSecretFlag
if *useMetadata {
cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT))
clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID))
clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET))
emailClientId = metadata.Must(metadata.ProjectGet(metadata.GMAIL_CLIENT_ID))
emailClientSecret = metadata.Must(metadata.ProjectGet(metadata.GMAIL_CLIENT_SECRET))
cachedGMailToken := metadata.Must(metadata.ProjectGet(metadata.GMAIL_CACHED_TOKEN))
err = ioutil.WriteFile(tokenFile, []byte(cachedGMailToken), os.ModePerm)
if err != nil {
glog.Fatalf("Failed to cache token: %s", err)
}
}
login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, false)
var emailAuth *email.GMail
if !*testing {
if !*useMetadata && (emailClientId == "" || emailClientSecret == "") {
glog.Fatal("If -use_metadata=false, you must provide -email_clientid and -email_clientsecret")
}
emailAuth, err = email.NewGMail(emailClientId, emailClientSecret, tokenFile)
if err != nil {
glog.Fatalf("Failed to create email auth: %v", err)
}
}
// Initialize the database.
if !*testing && *useMetadata {
if err := alertDBConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
if err := alertDBConf.InitDB(); err != nil {
glog.Fatal(err)
}
buildbotDb, err := buildbot.NewRemoteDB(*buildbotDbHost)
if err != nil {
glog.Fatal(err)
}
// Create the AlertManager.
alertManager, err = alerting.MakeAlertManager(parsedPollInterval, emailAuth)
if err != nil {
glog.Fatalf("Failed to create AlertManager: %v", err)
}
rulesList, err = rules.MakeRules(*alertsFile, dbClient, parsedPollInterval, alertManager, *testing)
if err != nil {
glog.Fatalf("Failed to set up rules: %v", err)
}
StartAlertRoutines(alertManager, 10*parsedPollInterval, dbClient, buildbotDb)
runServer(serverURL)
}
func main() {
defer common.LogPanic()
var err error
// Global init to initialize glog and parse arguments.
common.InitWithMetrics("internal", graphiteServer)
if !*local {
*targetList = metadata.Must(metadata.ProjectGet("datahopper_internal_targets"))
}
targets := strings.Split(*targetList, " ")
glog.Infof("Targets: %#v", targets)
codenameDB, err = leveldb.OpenFile(*codenameDbDir, nil)
if err != nil && errors.IsCorrupted(err) {
codenameDB, err = leveldb.RecoverFile(*codenameDbDir, nil)
}
if err != nil {
glog.Fatalf("Failed to open codename leveldb at %s: %s", *codenameDbDir, err)
}
// Initialize the buildbot database.
db, err = buildbot.NewRemoteDB(*buildbotDbHost)
if err != nil {
glog.Fatal(err)
}
var redirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port)
if !*local {
redirectURL = "https://internal.skia.org/oauth2callback/"
}
if err := login.InitFromMetadataOrJSON(redirectURL, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST); err != nil {
glog.Fatalf("Failed to initialize login system: %s", err)
}
if *local {
webhook.InitRequestSaltForTesting()
} else {
webhook.MustInitRequestSaltFromMetadata()
}
repos = gitinfo.NewRepoMap(*workdir)
// Ingest Android framework builds.
go func() {
glog.Infof("Starting.")
// In this case we don't want a backoff transport since the Apiary backend
// seems to fail a lot, so we basically want to fall back to polling if a
// call fails.
client, err := auth.NewJWTServiceAccountClient("", "", &http.Transport{Dial: util.DialTimeout}, androidbuildinternal.AndroidbuildInternalScope, storage.CloudPlatformScope)
if err != nil {
glog.Fatalf("Unable to create authenticated client: %s", err)
}
buildService, err := androidbuildinternal.New(client)
if err != nil {
glog.Fatalf("Failed to obtain Android build service: %v", err)
}
glog.Infof("Ready to start loop.")
step(targets, buildService)
for _ = range time.Tick(*period) {
step(targets, buildService)
}
}()
r := mux.NewRouter()
r.HandleFunc("/", indexHandler)
r.HandleFunc("/builders/{codename}/builds/{buildNumber}", redirectHandler)
r.HandleFunc("/builders/{codename}", builderRedirectHandler)
r.HandleFunc("/ingestBuild", ingestBuildHandler)
r.HandleFunc("/loginstatus/", login.StatusHandler)
r.HandleFunc("/logout/", login.LogoutHandler)
r.HandleFunc("/oauth2callback/", login.OAuth2CallbackHandler)
http.Handle("/", util.LoggingGzipRequestResponse(r))
glog.Fatal(http.ListenAndServe(*port, nil))
}