func main() { common.InitWithMetrics("push", graphiteServer) Init() // By default use a set of credentials setup for localhost access. var cookieSalt = "notverysecret" var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com" var clientSecret = "cw0IosPu4yjaG2KWmppj2guj" var redirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port) if !*local { cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT)) clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID)) clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET)) redirectURL = "https://push.skia.org/oauth2callback/" } login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, *local) r := mux.NewRouter() r.PathPrefix("/res/").HandlerFunc(util.MakeResourceHandler(*resourcesDir)) r.HandleFunc("/", mainHandler) r.HandleFunc("/_/change", changeHandler) r.HandleFunc("/_/state", stateHandler) r.HandleFunc("/_/status", statusHandler) r.HandleFunc("/loginstatus/", login.StatusHandler) r.HandleFunc("/logout/", login.LogoutHandler) r.HandleFunc("/oauth2callback/", login.OAuth2CallbackHandler) http.Handle("/", util.LoggingGzipRequestResponse(r)) glog.Infoln("Ready to serve.") glog.Fatal(http.ListenAndServe(*port, nil)) }
// getOAuthClient returns an oauth client. func getOAuthClient(cacheFilePath string) *http.Client { var client *http.Client var err error if *doOauth || *local { if *local { // Load client secrets file. The client_secret.json file must come from // project id: 470362608618, which is whitelisted for access to // gs://chromium-skia-gm. client, err = auth.NewClientWithTransport(true, cacheFilePath, *clientSecretFile, nil, auth.SCOPE_READ_ONLY) if err != nil { glog.Fatalf("Failed to auth: (Did you download the client_secret.json file from the project with ID: google.com:chrome-skia #470362608618 ?) %s", err) } } else { // Load client id and secret from metadata. clientId := metadata.Must(metadata.ProjectGet(metadata.CHROMIUM_SKIA_GM_CLIENT_ID)) clientSecret := metadata.Must(metadata.ProjectGet(metadata.CHROMIUM_SKIA_GM_CLIENT_SECRET)) client, err = auth.NewClientFromIdAndSecret(clientId, clientSecret, cacheFilePath, auth.SCOPE_READ_ONLY) if err != nil { glog.Fatalf("Failed to auth: %s", err) } } } return client }
// Run the BugChomper server. func main() { common.InitWithMetrics("bug_chomper", graphiteServer) v, err := skiaversion.GetVersion() if err != nil { glog.Fatal(err) } glog.Infof("Version %s, built at %s", v.Commit, v.Date) loadTemplates() if *testing { *useMetadata = false } serverURL := "https://" + *host if *testing { serverURL = "http://" + *host + *port } // By default use a set of credentials setup for localhost access. var cookieSalt = "notverysecret" var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com" var clientSecret = "cw0IosPu4yjaG2KWmppj2guj" var redirectURL = serverURL + "/oauth2callback/" if *useMetadata { cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT)) clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID)) clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET)) } login.Init(clientID, clientSecret, redirectURL, cookieSalt, strings.Join(issue_tracker.OAUTH_SCOPE, " "), login.DEFAULT_DOMAIN_WHITELIST, false) runServer(serverURL) }
// monitorIssueTracker reads the counts for all the types of issues in the skia // issue tracker (code.google.com/p/skia) and stuffs the counts into Graphite. func monitorIssueTracker() { c := &http.Client{ Transport: &http.Transport{ Dial: dialTimeout, }, } if *useMetadata { *apikey = metadata.Must(metadata.ProjectGet(metadata.APIKEY)) } // Create a new metrics registry for the issue tracker metrics. addr, err := net.ResolveTCPAddr("tcp", *graphiteServer) if err != nil { glog.Fatalln("Failed to resolve the Graphite server: ", err) } issueRegistry := metrics.NewRegistry() go graphite.Graphite(issueRegistry, common.SAMPLE_PERIOD, "issues", addr) // IssueStatus has all the info we need to capture and record a single issue status. I.e. capture // the count of all issues with a status of "New". type IssueStatus struct { Name string Metric metrics.Gauge URL string } allIssueStatusLabels := []string{ "New", "Accepted", "Unconfirmed", "Started", "Fixed", "Verified", "Invalid", "WontFix", "Done", "Available", "Assigned", } issueStatus := []*IssueStatus{} for _, issueName := range allIssueStatusLabels { issueStatus = append(issueStatus, &IssueStatus{ Name: issueName, Metric: metrics.NewRegisteredGauge(strings.ToLower(issueName), issueRegistry), URL: "https://www.googleapis.com/projecthosting/v2/projects/skia/issues?fields=totalResults&key=" + *apikey + "&status=" + issueName, }) } liveness := imetrics.NewLiveness("issue-tracker") for _ = range time.Tick(ISSUE_TRACKER_PERIOD) { for _, issue := range issueStatus { resp, err := c.Get(issue.URL) jsonResp := map[string]int64{} dec := json.NewDecoder(resp.Body) if err := dec.Decode(&jsonResp); err != nil { glog.Warningf("Failed to decode JSON response: %s", err) util.Close(resp.Body) continue } issue.Metric.Update(jsonResp["totalResults"]) glog.Infof("Num Issues: %s - %d", issue.Name, jsonResp["totalResults"]) if err == nil && resp.Body != nil { util.Close(resp.Body) } } liveness.Update() } }
func main() { defer common.LogPanic() common.InitWithMetrics("mathserv", graphiteServer) Init() // By default use a set of credentials setup for localhost access. var cookieSalt = "notverysecret" var clientID = "952643138919-5a692pfevie766aiog15io45kjpsh33v.apps.googleusercontent.com" var clientSecret = "QQfqRYU1ELkds90ku8xlIGl1" var redirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port) if !*local { cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT)) clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID)) clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET)) redirectURL = "https://mathinate.com/oauth2callback/" } login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, "", *local) r := mux.NewRouter() r.PathPrefix("/res/").HandlerFunc(util.MakeResourceHandler(*resourcesDir)) r.HandleFunc("/", mainHandler) r.HandleFunc("/loginstatus/", login.StatusHandler) r.HandleFunc("/logout/", login.LogoutHandler) r.HandleFunc("/oauth2callback/", login.OAuth2CallbackHandler) http.Handle("/", util.LoggingGzipRequestResponse(r)) AttemptLoadCertFromMetadata() glog.Infoln("Ready to serve.") if *certChainFile != "" { glog.Infof("Serving TLS") go func() { redir := mux.NewRouter() redir.HandleFunc("/", redirHandler) glog.Fatal(http.ListenAndServe(*httpPort, redir)) }() glog.Fatal(http.ListenAndServeTLS(*port, *certChainFile, *keyFile, nil)) } else { glog.Infof("Only serving HTTP") glog.Fatal(http.ListenAndServe(*port, nil)) } }
func main() { flag.Parse() Init() // Set up login var cookieSalt = "notverysecret" var clientID = "31977622648-ubjke2f3staq6ouas64r31h8f8tcbiqp.apps.googleusercontent.com" var clientSecret = "rK-kRY71CXmcg0v9I9KIgWci" var useRedirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", config.Config.FrontEnd.Port) if !config.Config.Common.Local { cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT)) clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID)) clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET)) useRedirectURL = config.Config.FrontEnd.RedirectURL } login.Init(clientID, clientSecret, useRedirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, config.Config.Common.Local) // Set up the login related resources. router.HandleFunc(OAUTH2_CALLBACK_PATH, login.OAuth2CallbackHandler) router.HandleFunc("/loginstatus/", login.StatusHandler) router.HandleFunc("/logout/", login.LogoutHandler) router.HandleFunc("/", mainHandler) router.HandleFunc("/failed", mainHandler) router.HandleFunc("/passed", mainHandler) router.PathPrefix("/res/").HandlerFunc(makeResourceHandler()) jsonRouter := router.PathPrefix("/_").Subrouter() jsonRouter.HandleFunc("/list", fuzzListHandler).Name("fuzzListHandler") rootHandler := util.LoggingGzipRequestResponse(router) if config.Config.FrontEnd.ForceLogin { rootHandler = login.ForceAuth(rootHandler, OAUTH2_CALLBACK_PATH) } http.Handle("/", rootHandler) glog.Fatal(http.ListenAndServe(config.Config.FrontEnd.Port, nil)) }
func main() { defer common.LogPanic() // Setup flags. influxdb.SetupFlags() common.InitWithMetrics("status", graphiteServer) v, err := skiaversion.GetVersion() if err != nil { glog.Fatal(err) } glog.Infof("Version %s, built at %s", v.Commit, v.Date) Init() if *testing { *useMetadata = false } serverURL := "https://" + *host if *testing { serverURL = "http://" + *host + *port } // Create buildbot remote DB. db, err = buildbot.NewRemoteDB(*buildbotDbHost) if err != nil { glog.Fatal(err) } // Setup InfluxDB client. dbClient, err = influxdb.NewClientFromFlagsAndMetadata(*testing) if err != nil { glog.Fatal(err) } // By default use a set of credentials setup for localhost access. var cookieSalt = "notverysecret" var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com" var clientSecret = "cw0IosPu4yjaG2KWmppj2guj" var redirectURL = serverURL + "/oauth2callback/" if *useMetadata { cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT)) clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID)) clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET)) } login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, false) // Check out source code. skiaRepo, err := gitinfo.CloneOrUpdate("https://skia.googlesource.com/skia.git", path.Join(*workdir, "skia"), true) if err != nil { glog.Fatalf("Failed to check out Skia: %v", err) } infraRepoPath := path.Join(*workdir, "infra") infraRepo, err := gitinfo.CloneOrUpdate("https://skia.googlesource.com/buildbot.git", infraRepoPath, true) if err != nil { glog.Fatalf("Failed to checkout Infra: %v", err) } glog.Info("CloneOrUpdate complete") // Create the commit caches. commitCaches = map[string]*commit_cache.CommitCache{} skiaCache, err := commit_cache.New(skiaRepo, path.Join(*workdir, "commit_cache.gob"), DEFAULT_COMMITS_TO_LOAD, db) if err != nil { glog.Fatalf("Failed to create commit cache: %v", err) } commitCaches[SKIA_REPO] = skiaCache infraCache, err := commit_cache.New(infraRepo, path.Join(*workdir, "commit_cache_infra.gob"), DEFAULT_COMMITS_TO_LOAD, db) if err != nil { glog.Fatalf("Failed to create commit cache: %v", err) } commitCaches[INFRA_REPO] = infraCache glog.Info("commit_cache complete") // Load Perf and Gold data in a loop. perfStatus, err = dbClient.Int64PollingStatus("select value from \"alerting.new.value\" where app='skiaperf' and host='skia-perf' order by time desc limit 1", time.Minute) if err != nil { glog.Fatalf("Failed to create polling Perf status: %v", err) } goldGMStatus, err = dbClient.Int64PollingStatus(fmt.Sprintf(GOLD_STATUS_QUERY_TMPL, "gm"), time.Minute) if err != nil { glog.Fatalf("Failed to create polling Gold status: %v", err) } goldImageStatus, err = dbClient.Int64PollingStatus(fmt.Sprintf(GOLD_STATUS_QUERY_TMPL, "image"), time.Minute) if err != nil { glog.Fatalf("Failed to create polling Gold status: %v", err) } // Load slave_hosts_cfg and device cfgs in a loop. slaveHosts, err = buildbot.SlaveHostsCfgPoller(infraRepoPath) if err != nil { glog.Fatal(err) } androidDevices, err = device_cfg.AndroidDeviceCfgPoller(*workdir) if err != nil { glog.Fatal(err) } sshDevices, err = device_cfg.SSHDeviceCfgPoller(*workdir) if err != nil { glog.Fatal(err) } runServer(serverURL) }
func main() { // Setup DB flags. dbConf := db.DBConfigFromFlags() common.InitWithMetrics("skiaperf", graphiteServer) Init() if !*local { if err := dbConf.GetPasswordFromMetadata(); err != nil { glog.Fatal(err) } } if err := dbConf.InitDB(); err != nil { glog.Fatal(err) } stats.Start(nanoTileStore, git) alerting.Start(nanoTileStore, *apikey) // By default use a set of credentials setup for localhost access. var cookieSalt = "notverysecret" var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com" var clientSecret = "cw0IosPu4yjaG2KWmppj2guj" var redirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port) if !*local { cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT)) clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID)) clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET)) redirectURL = "https://perf.skia.org/oauth2callback/" } login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, *local) glog.Infoln("Begin loading data.") // Resources are served directly. router := mux.NewRouter() router.PathPrefix("/res/").HandlerFunc(makeResourceHandler()) router.HandleFunc("/", mainHandler) router.HandleFunc("/shortcuts/", shortcutHandler) router.PathPrefix("/tiles/").HandlerFunc(tileHandler) router.PathPrefix("/single/").HandlerFunc(singleHandler) router.PathPrefix("/query/").HandlerFunc(queryHandler) router.HandleFunc("/commits/", commitsHandler) router.HandleFunc("/shortcommits/", shortCommitsHandler) router.HandleFunc("/trybots/", trybotHandler) router.HandleFunc("/clusters/", clustersHandler) router.HandleFunc("/clustering/", clusteringHandler) router.PathPrefix("/cl/").HandlerFunc(clHandler) router.PathPrefix("/activitylog/").HandlerFunc(activityHandler) router.HandleFunc("/alerts/", alertsHandler) router.HandleFunc("/alerting/", alertingHandler) router.HandleFunc("/alert_reset/", alertResetHandler) router.HandleFunc("/annotate/", annotate.Handler) router.HandleFunc("/compare/", compareHandler) router.HandleFunc("/calc/", calcHandler) router.HandleFunc("/help/", helpHandler) router.HandleFunc("/oauth2callback/", login.OAuth2CallbackHandler) router.HandleFunc("/logout/", login.LogoutHandler) router.HandleFunc("/loginstatus/", login.StatusHandler) http.Handle("/", util.LoggingGzipRequestResponse(router)) glog.Infoln("Ready to serve.") glog.Fatal(http.ListenAndServe(*port, nil)) }
func main() { defer common.LogPanic() alertDBConf := alerting.DBConfigFromFlags() buildbotDBConf := buildbot.DBConfigFromFlags() common.InitWithMetrics("alertserver", graphiteServer) v, err := skiaversion.GetVersion() if err != nil { glog.Fatal(err) } glog.Infof("Version %s, built at %s", v.Commit, v.Date) Init() if *validateAndExit { if _, err := rules.MakeRules(*alertsFile, nil, time.Second, nil, *testing); err != nil { glog.Fatalf("Failed to set up rules: %v", err) } return } parsedPollInterval, err := time.ParseDuration(*alertPollInterval) if err != nil { glog.Fatalf("Failed to parse -alertPollInterval: %s", *alertPollInterval) } if *testing { *useMetadata = false } if *useMetadata { *influxDbName = metadata.Must(metadata.ProjectGet(metadata.INFLUXDB_NAME)) *influxDbPassword = metadata.Must(metadata.ProjectGet(metadata.INFLUXDB_PASSWORD)) } dbClient, err := client.New(&client.ClientConfig{ Host: *influxDbHost, Username: *influxDbName, Password: *influxDbPassword, Database: *influxDbDatabase, HttpClient: nil, IsSecure: false, IsUDP: false, }) if err != nil { glog.Fatalf("Failed to initialize InfluxDB client: %s", err) } serverURL := "https://" + *host if *testing { serverURL = "http://" + *host + *port } usr, err := user.Current() if err != nil { glog.Fatal(err) } tokenFile, err := filepath.Abs(usr.HomeDir + "/" + GMAIL_TOKEN_CACHE_FILE) if err != nil { glog.Fatal(err) } // By default use a set of credentials setup for localhost access. var cookieSalt = "notverysecret" var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com" var clientSecret = "cw0IosPu4yjaG2KWmppj2guj" var redirectURL = serverURL + "/oauth2callback/" var emailClientId = *emailClientIdFlag var emailClientSecret = *emailClientSecretFlag if *useMetadata { cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT)) clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID)) clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET)) emailClientId = metadata.Must(metadata.ProjectGet(metadata.GMAIL_CLIENT_ID)) emailClientSecret = metadata.Must(metadata.ProjectGet(metadata.GMAIL_CLIENT_SECRET)) cachedGMailToken := metadata.Must(metadata.ProjectGet(metadata.GMAIL_CACHED_TOKEN)) err = ioutil.WriteFile(tokenFile, []byte(cachedGMailToken), os.ModePerm) if err != nil { glog.Fatalf("Failed to cache token: %s", err) } } login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, false) var emailAuth *email.GMail if !*testing { if !*useMetadata && (emailClientId == "" || emailClientSecret == "") { glog.Fatal("If -use_metadata=false, you must provide -email_clientid and -email_clientsecret") } emailAuth, err = email.NewGMail(emailClientId, emailClientSecret, tokenFile) if err != nil { glog.Fatalf("Failed to create email auth: %v", err) } } // Initialize the database. if !*testing && *useMetadata { if err := alertDBConf.GetPasswordFromMetadata(); err != nil { glog.Fatal(err) } if err := buildbotDBConf.GetPasswordFromMetadata(); err != nil { glog.Fatal(err) } } if err := alertDBConf.InitDB(); err != nil { glog.Fatal(err) } if err := buildbotDBConf.InitDB(); err != nil { glog.Fatal(err) } // Create the AlertManager. alertManager, err = alerting.MakeAlertManager(parsedPollInterval, emailAuth) if err != nil { glog.Fatalf("Failed to create AlertManager: %v", err) } rulesList, err = rules.MakeRules(*alertsFile, dbClient, parsedPollInterval, alertManager, *testing) if err != nil { glog.Fatalf("Failed to set up rules: %v", err) } StartAlertRoutines(alertManager, 10*parsedPollInterval, dbClient) runServer(serverURL) }
func main() { defer common.LogPanic() var err error mainTimer := timer.New("main init") // Setup DB flags. dbConf := database.ConfigFromFlags(db.PROD_DB_HOST, db.PROD_DB_PORT, database.USER_RW, db.PROD_DB_NAME, db.MigrationSteps()) // Global init to initialize common.InitWithMetrics("skiacorrectness", graphiteServer) v, err := skiaversion.GetVersion() if err != nil { glog.Fatalf("Unable to retrieve version: %s", err) } glog.Infof("Version %s, built at %s", v.Commit, v.Date) // Enable the memory profiler if memProfile was set. // TODO(stephana): This should be moved to a HTTP endpoint that // only responds to internal IP addresses/ports. if *memProfile > 0 { time.AfterFunc(*memProfile, func() { glog.Infof("Writing Memory Profile") f, err := ioutil.TempFile("./", "memory-profile") if err != nil { glog.Fatalf("Unable to create memory profile file: %s", err) } if err := pprof.WriteHeapProfile(f); err != nil { glog.Fatalf("Unable to write memory profile file: %v", err) } util.Close(f) glog.Infof("Memory profile written to %s", f.Name()) os.Exit(0) }) } if *cpuProfile > 0 { glog.Infof("Writing CPU Profile") f, err := ioutil.TempFile("./", "cpu-profile") if err != nil { glog.Fatalf("Unable to create cpu profile file: %s", err) } if err := pprof.StartCPUProfile(f); err != nil { glog.Fatalf("Unable to write cpu profile file: %v", err) } time.AfterFunc(*cpuProfile, func() { pprof.StopCPUProfile() util.Close(f) glog.Infof("CPU profile written to %s", f.Name()) os.Exit(0) }) } // Init this module. Init() // Initialize submodules. filediffstore.Init() // Set up login // TODO (stephana): Factor out to go/login/login.go and removed hard coded // values. var cookieSalt = "notverysecret" var clientID = "31977622648-ubjke2f3staq6ouas64r31h8f8tcbiqp.apps.googleusercontent.com" var clientSecret = "rK-kRY71CXmcg0v9I9KIgWci" var useRedirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port) if !*local { cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT)) clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID)) clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET)) useRedirectURL = *redirectURL } login.Init(clientID, clientSecret, useRedirectURL, cookieSalt, login.DEFAULT_SCOPE, *authWhiteList, *local) // get the Oauthclient if necessary. client := getOAuthClient(*oauthCacheFile) // Set up the cache implementation to use. cacheFactory := filediffstore.MemCacheFactory if *redisHost != "" { cacheFactory = func(uniqueId string, codec util.LRUCodec) util.LRUCache { return redisutil.NewRedisLRUCache(*redisHost, *redisDB, uniqueId, codec) } } // Get the expecations storage, the filediff storage and the tilestore. diffStore, err := filediffstore.NewFileDiffStore(client, *imageDir, *gsBucketName, filediffstore.DEFAULT_GS_IMG_DIR_NAME, cacheFactory, filediffstore.RECOMMENDED_WORKER_POOL_SIZE) if err != nil { glog.Fatalf("Allocating DiffStore failed: %s", err) } if !*local { if err := dbConf.GetPasswordFromMetadata(); err != nil { glog.Fatal(err) } } vdb, err := dbConf.NewVersionedDB() if err != nil { glog.Fatal(err) } if !vdb.IsLatestVersion() { glog.Fatal("Wrong DB version. Please updated to latest version.") } digestStore, err := digeststore.New(*storageDir) if err != nil { glog.Fatal(err) } eventBus := eventbus.New(nil) storages = &storage.Storage{ DiffStore: diffStore, ExpectationsStore: expstorage.NewCachingExpectationStore(expstorage.NewSQLExpectationStore(vdb), eventBus), IgnoreStore: ignore.NewSQLIgnoreStore(vdb), TileStore: filetilestore.NewFileTileStore(*tileStoreDir, config.DATASET_GOLD, 2*time.Minute), DigestStore: digestStore, NCommits: *nCommits, EventBus: eventBus, TrybotResults: trybot.NewTrybotResultStorage(vdb), RietveldAPI: rietveld.New(*rietveldURL, nil), } if err := history.Init(storages, *nTilesToBackfill); err != nil { glog.Fatalf("Unable to initialize history package: %s", err) } if blamer, err = blame.New(storages); err != nil { glog.Fatalf("Unable to create blamer: %s", err) } if err := ignore.Init(storages.IgnoreStore); err != nil { glog.Fatalf("Failed to start monitoring for expired ignore rules: %s", err) } // Enable the experimental features. tallies, err = tally.New(storages) if err != nil { glog.Fatalf("Failed to build tallies: %s", err) } paramsetSum = paramsets.New(tallies, storages) if !*local { *issueTrackerKey = metadata.Must(metadata.ProjectGet(metadata.APIKEY)) } issueTracker = issues.NewIssueTracker(*issueTrackerKey) summaries, err = summary.New(storages, tallies, blamer) if err != nil { glog.Fatalf("Failed to build summary: %s", err) } statusWatcher, err = status.New(storages) if err != nil { glog.Fatalf("Failed to initialize status watcher: %s", err) } imgFS := NewURLAwareFileServer(*imageDir, IMAGE_URL_PREFIX) pathToURLConverter = imgFS.GetURL if err := warmer.Init(storages, summaries, tallies); err != nil { glog.Fatalf("Failed to initialize the warmer: %s", err) } mainTimer.Stop() // Everything is wired up at this point. Run the self tests if requested. if *selfTest { search.SelfTest(storages, tallies, blamer, paramsetSum) } router := mux.NewRouter() // Set up the resource to serve the image files. router.PathPrefix(IMAGE_URL_PREFIX).Handler(imgFS.Handler) // New Polymer based UI endpoints. router.PathPrefix("/res/").HandlerFunc(makeResourceHandler()) // TODO(stephana): Remove the 'poly' prefix from all the handlers and clean // up main2.go by either merging it it into main.go or making it clearer that // it contains all the handlers. Make it clearer what variables are shared // between the different file. // All the handlers will be prefixed with poly to differentiate it from the // angular code until the angular code is removed. router.HandleFunc(OAUTH2_CALLBACK_PATH, login.OAuth2CallbackHandler) router.HandleFunc("/", byBlameHandler).Methods("GET") router.HandleFunc("/list", templateHandler("list.html")).Methods("GET") router.HandleFunc("/_/details", polyDetailsHandler).Methods("GET") router.HandleFunc("/_/diff", polyDiffJSONDigestHandler).Methods("GET") router.HandleFunc("/_/hashes", polyAllHashesHandler).Methods("GET") router.HandleFunc("/_/ignores", polyIgnoresJSONHandler).Methods("GET") router.HandleFunc("/_/ignores/add/", polyIgnoresAddHandler).Methods("POST") router.HandleFunc("/_/ignores/del/{id}", polyIgnoresDeleteHandler).Methods("POST") router.HandleFunc("/_/ignores/save/{id}", polyIgnoresUpdateHandler).Methods("POST") router.HandleFunc("/_/list", polyListTestsHandler).Methods("GET") router.HandleFunc("/_/paramset", polyParamsHandler).Methods("GET") router.HandleFunc("/_/nxn", nxnJSONHandler).Methods("GET") // TODO(stephana): Once /_/search3 is stable it will replace /_/search and the // /search*.html pages will be consolidated into one. router.HandleFunc("/_/search", polySearchJSONHandler).Methods("GET") router.HandleFunc("/_/search3", search3JSONHandler).Methods("GET") router.HandleFunc("/_/status/{test}", polyTestStatusHandler).Methods("GET") router.HandleFunc("/_/test", polyTestHandler).Methods("POST") router.HandleFunc("/_/triage", polyTriageHandler).Methods("POST") router.HandleFunc("/_/triagelog", polyTriageLogHandler).Methods("GET") router.HandleFunc("/_/triagelog/undo", triageUndoHandler).Methods("POST") router.HandleFunc("/_/failure", failureListJSONHandler).Methods("GET") router.HandleFunc("/_/failure/clear", failureClearJSONHandler).Methods("POST") router.HandleFunc("/_/trybot", listTrybotsJSONHandler).Methods("GET") router.HandleFunc("/byblame", byBlameHandler).Methods("GET") router.HandleFunc("/cluster", templateHandler("cluster.html")).Methods("GET") router.HandleFunc("/search2", search2Handler).Methods("GET") router.HandleFunc("/cmp/{test}", templateHandler("compare.html")).Methods("GET") router.HandleFunc("/detail", templateHandler("single.html")).Methods("GET") router.HandleFunc("/diff", templateHandler("diff.html")).Methods("GET") router.HandleFunc("/help", templateHandler("help.html")).Methods("GET") router.HandleFunc("/ignores", templateHandler("ignores.html")).Methods("GET") router.HandleFunc("/loginstatus/", login.StatusHandler) router.HandleFunc("/logout/", login.LogoutHandler) router.HandleFunc("/search", templateHandler("search.html")).Methods("GET") router.HandleFunc("/triagelog", templateHandler("triagelog.html")).Methods("GET") router.HandleFunc("/trybot", templateHandler("trybot.html")).Methods("GET") router.HandleFunc("/failures", templateHandler("failures.html")).Methods("GET") // Add the necessary middleware and have the router handle all requests. // By structuring the middleware this way we only log requests that are // authenticated. rootHandler := util.LoggingGzipRequestResponse(router) if *forceLogin { rootHandler = login.ForceAuth(rootHandler, OAUTH2_CALLBACK_PATH) } // The polyStatusHandler is being polled, so we exclude it from logging. http.HandleFunc("/_/trstatus", polyStatusHandler) http.Handle("/", rootHandler) // Start the server glog.Infoln("Serving on http://127.0.0.1" + *port) glog.Fatal(http.ListenAndServe(*port, nil)) }
func main() { defer common.LogPanic() common.InitWithMetrics("autoroll", graphiteServer) Init() v, err := skiaversion.GetVersion() if err != nil { glog.Fatal(err) } glog.Infof("Version %s, built at %s", v.Commit, v.Date) if *testing { *useMetadata = false } // Create the Rietveld client. client, err := auth.NewClientFromIdAndSecret(rietveld.CLIENT_ID, rietveld.CLIENT_SECRET, path.Join(*workdir, "oauth_cache"), rietveld.OAUTH_SCOPES...) if err != nil { glog.Fatal(err) } r := rietveld.New(RIETVELD_URL, client) // Retrieve the list of extra CQ trybots. // TODO(borenet): Make this editable on the web front-end. cqExtraTrybots, err := getCQExtraTrybots() if err != nil { glog.Fatal(err) } // Retrieve the initial email list. emails, err := getSheriff() if err != nil { glog.Fatal(err) } // Start the autoroller. arb, err = autoroller.NewAutoRoller(*workdir, cqExtraTrybots, emails, r, time.Minute, 15*time.Minute, *depot_tools) if err != nil { glog.Fatal(err) } // Update the current sheriff in a loop. go func() { for _ = range time.Tick(30 * time.Minute) { emails, err := getSheriff() if err != nil { glog.Errorf("Failed to retrieve current sheriff: %s", err) } else { arb.SetEmails(emails) } } }() serverURL := "https://" + *host if *testing { serverURL = "http://" + *host + *port } // By default use a set of credentials setup for localhost access. var cookieSalt = "notverysecret" var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com" var clientSecret = "cw0IosPu4yjaG2KWmppj2guj" var redirectURL = serverURL + "/oauth2callback/" if *useMetadata { cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT)) clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID)) clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET)) } login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, false) runServer(serverURL) }
func main() { defer common.LogPanic() var err error // Setup flags. dbConf := buildbot.DBConfigFromFlags() // Global init to initialize glog and parse arguments. // Global init to initialize glog and parse arguments. common.InitWithMetrics("internal", graphiteServer) if !*local { *targetList = metadata.Must(metadata.ProjectGet("datahopper_internal_targets")) } targets := strings.Split(*targetList, " ") glog.Infof("Targets: %#v", targets) codenameDB, err = leveldb.OpenFile(*codenameDbDir, nil) if err != nil && errors.IsCorrupted(err) { codenameDB, err = leveldb.RecoverFile(*codenameDbDir, nil) } if err != nil { glog.Fatalf("Failed to open codename leveldb at %s: %s", *codenameDbDir, err) } // Initialize the buildbot database. if !*local { if err := dbConf.GetPasswordFromMetadata(); err != nil { glog.Fatal(err) } } if err := dbConf.InitDB(); err != nil { glog.Fatal(err) } var cookieSalt = "notverysecret" var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com" var clientSecret = "cw0IosPu4yjaG2KWmppj2guj" var redirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port) if !*local { cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT)) clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID)) clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET)) redirectURL = "https://internal.skia.org/oauth2callback/" } login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, *local) // Ingest Android framework builds. go func() { glog.Infof("Starting.") repos := gitinfo.NewRepoMap(*workdir) // In this case we don't want a backoff transport since the Apiary backend // seems to fail a lot, so we basically want to fall back to polling if a // call fails. client, err := auth.NewClientWithTransport(*local, *oauthCacheFile, "", &http.Transport{Dial: util.DialTimeout}, androidbuildinternal.AndroidbuildInternalScope, storage.CloudPlatformScope) if err != nil { glog.Fatalf("Unable to create installed app oauth client:%s", err) } buildService, err := androidbuildinternal.New(client) if err != nil { glog.Fatalf("Failed to obtain Android build service: %v", err) } glog.Infof("Ready to start loop.") step(targets, buildService, repos) for _ = range time.Tick(*period) { step(targets, buildService, repos) } }() r := mux.NewRouter() r.HandleFunc("/", indexHandler) r.HandleFunc("/builders/{codename}/builds/{buildNumber}", redirectHandler) r.HandleFunc("/builders/{codename}", builderRedirectHandler) r.HandleFunc("/loginstatus/", login.StatusHandler) r.HandleFunc("/logout/", login.LogoutHandler) r.HandleFunc("/oauth2callback/", login.OAuth2CallbackHandler) http.Handle("/", util.LoggingGzipRequestResponse(r)) glog.Fatal(http.ListenAndServe(*port, nil)) }
// Call once at startup to read requestSalt from project metadata. func MustInitRequestSaltFromMetadata() { saltBase64 := metadata.Must(metadata.ProjectGet(metadata.WEBHOOK_REQUEST_SALT)) if err := setRequestSaltFromBase64([]byte(saltBase64)); err != nil { glog.Fatalf("Could not decode salt from %s: %v", metadata.WEBHOOK_REQUEST_SALT, err) } }
func main() { defer common.LogPanic() // Setup flags. dbConf := db.DBConfigFromFlags() ctfeutil.PreExecuteTemplateHook = func() { // Don't use cached templates in local mode. if *local { reloadTemplates() } } common.InitWithMetrics("ctfe", graphiteServer) v, err := skiaversion.GetVersion() if err != nil { glog.Fatal(err) } glog.Infof("Version %s, built at %s", v.Commit, v.Date) Init() serverURL := "https://" + *host if *local { serverURL = "http://" + *host + *port } // By default use a set of credentials setup for localhost access. var cookieSalt = "notverysecret" var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com" var clientSecret = "cw0IosPu4yjaG2KWmppj2guj" var redirectURL = serverURL + "/oauth2callback/" if !*local { cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT)) clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID)) clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET)) } login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, *local) if *local { webhook.InitRequestSaltForTesting() } else { webhook.MustInitRequestSaltFromMetadata() } glog.Info("CloneOrUpdate complete") // Initialize the ctfe database. if !*local { if err := dbConf.GetPasswordFromMetadata(); err != nil { glog.Fatal(err) } } if err := dbConf.InitDB(); err != nil { glog.Fatal(err) } startCtfeMetrics() // Start the repeated tasks scheduler. go repeatedTasksScheduler() runServer(serverURL) }
func main() { defer common.LogPanic() var err error // Global init to initialize glog and parse arguments. common.InitWithMetrics("internal", graphiteServer) if !*local { *targetList = metadata.Must(metadata.ProjectGet("datahopper_internal_targets")) } targets := strings.Split(*targetList, " ") glog.Infof("Targets: %#v", targets) codenameDB, err = leveldb.OpenFile(*codenameDbDir, nil) if err != nil && errors.IsCorrupted(err) { codenameDB, err = leveldb.RecoverFile(*codenameDbDir, nil) } if err != nil { glog.Fatalf("Failed to open codename leveldb at %s: %s", *codenameDbDir, err) } // Initialize the buildbot database. db, err = buildbot.NewRemoteDB(*buildbotDbHost) if err != nil { glog.Fatal(err) } var redirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port) if !*local { redirectURL = "https://internal.skia.org/oauth2callback/" } if err := login.InitFromMetadataOrJSON(redirectURL, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST); err != nil { glog.Fatalf("Failed to initialize login system: %s", err) } if *local { webhook.InitRequestSaltForTesting() } else { webhook.MustInitRequestSaltFromMetadata() } repos = gitinfo.NewRepoMap(*workdir) // Ingest Android framework builds. go func() { glog.Infof("Starting.") // In this case we don't want a backoff transport since the Apiary backend // seems to fail a lot, so we basically want to fall back to polling if a // call fails. client, err := auth.NewJWTServiceAccountClient("", "", &http.Transport{Dial: util.DialTimeout}, androidbuildinternal.AndroidbuildInternalScope, storage.CloudPlatformScope) if err != nil { glog.Fatalf("Unable to create authenticated client: %s", err) } buildService, err := androidbuildinternal.New(client) if err != nil { glog.Fatalf("Failed to obtain Android build service: %v", err) } glog.Infof("Ready to start loop.") step(targets, buildService) for _ = range time.Tick(*period) { step(targets, buildService) } }() r := mux.NewRouter() r.HandleFunc("/", indexHandler) r.HandleFunc("/builders/{codename}/builds/{buildNumber}", redirectHandler) r.HandleFunc("/builders/{codename}", builderRedirectHandler) r.HandleFunc("/ingestBuild", ingestBuildHandler) r.HandleFunc("/loginstatus/", login.StatusHandler) r.HandleFunc("/logout/", login.LogoutHandler) r.HandleFunc("/oauth2callback/", login.OAuth2CallbackHandler) http.Handle("/", util.LoggingGzipRequestResponse(r)) glog.Fatal(http.ListenAndServe(*port, nil)) }