// getOAuthClient returns an oauth client.
func getOAuthClient(cacheFilePath string) *http.Client {
var client *http.Client
var err error
if *doOauth || *local {
if *local {
// Load client secrets file. The client_secret.json file must come from
// project id: 470362608618, which is whitelisted for access to
// gs://chromium-skia-gm.
client, err = auth.NewClientWithTransport(true, cacheFilePath, *clientSecretFile, nil, auth.SCOPE_READ_ONLY)
if err != nil {
glog.Fatalf("Failed to auth: (Did you download the client_secret.json file from the project with ID: google.com:chrome-skia #470362608618 ?) %s", err)
}
} else {
// Load client id and secret from metadata.
clientId := metadata.Must(metadata.ProjectGet(metadata.CHROMIUM_SKIA_GM_CLIENT_ID))
clientSecret := metadata.Must(metadata.ProjectGet(metadata.CHROMIUM_SKIA_GM_CLIENT_SECRET))
client, err = auth.NewClientFromIdAndSecret(clientId, clientSecret, cacheFilePath, auth.SCOPE_READ_ONLY)
if err != nil {
glog.Fatalf("Failed to auth: %s", err)
}
}
}
return client
}
// Run the BugChomper server.
func main() {
common.InitWithMetrics("bug_chomper", graphiteServer)
v, err := skiaversion.GetVersion()
if err != nil {
glog.Fatal(err)
}
glog.Infof("Version %s, built at %s", v.Commit, v.Date)
loadTemplates()
if *testing {
*useMetadata = false
}
serverURL := "https://" + *host
if *testing {
serverURL = "http://" + *host + *port
}
// By default use a set of credentials setup for localhost access.
var cookieSalt = "notverysecret"
var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com"
var clientSecret = "cw0IosPu4yjaG2KWmppj2guj"
var redirectURL = serverURL + "/oauth2callback/"
if *useMetadata {
cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT))
clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID))
clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET))
}
login.Init(clientID, clientSecret, redirectURL, cookieSalt, strings.Join(issue_tracker.OAUTH_SCOPE, " "), login.DEFAULT_DOMAIN_WHITELIST, false)
runServer(serverURL)
}
func main() {
common.InitWithMetrics("push", graphiteServer)
Init()
// By default use a set of credentials setup for localhost access.
var cookieSalt = "notverysecret"
var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com"
var clientSecret = "cw0IosPu4yjaG2KWmppj2guj"
var redirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port)
if !*local {
cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT))
clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID))
clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET))
redirectURL = "https://push.skia.org/oauth2callback/"
}
login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, *local)
r := mux.NewRouter()
r.PathPrefix("/res/").HandlerFunc(util.MakeResourceHandler(*resourcesDir))
r.HandleFunc("/", mainHandler)
r.HandleFunc("/_/change", changeHandler)
r.HandleFunc("/_/state", stateHandler)
r.HandleFunc("/_/status", statusHandler)
r.HandleFunc("/loginstatus/", login.StatusHandler)
r.HandleFunc("/logout/", login.LogoutHandler)
r.HandleFunc("/oauth2callback/", login.OAuth2CallbackHandler)
http.Handle("/", util.LoggingGzipRequestResponse(r))
glog.Infoln("Ready to serve.")
glog.Fatal(http.ListenAndServe(*port, nil))
}
func AttemptLoadCertFromMetadata() {
if *certChainFile == "" {
// Try loading from GCE project level metadata.
certChainContents, err := metadata.ProjectGet("ca_cert_chain")
if err != nil {
glog.Errorf("Failed to load ca_cert_chain from metadata: %s", err)
return
}
keyContents, err := metadata.ProjectGet("ca_key")
if err != nil {
glog.Errorf("Failed to load ca_key from metadata: %s", err)
return
}
fullCertChainFilename := filepath.Join(*workDir, CA_CERT_CHAIN_FILENAME)
fullKeyFilename := filepath.Join(*workDir, CA_KEY_FILENAME)
if err := ioutil.WriteFile(fullCertChainFilename, []byte(certChainContents), 0600); err != nil {
glog.Errorf("Failed to write %s: %s", fullCertChainFilename, err)
return
}
if err := ioutil.WriteFile(fullKeyFilename, []byte(keyContents), 0600); err != nil {
glog.Errorf("Failed to write %s: %s", fullKeyFilename, err)
return
}
*keyFile = fullKeyFilename
*certChainFile = fullCertChainFilename
glog.Infof("SUCCESS: Loaded cert from metadata.")
}
}
// NewClientFromFlagsAndMetadata returns a Client with credentials obtained
// from a combination of flags and metadata, depending on whether the program
// is running in local mode.
func NewClientFromFlagsAndMetadata(local bool) (*Client, error) {
if !local {
userMeta, err := metadata.ProjectGet(metadata.INFLUXDB_NAME)
if err != nil {
return nil, err
}
passMeta, err := metadata.ProjectGet(metadata.INFLUXDB_PASSWORD)
if err != nil {
return nil, err
}
*user = userMeta
*password = passMeta
}
return NewClientFromFlags()
}
// tryLoadingFromMetadata tries to load the cookie salt, client id, and client
// secret from GCE project level metadata. If it fails then it returns the salt
// it was passed and the client id and secret are the empty string.
//
// Returns salt, clientID, clientSecret.
func tryLoadingFromMetadata() (string, string, string) {
cookieSalt, err := metadata.ProjectGet(metadata.COOKIESALT)
if err != nil {
return DEFAULT_COOKIE_SALT, "", ""
}
clientID, err := metadata.ProjectGet(metadata.CLIENT_ID)
if err != nil {
return DEFAULT_COOKIE_SALT, "", ""
}
clientSecret, err := metadata.ProjectGet(metadata.CLIENT_SECRET)
if err != nil {
return DEFAULT_COOKIE_SALT, "", ""
}
return cookieSalt, clientID, clientSecret
}
// monitorIssueTracker reads the counts for all the types of issues in the skia
// issue tracker (code.google.com/p/skia) and stuffs the counts into Graphite.
func monitorIssueTracker() {
c := &http.Client{
Transport: &http.Transport{
Dial: dialTimeout,
},
}
if *useMetadata {
*apikey = metadata.Must(metadata.ProjectGet(metadata.APIKEY))
}
// Create a new metrics registry for the issue tracker metrics.
addr, err := net.ResolveTCPAddr("tcp", *graphiteServer)
if err != nil {
glog.Fatalln("Failed to resolve the Graphite server: ", err)
}
issueRegistry := metrics.NewRegistry()
go graphite.Graphite(issueRegistry, common.SAMPLE_PERIOD, "issues", addr)
// IssueStatus has all the info we need to capture and record a single issue status. I.e. capture
// the count of all issues with a status of "New".
type IssueStatus struct {
Name string
Metric metrics.Gauge
URL string
}
allIssueStatusLabels := []string{
"New", "Accepted", "Unconfirmed", "Started", "Fixed", "Verified", "Invalid", "WontFix", "Done", "Available", "Assigned",
}
issueStatus := []*IssueStatus{}
for _, issueName := range allIssueStatusLabels {
issueStatus = append(issueStatus, &IssueStatus{
Name: issueName,
Metric: metrics.NewRegisteredGauge(strings.ToLower(issueName), issueRegistry),
URL: "https://www.googleapis.com/projecthosting/v2/projects/skia/issues?fields=totalResults&key=" + *apikey + "&status=" + issueName,
})
}
liveness := imetrics.NewLiveness("issue-tracker")
for _ = range time.Tick(ISSUE_TRACKER_PERIOD) {
for _, issue := range issueStatus {
resp, err := c.Get(issue.URL)
jsonResp := map[string]int64{}
dec := json.NewDecoder(resp.Body)
if err := dec.Decode(&jsonResp); err != nil {
glog.Warningf("Failed to decode JSON response: %s", err)
util.Close(resp.Body)
continue
}
issue.Metric.Update(jsonResp["totalResults"])
glog.Infof("Num Issues: %s - %d", issue.Name, jsonResp["totalResults"])
if err == nil && resp.Body != nil {
util.Close(resp.Body)
}
}
liveness.Update()
}
}
// InitRequestSaltFromMetadata reads requestSalt from project metadata and returns any error
// encountered. Should be called once at startup.
func InitRequestSaltFromMetadata() error {
saltBase64, err := metadata.ProjectGet(metadata.WEBHOOK_REQUEST_SALT)
if err != nil {
return err
}
if err := setRequestSaltFromBase64([]byte(saltBase64)); err != nil {
return fmt.Errorf("Could not decode salt from %s: %s", metadata.WEBHOOK_REQUEST_SALT, err)
}
return nil
}
func main() {
defer common.LogPanic()
common.InitWithMetrics("mathserv", graphiteServer)
Init()
// By default use a set of credentials setup for localhost access.
var cookieSalt = "notverysecret"
var clientID = "952643138919-5a692pfevie766aiog15io45kjpsh33v.apps.googleusercontent.com"
var clientSecret = "QQfqRYU1ELkds90ku8xlIGl1"
var redirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port)
if !*local {
cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT))
clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID))
clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET))
redirectURL = "https://mathinate.com/oauth2callback/"
}
login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, "", *local)
r := mux.NewRouter()
r.PathPrefix("/res/").HandlerFunc(util.MakeResourceHandler(*resourcesDir))
r.HandleFunc("/", mainHandler)
r.HandleFunc("/loginstatus/", login.StatusHandler)
r.HandleFunc("/logout/", login.LogoutHandler)
r.HandleFunc("/oauth2callback/", login.OAuth2CallbackHandler)
http.Handle("/", util.LoggingGzipRequestResponse(r))
AttemptLoadCertFromMetadata()
glog.Infoln("Ready to serve.")
if *certChainFile != "" {
glog.Infof("Serving TLS")
go func() {
redir := mux.NewRouter()
redir.HandleFunc("/", redirHandler)
glog.Fatal(http.ListenAndServe(*httpPort, redir))
}()
glog.Fatal(http.ListenAndServeTLS(*port, *certChainFile, *keyFile, nil))
} else {
glog.Infof("Only serving HTTP")
glog.Fatal(http.ListenAndServe(*port, nil))
}
}
// apiKeyFromFlag returns the key that it was passed if the key isn't empty,
// otherwise it tries to fetch the key from the metadata server.
//
// Returns the API Key, or "" if it failed to fetch the key from the metadata
// server.
func apiKeyFromFlag(apiKeyFlag string) string {
apiKey := apiKeyFlag
// If apiKey isn't passed in then read it from the metadata server.
if apiKey == "" {
var err error
if apiKey, err = metadata.ProjectGet(metadata.APIKEY); err != nil {
glog.Errorf("Retrieving API Key failed: %s", err)
return ""
}
}
return apiKey
}
func main() {
flag.Parse()
Init()
// Set up login
var cookieSalt = "notverysecret"
var clientID = "31977622648-ubjke2f3staq6ouas64r31h8f8tcbiqp.apps.googleusercontent.com"
var clientSecret = "rK-kRY71CXmcg0v9I9KIgWci"
var useRedirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", config.Config.FrontEnd.Port)
if !config.Config.Common.Local {
cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT))
clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID))
clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET))
useRedirectURL = config.Config.FrontEnd.RedirectURL
}
login.Init(clientID, clientSecret, useRedirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, config.Config.Common.Local)
// Set up the login related resources.
router.HandleFunc(OAUTH2_CALLBACK_PATH, login.OAuth2CallbackHandler)
router.HandleFunc("/loginstatus/", login.StatusHandler)
router.HandleFunc("/logout/", login.LogoutHandler)
router.HandleFunc("/", mainHandler)
router.HandleFunc("/failed", mainHandler)
router.HandleFunc("/passed", mainHandler)
router.PathPrefix("/res/").HandlerFunc(makeResourceHandler())
jsonRouter := router.PathPrefix("/_").Subrouter()
jsonRouter.HandleFunc("/list", fuzzListHandler).Name("fuzzListHandler")
rootHandler := util.LoggingGzipRequestResponse(router)
if config.Config.FrontEnd.ForceLogin {
rootHandler = login.ForceAuth(rootHandler, OAUTH2_CALLBACK_PATH)
}
http.Handle("/", rootHandler)
glog.Fatal(http.ListenAndServe(config.Config.FrontEnd.Port, nil))
}
// GetPasswordFromMetadata retrieve the password from metadata and sets the Password field.
func (c *DatabaseConfig) GetPasswordFromMetadata() error {
if err := c.validate(); err != nil {
return err
}
key := ""
if c.User == USER_RW {
key = metadata.DATABASE_RW_PASSWORD
} else if c.User == USER_ROOT {
key = metadata.DATABASE_ROOT_PASSWORD
}
if key == "" {
return fmt.Errorf("Unknown user %s; could not obtain password from metadata.", c.User)
}
password, err := metadata.ProjectGet(key)
if err != nil {
return fmt.Errorf("Failed to find metadata.")
}
c.Password = password
return nil
}
// NewJWTServiceAccountClient creates a new http.Client that is loaded by first
// attempting to load the JWT JSON Service Account data from GCE Project Level
// metadata, and if that fails falls back to loading the data from a local
// file.
//
// metadataname - The name of the GCE project level metadata key that holds the JWT JSON. If empty a default is used.
// filename - The name of the local file that holds the JWT JSON. If empty a default is used.
// transport - A transport. If nil then a default is used.
func NewJWTServiceAccountClient(metadataname, filename string, transport http.RoundTripper, scopes ...string) (*http.Client, error) {
if metadataname == "" {
metadataname = metadata.JWT_SERVICE_ACCOUNT
}
if filename == "" {
filename = DEFAULT_JWT_FILENAME
}
var body []byte
jwt, err := metadata.ProjectGet(metadataname)
if err != nil {
body, err = ioutil.ReadFile(filename)
if err != nil {
return nil, fmt.Errorf("Couldn't find JWT via metadata or in a local file.")
}
} else {
body = []byte(jwt)
}
if transport == nil {
transport = util.NewBackOffTransport()
}
jwtConfig, err := google.JWTConfigFromJSON(body, scopes...)
if err != nil {
return nil, err
}
tokenClient := &http.Client{
Transport: transport,
Timeout: util.REQUEST_TIMEOUT,
}
ctx := context.WithValue(context.Background(), oauth2.HTTPClient, tokenClient)
return &http.Client{
Transport: &oauth2.Transport{
Source: jwtConfig.TokenSource(ctx),
Base: transport,
},
Timeout: util.REQUEST_TIMEOUT,
}, nil
}
func main() {
defer common.LogPanic()
// Setup flags.
influxdb.SetupFlags()
common.InitWithMetrics("status", graphiteServer)
v, err := skiaversion.GetVersion()
if err != nil {
glog.Fatal(err)
}
glog.Infof("Version %s, built at %s", v.Commit, v.Date)
Init()
if *testing {
*useMetadata = false
}
serverURL := "https://" + *host
if *testing {
serverURL = "http://" + *host + *port
}
// Create buildbot remote DB.
db, err = buildbot.NewRemoteDB(*buildbotDbHost)
if err != nil {
glog.Fatal(err)
}
// Setup InfluxDB client.
dbClient, err = influxdb.NewClientFromFlagsAndMetadata(*testing)
if err != nil {
glog.Fatal(err)
}
// By default use a set of credentials setup for localhost access.
var cookieSalt = "notverysecret"
var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com"
var clientSecret = "cw0IosPu4yjaG2KWmppj2guj"
var redirectURL = serverURL + "/oauth2callback/"
if *useMetadata {
cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT))
clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID))
clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET))
}
login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, false)
// Check out source code.
skiaRepo, err := gitinfo.CloneOrUpdate("https://skia.googlesource.com/skia.git", path.Join(*workdir, "skia"), true)
if err != nil {
glog.Fatalf("Failed to check out Skia: %v", err)
}
infraRepoPath := path.Join(*workdir, "infra")
infraRepo, err := gitinfo.CloneOrUpdate("https://skia.googlesource.com/buildbot.git", infraRepoPath, true)
if err != nil {
glog.Fatalf("Failed to checkout Infra: %v", err)
}
glog.Info("CloneOrUpdate complete")
// Create the commit caches.
commitCaches = map[string]*commit_cache.CommitCache{}
skiaCache, err := commit_cache.New(skiaRepo, path.Join(*workdir, "commit_cache.gob"), DEFAULT_COMMITS_TO_LOAD, db)
if err != nil {
glog.Fatalf("Failed to create commit cache: %v", err)
}
commitCaches[SKIA_REPO] = skiaCache
infraCache, err := commit_cache.New(infraRepo, path.Join(*workdir, "commit_cache_infra.gob"), DEFAULT_COMMITS_TO_LOAD, db)
if err != nil {
glog.Fatalf("Failed to create commit cache: %v", err)
}
commitCaches[INFRA_REPO] = infraCache
glog.Info("commit_cache complete")
// Load Perf and Gold data in a loop.
perfStatus, err = dbClient.Int64PollingStatus("select value from \"alerting.new.value\" where app='skiaperf' and host='skia-perf' order by time desc limit 1", time.Minute)
if err != nil {
glog.Fatalf("Failed to create polling Perf status: %v", err)
}
goldGMStatus, err = dbClient.Int64PollingStatus(fmt.Sprintf(GOLD_STATUS_QUERY_TMPL, "gm"), time.Minute)
if err != nil {
glog.Fatalf("Failed to create polling Gold status: %v", err)
}
goldImageStatus, err = dbClient.Int64PollingStatus(fmt.Sprintf(GOLD_STATUS_QUERY_TMPL, "image"), time.Minute)
if err != nil {
glog.Fatalf("Failed to create polling Gold status: %v", err)
}
// Load slave_hosts_cfg and device cfgs in a loop.
slaveHosts, err = buildbot.SlaveHostsCfgPoller(infraRepoPath)
if err != nil {
glog.Fatal(err)
}
androidDevices, err = device_cfg.AndroidDeviceCfgPoller(*workdir)
if err != nil {
glog.Fatal(err)
}
sshDevices, err = device_cfg.SSHDeviceCfgPoller(*workdir)
if err != nil {
glog.Fatal(err)
}
runServer(serverURL)
}
func main() {
// Setup DB flags.
dbConf := db.DBConfigFromFlags()
common.InitWithMetrics("skiaperf", graphiteServer)
Init()
if !*local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
if err := dbConf.InitDB(); err != nil {
glog.Fatal(err)
}
stats.Start(nanoTileStore, git)
alerting.Start(nanoTileStore, *apikey)
// By default use a set of credentials setup for localhost access.
var cookieSalt = "notverysecret"
var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com"
var clientSecret = "cw0IosPu4yjaG2KWmppj2guj"
var redirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port)
if !*local {
cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT))
clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID))
clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET))
redirectURL = "https://perf.skia.org/oauth2callback/"
}
login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, *local)
glog.Infoln("Begin loading data.")
// Resources are served directly.
router := mux.NewRouter()
router.PathPrefix("/res/").HandlerFunc(makeResourceHandler())
router.HandleFunc("/", mainHandler)
router.HandleFunc("/shortcuts/", shortcutHandler)
router.PathPrefix("/tiles/").HandlerFunc(tileHandler)
router.PathPrefix("/single/").HandlerFunc(singleHandler)
router.PathPrefix("/query/").HandlerFunc(queryHandler)
router.HandleFunc("/commits/", commitsHandler)
router.HandleFunc("/shortcommits/", shortCommitsHandler)
router.HandleFunc("/trybots/", trybotHandler)
router.HandleFunc("/clusters/", clustersHandler)
router.HandleFunc("/clustering/", clusteringHandler)
router.PathPrefix("/cl/").HandlerFunc(clHandler)
router.PathPrefix("/activitylog/").HandlerFunc(activityHandler)
router.HandleFunc("/alerts/", alertsHandler)
router.HandleFunc("/alerting/", alertingHandler)
router.HandleFunc("/alert_reset/", alertResetHandler)
router.HandleFunc("/annotate/", annotate.Handler)
router.HandleFunc("/compare/", compareHandler)
router.HandleFunc("/calc/", calcHandler)
router.HandleFunc("/help/", helpHandler)
router.HandleFunc("/oauth2callback/", login.OAuth2CallbackHandler)
router.HandleFunc("/logout/", login.LogoutHandler)
router.HandleFunc("/loginstatus/", login.StatusHandler)
http.Handle("/", util.LoggingGzipRequestResponse(router))
glog.Infoln("Ready to serve.")
glog.Fatal(http.ListenAndServe(*port, nil))
}
func main() {
defer common.LogPanic()
alertDBConf := alerting.DBConfigFromFlags()
buildbotDBConf := buildbot.DBConfigFromFlags()
common.InitWithMetrics("alertserver", graphiteServer)
v, err := skiaversion.GetVersion()
if err != nil {
glog.Fatal(err)
}
glog.Infof("Version %s, built at %s", v.Commit, v.Date)
Init()
if *validateAndExit {
if _, err := rules.MakeRules(*alertsFile, nil, time.Second, nil, *testing); err != nil {
glog.Fatalf("Failed to set up rules: %v", err)
}
return
}
parsedPollInterval, err := time.ParseDuration(*alertPollInterval)
if err != nil {
glog.Fatalf("Failed to parse -alertPollInterval: %s", *alertPollInterval)
}
if *testing {
*useMetadata = false
}
if *useMetadata {
*influxDbName = metadata.Must(metadata.ProjectGet(metadata.INFLUXDB_NAME))
*influxDbPassword = metadata.Must(metadata.ProjectGet(metadata.INFLUXDB_PASSWORD))
}
dbClient, err := client.New(&client.ClientConfig{
Host: *influxDbHost,
Username: *influxDbName,
Password: *influxDbPassword,
Database: *influxDbDatabase,
HttpClient: nil,
IsSecure: false,
IsUDP: false,
})
if err != nil {
glog.Fatalf("Failed to initialize InfluxDB client: %s", err)
}
serverURL := "https://" + *host
if *testing {
serverURL = "http://" + *host + *port
}
usr, err := user.Current()
if err != nil {
glog.Fatal(err)
}
tokenFile, err := filepath.Abs(usr.HomeDir + "/" + GMAIL_TOKEN_CACHE_FILE)
if err != nil {
glog.Fatal(err)
}
// By default use a set of credentials setup for localhost access.
var cookieSalt = "notverysecret"
var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com"
var clientSecret = "cw0IosPu4yjaG2KWmppj2guj"
var redirectURL = serverURL + "/oauth2callback/"
var emailClientId = *emailClientIdFlag
var emailClientSecret = *emailClientSecretFlag
if *useMetadata {
cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT))
clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID))
clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET))
emailClientId = metadata.Must(metadata.ProjectGet(metadata.GMAIL_CLIENT_ID))
emailClientSecret = metadata.Must(metadata.ProjectGet(metadata.GMAIL_CLIENT_SECRET))
cachedGMailToken := metadata.Must(metadata.ProjectGet(metadata.GMAIL_CACHED_TOKEN))
err = ioutil.WriteFile(tokenFile, []byte(cachedGMailToken), os.ModePerm)
if err != nil {
glog.Fatalf("Failed to cache token: %s", err)
}
}
login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, false)
var emailAuth *email.GMail
if !*testing {
if !*useMetadata && (emailClientId == "" || emailClientSecret == "") {
glog.Fatal("If -use_metadata=false, you must provide -email_clientid and -email_clientsecret")
}
emailAuth, err = email.NewGMail(emailClientId, emailClientSecret, tokenFile)
if err != nil {
glog.Fatalf("Failed to create email auth: %v", err)
}
}
// Initialize the database.
if !*testing && *useMetadata {
if err := alertDBConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
if err := buildbotDBConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
if err := alertDBConf.InitDB(); err != nil {
glog.Fatal(err)
}
if err := buildbotDBConf.InitDB(); err != nil {
glog.Fatal(err)
}
// Create the AlertManager.
alertManager, err = alerting.MakeAlertManager(parsedPollInterval, emailAuth)
if err != nil {
glog.Fatalf("Failed to create AlertManager: %v", err)
}
rulesList, err = rules.MakeRules(*alertsFile, dbClient, parsedPollInterval, alertManager, *testing)
if err != nil {
glog.Fatalf("Failed to set up rules: %v", err)
}
StartAlertRoutines(alertManager, 10*parsedPollInterval, dbClient)
runServer(serverURL)
}
func main() {
defer common.LogPanic()
var err error
// Global init to initialize glog and parse arguments.
common.InitWithMetrics("internal", graphiteServer)
if !*local {
*targetList = metadata.Must(metadata.ProjectGet("datahopper_internal_targets"))
}
targets := strings.Split(*targetList, " ")
glog.Infof("Targets: %#v", targets)
codenameDB, err = leveldb.OpenFile(*codenameDbDir, nil)
if err != nil && errors.IsCorrupted(err) {
codenameDB, err = leveldb.RecoverFile(*codenameDbDir, nil)
}
if err != nil {
glog.Fatalf("Failed to open codename leveldb at %s: %s", *codenameDbDir, err)
}
// Initialize the buildbot database.
db, err = buildbot.NewRemoteDB(*buildbotDbHost)
if err != nil {
glog.Fatal(err)
}
var redirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port)
if !*local {
redirectURL = "https://internal.skia.org/oauth2callback/"
}
if err := login.InitFromMetadataOrJSON(redirectURL, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST); err != nil {
glog.Fatalf("Failed to initialize login system: %s", err)
}
if *local {
webhook.InitRequestSaltForTesting()
} else {
webhook.MustInitRequestSaltFromMetadata()
}
repos = gitinfo.NewRepoMap(*workdir)
// Ingest Android framework builds.
go func() {
glog.Infof("Starting.")
// In this case we don't want a backoff transport since the Apiary backend
// seems to fail a lot, so we basically want to fall back to polling if a
// call fails.
client, err := auth.NewJWTServiceAccountClient("", "", &http.Transport{Dial: util.DialTimeout}, androidbuildinternal.AndroidbuildInternalScope, storage.CloudPlatformScope)
if err != nil {
glog.Fatalf("Unable to create authenticated client: %s", err)
}
buildService, err := androidbuildinternal.New(client)
if err != nil {
glog.Fatalf("Failed to obtain Android build service: %v", err)
}
glog.Infof("Ready to start loop.")
step(targets, buildService)
for _ = range time.Tick(*period) {
step(targets, buildService)
}
}()
r := mux.NewRouter()
r.HandleFunc("/", indexHandler)
r.HandleFunc("/builders/{codename}/builds/{buildNumber}", redirectHandler)
r.HandleFunc("/builders/{codename}", builderRedirectHandler)
r.HandleFunc("/ingestBuild", ingestBuildHandler)
r.HandleFunc("/loginstatus/", login.StatusHandler)
r.HandleFunc("/logout/", login.LogoutHandler)
r.HandleFunc("/oauth2callback/", login.OAuth2CallbackHandler)
http.Handle("/", util.LoggingGzipRequestResponse(r))
glog.Fatal(http.ListenAndServe(*port, nil))
}
func main() {
defer common.LogPanic()
var err error
mainTimer := timer.New("main init")
// Setup DB flags.
dbConf := database.ConfigFromFlags(db.PROD_DB_HOST, db.PROD_DB_PORT, database.USER_RW, db.PROD_DB_NAME, db.MigrationSteps())
// Global init to initialize
common.InitWithMetrics("skiacorrectness", graphiteServer)
v, err := skiaversion.GetVersion()
if err != nil {
glog.Fatalf("Unable to retrieve version: %s", err)
}
glog.Infof("Version %s, built at %s", v.Commit, v.Date)
// Enable the memory profiler if memProfile was set.
// TODO(stephana): This should be moved to a HTTP endpoint that
// only responds to internal IP addresses/ports.
if *memProfile > 0 {
time.AfterFunc(*memProfile, func() {
glog.Infof("Writing Memory Profile")
f, err := ioutil.TempFile("./", "memory-profile")
if err != nil {
glog.Fatalf("Unable to create memory profile file: %s", err)
}
if err := pprof.WriteHeapProfile(f); err != nil {
glog.Fatalf("Unable to write memory profile file: %v", err)
}
util.Close(f)
glog.Infof("Memory profile written to %s", f.Name())
os.Exit(0)
})
}
if *cpuProfile > 0 {
glog.Infof("Writing CPU Profile")
f, err := ioutil.TempFile("./", "cpu-profile")
if err != nil {
glog.Fatalf("Unable to create cpu profile file: %s", err)
}
if err := pprof.StartCPUProfile(f); err != nil {
glog.Fatalf("Unable to write cpu profile file: %v", err)
}
time.AfterFunc(*cpuProfile, func() {
pprof.StopCPUProfile()
util.Close(f)
glog.Infof("CPU profile written to %s", f.Name())
os.Exit(0)
})
}
// Init this module.
Init()
// Initialize submodules.
filediffstore.Init()
// Set up login
// TODO (stephana): Factor out to go/login/login.go and removed hard coded
// values.
var cookieSalt = "notverysecret"
var clientID = "31977622648-ubjke2f3staq6ouas64r31h8f8tcbiqp.apps.googleusercontent.com"
var clientSecret = "rK-kRY71CXmcg0v9I9KIgWci"
var useRedirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port)
if !*local {
cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT))
clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID))
clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET))
useRedirectURL = *redirectURL
}
login.Init(clientID, clientSecret, useRedirectURL, cookieSalt, login.DEFAULT_SCOPE, *authWhiteList, *local)
// get the Oauthclient if necessary.
client := getOAuthClient(*oauthCacheFile)
// Set up the cache implementation to use.
cacheFactory := filediffstore.MemCacheFactory
if *redisHost != "" {
cacheFactory = func(uniqueId string, codec util.LRUCodec) util.LRUCache {
return redisutil.NewRedisLRUCache(*redisHost, *redisDB, uniqueId, codec)
}
}
// Get the expecations storage, the filediff storage and the tilestore.
diffStore, err := filediffstore.NewFileDiffStore(client, *imageDir, *gsBucketName, filediffstore.DEFAULT_GS_IMG_DIR_NAME, cacheFactory, filediffstore.RECOMMENDED_WORKER_POOL_SIZE)
if err != nil {
glog.Fatalf("Allocating DiffStore failed: %s", err)
}
if !*local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
vdb, err := dbConf.NewVersionedDB()
if err != nil {
glog.Fatal(err)
}
if !vdb.IsLatestVersion() {
glog.Fatal("Wrong DB version. Please updated to latest version.")
}
digestStore, err := digeststore.New(*storageDir)
if err != nil {
glog.Fatal(err)
}
eventBus := eventbus.New(nil)
storages = &storage.Storage{
DiffStore: diffStore,
ExpectationsStore: expstorage.NewCachingExpectationStore(expstorage.NewSQLExpectationStore(vdb), eventBus),
IgnoreStore: ignore.NewSQLIgnoreStore(vdb),
TileStore: filetilestore.NewFileTileStore(*tileStoreDir, config.DATASET_GOLD, 2*time.Minute),
DigestStore: digestStore,
NCommits: *nCommits,
EventBus: eventBus,
TrybotResults: trybot.NewTrybotResultStorage(vdb),
RietveldAPI: rietveld.New(*rietveldURL, nil),
}
if err := history.Init(storages, *nTilesToBackfill); err != nil {
glog.Fatalf("Unable to initialize history package: %s", err)
}
if blamer, err = blame.New(storages); err != nil {
glog.Fatalf("Unable to create blamer: %s", err)
}
if err := ignore.Init(storages.IgnoreStore); err != nil {
glog.Fatalf("Failed to start monitoring for expired ignore rules: %s", err)
}
// Enable the experimental features.
tallies, err = tally.New(storages)
if err != nil {
glog.Fatalf("Failed to build tallies: %s", err)
}
paramsetSum = paramsets.New(tallies, storages)
if !*local {
*issueTrackerKey = metadata.Must(metadata.ProjectGet(metadata.APIKEY))
}
issueTracker = issues.NewIssueTracker(*issueTrackerKey)
summaries, err = summary.New(storages, tallies, blamer)
if err != nil {
glog.Fatalf("Failed to build summary: %s", err)
}
statusWatcher, err = status.New(storages)
if err != nil {
glog.Fatalf("Failed to initialize status watcher: %s", err)
}
imgFS := NewURLAwareFileServer(*imageDir, IMAGE_URL_PREFIX)
pathToURLConverter = imgFS.GetURL
if err := warmer.Init(storages, summaries, tallies); err != nil {
glog.Fatalf("Failed to initialize the warmer: %s", err)
}
mainTimer.Stop()
// Everything is wired up at this point. Run the self tests if requested.
if *selfTest {
search.SelfTest(storages, tallies, blamer, paramsetSum)
}
router := mux.NewRouter()
// Set up the resource to serve the image files.
router.PathPrefix(IMAGE_URL_PREFIX).Handler(imgFS.Handler)
// New Polymer based UI endpoints.
router.PathPrefix("/res/").HandlerFunc(makeResourceHandler())
// TODO(stephana): Remove the 'poly' prefix from all the handlers and clean
// up main2.go by either merging it it into main.go or making it clearer that
// it contains all the handlers. Make it clearer what variables are shared
// between the different file.
// All the handlers will be prefixed with poly to differentiate it from the
// angular code until the angular code is removed.
router.HandleFunc(OAUTH2_CALLBACK_PATH, login.OAuth2CallbackHandler)
router.HandleFunc("/", byBlameHandler).Methods("GET")
router.HandleFunc("/list", templateHandler("list.html")).Methods("GET")
router.HandleFunc("/_/details", polyDetailsHandler).Methods("GET")
router.HandleFunc("/_/diff", polyDiffJSONDigestHandler).Methods("GET")
router.HandleFunc("/_/hashes", polyAllHashesHandler).Methods("GET")
router.HandleFunc("/_/ignores", polyIgnoresJSONHandler).Methods("GET")
router.HandleFunc("/_/ignores/add/", polyIgnoresAddHandler).Methods("POST")
router.HandleFunc("/_/ignores/del/{id}", polyIgnoresDeleteHandler).Methods("POST")
router.HandleFunc("/_/ignores/save/{id}", polyIgnoresUpdateHandler).Methods("POST")
router.HandleFunc("/_/list", polyListTestsHandler).Methods("GET")
router.HandleFunc("/_/paramset", polyParamsHandler).Methods("GET")
router.HandleFunc("/_/nxn", nxnJSONHandler).Methods("GET")
// TODO(stephana): Once /_/search3 is stable it will replace /_/search and the
// /search*.html pages will be consolidated into one.
router.HandleFunc("/_/search", polySearchJSONHandler).Methods("GET")
router.HandleFunc("/_/search3", search3JSONHandler).Methods("GET")
router.HandleFunc("/_/status/{test}", polyTestStatusHandler).Methods("GET")
router.HandleFunc("/_/test", polyTestHandler).Methods("POST")
router.HandleFunc("/_/triage", polyTriageHandler).Methods("POST")
router.HandleFunc("/_/triagelog", polyTriageLogHandler).Methods("GET")
router.HandleFunc("/_/triagelog/undo", triageUndoHandler).Methods("POST")
router.HandleFunc("/_/failure", failureListJSONHandler).Methods("GET")
router.HandleFunc("/_/failure/clear", failureClearJSONHandler).Methods("POST")
router.HandleFunc("/_/trybot", listTrybotsJSONHandler).Methods("GET")
router.HandleFunc("/byblame", byBlameHandler).Methods("GET")
router.HandleFunc("/cluster", templateHandler("cluster.html")).Methods("GET")
router.HandleFunc("/search2", search2Handler).Methods("GET")
router.HandleFunc("/cmp/{test}", templateHandler("compare.html")).Methods("GET")
router.HandleFunc("/detail", templateHandler("single.html")).Methods("GET")
router.HandleFunc("/diff", templateHandler("diff.html")).Methods("GET")
router.HandleFunc("/help", templateHandler("help.html")).Methods("GET")
router.HandleFunc("/ignores", templateHandler("ignores.html")).Methods("GET")
router.HandleFunc("/loginstatus/", login.StatusHandler)
router.HandleFunc("/logout/", login.LogoutHandler)
router.HandleFunc("/search", templateHandler("search.html")).Methods("GET")
router.HandleFunc("/triagelog", templateHandler("triagelog.html")).Methods("GET")
router.HandleFunc("/trybot", templateHandler("trybot.html")).Methods("GET")
router.HandleFunc("/failures", templateHandler("failures.html")).Methods("GET")
// Add the necessary middleware and have the router handle all requests.
// By structuring the middleware this way we only log requests that are
// authenticated.
rootHandler := util.LoggingGzipRequestResponse(router)
if *forceLogin {
rootHandler = login.ForceAuth(rootHandler, OAUTH2_CALLBACK_PATH)
}
// The polyStatusHandler is being polled, so we exclude it from logging.
http.HandleFunc("/_/trstatus", polyStatusHandler)
http.Handle("/", rootHandler)
// Start the server
glog.Infoln("Serving on http://127.0.0.1" + *port)
glog.Fatal(http.ListenAndServe(*port, nil))
}
func main() {
defer common.LogPanic()
// Setup flags.
dbConf := db.DBConfigFromFlags()
ctfeutil.PreExecuteTemplateHook = func() {
// Don't use cached templates in local mode.
if *local {
reloadTemplates()
}
}
common.InitWithMetrics("ctfe", graphiteServer)
v, err := skiaversion.GetVersion()
if err != nil {
glog.Fatal(err)
}
glog.Infof("Version %s, built at %s", v.Commit, v.Date)
Init()
serverURL := "https://" + *host
if *local {
serverURL = "http://" + *host + *port
}
// By default use a set of credentials setup for localhost access.
var cookieSalt = "notverysecret"
var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com"
var clientSecret = "cw0IosPu4yjaG2KWmppj2guj"
var redirectURL = serverURL + "/oauth2callback/"
if !*local {
cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT))
clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID))
clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET))
}
login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, *local)
if *local {
webhook.InitRequestSaltForTesting()
} else {
webhook.MustInitRequestSaltFromMetadata()
}
glog.Info("CloneOrUpdate complete")
// Initialize the ctfe database.
if !*local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
if err := dbConf.InitDB(); err != nil {
glog.Fatal(err)
}
startCtfeMetrics()
// Start the repeated tasks scheduler.
go repeatedTasksScheduler()
runServer(serverURL)
}
func main() {
defer common.LogPanic()
common.InitWithMetrics("autoroll", graphiteServer)
Init()
v, err := skiaversion.GetVersion()
if err != nil {
glog.Fatal(err)
}
glog.Infof("Version %s, built at %s", v.Commit, v.Date)
if *testing {
*useMetadata = false
}
// Create the Rietveld client.
client, err := auth.NewClientFromIdAndSecret(rietveld.CLIENT_ID, rietveld.CLIENT_SECRET, path.Join(*workdir, "oauth_cache"), rietveld.OAUTH_SCOPES...)
if err != nil {
glog.Fatal(err)
}
r := rietveld.New(RIETVELD_URL, client)
// Retrieve the list of extra CQ trybots.
// TODO(borenet): Make this editable on the web front-end.
cqExtraTrybots, err := getCQExtraTrybots()
if err != nil {
glog.Fatal(err)
}
// Retrieve the initial email list.
emails, err := getSheriff()
if err != nil {
glog.Fatal(err)
}
// Start the autoroller.
arb, err = autoroller.NewAutoRoller(*workdir, cqExtraTrybots, emails, r, time.Minute, 15*time.Minute, *depot_tools)
if err != nil {
glog.Fatal(err)
}
// Update the current sheriff in a loop.
go func() {
for _ = range time.Tick(30 * time.Minute) {
emails, err := getSheriff()
if err != nil {
glog.Errorf("Failed to retrieve current sheriff: %s", err)
} else {
arb.SetEmails(emails)
}
}
}()
serverURL := "https://" + *host
if *testing {
serverURL = "http://" + *host + *port
}
// By default use a set of credentials setup for localhost access.
var cookieSalt = "notverysecret"
var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com"
var clientSecret = "cw0IosPu4yjaG2KWmppj2guj"
var redirectURL = serverURL + "/oauth2callback/"
if *useMetadata {
cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT))
clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID))
clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET))
}
login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, false)
runServer(serverURL)
}
// Call once at startup to read requestSalt from project metadata.
func MustInitRequestSaltFromMetadata() {
saltBase64 := metadata.Must(metadata.ProjectGet(metadata.WEBHOOK_REQUEST_SALT))
if err := setRequestSaltFromBase64([]byte(saltBase64)); err != nil {
glog.Fatalf("Could not decode salt from %s: %v", metadata.WEBHOOK_REQUEST_SALT, err)
}
}
func main() {
defer common.LogPanic()
var err error
// Setup flags.
dbConf := buildbot.DBConfigFromFlags() // Global init to initialize glog and parse arguments.
// Global init to initialize glog and parse arguments.
common.InitWithMetrics("internal", graphiteServer)
if !*local {
*targetList = metadata.Must(metadata.ProjectGet("datahopper_internal_targets"))
}
targets := strings.Split(*targetList, " ")
glog.Infof("Targets: %#v", targets)
codenameDB, err = leveldb.OpenFile(*codenameDbDir, nil)
if err != nil && errors.IsCorrupted(err) {
codenameDB, err = leveldb.RecoverFile(*codenameDbDir, nil)
}
if err != nil {
glog.Fatalf("Failed to open codename leveldb at %s: %s", *codenameDbDir, err)
}
// Initialize the buildbot database.
if !*local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
if err := dbConf.InitDB(); err != nil {
glog.Fatal(err)
}
var cookieSalt = "notverysecret"
var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com"
var clientSecret = "cw0IosPu4yjaG2KWmppj2guj"
var redirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port)
if !*local {
cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT))
clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID))
clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET))
redirectURL = "https://internal.skia.org/oauth2callback/"
}
login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, *local)
// Ingest Android framework builds.
go func() {
glog.Infof("Starting.")
repos := gitinfo.NewRepoMap(*workdir)
// In this case we don't want a backoff transport since the Apiary backend
// seems to fail a lot, so we basically want to fall back to polling if a
// call fails.
client, err := auth.NewClientWithTransport(*local, *oauthCacheFile, "", &http.Transport{Dial: util.DialTimeout}, androidbuildinternal.AndroidbuildInternalScope, storage.CloudPlatformScope)
if err != nil {
glog.Fatalf("Unable to create installed app oauth client:%s", err)
}
buildService, err := androidbuildinternal.New(client)
if err != nil {
glog.Fatalf("Failed to obtain Android build service: %v", err)
}
glog.Infof("Ready to start loop.")
step(targets, buildService, repos)
for _ = range time.Tick(*period) {
step(targets, buildService, repos)
}
}()
r := mux.NewRouter()
r.HandleFunc("/", indexHandler)
r.HandleFunc("/builders/{codename}/builds/{buildNumber}", redirectHandler)
r.HandleFunc("/builders/{codename}", builderRedirectHandler)
r.HandleFunc("/loginstatus/", login.StatusHandler)
r.HandleFunc("/logout/", login.LogoutHandler)
r.HandleFunc("/oauth2callback/", login.OAuth2CallbackHandler)
http.Handle("/", util.LoggingGzipRequestResponse(r))
glog.Fatal(http.ListenAndServe(*port, nil))
}