示例#1
0
func main() {
	defer common.LogPanic()
	// Setup DB flags.
	dbConf := db.DBConfigFromFlags()

	common.InitWithMetricsCB("ingest", func() string {
		common.DecodeTomlFile(*configFilename, &config)
		return config.Common.GraphiteServer
	})

	// Initialize the database. We might not need the oauth dialog if it fails.
	if !config.Common.Local {
		if err := dbConf.GetPasswordFromMetadata(); err != nil {
			glog.Fatal(err)
		}
	}
	if err := dbConf.InitDB(); err != nil {
		glog.Fatal(err)
	}

	// Get a backoff transport.
	transport := util.NewBackOffTransport()

	// Determine the oauth scopes we are going to use. Storage is used by all
	// ingesters.
	scopes := []string{storage.CloudPlatformScope}
	for _, ingesterConfig := range config.Ingesters {
		if ingesterConfig.ConstructorName == gconfig.CONSTRUCTOR_ANDROID_GOLD {
			scopes = append(scopes, androidbuildinternal.AndroidbuildInternalScope)
		}
	}

	// Initialize the oauth client that is used to access all scopes.
	var client *http.Client
	var err error
	if config.Common.Local {
		if config.Common.DoOAuth {
			client, err = auth.InstalledAppClient(config.Common.OAuthCacheFile,
				config.Common.OAuthClientSecretFile,
				transport, scopes...)
			if err != nil {
				glog.Fatalf("Failed to auth: %s", err)
			}
		} else {
			client = nil
			// Add back service account access here when it's fixed.
		}
	} else {
		// Assume we are on a GCE instance.
		client = auth.GCEServiceAccountClient(transport)
	}

	// Initialize the ingester and gold ingester.
	ingester.Init(client)
	if _, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD]; ok {
		if err := goldingester.Init(client, filepath.Join(config.Ingesters["gold"].StatusDir, "android-build-info")); err != nil {
			glog.Fatalf("Unable to initialize GoldIngester: %s", err)
		}
	}

	// TODO(stephana): Cleanup the way trybots are instantiated so that each trybot has it's own
	// database connection and they are all instantiated the same way instead of the
	// one-off approaches.
	if ingesterConf, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD_TRYBOT]; ok {
		dbConf := &database.DatabaseConfig{
			Host:           ingesterConf.DBHost,
			Port:           ingesterConf.DBPort,
			User:           database.USER_RW,
			Name:           ingesterConf.DBName,
			MigrationSteps: golddb.MigrationSteps(),
		}

		if !config.Common.Local {
			if err := dbConf.GetPasswordFromMetadata(); err != nil {
				glog.Fatal(err)
			}
		}
		vdb, err := dbConf.NewVersionedDB()
		if err != nil {
			glog.Fatalf("Unable to open db connection: %s", err)
		}
		goldtrybot.Init(vdb)
	}

	git, err := gitinfo.NewGitInfo(config.Common.GitRepoDir, true, false)
	if err != nil {
		glog.Fatalf("Failed loading Git info: %s\n", err)
	}

	for dataset, ingesterConfig := range config.Ingesters {
		// Get duration equivalent to the number of days.
		minDuration := 24 * time.Hour * time.Duration(ingesterConfig.MinDays)

		constructorName := ingesterConfig.ConstructorName
		if constructorName == "" {
			constructorName = dataset
		}

		constructor := ingester.Constructor(constructorName)
		resultIngester := constructor()

		glog.Infof("Process name: %s", dataset)
		startProcess := NewIngestionProcess(git,
			config.Common.TileDir,
			dataset,
			resultIngester,
			ingesterConfig.ExtraParams,
			ingesterConfig.RunEvery.Duration,
			ingesterConfig.NCommits,
			minDuration,
			ingesterConfig.StatusDir,
			ingesterConfig.MetricName)
		startProcess()
	}

	select {}
}
示例#2
0
func main() {
	defer common.LogPanic()
	// Setup DB flags.
	dbConf := idb.DBConfigFromFlags()

	common.InitWithMetrics("skiaperf", graphiteServer)
	Init()
	if !*local {
		if err := dbConf.GetPasswordFromMetadata(); err != nil {
			glog.Fatal(err)
		}
	}
	if err := dbConf.InitDB(); err != nil {
		glog.Fatal(err)
	}

	stats.Start(masterTileBuilder, git)
	alerting.Start(masterTileBuilder)

	var redirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port)
	if !*local {
		redirectURL = "https://perf.skia.org/oauth2callback/"
	}
	if err := login.InitFromMetadataOrJSON(redirectURL, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST); err != nil {
		glog.Fatalf("Failed to initialize the login system: %s", err)
	}

	// Resources are served directly.
	router := mux.NewRouter()

	router.PathPrefix("/res/").HandlerFunc(makeResourceHandler())

	router.HandleFunc("/", templateHandler("index.html"))
	router.HandleFunc("/frame/", templateHandler("frame.html"))
	router.HandleFunc("/shortcuts/", shortcutHandler)
	router.PathPrefix("/tiles/").HandlerFunc(tileHandler)
	router.PathPrefix("/single/").HandlerFunc(singleHandler)
	router.PathPrefix("/query/").HandlerFunc(queryHandler)
	router.HandleFunc("/commits/", commitsHandler)
	router.HandleFunc("/_/commits/", commitsJSONHandler)
	router.HandleFunc("/shortcommits/", shortCommitsHandler)
	router.HandleFunc("/trybots/", trybotHandler)
	router.HandleFunc("/clusters/", templateHandler("clusters.html"))
	router.HandleFunc("/clustering/", clusteringHandler)
	router.PathPrefix("/cl/").HandlerFunc(clHandler)
	router.PathPrefix("/activitylog/").HandlerFunc(activityHandler)
	router.HandleFunc("/alerts/", templateHandler("alerting.html"))
	router.HandleFunc("/alerting/", alertingHandler)
	router.HandleFunc("/alert_reset/", alertResetHandler)
	router.HandleFunc("/annotate/", annotate.Handler)
	router.HandleFunc("/compare/", templateHandler("compare.html"))
	router.HandleFunc("/kernel/", templateHandler("kernel.html"))
	router.HandleFunc("/_/kernel/", kernelJSONHandler)
	router.HandleFunc("/calc/", calcHandler)
	router.HandleFunc("/help/", helpHandler)
	router.HandleFunc("/oauth2callback/", login.OAuth2CallbackHandler)
	router.HandleFunc("/logout/", login.LogoutHandler)
	router.HandleFunc("/loginstatus/", login.StatusHandler)

	http.Handle("/", util.LoggingGzipRequestResponse(router))

	glog.Infoln("Ready to serve.")
	glog.Fatal(http.ListenAndServe(*port, nil))
}
示例#3
0
func main() {
	// Setup DB flags.
	dbConf := db.DBConfigFromFlags()

	common.InitWithMetrics("skiaperf", graphiteServer)
	Init()
	if !*local {
		if err := dbConf.GetPasswordFromMetadata(); err != nil {
			glog.Fatal(err)
		}
	}
	if err := dbConf.InitDB(); err != nil {
		glog.Fatal(err)
	}

	stats.Start(nanoTileStore, git)
	alerting.Start(nanoTileStore, *apikey)

	// By default use a set of credentials setup for localhost access.
	var cookieSalt = "notverysecret"
	var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com"
	var clientSecret = "cw0IosPu4yjaG2KWmppj2guj"
	var redirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port)
	if !*local {
		cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT))
		clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID))
		clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET))
		redirectURL = "https://perf.skia.org/oauth2callback/"
	}
	login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, *local)
	glog.Infoln("Begin loading data.")

	// Resources are served directly.
	router := mux.NewRouter()

	router.PathPrefix("/res/").HandlerFunc(makeResourceHandler())

	router.HandleFunc("/", mainHandler)
	router.HandleFunc("/shortcuts/", shortcutHandler)
	router.PathPrefix("/tiles/").HandlerFunc(tileHandler)
	router.PathPrefix("/single/").HandlerFunc(singleHandler)
	router.PathPrefix("/query/").HandlerFunc(queryHandler)
	router.HandleFunc("/commits/", commitsHandler)
	router.HandleFunc("/shortcommits/", shortCommitsHandler)
	router.HandleFunc("/trybots/", trybotHandler)
	router.HandleFunc("/clusters/", clustersHandler)
	router.HandleFunc("/clustering/", clusteringHandler)
	router.PathPrefix("/cl/").HandlerFunc(clHandler)
	router.PathPrefix("/activitylog/").HandlerFunc(activityHandler)
	router.HandleFunc("/alerts/", alertsHandler)
	router.HandleFunc("/alerting/", alertingHandler)
	router.HandleFunc("/alert_reset/", alertResetHandler)
	router.HandleFunc("/annotate/", annotate.Handler)
	router.HandleFunc("/compare/", compareHandler)
	router.HandleFunc("/calc/", calcHandler)
	router.HandleFunc("/help/", helpHandler)
	router.HandleFunc("/oauth2callback/", login.OAuth2CallbackHandler)
	router.HandleFunc("/logout/", login.LogoutHandler)
	router.HandleFunc("/loginstatus/", login.StatusHandler)

	http.Handle("/", util.LoggingGzipRequestResponse(router))

	glog.Infoln("Ready to serve.")
	glog.Fatal(http.ListenAndServe(*port, nil))
}