func TestTrybotIngester(t *testing.T) {
// Set up the test database.
testDb := testutil.SetupMySQLTestDatabase(t, db.MigrationSteps())
defer testDb.Close(t)
conf := testutil.LocalTestDatabaseConfig(db.MigrationSteps())
vdb, err := conf.NewVersionedDB()
assert.Nil(t, err)
Init(vdb)
resultStore := NewTrybotResultStorage(vdb)
// Get the constructor and create an instance of gold-trybot ingester.
tbIngester := ingester.Constructor(config.CONSTRUCTOR_GOLD_TRYBOT)()
_ = ingestFile(t, tbIngester, TEST_FILE)
tries, err := resultStore.Get(TEST_ISSUE)
assert.Nil(t, err)
assert.Equal(t, 1, len(tries.Bots))
var targetBot *BotResults
for _, v := range tries.Bots {
assert.Equal(t, len(TEST_DIGESTS), len(v.TestResults))
targetBot = v
}
for _, entry := range targetBot.TestResults {
assert.True(t, TEST_DIGESTS[tries.Digests[entry.DigestIdx]])
}
allIssues, _, err := resultStore.List(0, 10)
assert.Nil(t, err)
assert.Equal(t, 1, len(allIssues))
assert.Equal(t, TEST_ISSUE, allIssues[0].Issue)
time.Sleep(time.Second)
_ = ingestFile(t, tbIngester, TEST_FILE_2)
tries, err = resultStore.Get(TEST_ISSUE)
assert.Nil(t, err)
assert.Equal(t, 1, len(tries.Bots))
assert.Equal(t, 1, len(tries.Digests))
assert.Equal(t, MODIFIED_DIGEST, tries.Digests[0])
for _, bot := range tries.Bots {
for _, result := range bot.TestResults {
assert.Equal(t, MODIFIED_DIGEST, result.digest)
assert.Equal(t, 0, result.DigestIdx)
}
}
}
func TestMySQLExpectationsStore(t *testing.T) {
// Set up the test database.
testDb := testutil.SetupMySQLTestDatabase(t, db.MigrationSteps())
defer testDb.Close(t)
conf := testutil.LocalTestDatabaseConfig(db.MigrationSteps())
vdb, err := conf.NewVersionedDB()
assert.Nil(t, err)
// Test the MySQL backed store
sqlStore := NewSQLExpectationStore(vdb)
testExpectationStore(t, sqlStore, nil)
// Test the caching version of the MySQL store.
eventBus := eventbus.New(nil)
cachingStore := NewCachingExpectationStore(sqlStore, eventBus)
testExpectationStore(t, cachingStore, eventBus)
}
func TestSQLIgnoreStore(t *testing.T) {
// Set up the database. This also locks the db until this test is finished
// causing similar tests to wait.
migrationSteps := db.MigrationSteps()
mysqlDB := testutil.SetupMySQLTestDatabase(t, migrationSteps)
defer mysqlDB.Close(t)
vdb, err := testutil.LocalTestDatabaseConfig(migrationSteps).NewVersionedDB()
assert.Nil(t, err)
defer testutils.AssertCloses(t, vdb)
store := NewSQLIgnoreStore(vdb)
testIgnoreStore(t, store)
}
func TestTrybotIngester(t *testing.T) {
// Set up the test database.
testDb := testutil.SetupMySQLTestDatabase(t, db.MigrationSteps())
defer testDb.Close(t)
conf := testutil.LocalTestDatabaseConfig(db.MigrationSteps())
vdb, err := conf.NewVersionedDB()
assert.Nil(t, err)
Init(vdb)
// Get the constructor and create an instance of gold-trybot ingester.
tbIngester := ingester.Constructor(config.CONSTRUCTOR_GOLD_TRYBOT)()
_ = ingestFile(t, tbIngester, TEST_FILE)
tries, err := Get(vdb, TEST_ISSUE)
assert.Nil(t, err)
assert.Equal(t, len(TEST_DIGESTS), len(tries))
for _, entry := range tries {
assert.True(t, TEST_DIGESTS[entry.Digest])
}
allIssues, err := List(vdb, 10)
assert.Nil(t, err)
assert.Equal(t, 1, len(allIssues))
assert.Equal(t, TEST_ISSUE, allIssues[0])
time.Sleep(time.Second)
_ = ingestFile(t, tbIngester, TEST_FILE_2)
tries, err = Get(vdb, TEST_ISSUE)
assert.Nil(t, err)
assert.Equal(t, len(TEST_DIGESTS), len(tries))
for _, entry := range tries {
assert.Equal(t, MODIFIED_DIGEST, entry.Digest)
}
}
func main() {
defer common.LogPanic()
// Setup DB flags.
dbConf := db.DBConfigFromFlags()
common.InitWithMetricsCB("ingest", func() string {
common.DecodeTomlFile(*configFilename, &config)
return config.Common.GraphiteServer
})
// Initialize the database. We might not need the oauth dialog if it fails.
if !config.Common.Local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
if err := dbConf.InitDB(); err != nil {
glog.Fatal(err)
}
// Get a backoff transport.
transport := util.NewBackOffTransport()
// Determine the oauth scopes we are going to use. Storage is used by all
// ingesters.
scopes := []string{storage.CloudPlatformScope}
for _, ingesterConfig := range config.Ingesters {
if ingesterConfig.ConstructorName == gconfig.CONSTRUCTOR_ANDROID_GOLD {
scopes = append(scopes, androidbuildinternal.AndroidbuildInternalScope)
}
}
// Initialize the oauth client that is used to access all scopes.
var client *http.Client
var err error
if config.Common.Local {
if config.Common.DoOAuth {
client, err = auth.InstalledAppClient(config.Common.OAuthCacheFile,
config.Common.OAuthClientSecretFile,
transport, scopes...)
if err != nil {
glog.Fatalf("Failed to auth: %s", err)
}
} else {
client = nil
// Add back service account access here when it's fixed.
}
} else {
// Assume we are on a GCE instance.
client = auth.GCEServiceAccountClient(transport)
}
// Initialize the ingester and gold ingester.
ingester.Init(client)
if _, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD]; ok {
if err := goldingester.Init(client, filepath.Join(config.Ingesters["gold"].StatusDir, "android-build-info")); err != nil {
glog.Fatalf("Unable to initialize GoldIngester: %s", err)
}
}
// TODO(stephana): Cleanup the way trybots are instantiated so that each trybot has it's own
// database connection and they are all instantiated the same way instead of the
// one-off approaches.
if ingesterConf, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD_TRYBOT]; ok {
dbConf := &database.DatabaseConfig{
Host: ingesterConf.DBHost,
Port: ingesterConf.DBPort,
User: database.USER_RW,
Name: ingesterConf.DBName,
MigrationSteps: golddb.MigrationSteps(),
}
if !config.Common.Local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
vdb, err := dbConf.NewVersionedDB()
if err != nil {
glog.Fatalf("Unable to open db connection: %s", err)
}
goldtrybot.Init(vdb)
}
git, err := gitinfo.NewGitInfo(config.Common.GitRepoDir, true, false)
if err != nil {
glog.Fatalf("Failed loading Git info: %s\n", err)
}
for dataset, ingesterConfig := range config.Ingesters {
// Get duration equivalent to the number of days.
minDuration := 24 * time.Hour * time.Duration(ingesterConfig.MinDays)
constructorName := ingesterConfig.ConstructorName
if constructorName == "" {
constructorName = dataset
}
constructor := ingester.Constructor(constructorName)
resultIngester := constructor()
glog.Infof("Process name: %s", dataset)
startProcess := NewIngestionProcess(git,
config.Common.TileDir,
dataset,
resultIngester,
ingesterConfig.ExtraParams,
ingesterConfig.RunEvery.Duration,
ingesterConfig.NCommits,
minDuration,
ingesterConfig.StatusDir,
ingesterConfig.MetricName)
startProcess()
}
select {}
}
func main() {
defer common.LogPanic()
// Set up flags.
dbConf := database.ConfigFromFlags(db.PROD_DB_HOST, db.PROD_DB_PORT, database.USER_ROOT, db.PROD_DB_NAME, db.MigrationSteps())
// Global init to initialize glog and parse arguments.
common.Init()
v, err := skiaversion.GetVersion()
if err != nil {
glog.Fatalf("Unable to retrieve version: %s", err)
}
glog.Infof("Version %s, built at %s", v.Commit, v.Date)
if *promptPassword {
if err := dbConf.PromptForPassword(); err != nil {
glog.Fatal(err)
}
}
vdb, err := dbConf.NewVersionedDB()
if err != nil {
glog.Fatal(err)
}
// Get the current database version
maxDBVersion := vdb.MaxDBVersion()
glog.Infof("Latest database version: %d", maxDBVersion)
dbVersion, err := vdb.DBVersion()
if err != nil {
glog.Fatalf("Unable to retrieve database version. Error: %s", err)
}
glog.Infof("Current database version: %d", dbVersion)
if dbVersion < maxDBVersion {
glog.Infof("Migrating to version: %d", maxDBVersion)
err = vdb.Migrate(maxDBVersion)
if err != nil {
glog.Fatalf("Unable to retrieve database version. Error: %s", err)
}
}
glog.Infoln("Database migration finished.")
}
func main() {
defer common.LogPanic()
var err error
mainTimer := timer.New("main init")
// Setup DB flags.
dbConf := database.ConfigFromFlags(db.PROD_DB_HOST, db.PROD_DB_PORT, database.USER_RW, db.PROD_DB_NAME, db.MigrationSteps())
// Global init to initialize
common.InitWithMetrics("skiacorrectness", graphiteServer)
v, err := skiaversion.GetVersion()
if err != nil {
glog.Fatalf("Unable to retrieve version: %s", err)
}
glog.Infof("Version %s, built at %s", v.Commit, v.Date)
// Enable the memory profiler if memProfile was set.
// TODO(stephana): This should be moved to a HTTP endpoint that
// only responds to internal IP addresses/ports.
if *memProfile > 0 {
time.AfterFunc(*memProfile, func() {
glog.Infof("Writing Memory Profile")
f, err := ioutil.TempFile("./", "memory-profile")
if err != nil {
glog.Fatalf("Unable to create memory profile file: %s", err)
}
if err := pprof.WriteHeapProfile(f); err != nil {
glog.Fatalf("Unable to write memory profile file: %v", err)
}
util.Close(f)
glog.Infof("Memory profile written to %s", f.Name())
os.Exit(0)
})
}
if *cpuProfile > 0 {
glog.Infof("Writing CPU Profile")
f, err := ioutil.TempFile("./", "cpu-profile")
if err != nil {
glog.Fatalf("Unable to create cpu profile file: %s", err)
}
if err := pprof.StartCPUProfile(f); err != nil {
glog.Fatalf("Unable to write cpu profile file: %v", err)
}
time.AfterFunc(*cpuProfile, func() {
pprof.StopCPUProfile()
util.Close(f)
glog.Infof("CPU profile written to %s", f.Name())
os.Exit(0)
})
}
// Init this module.
Init()
// Initialize submodules.
filediffstore.Init()
// Set up login
// TODO (stephana): Factor out to go/login/login.go and removed hard coded
// values.
var cookieSalt = "notverysecret"
var clientID = "31977622648-ubjke2f3staq6ouas64r31h8f8tcbiqp.apps.googleusercontent.com"
var clientSecret = "rK-kRY71CXmcg0v9I9KIgWci"
var useRedirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port)
if !*local {
cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT))
clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID))
clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET))
useRedirectURL = *redirectURL
}
login.Init(clientID, clientSecret, useRedirectURL, cookieSalt, login.DEFAULT_SCOPE, *authWhiteList, *local)
// get the Oauthclient if necessary.
client := getOAuthClient(*oauthCacheFile)
// Set up the cache implementation to use.
cacheFactory := filediffstore.MemCacheFactory
if *redisHost != "" {
cacheFactory = func(uniqueId string, codec util.LRUCodec) util.LRUCache {
return redisutil.NewRedisLRUCache(*redisHost, *redisDB, uniqueId, codec)
}
}
// Get the expecations storage, the filediff storage and the tilestore.
diffStore, err := filediffstore.NewFileDiffStore(client, *imageDir, *gsBucketName, filediffstore.DEFAULT_GS_IMG_DIR_NAME, cacheFactory, filediffstore.RECOMMENDED_WORKER_POOL_SIZE)
if err != nil {
glog.Fatalf("Allocating DiffStore failed: %s", err)
}
if !*local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
vdb, err := dbConf.NewVersionedDB()
if err != nil {
glog.Fatal(err)
}
if !vdb.IsLatestVersion() {
glog.Fatal("Wrong DB version. Please updated to latest version.")
}
digestStore, err := digeststore.New(*storageDir)
if err != nil {
glog.Fatal(err)
}
eventBus := eventbus.New(nil)
storages = &storage.Storage{
DiffStore: diffStore,
ExpectationsStore: expstorage.NewCachingExpectationStore(expstorage.NewSQLExpectationStore(vdb), eventBus),
IgnoreStore: ignore.NewSQLIgnoreStore(vdb),
TileStore: filetilestore.NewFileTileStore(*tileStoreDir, config.DATASET_GOLD, 2*time.Minute),
DigestStore: digestStore,
NCommits: *nCommits,
EventBus: eventBus,
TrybotResults: trybot.NewTrybotResultStorage(vdb),
RietveldAPI: rietveld.New(*rietveldURL, nil),
}
if err := history.Init(storages, *nTilesToBackfill); err != nil {
glog.Fatalf("Unable to initialize history package: %s", err)
}
if blamer, err = blame.New(storages); err != nil {
glog.Fatalf("Unable to create blamer: %s", err)
}
if err := ignore.Init(storages.IgnoreStore); err != nil {
glog.Fatalf("Failed to start monitoring for expired ignore rules: %s", err)
}
// Enable the experimental features.
tallies, err = tally.New(storages)
if err != nil {
glog.Fatalf("Failed to build tallies: %s", err)
}
paramsetSum = paramsets.New(tallies, storages)
if !*local {
*issueTrackerKey = metadata.Must(metadata.ProjectGet(metadata.APIKEY))
}
issueTracker = issues.NewIssueTracker(*issueTrackerKey)
summaries, err = summary.New(storages, tallies, blamer)
if err != nil {
glog.Fatalf("Failed to build summary: %s", err)
}
statusWatcher, err = status.New(storages)
if err != nil {
glog.Fatalf("Failed to initialize status watcher: %s", err)
}
imgFS := NewURLAwareFileServer(*imageDir, IMAGE_URL_PREFIX)
pathToURLConverter = imgFS.GetURL
if err := warmer.Init(storages, summaries, tallies); err != nil {
glog.Fatalf("Failed to initialize the warmer: %s", err)
}
mainTimer.Stop()
// Everything is wired up at this point. Run the self tests if requested.
if *selfTest {
search.SelfTest(storages, tallies, blamer, paramsetSum)
}
router := mux.NewRouter()
// Set up the resource to serve the image files.
router.PathPrefix(IMAGE_URL_PREFIX).Handler(imgFS.Handler)
// New Polymer based UI endpoints.
router.PathPrefix("/res/").HandlerFunc(makeResourceHandler())
// TODO(stephana): Remove the 'poly' prefix from all the handlers and clean
// up main2.go by either merging it it into main.go or making it clearer that
// it contains all the handlers. Make it clearer what variables are shared
// between the different file.
// All the handlers will be prefixed with poly to differentiate it from the
// angular code until the angular code is removed.
router.HandleFunc(OAUTH2_CALLBACK_PATH, login.OAuth2CallbackHandler)
router.HandleFunc("/", byBlameHandler).Methods("GET")
router.HandleFunc("/list", templateHandler("list.html")).Methods("GET")
router.HandleFunc("/_/details", polyDetailsHandler).Methods("GET")
router.HandleFunc("/_/diff", polyDiffJSONDigestHandler).Methods("GET")
router.HandleFunc("/_/hashes", polyAllHashesHandler).Methods("GET")
router.HandleFunc("/_/ignores", polyIgnoresJSONHandler).Methods("GET")
router.HandleFunc("/_/ignores/add/", polyIgnoresAddHandler).Methods("POST")
router.HandleFunc("/_/ignores/del/{id}", polyIgnoresDeleteHandler).Methods("POST")
router.HandleFunc("/_/ignores/save/{id}", polyIgnoresUpdateHandler).Methods("POST")
router.HandleFunc("/_/list", polyListTestsHandler).Methods("GET")
router.HandleFunc("/_/paramset", polyParamsHandler).Methods("GET")
router.HandleFunc("/_/nxn", nxnJSONHandler).Methods("GET")
// TODO(stephana): Once /_/search3 is stable it will replace /_/search and the
// /search*.html pages will be consolidated into one.
router.HandleFunc("/_/search", polySearchJSONHandler).Methods("GET")
router.HandleFunc("/_/search3", search3JSONHandler).Methods("GET")
router.HandleFunc("/_/status/{test}", polyTestStatusHandler).Methods("GET")
router.HandleFunc("/_/test", polyTestHandler).Methods("POST")
router.HandleFunc("/_/triage", polyTriageHandler).Methods("POST")
router.HandleFunc("/_/triagelog", polyTriageLogHandler).Methods("GET")
router.HandleFunc("/_/triagelog/undo", triageUndoHandler).Methods("POST")
router.HandleFunc("/_/failure", failureListJSONHandler).Methods("GET")
router.HandleFunc("/_/failure/clear", failureClearJSONHandler).Methods("POST")
router.HandleFunc("/_/trybot", listTrybotsJSONHandler).Methods("GET")
router.HandleFunc("/byblame", byBlameHandler).Methods("GET")
router.HandleFunc("/cluster", templateHandler("cluster.html")).Methods("GET")
router.HandleFunc("/search2", search2Handler).Methods("GET")
router.HandleFunc("/cmp/{test}", templateHandler("compare.html")).Methods("GET")
router.HandleFunc("/detail", templateHandler("single.html")).Methods("GET")
router.HandleFunc("/diff", templateHandler("diff.html")).Methods("GET")
router.HandleFunc("/help", templateHandler("help.html")).Methods("GET")
router.HandleFunc("/ignores", templateHandler("ignores.html")).Methods("GET")
router.HandleFunc("/loginstatus/", login.StatusHandler)
router.HandleFunc("/logout/", login.LogoutHandler)
router.HandleFunc("/search", templateHandler("search.html")).Methods("GET")
router.HandleFunc("/triagelog", templateHandler("triagelog.html")).Methods("GET")
router.HandleFunc("/trybot", templateHandler("trybot.html")).Methods("GET")
router.HandleFunc("/failures", templateHandler("failures.html")).Methods("GET")
// Add the necessary middleware and have the router handle all requests.
// By structuring the middleware this way we only log requests that are
// authenticated.
rootHandler := util.LoggingGzipRequestResponse(router)
if *forceLogin {
rootHandler = login.ForceAuth(rootHandler, OAUTH2_CALLBACK_PATH)
}
// The polyStatusHandler is being polled, so we exclude it from logging.
http.HandleFunc("/_/trstatus", polyStatusHandler)
http.Handle("/", rootHandler)
// Start the server
glog.Infoln("Serving on http://127.0.0.1" + *port)
glog.Fatal(http.ListenAndServe(*port, nil))
}
