func TestTrybotIngester(t *testing.T) { // Set up the test database. testDb := testutil.SetupMySQLTestDatabase(t, db.MigrationSteps()) defer testDb.Close(t) conf := testutil.LocalTestDatabaseConfig(db.MigrationSteps()) vdb, err := conf.NewVersionedDB() assert.Nil(t, err) Init(vdb) resultStore := NewTrybotResultStorage(vdb) // Get the constructor and create an instance of gold-trybot ingester. tbIngester := ingester.Constructor(config.CONSTRUCTOR_GOLD_TRYBOT)() _ = ingestFile(t, tbIngester, TEST_FILE) tries, err := resultStore.Get(TEST_ISSUE) assert.Nil(t, err) assert.Equal(t, 1, len(tries.Bots)) var targetBot *BotResults for _, v := range tries.Bots { assert.Equal(t, len(TEST_DIGESTS), len(v.TestResults)) targetBot = v } for _, entry := range targetBot.TestResults { assert.True(t, TEST_DIGESTS[tries.Digests[entry.DigestIdx]]) } allIssues, _, err := resultStore.List(0, 10) assert.Nil(t, err) assert.Equal(t, 1, len(allIssues)) assert.Equal(t, TEST_ISSUE, allIssues[0].Issue) time.Sleep(time.Second) _ = ingestFile(t, tbIngester, TEST_FILE_2) tries, err = resultStore.Get(TEST_ISSUE) assert.Nil(t, err) assert.Equal(t, 1, len(tries.Bots)) assert.Equal(t, 1, len(tries.Digests)) assert.Equal(t, MODIFIED_DIGEST, tries.Digests[0]) for _, bot := range tries.Bots { for _, result := range bot.TestResults { assert.Equal(t, MODIFIED_DIGEST, result.digest) assert.Equal(t, 0, result.DigestIdx) } } }
func TestTrybotIngester(t *testing.T) { // Set up the test database. testDb := testutil.SetupMySQLTestDatabase(t, db.MigrationSteps()) defer testDb.Close(t) conf := testutil.LocalTestDatabaseConfig(db.MigrationSteps()) vdb, err := conf.NewVersionedDB() assert.Nil(t, err) Init(vdb) // Get the constructor and create an instance of gold-trybot ingester. tbIngester := ingester.Constructor(config.CONSTRUCTOR_GOLD_TRYBOT)() _ = ingestFile(t, tbIngester, TEST_FILE) tries, err := Get(vdb, TEST_ISSUE) assert.Nil(t, err) assert.Equal(t, len(TEST_DIGESTS), len(tries)) for _, entry := range tries { assert.True(t, TEST_DIGESTS[entry.Digest]) } allIssues, err := List(vdb, 10) assert.Nil(t, err) assert.Equal(t, 1, len(allIssues)) assert.Equal(t, TEST_ISSUE, allIssues[0]) time.Sleep(time.Second) _ = ingestFile(t, tbIngester, TEST_FILE_2) tries, err = Get(vdb, TEST_ISSUE) assert.Nil(t, err) assert.Equal(t, len(TEST_DIGESTS), len(tries)) for _, entry := range tries { assert.Equal(t, MODIFIED_DIGEST, entry.Digest) } }
func main() { defer common.LogPanic() // Setup DB flags. dbConf := db.DBConfigFromFlags() common.InitWithMetricsCB("ingest", func() string { common.DecodeTomlFile(*configFilename, &config) return config.Common.GraphiteServer }) // Initialize the database. We might not need the oauth dialog if it fails. if !config.Common.Local { if err := dbConf.GetPasswordFromMetadata(); err != nil { glog.Fatal(err) } } if err := dbConf.InitDB(); err != nil { glog.Fatal(err) } // Get a backoff transport. transport := util.NewBackOffTransport() // Determine the oauth scopes we are going to use. Storage is used by all // ingesters. scopes := []string{storage.CloudPlatformScope} for _, ingesterConfig := range config.Ingesters { if ingesterConfig.ConstructorName == gconfig.CONSTRUCTOR_ANDROID_GOLD { scopes = append(scopes, androidbuildinternal.AndroidbuildInternalScope) } } // Initialize the oauth client that is used to access all scopes. var client *http.Client var err error if config.Common.Local { if config.Common.DoOAuth { client, err = auth.InstalledAppClient(config.Common.OAuthCacheFile, config.Common.OAuthClientSecretFile, transport, scopes...) if err != nil { glog.Fatalf("Failed to auth: %s", err) } } else { client = nil // Add back service account access here when it's fixed. } } else { // Assume we are on a GCE instance. client = auth.GCEServiceAccountClient(transport) } // Initialize the ingester and gold ingester. ingester.Init(client) if _, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD]; ok { if err := goldingester.Init(client, filepath.Join(config.Ingesters["gold"].StatusDir, "android-build-info")); err != nil { glog.Fatalf("Unable to initialize GoldIngester: %s", err) } } // TODO(stephana): Cleanup the way trybots are instantiated so that each trybot has it's own // database connection and they are all instantiated the same way instead of the // one-off approaches. if ingesterConf, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD_TRYBOT]; ok { dbConf := &database.DatabaseConfig{ Host: ingesterConf.DBHost, Port: ingesterConf.DBPort, User: database.USER_RW, Name: ingesterConf.DBName, MigrationSteps: golddb.MigrationSteps(), } if !config.Common.Local { if err := dbConf.GetPasswordFromMetadata(); err != nil { glog.Fatal(err) } } vdb, err := dbConf.NewVersionedDB() if err != nil { glog.Fatalf("Unable to open db connection: %s", err) } goldtrybot.Init(vdb) } git, err := gitinfo.NewGitInfo(config.Common.GitRepoDir, true, false) if err != nil { glog.Fatalf("Failed loading Git info: %s\n", err) } for dataset, ingesterConfig := range config.Ingesters { // Get duration equivalent to the number of days. minDuration := 24 * time.Hour * time.Duration(ingesterConfig.MinDays) constructorName := ingesterConfig.ConstructorName if constructorName == "" { constructorName = dataset } constructor := ingester.Constructor(constructorName) resultIngester := constructor() glog.Infof("Process name: %s", dataset) startProcess := NewIngestionProcess(git, config.Common.TileDir, dataset, resultIngester, ingesterConfig.ExtraParams, ingesterConfig.RunEvery.Duration, ingesterConfig.NCommits, minDuration, ingesterConfig.StatusDir, ingesterConfig.MetricName) startProcess() } select {} }