// getOAuthClient returns an oauth client.
func getOAuthClient(cacheFilePath string) *http.Client {
var client *http.Client
var err error
if *doOauth || *local {
if *local {
// Load client secrets file. The client_secret.json file must come from
// project id: 470362608618, which is whitelisted for access to
// gs://chromium-skia-gm.
client, err = auth.NewClientWithTransport(true, cacheFilePath, *clientSecretFile, nil, auth.SCOPE_READ_ONLY)
if err != nil {
glog.Fatalf("Failed to auth: (Did you download the client_secret.json file from the project with ID: google.com:chrome-skia #470362608618 ?) %s", err)
}
} else {
// Load client id and secret from metadata.
clientId := metadata.Must(metadata.ProjectGet(metadata.CHROMIUM_SKIA_GM_CLIENT_ID))
clientSecret := metadata.Must(metadata.ProjectGet(metadata.CHROMIUM_SKIA_GM_CLIENT_SECRET))
client, err = auth.NewClientFromIdAndSecret(clientId, clientSecret, cacheFilePath, auth.SCOPE_READ_ONLY)
if err != nil {
glog.Fatalf("Failed to auth: %s", err)
}
}
}
return client
}
// NewGsUtil initializes and returns a utility for CT interations with Google
// Storage. If client is nil then auth.NewClient is invoked.
func NewGsUtil(client *http.Client) (*GsUtil, error) {
if client == nil {
var err error
client, err = auth.NewClientWithTransport(true, GSTokenPath, ClientSecretPath, nil, auth.SCOPE_FULL_CONTROL)
if err != nil {
return nil, err
}
}
service, err := storage.New(client)
if err != nil {
return nil, fmt.Errorf("Failed to create interface to Google Storage: %s", err)
}
return &GsUtil{client: client, service: service}, nil
}
func main() {
defer common.LogPanic()
common.Init()
args := flag.Args()
if len(args) != 3 {
glog.Errorf("Expected arguments: branch target buildID")
glog.Errorf("i.e.: git_master-skia razor-userdebug 1772442")
os.Exit(1)
}
// Set the arguments necessary to lookup the git hash.
branch := args[0]
target := args[1]
buildID := args[2]
glog.Infof("Branch, target, buildID: %s, %s, %s", branch, target, buildID)
// In this case we don't want a backoff transport since the Apiary backend
// seems to fail a lot, so we basically want to fall back to polling if a
// call fails.
client, err := auth.NewClientWithTransport(*local, OAUTH_CACHE_FILEPATH, CLIENT_SECRET_FILEPATH, &http.Transport{Dial: util.DialTimeout}, androidbuildinternal.AndroidbuildInternalScope, storage.CloudPlatformScope)
if err != nil {
glog.Fatalf("Unable to create installed app oauth client:%s", err)
}
f, err := androidbuild.New("/tmp/android-gold-ingest", client)
if err != nil {
glog.Fatalf("Failed to construct client: %s", err)
}
for {
r, err := f.Get(branch, target, buildID)
if err != nil {
glog.Errorf("Failed to get requested info: %s", err)
time.Sleep(1 * time.Minute)
continue
}
if r != nil {
glog.Infof("Successfully found: %#v", *r)
}
time.Sleep(1 * time.Minute)
}
}
func main() {
defer common.LogPanic()
var err error
// Setup flags.
dbConf := buildbot.DBConfigFromFlags() // Global init to initialize glog and parse arguments.
// Global init to initialize glog and parse arguments.
common.InitWithMetrics("internal", graphiteServer)
if !*local {
*targetList = metadata.Must(metadata.ProjectGet("datahopper_internal_targets"))
}
targets := strings.Split(*targetList, " ")
glog.Infof("Targets: %#v", targets)
codenameDB, err = leveldb.OpenFile(*codenameDbDir, nil)
if err != nil && errors.IsCorrupted(err) {
codenameDB, err = leveldb.RecoverFile(*codenameDbDir, nil)
}
if err != nil {
glog.Fatalf("Failed to open codename leveldb at %s: %s", *codenameDbDir, err)
}
// Initialize the buildbot database.
if !*local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
if err := dbConf.InitDB(); err != nil {
glog.Fatal(err)
}
var cookieSalt = "notverysecret"
var clientID = "31977622648-1873k0c1e5edaka4adpv1ppvhr5id3qm.apps.googleusercontent.com"
var clientSecret = "cw0IosPu4yjaG2KWmppj2guj"
var redirectURL = fmt.Sprintf("http://localhost%s/oauth2callback/", *port)
if !*local {
cookieSalt = metadata.Must(metadata.ProjectGet(metadata.COOKIESALT))
clientID = metadata.Must(metadata.ProjectGet(metadata.CLIENT_ID))
clientSecret = metadata.Must(metadata.ProjectGet(metadata.CLIENT_SECRET))
redirectURL = "https://internal.skia.org/oauth2callback/"
}
login.Init(clientID, clientSecret, redirectURL, cookieSalt, login.DEFAULT_SCOPE, login.DEFAULT_DOMAIN_WHITELIST, *local)
// Ingest Android framework builds.
go func() {
glog.Infof("Starting.")
repos := gitinfo.NewRepoMap(*workdir)
// In this case we don't want a backoff transport since the Apiary backend
// seems to fail a lot, so we basically want to fall back to polling if a
// call fails.
client, err := auth.NewClientWithTransport(*local, *oauthCacheFile, "", &http.Transport{Dial: util.DialTimeout}, androidbuildinternal.AndroidbuildInternalScope, storage.CloudPlatformScope)
if err != nil {
glog.Fatalf("Unable to create installed app oauth client:%s", err)
}
buildService, err := androidbuildinternal.New(client)
if err != nil {
glog.Fatalf("Failed to obtain Android build service: %v", err)
}
glog.Infof("Ready to start loop.")
step(targets, buildService, repos)
for _ = range time.Tick(*period) {
step(targets, buildService, repos)
}
}()
r := mux.NewRouter()
r.HandleFunc("/", indexHandler)
r.HandleFunc("/builders/{codename}/builds/{buildNumber}", redirectHandler)
r.HandleFunc("/builders/{codename}", builderRedirectHandler)
r.HandleFunc("/loginstatus/", login.StatusHandler)
r.HandleFunc("/logout/", login.LogoutHandler)
r.HandleFunc("/oauth2callback/", login.OAuth2CallbackHandler)
http.Handle("/", util.LoggingGzipRequestResponse(r))
glog.Fatal(http.ListenAndServe(*port, nil))
}
func main() {
defer common.LogPanic()
// Setup DB flags.
dbConf := db.DBConfigFromFlags()
common.InitWithMetricsCB("ingest", func() string {
common.DecodeTomlFile(*configFilename, &config)
return config.Common.GraphiteServer
})
// Initialize the database. We might not need the oauth dialog if it fails.
if !config.Common.Local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
if err := dbConf.InitDB(); err != nil {
glog.Fatal(err)
}
// Determine the oauth scopes we are going to use. Storage is used by all ingesters.
scopes := []string{storage.CloudPlatformScope}
for _, ingesterConfig := range config.Ingesters {
if ingesterConfig.ConstructorName == gconfig.CONSTRUCTOR_ANDROID_GOLD {
scopes = append(scopes, androidbuildinternal.AndroidbuildInternalScope)
}
}
var client *http.Client = nil
if config.Common.DoOAuth || config.Common.Local {
var err error
client, err = auth.NewClientWithTransport(config.Common.Local, config.Common.OAuthCacheFile, config.Common.OAuthClientSecretFile, nil, scopes...)
if err != nil {
glog.Fatalf("Failed to auth: %s", err)
}
}
// Initialize the ingester and gold ingester.
ingester.Init(client)
if _, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD]; ok {
if err := goldingester.Init(client, filepath.Join(config.Ingesters["gold"].StatusDir, "android-build-info")); err != nil {
glog.Fatalf("Unable to initialize GoldIngester: %s", err)
}
}
// TODO(stephana): Cleanup the way trybots are instantiated so that each trybot has it's own
// database connection and they are all instantiated the same way instead of the
// one-off approaches.
if ingesterConf, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD_TRYBOT]; ok {
dbConf := &database.DatabaseConfig{
Host: ingesterConf.DBHost,
Port: ingesterConf.DBPort,
User: database.USER_RW,
Name: ingesterConf.DBName,
MigrationSteps: golddb.MigrationSteps(),
}
if !config.Common.Local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
vdb, err := dbConf.NewVersionedDB()
if err != nil {
glog.Fatalf("Unable to open db connection: %s", err)
}
goldtrybot.Init(vdb)
}
git, err := gitinfo.NewGitInfo(config.Common.GitRepoDir, true, false)
if err != nil {
glog.Fatalf("Failed loading Git info: %s\n", err)
}
for dataset, ingesterConfig := range config.Ingesters {
// Get duration equivalent to the number of days.
minDuration := 24 * time.Hour * time.Duration(ingesterConfig.MinDays)
constructorName := ingesterConfig.ConstructorName
if constructorName == "" {
constructorName = dataset
}
constructor := ingester.Constructor(constructorName)
resultIngester := constructor()
glog.Infof("Process name: %s", dataset)
startProcess := NewIngestionProcess(git,
config.Common.TileDir,
dataset,
resultIngester,
ingesterConfig.ExtraParams,
ingesterConfig.RunEvery.Duration,
ingesterConfig.NCommits,
minDuration,
ingesterConfig.StatusDir,
ingesterConfig.MetricName)
startProcess()
}
select {}
}
