func main() { flag.Parse() setDefaults() common.DecodeTomlFile(*configFilename, &config.Config) var err error if client, err = auth.NewClient(config.Config.Common.DoOAuth, config.Config.Common.OAuthCacheFile, storage.DevstorageFull_controlScope); err != nil { glog.Fatalf("Failed to create authenticated HTTP client: %s", err) } if store, err = storage.New(client); err != nil { glog.Fatalf("Failed to create storage service client: %s", err) } resourcePath, err := setup() if err != nil { glog.Fatalf("Couldn't setup: %s", err) } checkCPPTemplate(resourcePath) for { fuzz, err := generator.Fuzz() if err != nil { glog.Fatalf("Couldn't create a fuzz: %s", err) } hash, err := writeFuzz(fuzz) if err != nil { glog.Fatalf("Couldn't create the fuzz hash: %s", err) } runtimeErr := runFuzz(hash) if runtimeErr != nil { glog.Errorf("Looks like the fuzz %s failed: %s", hash, runtimeErr) } if err := uploadFuzz(hash, fuzz, runtimeErr); err != nil { glog.Fatalf("Couldn't upload the fuzz results (%s): %s", hash, err) } } }
func Init() { defer common.LogPanic() rand.Seed(time.Now().UnixNano()) common.InitWithMetricsCB("fuzzer", func() string { common.DecodeTomlFile(*configFilename, &config.Config) return config.Config.FrontEnd.GraphiteServer }) if config.Config.Common.ResourcePath == "" { _, filename, _, _ := runtime.Caller(0) config.Config.Common.ResourcePath = filepath.Join(filepath.Dir(filename), "../..") } path, err := filepath.Abs(config.Config.Common.ResourcePath) if err != nil { glog.Fatalf("Couldn't get absolute path to fuzzer resources: %s", err) } if err := os.Chdir(path); err != nil { glog.Fatal(err) } indexTemplate = htemplate.Must(htemplate.ParseFiles( filepath.Join(path, "templates/index.html"), filepath.Join(path, "templates/header.html"), filepath.Join(path, "templates/titlebar.html"), filepath.Join(path, "templates/footer.html"), )) if client, err = auth.NewClient(config.Config.Common.DoOAuth, config.Config.Common.OAuthCacheFile, storage.DevstorageFullControlScope); err != nil { glog.Fatalf("Failed to create authenticated HTTP client: %s", err) } if store, err = storage.New(client); err != nil { glog.Fatalf("Failed to create storage service client: %s", err) } }
func main() { defer common.LogPanic() // Setup DB flags. dbConf := db.DBConfigFromFlags() common.InitWithMetricsCB("ingest", func() string { common.DecodeTomlFile(*configFilename, &config) return config.Common.GraphiteServer }) // Initialize the database. We might not need the oauth dialog if it fails. if !config.Common.Local { if err := dbConf.GetPasswordFromMetadata(); err != nil { glog.Fatal(err) } } if err := dbConf.InitDB(); err != nil { glog.Fatal(err) } // Get a backoff transport. transport := util.NewBackOffTransport() // Determine the oauth scopes we are going to use. Storage is used by all // ingesters. scopes := []string{storage.CloudPlatformScope} for _, ingesterConfig := range config.Ingesters { if ingesterConfig.ConstructorName == gconfig.CONSTRUCTOR_ANDROID_GOLD { scopes = append(scopes, androidbuildinternal.AndroidbuildInternalScope) } } // Initialize the oauth client that is used to access all scopes. var client *http.Client var err error if config.Common.Local { if config.Common.DoOAuth { client, err = auth.InstalledAppClient(config.Common.OAuthCacheFile, config.Common.OAuthClientSecretFile, transport, scopes...) if err != nil { glog.Fatalf("Failed to auth: %s", err) } } else { client = nil // Add back service account access here when it's fixed. } } else { // Assume we are on a GCE instance. client = auth.GCEServiceAccountClient(transport) } // Initialize the ingester and gold ingester. ingester.Init(client) if _, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD]; ok { if err := goldingester.Init(client, filepath.Join(config.Ingesters["gold"].StatusDir, "android-build-info")); err != nil { glog.Fatalf("Unable to initialize GoldIngester: %s", err) } } // TODO(stephana): Cleanup the way trybots are instantiated so that each trybot has it's own // database connection and they are all instantiated the same way instead of the // one-off approaches. if ingesterConf, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD_TRYBOT]; ok { dbConf := &database.DatabaseConfig{ Host: ingesterConf.DBHost, Port: ingesterConf.DBPort, User: database.USER_RW, Name: ingesterConf.DBName, MigrationSteps: golddb.MigrationSteps(), } if !config.Common.Local { if err := dbConf.GetPasswordFromMetadata(); err != nil { glog.Fatal(err) } } vdb, err := dbConf.NewVersionedDB() if err != nil { glog.Fatalf("Unable to open db connection: %s", err) } goldtrybot.Init(vdb) } git, err := gitinfo.NewGitInfo(config.Common.GitRepoDir, true, false) if err != nil { glog.Fatalf("Failed loading Git info: %s\n", err) } for dataset, ingesterConfig := range config.Ingesters { // Get duration equivalent to the number of days. minDuration := 24 * time.Hour * time.Duration(ingesterConfig.MinDays) constructorName := ingesterConfig.ConstructorName if constructorName == "" { constructorName = dataset } constructor := ingester.Constructor(constructorName) resultIngester := constructor() glog.Infof("Process name: %s", dataset) startProcess := NewIngestionProcess(git, config.Common.TileDir, dataset, resultIngester, ingesterConfig.ExtraParams, ingesterConfig.RunEvery.Duration, ingesterConfig.NCommits, minDuration, ingesterConfig.StatusDir, ingesterConfig.MetricName) startProcess() } select {} }
func Init() { defer common.LogPanic() rand.Seed(time.Now().UnixNano()) config.Fiddle.UseChroot = false config.Fiddle.Port = ":8000" config.Fiddle.ResourcePath = "" config.Fiddle.CachePath = "../../cache" config.Fiddle.InoutPath = "../../inout" config.Fiddle.UseMetadata = true common.DecodeTomlFile(*configFilename, &config.Fiddle) path, err := filepath.Abs(config.Fiddle.ResourcePath) if err != nil { glog.Fatal(err) } if err := os.Chdir(path); err != nil { glog.Fatal(err) } codeTemplate = template.Must(template.ParseFiles(filepath.Join(path, "templates/template.cpp"))) gypTemplate = template.Must(template.ParseFiles(filepath.Join(path, "templates/template.gyp"))) indexTemplate = htemplate.Must(htemplate.ParseFiles( filepath.Join(path, "templates/index.html"), filepath.Join(path, "templates/titlebar.html"), filepath.Join(path, "templates/content.html"), filepath.Join(path, "templates/header.html"), filepath.Join(path, "templates/footer.html"), )) iframeTemplate = htemplate.Must(htemplate.ParseFiles( filepath.Join(path, "templates/iframe.html"), filepath.Join(path, "templates/content.html"), filepath.Join(path, "templates/header.html"), filepath.Join(path, "templates/footer.html"), )) recentTemplate = htemplate.Must(htemplate.ParseFiles( filepath.Join(path, "templates/recent.html"), filepath.Join(path, "templates/titlebar.html"), filepath.Join(path, "templates/header.html"), filepath.Join(path, "templates/footer.html"), )) workspaceTemplate = htemplate.Must(htemplate.ParseFiles( filepath.Join(path, "templates/workspace.html"), filepath.Join(path, "templates/titlebar.html"), filepath.Join(path, "templates/content.html"), filepath.Join(path, "templates/header.html"), filepath.Join(path, "templates/footer.html"), )) // The git command returns output of the format: // // f672cead70404080a991ebfb86c38316a4589b23 2014-04-27 19:21:51 +0000 // logOutput, err := exec.RunSimple(`git log --format=%H%x20%ai HEAD^..HEAD`) if err != nil { panic(err) } logInfo := strings.Split(logOutput, " ") gitHash = logInfo[0] gitInfo = logInfo[1] + " " + logInfo[2] + " " + logInfo[0][0:6] if config.Fiddle.UseMetadata { password := metadata.MustGet(PASSWORD_METADATA_KEY) // The IP address of the database is found here: // https://console.developers.google.com/project/31977622648/sql/instances/webtry/overview // And 3306 is the default port for MySQL. db, err = sql.Open("mysql", fmt.Sprintf("webtry:%s@tcp(173.194.83.52:3306)/webtry?parseTime=true", password)) if err != nil { glog.Fatalf("ERROR: Failed to open connection to SQL server: %q\n", err) } } else { // Fallback to sqlite for local use. db, err = sql.Open("sqlite3", "./webtry.db") if err != nil { glog.Errorf("Failed to open: %q\n", err) panic(err) } sql := `CREATE TABLE IF NOT EXISTS source_images ( id INTEGER PRIMARY KEY NOT NULL, image MEDIUMBLOB DEFAULT '' NOT NULL, -- formatted as a PNG. width INTEGER DEFAULT 0 NOT NULL, height INTEGER DEFAULT 0 NOT NULL, create_ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, hidden INTEGER DEFAULT 0 NOT NULL )` _, err = db.Exec(sql) if err != nil { glog.Errorf("Creating source_images table failed: %s", err) } sql = `CREATE TABLE IF NOT EXISTS webtry ( code TEXT DEFAULT '' NOT NULL, create_ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, hash CHAR(64) DEFAULT '' NOT NULL, width INTEGER DEFAULT 256 NOT NULL, height INTEGER DEFAULT 256 NOT NULL, source_image_id INTEGER DEFAULT 0 NOT NULL, PRIMARY KEY(hash) )` _, err = db.Exec(sql) if err != nil { glog.Errorf("Creating webtry table failed: %s", err) } sql = `CREATE TABLE IF NOT EXISTS workspace ( name CHAR(64) DEFAULT '' NOT NULL, create_ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, PRIMARY KEY(name) )` _, err = db.Exec(sql) if err != nil { glog.Errorf("Creating workspace table failed: %s", err) } sql = `CREATE TABLE IF NOT EXISTS workspacetry ( name CHAR(64) DEFAULT '' NOT NULL, create_ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, hash CHAR(64) DEFAULT '' NOT NULL, width INTEGER DEFAULT 256 NOT NULL, height INTEGER DEFAULT 256 NOT NULL, hidden INTEGER DEFAULT 0 NOT NULL, source_image_id INTEGER DEFAULT 0 NOT NULL, FOREIGN KEY (name) REFERENCES workspace(name) )` _, err = db.Exec(sql) if err != nil { glog.Errorf("Creating workspacetry table failed: %s", err) } } // Ping the database to keep the connection fresh. go func() { c := time.Tick(1 * time.Minute) for _ = range c { if err := db.Ping(); err != nil { glog.Errorf("Database failed to respond: %q\n", err) } } }() common.InitWithMetrics("webtry", graphiteServer) writeOutAllSourceImages() }