func main() {
flag.Parse()
setDefaults()
common.DecodeTomlFile(*configFilename, &config.Config)
var err error
if client, err = auth.NewClient(config.Config.Common.DoOAuth, config.Config.Common.OAuthCacheFile, storage.DevstorageFull_controlScope); err != nil {
glog.Fatalf("Failed to create authenticated HTTP client: %s", err)
}
if store, err = storage.New(client); err != nil {
glog.Fatalf("Failed to create storage service client: %s", err)
}
resourcePath, err := setup()
if err != nil {
glog.Fatalf("Couldn't setup: %s", err)
}
checkCPPTemplate(resourcePath)
for {
fuzz, err := generator.Fuzz()
if err != nil {
glog.Fatalf("Couldn't create a fuzz: %s", err)
}
hash, err := writeFuzz(fuzz)
if err != nil {
glog.Fatalf("Couldn't create the fuzz hash: %s", err)
}
runtimeErr := runFuzz(hash)
if runtimeErr != nil {
glog.Errorf("Looks like the fuzz %s failed: %s", hash, runtimeErr)
}
if err := uploadFuzz(hash, fuzz, runtimeErr); err != nil {
glog.Fatalf("Couldn't upload the fuzz results (%s): %s", hash, err)
}
}
}
func Init() {
defer common.LogPanic()
rand.Seed(time.Now().UnixNano())
common.InitWithMetricsCB("fuzzer", func() string {
common.DecodeTomlFile(*configFilename, &config.Config)
return config.Config.FrontEnd.GraphiteServer
})
if config.Config.Common.ResourcePath == "" {
_, filename, _, _ := runtime.Caller(0)
config.Config.Common.ResourcePath = filepath.Join(filepath.Dir(filename), "../..")
}
path, err := filepath.Abs(config.Config.Common.ResourcePath)
if err != nil {
glog.Fatalf("Couldn't get absolute path to fuzzer resources: %s", err)
}
if err := os.Chdir(path); err != nil {
glog.Fatal(err)
}
indexTemplate = htemplate.Must(htemplate.ParseFiles(
filepath.Join(path, "templates/index.html"),
filepath.Join(path, "templates/header.html"),
filepath.Join(path, "templates/titlebar.html"),
filepath.Join(path, "templates/footer.html"),
))
if client, err = auth.NewClient(config.Config.Common.DoOAuth, config.Config.Common.OAuthCacheFile, storage.DevstorageFullControlScope); err != nil {
glog.Fatalf("Failed to create authenticated HTTP client: %s", err)
}
if store, err = storage.New(client); err != nil {
glog.Fatalf("Failed to create storage service client: %s", err)
}
}
func main() {
defer common.LogPanic()
// Setup DB flags.
dbConf := db.DBConfigFromFlags()
common.InitWithMetricsCB("ingest", func() string {
common.DecodeTomlFile(*configFilename, &config)
return config.Common.GraphiteServer
})
// Initialize the database. We might not need the oauth dialog if it fails.
if !config.Common.Local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
if err := dbConf.InitDB(); err != nil {
glog.Fatal(err)
}
// Get a backoff transport.
transport := util.NewBackOffTransport()
// Determine the oauth scopes we are going to use. Storage is used by all
// ingesters.
scopes := []string{storage.CloudPlatformScope}
for _, ingesterConfig := range config.Ingesters {
if ingesterConfig.ConstructorName == gconfig.CONSTRUCTOR_ANDROID_GOLD {
scopes = append(scopes, androidbuildinternal.AndroidbuildInternalScope)
}
}
// Initialize the oauth client that is used to access all scopes.
var client *http.Client
var err error
if config.Common.Local {
if config.Common.DoOAuth {
client, err = auth.InstalledAppClient(config.Common.OAuthCacheFile,
config.Common.OAuthClientSecretFile,
transport, scopes...)
if err != nil {
glog.Fatalf("Failed to auth: %s", err)
}
} else {
client = nil
// Add back service account access here when it's fixed.
}
} else {
// Assume we are on a GCE instance.
client = auth.GCEServiceAccountClient(transport)
}
// Initialize the ingester and gold ingester.
ingester.Init(client)
if _, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD]; ok {
if err := goldingester.Init(client, filepath.Join(config.Ingesters["gold"].StatusDir, "android-build-info")); err != nil {
glog.Fatalf("Unable to initialize GoldIngester: %s", err)
}
}
// TODO(stephana): Cleanup the way trybots are instantiated so that each trybot has it's own
// database connection and they are all instantiated the same way instead of the
// one-off approaches.
if ingesterConf, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD_TRYBOT]; ok {
dbConf := &database.DatabaseConfig{
Host: ingesterConf.DBHost,
Port: ingesterConf.DBPort,
User: database.USER_RW,
Name: ingesterConf.DBName,
MigrationSteps: golddb.MigrationSteps(),
}
if !config.Common.Local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
vdb, err := dbConf.NewVersionedDB()
if err != nil {
glog.Fatalf("Unable to open db connection: %s", err)
}
goldtrybot.Init(vdb)
}
git, err := gitinfo.NewGitInfo(config.Common.GitRepoDir, true, false)
if err != nil {
glog.Fatalf("Failed loading Git info: %s\n", err)
}
for dataset, ingesterConfig := range config.Ingesters {
// Get duration equivalent to the number of days.
minDuration := 24 * time.Hour * time.Duration(ingesterConfig.MinDays)
constructorName := ingesterConfig.ConstructorName
if constructorName == "" {
constructorName = dataset
}
constructor := ingester.Constructor(constructorName)
resultIngester := constructor()
glog.Infof("Process name: %s", dataset)
startProcess := NewIngestionProcess(git,
config.Common.TileDir,
dataset,
resultIngester,
ingesterConfig.ExtraParams,
ingesterConfig.RunEvery.Duration,
ingesterConfig.NCommits,
minDuration,
ingesterConfig.StatusDir,
ingesterConfig.MetricName)
startProcess()
}
select {}
}
func Init() {
defer common.LogPanic()
rand.Seed(time.Now().UnixNano())
config.Fiddle.UseChroot = false
config.Fiddle.Port = ":8000"
config.Fiddle.ResourcePath = ""
config.Fiddle.CachePath = "../../cache"
config.Fiddle.InoutPath = "../../inout"
config.Fiddle.UseMetadata = true
common.DecodeTomlFile(*configFilename, &config.Fiddle)
path, err := filepath.Abs(config.Fiddle.ResourcePath)
if err != nil {
glog.Fatal(err)
}
if err := os.Chdir(path); err != nil {
glog.Fatal(err)
}
codeTemplate = template.Must(template.ParseFiles(filepath.Join(path, "templates/template.cpp")))
gypTemplate = template.Must(template.ParseFiles(filepath.Join(path, "templates/template.gyp")))
indexTemplate = htemplate.Must(htemplate.ParseFiles(
filepath.Join(path, "templates/index.html"),
filepath.Join(path, "templates/titlebar.html"),
filepath.Join(path, "templates/content.html"),
filepath.Join(path, "templates/header.html"),
filepath.Join(path, "templates/footer.html"),
))
iframeTemplate = htemplate.Must(htemplate.ParseFiles(
filepath.Join(path, "templates/iframe.html"),
filepath.Join(path, "templates/content.html"),
filepath.Join(path, "templates/header.html"),
filepath.Join(path, "templates/footer.html"),
))
recentTemplate = htemplate.Must(htemplate.ParseFiles(
filepath.Join(path, "templates/recent.html"),
filepath.Join(path, "templates/titlebar.html"),
filepath.Join(path, "templates/header.html"),
filepath.Join(path, "templates/footer.html"),
))
workspaceTemplate = htemplate.Must(htemplate.ParseFiles(
filepath.Join(path, "templates/workspace.html"),
filepath.Join(path, "templates/titlebar.html"),
filepath.Join(path, "templates/content.html"),
filepath.Join(path, "templates/header.html"),
filepath.Join(path, "templates/footer.html"),
))
// The git command returns output of the format:
//
// f672cead70404080a991ebfb86c38316a4589b23 2014-04-27 19:21:51 +0000
//
logOutput, err := exec.RunSimple(`git log --format=%H%x20%ai HEAD^..HEAD`)
if err != nil {
panic(err)
}
logInfo := strings.Split(logOutput, " ")
gitHash = logInfo[0]
gitInfo = logInfo[1] + " " + logInfo[2] + " " + logInfo[0][0:6]
if config.Fiddle.UseMetadata {
password := metadata.MustGet(PASSWORD_METADATA_KEY)
// The IP address of the database is found here:
// https://console.developers.google.com/project/31977622648/sql/instances/webtry/overview
// And 3306 is the default port for MySQL.
db, err = sql.Open("mysql", fmt.Sprintf("webtry:%s@tcp(173.194.83.52:3306)/webtry?parseTime=true", password))
if err != nil {
glog.Fatalf("ERROR: Failed to open connection to SQL server: %q\n", err)
}
} else {
// Fallback to sqlite for local use.
db, err = sql.Open("sqlite3", "./webtry.db")
if err != nil {
glog.Errorf("Failed to open: %q\n", err)
panic(err)
}
sql := `CREATE TABLE IF NOT EXISTS source_images (
id INTEGER PRIMARY KEY NOT NULL,
image MEDIUMBLOB DEFAULT '' NOT NULL, -- formatted as a PNG.
width INTEGER DEFAULT 0 NOT NULL,
height INTEGER DEFAULT 0 NOT NULL,
create_ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
hidden INTEGER DEFAULT 0 NOT NULL
)`
_, err = db.Exec(sql)
if err != nil {
glog.Errorf("Creating source_images table failed: %s", err)
}
sql = `CREATE TABLE IF NOT EXISTS webtry (
code TEXT DEFAULT '' NOT NULL,
create_ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
hash CHAR(64) DEFAULT '' NOT NULL,
width INTEGER DEFAULT 256 NOT NULL,
height INTEGER DEFAULT 256 NOT NULL,
source_image_id INTEGER DEFAULT 0 NOT NULL,
PRIMARY KEY(hash)
)`
_, err = db.Exec(sql)
if err != nil {
glog.Errorf("Creating webtry table failed: %s", err)
}
sql = `CREATE TABLE IF NOT EXISTS workspace (
name CHAR(64) DEFAULT '' NOT NULL,
create_ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
PRIMARY KEY(name)
)`
_, err = db.Exec(sql)
if err != nil {
glog.Errorf("Creating workspace table failed: %s", err)
}
sql = `CREATE TABLE IF NOT EXISTS workspacetry (
name CHAR(64) DEFAULT '' NOT NULL,
create_ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
hash CHAR(64) DEFAULT '' NOT NULL,
width INTEGER DEFAULT 256 NOT NULL,
height INTEGER DEFAULT 256 NOT NULL,
hidden INTEGER DEFAULT 0 NOT NULL,
source_image_id INTEGER DEFAULT 0 NOT NULL,
FOREIGN KEY (name) REFERENCES workspace(name)
)`
_, err = db.Exec(sql)
if err != nil {
glog.Errorf("Creating workspacetry table failed: %s", err)
}
}
// Ping the database to keep the connection fresh.
go func() {
c := time.Tick(1 * time.Minute)
for _ = range c {
if err := db.Ping(); err != nil {
glog.Errorf("Database failed to respond: %q\n", err)
}
}
}()
common.InitWithMetrics("webtry", graphiteServer)
writeOutAllSourceImages()
}