示例#1
0
func main() {
	flag.Parse()

	setDefaults()

	common.DecodeTomlFile(*configFilename, &config.Config)

	var err error
	if client, err = auth.NewClient(config.Config.Common.DoOAuth, config.Config.Common.OAuthCacheFile, storage.DevstorageFull_controlScope); err != nil {
		glog.Fatalf("Failed to create authenticated HTTP client: %s", err)
	}

	if store, err = storage.New(client); err != nil {
		glog.Fatalf("Failed to create storage service client: %s", err)
	}

	resourcePath, err := setup()
	if err != nil {
		glog.Fatalf("Couldn't setup: %s", err)
	}

	checkCPPTemplate(resourcePath)

	for {
		fuzz, err := generator.Fuzz()
		if err != nil {
			glog.Fatalf("Couldn't create a fuzz: %s", err)
		}

		hash, err := writeFuzz(fuzz)
		if err != nil {
			glog.Fatalf("Couldn't create the fuzz hash: %s", err)
		}

		runtimeErr := runFuzz(hash)
		if runtimeErr != nil {
			glog.Errorf("Looks like the fuzz %s failed: %s", hash, runtimeErr)
		}

		if err := uploadFuzz(hash, fuzz, runtimeErr); err != nil {
			glog.Fatalf("Couldn't upload the fuzz results (%s): %s", hash, err)
		}
	}
}
示例#2
0
func Init() {
	defer common.LogPanic()

	rand.Seed(time.Now().UnixNano())

	common.InitWithMetricsCB("fuzzer", func() string {
		common.DecodeTomlFile(*configFilename, &config.Config)
		return config.Config.FrontEnd.GraphiteServer
	})

	if config.Config.Common.ResourcePath == "" {
		_, filename, _, _ := runtime.Caller(0)
		config.Config.Common.ResourcePath = filepath.Join(filepath.Dir(filename), "../..")
	}

	path, err := filepath.Abs(config.Config.Common.ResourcePath)
	if err != nil {
		glog.Fatalf("Couldn't get absolute path to fuzzer resources: %s", err)
	}
	if err := os.Chdir(path); err != nil {
		glog.Fatal(err)
	}

	indexTemplate = htemplate.Must(htemplate.ParseFiles(
		filepath.Join(path, "templates/index.html"),
		filepath.Join(path, "templates/header.html"),
		filepath.Join(path, "templates/titlebar.html"),
		filepath.Join(path, "templates/footer.html"),
	))

	if client, err = auth.NewClient(config.Config.Common.DoOAuth, config.Config.Common.OAuthCacheFile, storage.DevstorageFullControlScope); err != nil {
		glog.Fatalf("Failed to create authenticated HTTP client: %s", err)
	}

	if store, err = storage.New(client); err != nil {
		glog.Fatalf("Failed to create storage service client: %s", err)
	}
}
示例#3
0
func main() {
	defer common.LogPanic()
	// Setup DB flags.
	dbConf := db.DBConfigFromFlags()

	common.InitWithMetricsCB("ingest", func() string {
		common.DecodeTomlFile(*configFilename, &config)
		return config.Common.GraphiteServer
	})

	// Initialize the database. We might not need the oauth dialog if it fails.
	if !config.Common.Local {
		if err := dbConf.GetPasswordFromMetadata(); err != nil {
			glog.Fatal(err)
		}
	}
	if err := dbConf.InitDB(); err != nil {
		glog.Fatal(err)
	}

	// Get a backoff transport.
	transport := util.NewBackOffTransport()

	// Determine the oauth scopes we are going to use. Storage is used by all
	// ingesters.
	scopes := []string{storage.CloudPlatformScope}
	for _, ingesterConfig := range config.Ingesters {
		if ingesterConfig.ConstructorName == gconfig.CONSTRUCTOR_ANDROID_GOLD {
			scopes = append(scopes, androidbuildinternal.AndroidbuildInternalScope)
		}
	}

	// Initialize the oauth client that is used to access all scopes.
	var client *http.Client
	var err error
	if config.Common.Local {
		if config.Common.DoOAuth {
			client, err = auth.InstalledAppClient(config.Common.OAuthCacheFile,
				config.Common.OAuthClientSecretFile,
				transport, scopes...)
			if err != nil {
				glog.Fatalf("Failed to auth: %s", err)
			}
		} else {
			client = nil
			// Add back service account access here when it's fixed.
		}
	} else {
		// Assume we are on a GCE instance.
		client = auth.GCEServiceAccountClient(transport)
	}

	// Initialize the ingester and gold ingester.
	ingester.Init(client)
	if _, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD]; ok {
		if err := goldingester.Init(client, filepath.Join(config.Ingesters["gold"].StatusDir, "android-build-info")); err != nil {
			glog.Fatalf("Unable to initialize GoldIngester: %s", err)
		}
	}

	// TODO(stephana): Cleanup the way trybots are instantiated so that each trybot has it's own
	// database connection and they are all instantiated the same way instead of the
	// one-off approaches.
	if ingesterConf, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD_TRYBOT]; ok {
		dbConf := &database.DatabaseConfig{
			Host:           ingesterConf.DBHost,
			Port:           ingesterConf.DBPort,
			User:           database.USER_RW,
			Name:           ingesterConf.DBName,
			MigrationSteps: golddb.MigrationSteps(),
		}

		if !config.Common.Local {
			if err := dbConf.GetPasswordFromMetadata(); err != nil {
				glog.Fatal(err)
			}
		}
		vdb, err := dbConf.NewVersionedDB()
		if err != nil {
			glog.Fatalf("Unable to open db connection: %s", err)
		}
		goldtrybot.Init(vdb)
	}

	git, err := gitinfo.NewGitInfo(config.Common.GitRepoDir, true, false)
	if err != nil {
		glog.Fatalf("Failed loading Git info: %s\n", err)
	}

	for dataset, ingesterConfig := range config.Ingesters {
		// Get duration equivalent to the number of days.
		minDuration := 24 * time.Hour * time.Duration(ingesterConfig.MinDays)

		constructorName := ingesterConfig.ConstructorName
		if constructorName == "" {
			constructorName = dataset
		}

		constructor := ingester.Constructor(constructorName)
		resultIngester := constructor()

		glog.Infof("Process name: %s", dataset)
		startProcess := NewIngestionProcess(git,
			config.Common.TileDir,
			dataset,
			resultIngester,
			ingesterConfig.ExtraParams,
			ingesterConfig.RunEvery.Duration,
			ingesterConfig.NCommits,
			minDuration,
			ingesterConfig.StatusDir,
			ingesterConfig.MetricName)
		startProcess()
	}

	select {}
}
示例#4
0
func Init() {
	defer common.LogPanic()
	rand.Seed(time.Now().UnixNano())

	config.Fiddle.UseChroot = false
	config.Fiddle.Port = ":8000"
	config.Fiddle.ResourcePath = ""
	config.Fiddle.CachePath = "../../cache"
	config.Fiddle.InoutPath = "../../inout"
	config.Fiddle.UseMetadata = true

	common.DecodeTomlFile(*configFilename, &config.Fiddle)

	path, err := filepath.Abs(config.Fiddle.ResourcePath)
	if err != nil {
		glog.Fatal(err)
	}
	if err := os.Chdir(path); err != nil {
		glog.Fatal(err)
	}

	codeTemplate = template.Must(template.ParseFiles(filepath.Join(path, "templates/template.cpp")))
	gypTemplate = template.Must(template.ParseFiles(filepath.Join(path, "templates/template.gyp")))
	indexTemplate = htemplate.Must(htemplate.ParseFiles(
		filepath.Join(path, "templates/index.html"),
		filepath.Join(path, "templates/titlebar.html"),
		filepath.Join(path, "templates/content.html"),
		filepath.Join(path, "templates/header.html"),
		filepath.Join(path, "templates/footer.html"),
	))
	iframeTemplate = htemplate.Must(htemplate.ParseFiles(
		filepath.Join(path, "templates/iframe.html"),
		filepath.Join(path, "templates/content.html"),
		filepath.Join(path, "templates/header.html"),
		filepath.Join(path, "templates/footer.html"),
	))
	recentTemplate = htemplate.Must(htemplate.ParseFiles(
		filepath.Join(path, "templates/recent.html"),
		filepath.Join(path, "templates/titlebar.html"),
		filepath.Join(path, "templates/header.html"),
		filepath.Join(path, "templates/footer.html"),
	))
	workspaceTemplate = htemplate.Must(htemplate.ParseFiles(
		filepath.Join(path, "templates/workspace.html"),
		filepath.Join(path, "templates/titlebar.html"),
		filepath.Join(path, "templates/content.html"),
		filepath.Join(path, "templates/header.html"),
		filepath.Join(path, "templates/footer.html"),
	))

	// The git command returns output of the format:
	//
	//   f672cead70404080a991ebfb86c38316a4589b23 2014-04-27 19:21:51 +0000
	//
	logOutput, err := exec.RunSimple(`git log --format=%H%x20%ai HEAD^..HEAD`)
	if err != nil {
		panic(err)
	}
	logInfo := strings.Split(logOutput, " ")
	gitHash = logInfo[0]
	gitInfo = logInfo[1] + " " + logInfo[2] + " " + logInfo[0][0:6]

	if config.Fiddle.UseMetadata {
		password := metadata.MustGet(PASSWORD_METADATA_KEY)

		// The IP address of the database is found here:
		//    https://console.developers.google.com/project/31977622648/sql/instances/webtry/overview
		// And 3306 is the default port for MySQL.
		db, err = sql.Open("mysql", fmt.Sprintf("webtry:%s@tcp(173.194.83.52:3306)/webtry?parseTime=true", password))
		if err != nil {
			glog.Fatalf("ERROR: Failed to open connection to SQL server: %q\n", err)
		}
	} else {
		// Fallback to sqlite for local use.
		db, err = sql.Open("sqlite3", "./webtry.db")
		if err != nil {
			glog.Errorf("Failed to open: %q\n", err)
			panic(err)
		}
		sql := `CREATE TABLE IF NOT EXISTS source_images (
             id        INTEGER     PRIMARY KEY                NOT NULL,
             image     MEDIUMBLOB  DEFAULT ''                 NOT NULL, -- formatted as a PNG.
             width     INTEGER     DEFAULT 0                  NOT NULL,
             height    INTEGER     DEFAULT 0                  NOT NULL,
             create_ts TIMESTAMP   DEFAULT CURRENT_TIMESTAMP  NOT NULL,
             hidden    INTEGER     DEFAULT 0                  NOT NULL
             )`
		_, err = db.Exec(sql)
		if err != nil {
			glog.Errorf("Creating source_images table failed: %s", err)
		}

		sql = `CREATE TABLE IF NOT EXISTS webtry (
             code               TEXT      DEFAULT ''                 NOT NULL,
             create_ts          TIMESTAMP DEFAULT CURRENT_TIMESTAMP  NOT NULL,
             hash               CHAR(64)  DEFAULT ''                 NOT NULL,
             width              INTEGER   DEFAULT 256                NOT NULL,
             height             INTEGER   DEFAULT 256                NOT NULL,
             source_image_id    INTEGER   DEFAULT 0                  NOT NULL,

             PRIMARY KEY(hash)
            )`
		_, err = db.Exec(sql)
		if err != nil {
			glog.Errorf("Creating webtry table failed: %s", err)
		}

		sql = `CREATE TABLE IF NOT EXISTS workspace (
          name      CHAR(64)  DEFAULT ''                 NOT NULL,
          create_ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP  NOT NULL,
          PRIMARY KEY(name)
        )`
		_, err = db.Exec(sql)
		if err != nil {
			glog.Errorf("Creating workspace table failed: %s", err)
		}

		sql = `CREATE TABLE IF NOT EXISTS workspacetry (
          name               CHAR(64)  DEFAULT ''                 NOT NULL,
          create_ts          TIMESTAMP DEFAULT CURRENT_TIMESTAMP  NOT NULL,
          hash               CHAR(64)  DEFAULT ''                 NOT NULL,
          width              INTEGER   DEFAULT 256                NOT NULL,
          height             INTEGER   DEFAULT 256                NOT NULL,
          hidden             INTEGER   DEFAULT 0                  NOT NULL,
          source_image_id    INTEGER   DEFAULT 0                  NOT NULL,

          FOREIGN KEY (name)   REFERENCES workspace(name)
        )`
		_, err = db.Exec(sql)
		if err != nil {
			glog.Errorf("Creating workspacetry table failed: %s", err)
		}
	}

	// Ping the database to keep the connection fresh.
	go func() {
		c := time.Tick(1 * time.Minute)
		for _ = range c {
			if err := db.Ping(); err != nil {
				glog.Errorf("Database failed to respond: %q\n", err)
			}
		}
	}()

	common.InitWithMetrics("webtry", graphiteServer)

	writeOutAllSourceImages()
}