func pullInit() {
hostname, err := os.Hostname()
if err != nil {
// Never call glog before common.Init*.
os.Exit(1)
}
common.InitWithMetrics("pulld."+hostname, graphiteServer)
glog.Infof("Running with hostname: %s", hostname)
client, err := auth.NewClient(*doOauth, *oauthCacheFile,
storage.DevstorageFullControlScope,
compute.ComputeReadonlyScope)
if err != nil {
glog.Fatalf("Failed to create authenticated HTTP client: %s", err)
}
glog.Info("Got authenticated client.")
store, err = storage.New(client)
if err != nil {
glog.Fatalf("Failed to create storage service client: %s", err)
}
step(client, store, hostname)
timeCh := time.Tick(time.Second * 60)
go func() {
for {
select {
case <-timeCh:
case <-httpTriggerCh:
}
step(client, store, hostname)
}
}()
}
func main() {
defer common.LogPanic()
// Setup flags.
dbConf := buildbot.DBConfigFromFlags()
// Global init.
common.InitWithMetrics(APP_NAME, graphiteServer)
// Parse the time period.
period, err := human.ParseDuration(*timePeriod)
if err != nil {
glog.Fatal(err)
}
// Initialize the buildbot database.
if !*local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
if err := dbConf.InitDB(); err != nil {
glog.Fatal(err)
}
// Initialize the BuildBucket client.
c, err := auth.NewClient(*local, path.Join(*workdir, "oauth_token_cache"), buildbucket.DEFAULT_SCOPES...)
if err != nil {
glog.Fatal(err)
}
bb := buildbucket.NewClient(c)
// Build the queue.
repos := gitinfo.NewRepoMap(*workdir)
for _, r := range REPOS {
if _, err := repos.Repo(r); err != nil {
glog.Fatal(err)
}
}
q, err := build_queue.NewBuildQueue(period, repos, *scoreThreshold, *scoreDecay24Hr, BOT_BLACKLIST)
if err != nil {
glog.Fatal(err)
}
// Start scheduling builds in a loop.
liveness := metrics.NewLiveness(APP_NAME)
if err := scheduleBuilds(q, bb); err != nil {
glog.Errorf("Failed to schedule builds: %v", err)
}
for _ = range time.Tick(time.Minute) {
liveness.Update()
if err := scheduleBuilds(q, bb); err != nil {
glog.Errorf("Failed to schedule builds: %v", err)
}
}
}
// getClient returns an authorized storage.Service
func getClient() (storageClient, error) {
client, err := auth.NewClient(*local, *oauthCacheFile, auth.SCOPE_FULL_CONTROL)
if err != nil {
return storageClient{}, fmt.Errorf("Failed to create an authorized client: %s", err)
}
gsService, err := storage.New(client)
if err != nil {
return storageClient{}, fmt.Errorf("Failed to create a Google Storage API client: %s", err)
}
return storageClient{httpClient: client, storageService: gsService}, nil
}
func main() {
flag.Parse()
setDefaults()
common.DecodeTomlFile(*configFilename, &config.Config)
var err error
if client, err = auth.NewClient(config.Config.Common.DoOAuth, config.Config.Common.OAuthCacheFile, storage.DevstorageFull_controlScope); err != nil {
glog.Fatalf("Failed to create authenticated HTTP client: %s", err)
}
if store, err = storage.New(client); err != nil {
glog.Fatalf("Failed to create storage service client: %s", err)
}
resourcePath, err := setup()
if err != nil {
glog.Fatalf("Couldn't setup: %s", err)
}
checkCPPTemplate(resourcePath)
for {
fuzz, err := generator.Fuzz()
if err != nil {
glog.Fatalf("Couldn't create a fuzz: %s", err)
}
hash, err := writeFuzz(fuzz)
if err != nil {
glog.Fatalf("Couldn't create the fuzz hash: %s", err)
}
runtimeErr := runFuzz(hash)
if runtimeErr != nil {
glog.Errorf("Looks like the fuzz %s failed: %s", hash, runtimeErr)
}
if err := uploadFuzz(hash, fuzz, runtimeErr); err != nil {
glog.Fatalf("Couldn't upload the fuzz results (%s): %s", hash, err)
}
}
}
func Init() {
if *resourcesDir == "" {
_, filename, _, _ := runtime.Caller(0)
*resourcesDir = filepath.Join(filepath.Dir(filename), "../..")
}
loadTemplates()
// Read toml config file.
if _, err := toml.DecodeFile(*configFilename, &config); err != nil {
glog.Fatalf("Failed to decode config file: %s", err)
}
serverNames = make([]string, 0, len(config.Servers))
for k, _ := range config.Servers {
serverNames = append(serverNames, k)
}
var err error
if client, err = auth.NewClient(*doOauth, *oauthCacheFile, auth.SCOPE_FULL_CONTROL, auth.SCOPE_GCE); err != nil {
glog.Fatalf("Failed to create authenticated HTTP client: %s", err)
}
if store, err = storage.New(client); err != nil {
glog.Fatalf("Failed to create storage service client: %s", err)
}
if comp, err = compute.New(client); err != nil {
glog.Fatalf("Failed to create compute service client: %s", err)
}
ip, err = NewIPAddresses(comp)
if err != nil {
glog.Fatalf("Failed to load IP addresses at startup: %s", err)
}
packageInfo, err = packages.NewAllInfo(client, store, serverNames)
if err != nil {
glog.Fatalf("Failed to create packages.AllInfo at startup: %s", err)
}
}
func Init() {
defer common.LogPanic()
rand.Seed(time.Now().UnixNano())
common.InitWithMetricsCB("fuzzer", func() string {
common.DecodeTomlFile(*configFilename, &config.Config)
return config.Config.FrontEnd.GraphiteServer
})
if config.Config.Common.ResourcePath == "" {
_, filename, _, _ := runtime.Caller(0)
config.Config.Common.ResourcePath = filepath.Join(filepath.Dir(filename), "../..")
}
path, err := filepath.Abs(config.Config.Common.ResourcePath)
if err != nil {
glog.Fatalf("Couldn't get absolute path to fuzzer resources: %s", err)
}
if err := os.Chdir(path); err != nil {
glog.Fatal(err)
}
indexTemplate = htemplate.Must(htemplate.ParseFiles(
filepath.Join(path, "templates/index.html"),
filepath.Join(path, "templates/header.html"),
filepath.Join(path, "templates/titlebar.html"),
filepath.Join(path, "templates/footer.html"),
))
if client, err = auth.NewClient(config.Config.Common.DoOAuth, config.Config.Common.OAuthCacheFile, storage.DevstorageFullControlScope); err != nil {
glog.Fatalf("Failed to create authenticated HTTP client: %s", err)
}
if store, err = storage.New(client); err != nil {
glog.Fatalf("Failed to create storage service client: %s", err)
}
}
func main() {
if *hostname == "" {
var err error
*hostname, err = os.Hostname()
if err != nil {
// Never call glog before common.Init*.
os.Exit(1)
}
}
common.InitWithMetrics("pull."+*hostname, graphiteServer)
glog.Infof("Running with hostname: %s", *hostname)
client, err := auth.NewClient(*doOauth, *oauthCacheFile,
storage.DevstorageFull_controlScope,
compute.ComputeReadonlyScope)
if err != nil {
glog.Fatalf("Failed to create authenticated HTTP client: %s", err)
}
glog.Info("Got authenticated client.")
store, err := storage.New(client)
if err != nil {
glog.Fatalf("Failed to create storage service client: %s", err)
}
for _ = range time.Tick(time.Second * 15) {
before, err := filepath.Glob("/etc/monit/conf.d/*")
if err != nil {
glog.Errorf("Failed to list all monit config files: %s", err)
continue
}
glog.Info("About to read package list.")
// Read the old and new packages from their respective storage locations.
serverList, err := packages.InstalledForServer(client, store, *hostname)
if err != nil {
glog.Errorf("Failed to retrieve remote package list: %s", err)
continue
}
localList, err := packages.FromLocalFile(*installedPackagesFile)
if err != nil {
glog.Errorf("Failed to retrieve local package list: %s", err)
continue
}
glog.Info("Comparing against currently installed packages.")
// Install any new or updated packages.
newPackages, installedPackages := differences(serverList.Names, localList)
for _, p := range newPackages {
glog.Infof("New Package:%s", p)
}
for _, p := range installedPackages {
glog.Infof("Installed Package:%s", p)
}
save := false
for _, name := range newPackages {
// If just an appname appears w/o a packge name then that means
// that package hasn't been selected, so just skip it for now.
if len(strings.Split(name, "/")) == 1 {
continue
}
if err := packages.Install(client, store, name); err != nil {
glog.Errorf("Failed to install package %s: %s", name, err)
continue
}
installedPackages = append(installedPackages, name)
save = true
}
// Only write out the file if any new packages were installed.
if !save {
continue
}
if err := packages.ToLocalFile(installedPackages, *installedPackagesFile); err != nil {
glog.Errorf("Failed to write local package list: %s", err)
}
after, err := filepath.Glob("/etc/monit/conf.d/*")
if err != nil {
glog.Errorf("Failed to list all monit config files: %s", err)
continue
}
// Tell monit to reload if the name or number of files under /etc/monit/conf.d have changed.
if !util.SSliceEqual(before, after) {
cmd := exec.Command("sudo", "monit", "reload")
var out bytes.Buffer
cmd.Stdout = &out
if err := cmd.Run(); err != nil {
glog.Errorf("Failed to reload monit: %s", err)
glog.Errorf("Failed to reload monit (stdout): %s", out.String())
break
}
}
// The pull application is special and not monitored by monit to restart on
// timestamp changes because that might kill pull while it was updating
// itself. Instead pull will just exit when it notices that it has been
// updated and count on monit to restart pull.
if containsPull(newPackages) {
glog.Info("The pull package has been updated, exiting to allow a restart.")
os.Exit(0)
}
}
}
