Exemplo n.º 1
0
func pullInit() {
	hostname, err := os.Hostname()
	if err != nil {
		// Never call glog before common.Init*.
		os.Exit(1)
	}
	common.InitWithMetrics("pulld."+hostname, graphiteServer)
	glog.Infof("Running with hostname: %s", hostname)

	client, err := auth.NewClient(*doOauth, *oauthCacheFile,
		storage.DevstorageFullControlScope,

		compute.ComputeReadonlyScope)
	if err != nil {
		glog.Fatalf("Failed to create authenticated HTTP client: %s", err)
	}
	glog.Info("Got authenticated client.")

	store, err = storage.New(client)
	if err != nil {
		glog.Fatalf("Failed to create storage service client: %s", err)
	}

	step(client, store, hostname)
	timeCh := time.Tick(time.Second * 60)
	go func() {
		for {
			select {
			case <-timeCh:
			case <-httpTriggerCh:
			}
			step(client, store, hostname)
		}
	}()
}
Exemplo n.º 2
0
func main() {
	defer common.LogPanic()
	// Setup flags.
	dbConf := buildbot.DBConfigFromFlags()

	// Global init.
	common.InitWithMetrics(APP_NAME, graphiteServer)

	// Parse the time period.
	period, err := human.ParseDuration(*timePeriod)
	if err != nil {
		glog.Fatal(err)
	}

	// Initialize the buildbot database.
	if !*local {
		if err := dbConf.GetPasswordFromMetadata(); err != nil {
			glog.Fatal(err)
		}
	}
	if err := dbConf.InitDB(); err != nil {
		glog.Fatal(err)
	}

	// Initialize the BuildBucket client.
	c, err := auth.NewClient(*local, path.Join(*workdir, "oauth_token_cache"), buildbucket.DEFAULT_SCOPES...)
	if err != nil {
		glog.Fatal(err)
	}
	bb := buildbucket.NewClient(c)

	// Build the queue.
	repos := gitinfo.NewRepoMap(*workdir)
	for _, r := range REPOS {
		if _, err := repos.Repo(r); err != nil {
			glog.Fatal(err)
		}
	}
	q, err := build_queue.NewBuildQueue(period, repos, *scoreThreshold, *scoreDecay24Hr, BOT_BLACKLIST)
	if err != nil {
		glog.Fatal(err)
	}

	// Start scheduling builds in a loop.
	liveness := metrics.NewLiveness(APP_NAME)
	if err := scheduleBuilds(q, bb); err != nil {
		glog.Errorf("Failed to schedule builds: %v", err)
	}
	for _ = range time.Tick(time.Minute) {
		liveness.Update()
		if err := scheduleBuilds(q, bb); err != nil {
			glog.Errorf("Failed to schedule builds: %v", err)
		}
	}
}
Exemplo n.º 3
0
// getClient returns an authorized storage.Service
func getClient() (storageClient, error) {
	client, err := auth.NewClient(*local, *oauthCacheFile, auth.SCOPE_FULL_CONTROL)
	if err != nil {
		return storageClient{}, fmt.Errorf("Failed to create an authorized client: %s", err)
	}
	gsService, err := storage.New(client)
	if err != nil {
		return storageClient{}, fmt.Errorf("Failed to create a Google Storage API client: %s", err)
	}
	return storageClient{httpClient: client, storageService: gsService}, nil
}
Exemplo n.º 4
0
func main() {
	flag.Parse()

	setDefaults()

	common.DecodeTomlFile(*configFilename, &config.Config)

	var err error
	if client, err = auth.NewClient(config.Config.Common.DoOAuth, config.Config.Common.OAuthCacheFile, storage.DevstorageFull_controlScope); err != nil {
		glog.Fatalf("Failed to create authenticated HTTP client: %s", err)
	}

	if store, err = storage.New(client); err != nil {
		glog.Fatalf("Failed to create storage service client: %s", err)
	}

	resourcePath, err := setup()
	if err != nil {
		glog.Fatalf("Couldn't setup: %s", err)
	}

	checkCPPTemplate(resourcePath)

	for {
		fuzz, err := generator.Fuzz()
		if err != nil {
			glog.Fatalf("Couldn't create a fuzz: %s", err)
		}

		hash, err := writeFuzz(fuzz)
		if err != nil {
			glog.Fatalf("Couldn't create the fuzz hash: %s", err)
		}

		runtimeErr := runFuzz(hash)
		if runtimeErr != nil {
			glog.Errorf("Looks like the fuzz %s failed: %s", hash, runtimeErr)
		}

		if err := uploadFuzz(hash, fuzz, runtimeErr); err != nil {
			glog.Fatalf("Couldn't upload the fuzz results (%s): %s", hash, err)
		}
	}
}
Exemplo n.º 5
0
func Init() {
	if *resourcesDir == "" {
		_, filename, _, _ := runtime.Caller(0)
		*resourcesDir = filepath.Join(filepath.Dir(filename), "../..")
	}
	loadTemplates()

	// Read toml config file.
	if _, err := toml.DecodeFile(*configFilename, &config); err != nil {
		glog.Fatalf("Failed to decode config file: %s", err)
	}

	serverNames = make([]string, 0, len(config.Servers))
	for k, _ := range config.Servers {
		serverNames = append(serverNames, k)
	}

	var err error
	if client, err = auth.NewClient(*doOauth, *oauthCacheFile, auth.SCOPE_FULL_CONTROL, auth.SCOPE_GCE); err != nil {
		glog.Fatalf("Failed to create authenticated HTTP client: %s", err)
	}

	if store, err = storage.New(client); err != nil {
		glog.Fatalf("Failed to create storage service client: %s", err)
	}
	if comp, err = compute.New(client); err != nil {
		glog.Fatalf("Failed to create compute service client: %s", err)
	}
	ip, err = NewIPAddresses(comp)
	if err != nil {
		glog.Fatalf("Failed to load IP addresses at startup: %s", err)
	}

	packageInfo, err = packages.NewAllInfo(client, store, serverNames)
	if err != nil {
		glog.Fatalf("Failed to create packages.AllInfo at startup: %s", err)
	}
}
Exemplo n.º 6
0
func Init() {
	defer common.LogPanic()

	rand.Seed(time.Now().UnixNano())

	common.InitWithMetricsCB("fuzzer", func() string {
		common.DecodeTomlFile(*configFilename, &config.Config)
		return config.Config.FrontEnd.GraphiteServer
	})

	if config.Config.Common.ResourcePath == "" {
		_, filename, _, _ := runtime.Caller(0)
		config.Config.Common.ResourcePath = filepath.Join(filepath.Dir(filename), "../..")
	}

	path, err := filepath.Abs(config.Config.Common.ResourcePath)
	if err != nil {
		glog.Fatalf("Couldn't get absolute path to fuzzer resources: %s", err)
	}
	if err := os.Chdir(path); err != nil {
		glog.Fatal(err)
	}

	indexTemplate = htemplate.Must(htemplate.ParseFiles(
		filepath.Join(path, "templates/index.html"),
		filepath.Join(path, "templates/header.html"),
		filepath.Join(path, "templates/titlebar.html"),
		filepath.Join(path, "templates/footer.html"),
	))

	if client, err = auth.NewClient(config.Config.Common.DoOAuth, config.Config.Common.OAuthCacheFile, storage.DevstorageFullControlScope); err != nil {
		glog.Fatalf("Failed to create authenticated HTTP client: %s", err)
	}

	if store, err = storage.New(client); err != nil {
		glog.Fatalf("Failed to create storage service client: %s", err)
	}
}
Exemplo n.º 7
0
func main() {
	if *hostname == "" {
		var err error
		*hostname, err = os.Hostname()
		if err != nil {
			// Never call glog before common.Init*.
			os.Exit(1)
		}
	}
	common.InitWithMetrics("pull."+*hostname, graphiteServer)
	glog.Infof("Running with hostname: %s", *hostname)

	client, err := auth.NewClient(*doOauth, *oauthCacheFile,
		storage.DevstorageFull_controlScope,
		compute.ComputeReadonlyScope)

	if err != nil {
		glog.Fatalf("Failed to create authenticated HTTP client: %s", err)
	}
	glog.Info("Got authenticated client.")

	store, err := storage.New(client)
	if err != nil {
		glog.Fatalf("Failed to create storage service client: %s", err)
	}

	for _ = range time.Tick(time.Second * 15) {
		before, err := filepath.Glob("/etc/monit/conf.d/*")
		if err != nil {
			glog.Errorf("Failed to list all monit config files: %s", err)
			continue
		}

		glog.Info("About to read package list.")
		// Read the old and new packages from their respective storage locations.
		serverList, err := packages.InstalledForServer(client, store, *hostname)
		if err != nil {
			glog.Errorf("Failed to retrieve remote package list: %s", err)
			continue
		}
		localList, err := packages.FromLocalFile(*installedPackagesFile)
		if err != nil {
			glog.Errorf("Failed to retrieve local package list: %s", err)
			continue
		}

		glog.Info("Comparing against currently installed packages.")
		// Install any new or updated packages.
		newPackages, installedPackages := differences(serverList.Names, localList)
		for _, p := range newPackages {
			glog.Infof("New Package:%s", p)
		}
		for _, p := range installedPackages {
			glog.Infof("Installed Package:%s", p)
		}

		save := false
		for _, name := range newPackages {
			// If just an appname appears w/o a packge name then that means
			// that package hasn't been selected, so just skip it for now.
			if len(strings.Split(name, "/")) == 1 {
				continue
			}
			if err := packages.Install(client, store, name); err != nil {
				glog.Errorf("Failed to install package %s: %s", name, err)
				continue
			}
			installedPackages = append(installedPackages, name)
			save = true
		}
		// Only write out the file if any new packages were installed.
		if !save {
			continue
		}
		if err := packages.ToLocalFile(installedPackages, *installedPackagesFile); err != nil {
			glog.Errorf("Failed to write local package list: %s", err)
		}

		after, err := filepath.Glob("/etc/monit/conf.d/*")
		if err != nil {
			glog.Errorf("Failed to list all monit config files: %s", err)
			continue
		}

		// Tell monit to reload if the name or number of files under /etc/monit/conf.d have changed.
		if !util.SSliceEqual(before, after) {
			cmd := exec.Command("sudo", "monit", "reload")
			var out bytes.Buffer
			cmd.Stdout = &out
			if err := cmd.Run(); err != nil {
				glog.Errorf("Failed to reload monit: %s", err)
				glog.Errorf("Failed to reload monit (stdout): %s", out.String())
				break
			}
		}

		// The pull application is special and not monitored by monit to restart on
		// timestamp changes because that might kill pull while it was updating
		// itself. Instead pull will just exit when it notices that it has been
		// updated and count on monit to restart pull.
		if containsPull(newPackages) {
			glog.Info("The pull package has been updated, exiting to allow a restart.")
			os.Exit(0)
		}
	}
}