Exemple #1
0
func main() {
	flag.Parse()
	if os.Geteuid() == 0 {
		fmt.Println("Do not run the Image Server as root")
		os.Exit(1)
	}
	circularBuffer := logbuf.New(*logbufLines)
	logger := log.New(circularBuffer, "", log.LstdFlags)
	objSrv, err := filesystem.NewObjectServer(*objectDir, logger)
	if err != nil {
		fmt.Printf("Cannot create ObjectServer\t%s\n", err)
		os.Exit(1)
	}
	imdb, err := scanner.LoadImageDataBase(*imageDir, objSrv)
	if err != nil {
		fmt.Printf("Cannot load image database\t%s\n", err)
		os.Exit(1)
	}
	imageserverRpcd.Setup(imdb, logger)
	rpcHtmlWriter := objectserverRpcd.Setup(objSrv, logger)
	rpc.HandleHTTP()
	httpd.AddHtmlWriter(imdb)
	httpd.AddHtmlWriter(rpcHtmlWriter)
	httpd.AddHtmlWriter(circularBuffer)
	err = httpd.StartServer(*portNum, imdb, false)
	if err != nil {
		fmt.Printf("Unable to create http server\t%s\n", err)
		os.Exit(1)
	}
}
Exemple #2
0
func main() {
	flag.Parse()
	tricorder.RegisterFlags()
	if os.Geteuid() == 0 {
		fmt.Fprintln(os.Stderr, "Do not run the Image Server as root")
		os.Exit(1)
	}
	setupTls(*caFile, *certFile, *keyFile)
	circularBuffer := logbuf.New(*logbufLines)
	logger := log.New(circularBuffer, "", log.LstdFlags)
	objSrv, err := filesystem.NewObjectServer(*objectDir, logger)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot create ObjectServer\t%s\n", err)
		os.Exit(1)
	}
	imdb, err := scanner.LoadImageDataBase(*imageDir, objSrv, logger)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot load image database\t%s\n", err)
		os.Exit(1)
	}
	tricorder.RegisterMetric("/image-count",
		func() uint { return imdb.CountImages() },
		units.None, "number of images")
	imageserverRpcd.Setup(imdb, logger)
	rpcHtmlWriter := objectserverRpcd.Setup(objSrv, logger)
	rpc.HandleHTTP()
	httpd.AddHtmlWriter(imdb)
	httpd.AddHtmlWriter(rpcHtmlWriter)
	httpd.AddHtmlWriter(circularBuffer)
	if err = httpd.StartServer(*portNum, imdb, false); err != nil {
		fmt.Fprintf(os.Stderr, "Unable to create http server\t%s\n", err)
		os.Exit(1)
	}
}
Exemple #3
0
func main() {
	flag.Parse()
	tricorder.RegisterFlags()
	if os.Geteuid() == 0 {
		fmt.Fprintln(os.Stderr, "Do not run the Image Server as root")
		os.Exit(1)
	}
	if *archiveMode && *imageServerHostname == "" {
		fmt.Fprintln(os.Stderr, "-imageServerHostname required in archive mode")
		os.Exit(1)
	}
	circularBuffer := logbuf.New()
	logger := log.New(circularBuffer, "", log.LstdFlags)
	if err := setupserver.SetupTls(); err != nil {
		logger.Println(err)
		circularBuffer.Flush()
		if !*permitInsecureMode {
			os.Exit(1)
		}
	}
	objSrv, err := filesystem.NewObjectServer(*objectDir, logger)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot create ObjectServer: %s\n", err)
		os.Exit(1)
	}
	cleanupUnreferencedObjects := true
	if *imageServerHostname != "" {
		cleanupUnreferencedObjects = false
	}
	imdb, err := scanner.LoadImageDataBase(*imageDir, objSrv,
		cleanupUnreferencedObjects, logger)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot load image database: %s\n", err)
		os.Exit(1)
	}
	tricorder.RegisterMetric("/image-count",
		func() uint { return imdb.CountImages() },
		units.None, "number of images")
	imgSrvRpcHtmlWriter := imageserverRpcd.Setup(imdb, *imageServerHostname,
		logger)
	objSrvRpcHtmlWriter := objectserverRpcd.Setup(objSrv, logger)
	httpd.AddHtmlWriter(imdb)
	httpd.AddHtmlWriter(&imageObjectServersType{imdb, objSrv})
	httpd.AddHtmlWriter(imgSrvRpcHtmlWriter)
	httpd.AddHtmlWriter(objSrvRpcHtmlWriter)
	httpd.AddHtmlWriter(circularBuffer)
	if *imageServerHostname != "" {
		go replicator(fmt.Sprintf("%s:%d", *imageServerHostname,
			*imageServerPortNum), imdb, objSrv, *archiveMode, logger)
	}
	if err = httpd.StartServer(*portNum, imdb, objSrv, false); err != nil {
		fmt.Fprintf(os.Stderr, "Unable to create http server: %s\n", err)
		os.Exit(1)
	}
}
Exemple #4
0
func main() {
	flag.Parse()
	if os.Geteuid() == 0 {
		fmt.Fprintln(os.Stderr, "Do not run the Dominator as root")
		os.Exit(1)
	}
	fi, err := os.Lstat(*stateDir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot stat: %s\t%s\n", *stateDir, err)
		os.Exit(1)
	}
	if !fi.IsDir() {
		fmt.Fprintf(os.Stderr, "%s is not a directory\n", *stateDir)
		os.Exit(1)
	}
	interval := time.Duration(*minInterval) * time.Second
	circularBuffer := logbuf.New(*logbufLines)
	logger := log.New(circularBuffer, "", log.LstdFlags)
	mdbChannel := mdb.StartMdbDaemon(path.Join(*stateDir, "mdb"), logger)
	herd := herd.NewHerd(fmt.Sprintf("%s:%d", *imageServerHostname,
		*imageServerPortNum), logger)
	herd.AddHtmlWriter(circularBuffer)
	if err = herd.StartServer(*portNum, true); err != nil {
		fmt.Fprintf(os.Stderr, "Unable to create http server\t%s\n", err)
		os.Exit(1)
	}
	nextCycleStopTime := time.Now().Add(interval)
	for {
		select {
		case mdb := <-mdbChannel:
			herd.MdbUpdate(mdb)
			if *debug {
				showMdb(mdb)
			}
			runtime.GC() // An opportune time to take out the garbage.
		default:
			// Do work.
			if herd.PollNextSub() {
				if *debug {
					fmt.Print(".")
				}
				sleepTime := nextCycleStopTime.Sub(time.Now())
				time.Sleep(sleepTime)
				nextCycleStopTime = time.Now().Add(interval)
				if sleepTime < 0 { // There was no time to rest.
					runtime.GC() // An opportune time to take out the garbage.
				}
			}
		}
	}
}
Exemple #5
0
func main() {
	flag.Usage = printUsage
	flag.Parse()
	tricorder.RegisterFlags()
	circularBuffer := logbuf.New()
	logger := log.New(circularBuffer, "", log.LstdFlags)
	// We have to have inputs.
	if *sourcesFile == "" {
		printUsage()
		os.Exit(2)
	}
	handleSignals(logger)
	var generators []generator
	readerChannel := fsutil.WatchFile(*sourcesFile, logger)
	file, err := os.Open(*sourcesFile)
	if err != nil {
		showErrorAndDie(err)
	}
	(<-readerChannel).Close()
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		fields := strings.Fields(scanner.Text())
		if len(fields) == 0 || len(fields[0]) == 0 || fields[0][0] == '#' {
			continue
		}
		gen, err := getSource(fields)
		if err != nil {
			showErrorAndDie(err)
		}
		generators = append(generators, gen)
	}
	if err := scanner.Err(); err != nil {
		showErrorAndDie(err)
	}
	file.Close()
	httpSrv, err := startHttpServer(*portNum)
	if err != nil {
		showErrorAndDie(err)
	}
	httpSrv.AddHtmlWriter(circularBuffer)
	updateFunc := startRpcd(logger)
	go runDaemon(generators, *mdbFile, *hostnameRegex, *datacentre,
		*fetchInterval, updateFunc, logger, *debug)
	<-readerChannel
	fsutil.WatchFileStop()
	if err := syscall.Exec(os.Args[0], os.Args, os.Environ()); err != nil {
		logger.Printf("Unable to Exec:%s: %s\n", os.Args[0], err)
	}
}
Exemple #6
0
func main() {
	flag.Usage = printUsage
	flag.Parse()
	tricorder.RegisterFlags()
	if os.Geteuid() == 0 {
		fmt.Fprintln(os.Stderr, "Do not run the filegen server as root")
		os.Exit(1)
	}
	circularBuffer := logbuf.New()
	logger := log.New(circularBuffer, "", log.LstdFlags)
	if err := setupserver.SetupTls(); err != nil {
		logger.Println(err)
		circularBuffer.Flush()
		if !*permitInsecureMode {
			os.Exit(1)
		}
	}
	manager := filegen.New(logger)
	if *configFile != "" {
		if err := util.LoadConfiguration(manager, *configFile); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
		ch := fsutil.WatchFile(*configFile, nil)
		(<-ch).Close() // Drain the first event.
		go func() {
			<-ch
			err := syscall.Exec(os.Args[0], os.Args, os.Environ())
			if err != nil {
				logger.Printf("Unable to Exec:%s: %s\n", os.Args[0], err)
			}
		}()
	}
	httpd.AddHtmlWriter(manager)
	httpd.AddHtmlWriter(circularBuffer)
	for _, pathname := range flag.Args() {
		if err := registerSourceDirectory(manager, pathname, "/"); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}
	if err := httpd.StartServer(*portNum, manager, false); err != nil {
		fmt.Fprintf(os.Stderr, "Unable to create http server: %s\n", err)
		os.Exit(1)
	}
}
Exemple #7
0
func doMain() error {
	flag.Parse()
	runtime.GOMAXPROCS(int(*maxThreads))
	runtime.LockOSThread()
	circularBuffer := logbuf.New(*logbufLines)
	logger := log.New(circularBuffer, "", log.LstdFlags)
	probers, err := setupProbers()
	if err != nil {
		return err
	}
	latencyBucketer := tricorder.NewGeometricBucketer(0.1, 100e3)
	scanTimeDistribution := latencyBucketer.NewDistribution()
	var scanStartTime time.Time
	if err := tricorder.RegisterMetric("scan-duration",
		scanTimeDistribution, units.Millisecond,
		"duration of last probe"); err != nil {
		return err
	}
	if err := tricorder.RegisterMetric("scan-start-time", &scanStartTime,
		units.None, "start time of last probe"); err != nil {
		return err
	}
	rpc.HandleHTTP()
	httpd.AddHtmlWriter(circularBuffer)
	if err := httpd.StartServer(*portNum); err != nil {
		return err
	}
	for {
		scanStartTime = time.Now()
		for _, p := range probers {
			if err := p.Probe(); err != nil {
				logger.Println(err)
			}
		}
		scanDuration := time.Since(scanStartTime)
		scanTimeDistribution.Add(scanDuration)
		time.Sleep(time.Second*time.Duration(*probeInterval) - scanDuration)
	}
	_ = logger
	return nil
}
Exemple #8
0
func doMain() error {
	flag.Parse()
	runtime.GOMAXPROCS(int(*maxThreads))
	runtime.LockOSThread()
	circularBuffer := logbuf.New()
	logger := log.New(circularBuffer, "", log.LstdFlags)
	proberList, err := setupProbers()
	if err != nil {
		return err
	}
	if err := setupHealthchecks(*configDir, proberList, logger); err != nil {
		logger.Printf("Error occured while setting up Healthchecks")
		return err
	}
	httpd.AddHtmlWriter(proberList)
	httpd.AddHtmlWriter(circularBuffer)
	sighupChannel := make(chan os.Signal)
	signal.Notify(sighupChannel, syscall.SIGHUP)
	sigtermChannel := make(chan os.Signal)
	signal.Notify(sigtermChannel, syscall.SIGTERM, syscall.SIGINT)
	rpc.HandleHTTP()
	if err := httpd.StartServer(*portNum); err != nil {
		return err
	}
	writePidfile()
	proberList.StartProbing(*probeInterval, logger)
	for {
		select {
		case <-sighupChannel:
			err = syscall.Exec(os.Args[0], os.Args, os.Environ())
			if err != nil {
				logger.Printf("Unable to Exec:%s\t%s\n", os.Args[0], err)
			}
		case <-sigtermChannel:
			gracefulCleanup()
		}
	}
	return nil
}
Exemple #9
0
func main() {
	tricorder.RegisterFlags()
	flag.Parse()
	circularBuffer := logbuf.New()
	logger := log.New(circularBuffer, "", log.LstdFlags)
	handleSignals(logger)
	// Read configs early so that we will fail fast.
	maybeNilMemoryManager := maybeCreateMemoryManager(logger)
	consumerBuilders, err := newPStoreConsumers(maybeNilMemoryManager)
	if err != nil {
		log.Println("Pstore config file error:", err)
		logger.Println("Pstore config file error:", err)
	}
	metricNameEngine := suggest.NewEngine()
	metricNameAdder := newTsdbAdder(metricNameEngine)
	tagkEngine := suggest.NewSuggester("appname", "HostName")
	tagvEngine := suggest.NewEngine()
	tagvAdder := newTsdbAdder(tagvEngine)

	appList := createApplicationList()
	for _, app := range appList.All() {
		tagvAdder.Add(app.Name())
	}
	applicationStats := createApplicationStats(
		appList, logger, tagvAdder, maybeNilMemoryManager)
	rpc.RegisterName(
		"Scotty",
		&rpcType{AS: applicationStats},
	)
	rpc.HandleHTTP()
	connectionErrors := newConnectionErrorsType()
	if consumerBuilders == nil {
		startCollector(
			applicationStats,
			connectionErrors,
			totalCountCollectionType(nil),
			metricNameAdder,
			&maybeNilMemoryManagerWrapperType{maybeNilMemoryManager})
	} else {
		var coord coordinatorBuilderType
		if *fCoord != "" {
			var err error
			coord, err = consul.NewCoordinator(logger)
			if err != nil {
				logger.Println(err)
				coord = &blockingCoordinatorType{}
			}
		}
		totalCounts := startPStoreLoops(
			applicationStats,
			consumerBuilders,
			logger,
			coord)
		startCollector(
			applicationStats,
			connectionErrors,
			totalCountCollectionType(totalCounts),
			metricNameAdder,
			&maybeNilMemoryManagerWrapperType{maybeNilMemoryManager})
	}

	http.Handle(
		"/",
		gzipHandler{&splash.Handler{
			AS:  applicationStats,
			Log: circularBuffer,
		}})
	http.Handle(
		"/showAllApps",
		gzipHandler{&showallapps.Handler{
			AS:             applicationStats,
			CollectionFreq: *fCollectionFrequency,
		}})
	http.Handle(
		"/api/hosts/",
		http.StripPrefix(
			"/api/hosts/",
			gzipHandler{&byEndpointHandler{
				AS: applicationStats,
			}}))
	http.Handle(
		"/api/latest/",
		http.StripPrefix(
			"/api/latest/",
			gzipHandler{&latestHandler{
				AS: applicationStats,
			}}))

	http.Handle(
		"/api/errors/",
		gzipHandler{&errorHandler{
			ConnectionErrors: connectionErrors,
		}},
	)

	tsdbServeMux := http.NewServeMux()

	tsdbServeMux.Handle(
		"/api/query",
		tsdbexec.NewHandler(
			func(r *tsdbjson.QueryRequest) ([]tsdbjson.TimeSeries, error) {
				return tsdbexec.Query(
					r, applicationStats, *fCollectionFrequency)
			}))
	tsdbServeMux.Handle(
		"/api/suggest",
		tsdbexec.NewHandler(
			func(req url.Values) ([]string, error) {
				return tsdbexec.Suggest(
					req,
					map[string]suggest.Suggester{
						"metrics": metricNameEngine,
						"tagk":    tagkEngine,
						"tagv":    tagvEngine,
					})
			},
		))
	tsdbServeMux.Handle(
		"/api/aggregators",
		tsdbexec.NewHandler(
			func(req url.Values) ([]string, error) {
				return aggregators.Names(), nil
			},
		))
	tsdbServeMux.Handle(
		"/api/version",
		tsdbexec.NewHandler(
			func(req url.Values) (map[string]string, error) {
				return map[string]string{
					"version": "1.0",
				}, nil
			},
		))
	tsdbServeMux.Handle(
		"/api/config",
		tsdbexec.NewHandler(
			func(req url.Values) (map[string]string, error) {
				return map[string]string{
					"tsd.ore.auto_create_metrics": "true",
					"tsd.ore.auto_create_tagks":   "true",
					"tsd.ore.auto_create_tagvs":   "true",
				}, nil
			},
		))
	tsdbServeMux.Handle(
		"/api/config/filters",
		tsdbexec.NewHandler(
			func(req url.Values) (interface{}, error) {
				return tsdbjson.AllFilterDescriptions(), nil
			},
		))
	tsdbServeMux.Handle(
		"/api/dropcaches",
		tsdbexec.NewHandler(
			func(req url.Values) (map[string]string, error) {
				return map[string]string{
					"message": "Caches dropped",
					"status":  "200",
				}, nil
			},
		))
	tsdbServeMux.Handle(
		"/api",
		tsdbexec.NotFoundHandler,
	)

	go func() {
		if err := http.ListenAndServe(fmt.Sprintf(":%d", *fTsdbPort), tsdbServeMux); err != nil {
			log.Fatal(err)
		}
	}()

	if err := http.ListenAndServe(fmt.Sprintf(":%d", *fPort), nil); err != nil {
		log.Fatal(err)
	}
}
Exemple #10
0
func main() {
	flag.Parse()
	workingRootDir := path.Join(*subdDir, "root")
	objectsDir := path.Join(*subdDir, "objects")
	tmpDir := path.Join(*subdDir, "tmp")
	netbenchFilename := path.Join(*subdDir, "netbench")
	oldTriggersFilename := path.Join(*subdDir, "triggers.previous")
	if !createDirectory(workingRootDir) {
		os.Exit(1)
	}
	if !sanityCheck() {
		os.Exit(1)
	}
	if !createDirectory(objectsDir) {
		os.Exit(1)
	}
	if !createDirectory(tmpDir) {
		os.Exit(1)
	}
	if !mountTmpfs(tmpDir) {
		os.Exit(1)
	}
	if !unshareAndBind(workingRootDir) {
		os.Exit(1)
	}
	runtime.GOMAXPROCS(int(*maxThreads))
	bytesPerSecond, blocksPerSecond, firstScan, ok := getCachedFsSpeed(
		workingRootDir, tmpDir)
	if !ok {
		os.Exit(1)
	}
	circularBuffer := logbuf.New(*logbufLines)
	logger := log.New(circularBuffer, "", log.LstdFlags)
	var configuration scanner.Configuration
	var err error
	configuration.ScanFilter, err = filter.NewFilter(constants.ScanExcludeList)
	if err != nil {
		fmt.Printf("Unable to set default scan exclusions\t%s\n", err)
		os.Exit(1)
	}
	configuration.FsScanContext = fsrateio.NewReaderContext(bytesPerSecond,
		blocksPerSecond, 0)
	defaultSpeed := configuration.FsScanContext.GetContext().SpeedPercent()
	if firstScan {
		configuration.FsScanContext.GetContext().SetSpeedPercent(100)
	}
	if *showStats {
		fmt.Println(configuration.FsScanContext)
	}
	var fsh scanner.FileSystemHistory
	fsChannel := scanner.StartScannerDaemon(workingRootDir, objectsDir,
		&configuration, logger)
	networkReaderContext := rateio.NewReaderContext(
		getCachedNetworkSpeed(netbenchFilename),
		constants.DefaultNetworkSpeedPercent, &rateio.ReadMeasurer{})
	configuration.NetworkReaderContext = networkReaderContext
	rescanObjectCacheChannel := rpcd.Setup(&configuration, &fsh, objectsDir,
		networkReaderContext, netbenchFilename, oldTriggersFilename, logger)
	httpd.AddHtmlWriter(&fsh)
	httpd.AddHtmlWriter(&configuration)
	httpd.AddHtmlWriter(circularBuffer)
	html.RegisterHtmlWriterForPattern("/dumpFileSystem", "Scanned File System",
		&DumpableFileSystemHistory{&fsh})
	err = httpd.StartServer(*portNum)
	if err != nil {
		fmt.Printf("Unable to create http server\t%s\n", err)
		os.Exit(1)
	}
	fsh.Update(nil)
	invalidateNextScanObjectCache := false
	sighupChannel := make(chan os.Signal)
	signal.Notify(sighupChannel, syscall.SIGHUP)
	sigtermChannel := make(chan os.Signal)
	signal.Notify(sigtermChannel, syscall.SIGTERM, syscall.SIGINT)
	writePidfile()
	for iter := 0; true; {
		select {
		case <-sighupChannel:
			err = syscall.Exec(os.Args[0], os.Args, os.Environ())
			if err != nil {
				logger.Printf("Unable to Exec:%s\t%s\n", os.Args[0], err)
			}
		case <-sigtermChannel:
			gracefulCleanup()
		case fs := <-fsChannel:
			if *showStats {
				fmt.Printf("Completed cycle: %d\n", iter)
			}
			if invalidateNextScanObjectCache {
				fs.ScanObjectCache()
				invalidateNextScanObjectCache = false
			}
			fsh.Update(fs)
			iter++
			runtime.GC() // An opportune time to take out the garbage.
			if *showStats {
				fmt.Print(fsh)
				fmt.Print(fsh.FileSystem())
				memstats.WriteMemoryStats(os.Stdout)
				fmt.Println()
			}
			if firstScan {
				configuration.FsScanContext.GetContext().SetSpeedPercent(
					defaultSpeed)
				firstScan = false
				if *showStats {
					fmt.Println(configuration.FsScanContext)
				}
			}
		case <-rescanObjectCacheChannel:
			invalidateNextScanObjectCache = true
			fsh.UpdateObjectCacheOnly()
		}
	}
}
Exemple #11
0
func main() {
	flag.Parse()
	tricorder.RegisterFlags()
	circularBuffer := logbuf.New()
	logger := log.New(circularBuffer, "", log.LstdFlags)
	if err := setupserver.SetupTls(); err != nil {
		logger.Println(err)
		circularBuffer.Flush()
		if !*permitInsecureMode {
			os.Exit(1)
		}
	}
	rlim := syscall.Rlimit{*fdLimit, *fdLimit}
	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlim); err != nil {
		fmt.Fprintf(os.Stderr, "Cannot set FD limit\t%s\n", err)
		os.Exit(1)
	}
	if os.Geteuid() == 0 {
		if err := setUser(*username); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}
	fi, err := os.Lstat(*stateDir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot stat: %s\t%s\n", *stateDir, err)
		os.Exit(1)
	}
	if !fi.IsDir() {
		fmt.Fprintf(os.Stderr, "%s is not a directory\n", *stateDir)
		os.Exit(1)
	}
	interval := time.Duration(*minInterval) * time.Second
	mdbChannel := mdbd.StartMdbDaemon(path.Join(*stateDir, *mdbFile), logger)
	objectServer, err := newObjectServer(path.Join(*stateDir, *objectsDir),
		logger)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot load objectcache: %s\n", err)
		os.Exit(1)
	}
	herd := herd.NewHerd(fmt.Sprintf("%s:%d", *imageServerHostname,
		*imageServerPortNum), objectServer, logger)
	herd.AddHtmlWriter(circularBuffer)
	rpcd.Setup(herd, logger)
	if err = herd.StartServer(*portNum, true); err != nil {
		fmt.Fprintf(os.Stderr, "Unable to create http server\t%s\n", err)
		os.Exit(1)
	}
	scanTokenChannel := make(chan bool, 1)
	scanTokenChannel <- true
	nextCycleStopTime := time.Now().Add(interval)
	for {
		select {
		case mdb := <-mdbChannel:
			herd.MdbUpdate(mdb)
			if *debug {
				showMdb(mdb)
			}
		case <-scanTokenChannel:
			// Scan one sub.
			if herd.PollNextSub() { // We've reached the end of a scan cycle.
				if *debug {
					fmt.Print(".")
				}
				go func(sleepDuration time.Duration) {
					time.Sleep(sleepDuration)
					nextCycleStopTime = time.Now().Add(interval)
					scanTokenChannel <- true
				}(nextCycleStopTime.Sub(time.Now()))
			} else {
				scanTokenChannel <- true
			}
		}
	}
}
Exemple #12
0
func main() {
	flag.Parse()
	tricorder.RegisterFlags()
	setupTls(*caFile,
		pathJoin(*certDir, *certFile), pathJoin(*certDir, *keyFile))
	rlim := syscall.Rlimit{*fdLimit, *fdLimit}
	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlim); err != nil {
		fmt.Fprintf(os.Stderr, "Cannot set FD limit\t%s\n", err)
		os.Exit(1)
	}
	if os.Geteuid() == 0 {
		if err := setUser(*username); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}
	fi, err := os.Lstat(*stateDir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot stat: %s\t%s\n", *stateDir, err)
		os.Exit(1)
	}
	if !fi.IsDir() {
		fmt.Fprintf(os.Stderr, "%s is not a directory\n", *stateDir)
		os.Exit(1)
	}
	interval := time.Duration(*minInterval) * time.Second
	circularBuffer := logbuf.New(*logbufLines)
	logger := log.New(circularBuffer, "", log.LstdFlags)
	mdbChannel := mdbd.StartMdbDaemon(path.Join(*stateDir, *mdbFile), logger)
	herd := herd.NewHerd(fmt.Sprintf("%s:%d", *imageServerHostname,
		*imageServerPortNum), logger)
	herd.AddHtmlWriter(circularBuffer)
	if err = herd.StartServer(*portNum, true); err != nil {
		fmt.Fprintf(os.Stderr, "Unable to create http server\t%s\n", err)
		os.Exit(1)
	}
	nextCycleStopTime := time.Now().Add(interval)
	for {
		select {
		case mdb := <-mdbChannel:
			herd.MdbUpdate(mdb)
			if *debug {
				showMdb(mdb)
			}
			runtime.GC() // An opportune time to take out the garbage.
		default:
			// Do work.
			if herd.PollNextSub() {
				if *debug {
					fmt.Print(".")
				}
				sleepTime := nextCycleStopTime.Sub(time.Now())
				time.Sleep(sleepTime)
				nextCycleStopTime = time.Now().Add(interval)
				if sleepTime < 0 { // There was no time to rest.
					runtime.GC() // An opportune time to take out the garbage.
				}
			}
		}
	}
}
Exemple #13
0
func main() {
	flag.Parse()
	tricorder.RegisterFlags()
	subdDirPathname := path.Join(*rootDir, *subdDir)
	workingRootDir := path.Join(subdDirPathname, "root")
	objectsDir := path.Join(workingRootDir, *subdDir, "objects")
	tmpDir := path.Join(subdDirPathname, "tmp")
	netbenchFilename := path.Join(subdDirPathname, "netbench")
	oldTriggersFilename := path.Join(subdDirPathname, "triggers.previous")
	if !createDirectory(workingRootDir) {
		os.Exit(1)
	}
	if !sanityCheck() {
		os.Exit(1)
	}
	if !createDirectory(tmpDir) {
		os.Exit(1)
	}
	if !mountTmpfs(tmpDir) {
		os.Exit(1)
	}
	if !unshareAndBind(workingRootDir) {
		os.Exit(1)
	}
	if !createDirectory(objectsDir) {
		os.Exit(1)
	}
	runtime.GOMAXPROCS(int(*maxThreads))
	circularBuffer := logbuf.New()
	logger := log.New(circularBuffer, "", log.LstdFlags)
	if err := setupserver.SetupTls(); err != nil {
		logger.Println(err)
		circularBuffer.Flush()
		if !*permitInsecureMode {
			os.Exit(1)
		}
	}
	bytesPerSecond, blocksPerSecond, firstScan, ok := getCachedFsSpeed(
		workingRootDir, tmpDir)
	if !ok {
		os.Exit(1)
	}
	publishFsSpeed(bytesPerSecond, blocksPerSecond)
	var configuration scanner.Configuration
	var err error
	configuration.ScanFilter, err = filter.New(scanExcludeList)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to set default scan exclusions: %s\n",
			err)
		os.Exit(1)
	}
	configuration.FsScanContext = fsrateio.NewReaderContext(bytesPerSecond,
		blocksPerSecond, *defaultScanSpeedPercent)
	defaultSpeed := configuration.FsScanContext.GetContext().SpeedPercent()
	if firstScan {
		configuration.FsScanContext.GetContext().SetSpeedPercent(100)
	}
	if *showStats {
		fmt.Println(configuration.FsScanContext)
	}
	var fsh scanner.FileSystemHistory
	mainFunc := func(fsChannel <-chan *scanner.FileSystem,
		disableScanner func(disableScanner bool)) {
		networkReaderContext := rateio.NewReaderContext(
			getCachedNetworkSpeed(netbenchFilename),
			*defaultNetworkSpeedPercent, &rateio.ReadMeasurer{})
		configuration.NetworkReaderContext = networkReaderContext
		invalidateNextScanObjectCache := false
		rpcdHtmlWriter :=
			rpcd.Setup(&configuration, &fsh, objectsDir,
				workingRootDir, networkReaderContext, netbenchFilename,
				oldTriggersFilename, disableScanner,
				func() {
					invalidateNextScanObjectCache = true
					fsh.UpdateObjectCacheOnly()
				},
				logger)
		configMetricsDir, err := tricorder.RegisterDirectory("/config")
		if err != nil {
			fmt.Fprintf(os.Stderr,
				"Unable to create /config metrics directory: %s\n",
				err)
			os.Exit(1)
		}
		configuration.RegisterMetrics(configMetricsDir)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Unable to create config metrics: %s\n", err)
			os.Exit(1)
		}
		httpd.AddHtmlWriter(rpcdHtmlWriter)
		httpd.AddHtmlWriter(&fsh)
		httpd.AddHtmlWriter(&configuration)
		httpd.AddHtmlWriter(circularBuffer)
		html.RegisterHtmlWriterForPattern("/dumpFileSystem",
			"Scanned File System",
			&DumpableFileSystemHistory{&fsh})
		if err = httpd.StartServer(*portNum); err != nil {
			fmt.Fprintf(os.Stderr, "Unable to create http server: %s\n", err)
			os.Exit(1)
		}
		fsh.Update(nil)
		sighupChannel := make(chan os.Signal)
		signal.Notify(sighupChannel, syscall.SIGHUP)
		sigtermChannel := make(chan os.Signal)
		signal.Notify(sigtermChannel, syscall.SIGTERM, syscall.SIGINT)
		writePidfile()
		for iter := 0; true; {
			select {
			case <-sighupChannel:
				logger.Printf("Caught SIGHUP: re-execing with: %v\n", os.Args)
				circularBuffer.Flush()
				err = syscall.Exec(os.Args[0], os.Args, os.Environ())
				if err != nil {
					logger.Printf("Unable to Exec:%s: %s\n", os.Args[0], err)
				}
			case <-sigtermChannel:
				logger.Printf("Caught SIGTERM: performing graceful cleanup\n")
				circularBuffer.Flush()
				gracefulCleanup()
			case fs := <-fsChannel:
				if *showStats {
					fmt.Printf("Completed cycle: %d\n", iter)
				}
				if invalidateNextScanObjectCache {
					fs.ScanObjectCache()
					invalidateNextScanObjectCache = false
				}
				fsh.Update(fs)
				iter++
				runtime.GC() // An opportune time to take out the garbage.
				if *showStats {
					fmt.Print(fsh)
					fmt.Print(fsh.FileSystem())
					memstats.WriteMemoryStats(os.Stdout)
					fmt.Println()
				}
				if firstScan {
					configuration.FsScanContext.GetContext().SetSpeedPercent(
						defaultSpeed)
					firstScan = false
					if *showStats {
						fmt.Println(configuration.FsScanContext)
					}
				}
			}
		}
	}
	scanner.StartScanning(workingRootDir, objectsDir, &configuration, logger,
		mainFunc)
}