func main() { flag.Parse() tricorder.RegisterFlags() if os.Geteuid() == 0 { fmt.Fprintln(os.Stderr, "Do not run the Image Server as root") os.Exit(1) } setupTls(*caFile, *certFile, *keyFile) circularBuffer := logbuf.New(*logbufLines) logger := log.New(circularBuffer, "", log.LstdFlags) objSrv, err := filesystem.NewObjectServer(*objectDir, logger) if err != nil { fmt.Fprintf(os.Stderr, "Cannot create ObjectServer\t%s\n", err) os.Exit(1) } imdb, err := scanner.LoadImageDataBase(*imageDir, objSrv, logger) if err != nil { fmt.Fprintf(os.Stderr, "Cannot load image database\t%s\n", err) os.Exit(1) } tricorder.RegisterMetric("/image-count", func() uint { return imdb.CountImages() }, units.None, "number of images") imageserverRpcd.Setup(imdb, logger) rpcHtmlWriter := objectserverRpcd.Setup(objSrv, logger) rpc.HandleHTTP() httpd.AddHtmlWriter(imdb) httpd.AddHtmlWriter(rpcHtmlWriter) httpd.AddHtmlWriter(circularBuffer) if err = httpd.StartServer(*portNum, imdb, false); err != nil { fmt.Fprintf(os.Stderr, "Unable to create http server\t%s\n", err) os.Exit(1) } }
func main() { flag.Parse() tricorder.RegisterFlags() if os.Geteuid() == 0 { fmt.Fprintln(os.Stderr, "Do not run the Image Server as root") os.Exit(1) } if *archiveMode && *imageServerHostname == "" { fmt.Fprintln(os.Stderr, "-imageServerHostname required in archive mode") os.Exit(1) } circularBuffer := logbuf.New() logger := log.New(circularBuffer, "", log.LstdFlags) if err := setupserver.SetupTls(); err != nil { logger.Println(err) circularBuffer.Flush() if !*permitInsecureMode { os.Exit(1) } } objSrv, err := filesystem.NewObjectServer(*objectDir, logger) if err != nil { fmt.Fprintf(os.Stderr, "Cannot create ObjectServer: %s\n", err) os.Exit(1) } cleanupUnreferencedObjects := true if *imageServerHostname != "" { cleanupUnreferencedObjects = false } imdb, err := scanner.LoadImageDataBase(*imageDir, objSrv, cleanupUnreferencedObjects, logger) if err != nil { fmt.Fprintf(os.Stderr, "Cannot load image database: %s\n", err) os.Exit(1) } tricorder.RegisterMetric("/image-count", func() uint { return imdb.CountImages() }, units.None, "number of images") imgSrvRpcHtmlWriter := imageserverRpcd.Setup(imdb, *imageServerHostname, logger) objSrvRpcHtmlWriter := objectserverRpcd.Setup(objSrv, logger) httpd.AddHtmlWriter(imdb) httpd.AddHtmlWriter(&imageObjectServersType{imdb, objSrv}) httpd.AddHtmlWriter(imgSrvRpcHtmlWriter) httpd.AddHtmlWriter(objSrvRpcHtmlWriter) httpd.AddHtmlWriter(circularBuffer) if *imageServerHostname != "" { go replicator(fmt.Sprintf("%s:%d", *imageServerHostname, *imageServerPortNum), imdb, objSrv, *archiveMode, logger) } if err = httpd.StartServer(*portNum, imdb, objSrv, false); err != nil { fmt.Fprintf(os.Stderr, "Unable to create http server: %s\n", err) os.Exit(1) } }
func main() { flag.Usage = printUsage flag.Parse() tricorder.RegisterFlags() circularBuffer := logbuf.New() logger := log.New(circularBuffer, "", log.LstdFlags) // We have to have inputs. if *sourcesFile == "" { printUsage() os.Exit(2) } handleSignals(logger) var generators []generator readerChannel := fsutil.WatchFile(*sourcesFile, logger) file, err := os.Open(*sourcesFile) if err != nil { showErrorAndDie(err) } (<-readerChannel).Close() scanner := bufio.NewScanner(file) for scanner.Scan() { fields := strings.Fields(scanner.Text()) if len(fields) == 0 || len(fields[0]) == 0 || fields[0][0] == '#' { continue } gen, err := getSource(fields) if err != nil { showErrorAndDie(err) } generators = append(generators, gen) } if err := scanner.Err(); err != nil { showErrorAndDie(err) } file.Close() httpSrv, err := startHttpServer(*portNum) if err != nil { showErrorAndDie(err) } httpSrv.AddHtmlWriter(circularBuffer) updateFunc := startRpcd(logger) go runDaemon(generators, *mdbFile, *hostnameRegex, *datacentre, *fetchInterval, updateFunc, logger, *debug) <-readerChannel fsutil.WatchFileStop() if err := syscall.Exec(os.Args[0], os.Args, os.Environ()); err != nil { logger.Printf("Unable to Exec:%s: %s\n", os.Args[0], err) } }
func main() { flag.Usage = printUsage flag.Parse() tricorder.RegisterFlags() if os.Geteuid() == 0 { fmt.Fprintln(os.Stderr, "Do not run the filegen server as root") os.Exit(1) } circularBuffer := logbuf.New() logger := log.New(circularBuffer, "", log.LstdFlags) if err := setupserver.SetupTls(); err != nil { logger.Println(err) circularBuffer.Flush() if !*permitInsecureMode { os.Exit(1) } } manager := filegen.New(logger) if *configFile != "" { if err := util.LoadConfiguration(manager, *configFile); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } ch := fsutil.WatchFile(*configFile, nil) (<-ch).Close() // Drain the first event. go func() { <-ch err := syscall.Exec(os.Args[0], os.Args, os.Environ()) if err != nil { logger.Printf("Unable to Exec:%s: %s\n", os.Args[0], err) } }() } httpd.AddHtmlWriter(manager) httpd.AddHtmlWriter(circularBuffer) for _, pathname := range flag.Args() { if err := registerSourceDirectory(manager, pathname, "/"); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } } if err := httpd.StartServer(*portNum, manager, false); err != nil { fmt.Fprintf(os.Stderr, "Unable to create http server: %s\n", err) os.Exit(1) } }
func main() { tricorder.RegisterFlags() flag.Parse() circularBuffer := logbuf.New() logger := log.New(circularBuffer, "", log.LstdFlags) handleSignals(logger) // Read configs early so that we will fail fast. maybeNilMemoryManager := maybeCreateMemoryManager(logger) consumerBuilders, err := newPStoreConsumers(maybeNilMemoryManager) if err != nil { log.Println("Pstore config file error:", err) logger.Println("Pstore config file error:", err) } metricNameEngine := suggest.NewEngine() metricNameAdder := newTsdbAdder(metricNameEngine) tagkEngine := suggest.NewSuggester("appname", "HostName") tagvEngine := suggest.NewEngine() tagvAdder := newTsdbAdder(tagvEngine) appList := createApplicationList() for _, app := range appList.All() { tagvAdder.Add(app.Name()) } applicationStats := createApplicationStats( appList, logger, tagvAdder, maybeNilMemoryManager) rpc.RegisterName( "Scotty", &rpcType{AS: applicationStats}, ) rpc.HandleHTTP() connectionErrors := newConnectionErrorsType() if consumerBuilders == nil { startCollector( applicationStats, connectionErrors, totalCountCollectionType(nil), metricNameAdder, &maybeNilMemoryManagerWrapperType{maybeNilMemoryManager}) } else { var coord coordinatorBuilderType if *fCoord != "" { var err error coord, err = consul.NewCoordinator(logger) if err != nil { logger.Println(err) coord = &blockingCoordinatorType{} } } totalCounts := startPStoreLoops( applicationStats, consumerBuilders, logger, coord) startCollector( applicationStats, connectionErrors, totalCountCollectionType(totalCounts), metricNameAdder, &maybeNilMemoryManagerWrapperType{maybeNilMemoryManager}) } http.Handle( "/", gzipHandler{&splash.Handler{ AS: applicationStats, Log: circularBuffer, }}) http.Handle( "/showAllApps", gzipHandler{&showallapps.Handler{ AS: applicationStats, CollectionFreq: *fCollectionFrequency, }}) http.Handle( "/api/hosts/", http.StripPrefix( "/api/hosts/", gzipHandler{&byEndpointHandler{ AS: applicationStats, }})) http.Handle( "/api/latest/", http.StripPrefix( "/api/latest/", gzipHandler{&latestHandler{ AS: applicationStats, }})) http.Handle( "/api/errors/", gzipHandler{&errorHandler{ ConnectionErrors: connectionErrors, }}, ) tsdbServeMux := http.NewServeMux() tsdbServeMux.Handle( "/api/query", tsdbexec.NewHandler( func(r *tsdbjson.QueryRequest) ([]tsdbjson.TimeSeries, error) { return tsdbexec.Query( r, applicationStats, *fCollectionFrequency) })) tsdbServeMux.Handle( "/api/suggest", tsdbexec.NewHandler( func(req url.Values) ([]string, error) { return tsdbexec.Suggest( req, map[string]suggest.Suggester{ "metrics": metricNameEngine, "tagk": tagkEngine, "tagv": tagvEngine, }) }, )) tsdbServeMux.Handle( "/api/aggregators", tsdbexec.NewHandler( func(req url.Values) ([]string, error) { return aggregators.Names(), nil }, )) tsdbServeMux.Handle( "/api/version", tsdbexec.NewHandler( func(req url.Values) (map[string]string, error) { return map[string]string{ "version": "1.0", }, nil }, )) tsdbServeMux.Handle( "/api/config", tsdbexec.NewHandler( func(req url.Values) (map[string]string, error) { return map[string]string{ "tsd.ore.auto_create_metrics": "true", "tsd.ore.auto_create_tagks": "true", "tsd.ore.auto_create_tagvs": "true", }, nil }, )) tsdbServeMux.Handle( "/api/config/filters", tsdbexec.NewHandler( func(req url.Values) (interface{}, error) { return tsdbjson.AllFilterDescriptions(), nil }, )) tsdbServeMux.Handle( "/api/dropcaches", tsdbexec.NewHandler( func(req url.Values) (map[string]string, error) { return map[string]string{ "message": "Caches dropped", "status": "200", }, nil }, )) tsdbServeMux.Handle( "/api", tsdbexec.NotFoundHandler, ) go func() { if err := http.ListenAndServe(fmt.Sprintf(":%d", *fTsdbPort), tsdbServeMux); err != nil { log.Fatal(err) } }() if err := http.ListenAndServe(fmt.Sprintf(":%d", *fPort), nil); err != nil { log.Fatal(err) } }
func main() { flag.Parse() tricorder.RegisterFlags() circularBuffer := logbuf.New() logger := log.New(circularBuffer, "", log.LstdFlags) if err := setupserver.SetupTls(); err != nil { logger.Println(err) circularBuffer.Flush() if !*permitInsecureMode { os.Exit(1) } } rlim := syscall.Rlimit{*fdLimit, *fdLimit} if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlim); err != nil { fmt.Fprintf(os.Stderr, "Cannot set FD limit\t%s\n", err) os.Exit(1) } if os.Geteuid() == 0 { if err := setUser(*username); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } } fi, err := os.Lstat(*stateDir) if err != nil { fmt.Fprintf(os.Stderr, "Cannot stat: %s\t%s\n", *stateDir, err) os.Exit(1) } if !fi.IsDir() { fmt.Fprintf(os.Stderr, "%s is not a directory\n", *stateDir) os.Exit(1) } interval := time.Duration(*minInterval) * time.Second mdbChannel := mdbd.StartMdbDaemon(path.Join(*stateDir, *mdbFile), logger) objectServer, err := newObjectServer(path.Join(*stateDir, *objectsDir), logger) if err != nil { fmt.Fprintf(os.Stderr, "Cannot load objectcache: %s\n", err) os.Exit(1) } herd := herd.NewHerd(fmt.Sprintf("%s:%d", *imageServerHostname, *imageServerPortNum), objectServer, logger) herd.AddHtmlWriter(circularBuffer) rpcd.Setup(herd, logger) if err = herd.StartServer(*portNum, true); err != nil { fmt.Fprintf(os.Stderr, "Unable to create http server\t%s\n", err) os.Exit(1) } scanTokenChannel := make(chan bool, 1) scanTokenChannel <- true nextCycleStopTime := time.Now().Add(interval) for { select { case mdb := <-mdbChannel: herd.MdbUpdate(mdb) if *debug { showMdb(mdb) } case <-scanTokenChannel: // Scan one sub. if herd.PollNextSub() { // We've reached the end of a scan cycle. if *debug { fmt.Print(".") } go func(sleepDuration time.Duration) { time.Sleep(sleepDuration) nextCycleStopTime = time.Now().Add(interval) scanTokenChannel <- true }(nextCycleStopTime.Sub(time.Now())) } else { scanTokenChannel <- true } } } }
func main() { flag.Parse() tricorder.RegisterFlags() setupTls(*caFile, *certFile, *keyFile) subdDirPathname := path.Join(*rootDir, *subdDir) workingRootDir := path.Join(subdDirPathname, "root") objectsDir := path.Join(workingRootDir, *subdDir, "objects") tmpDir := path.Join(subdDirPathname, "tmp") netbenchFilename := path.Join(subdDirPathname, "netbench") oldTriggersFilename := path.Join(subdDirPathname, "triggers.previous") if !createDirectory(workingRootDir) { os.Exit(1) } if !sanityCheck() { os.Exit(1) } if !createDirectory(tmpDir) { os.Exit(1) } if !mountTmpfs(tmpDir) { os.Exit(1) } if !unshareAndBind(workingRootDir) { os.Exit(1) } if !createDirectory(objectsDir) { os.Exit(1) } runtime.GOMAXPROCS(int(*maxThreads)) bytesPerSecond, blocksPerSecond, firstScan, ok := getCachedFsSpeed( workingRootDir, tmpDir) if !ok { os.Exit(1) } circularBuffer := logbuf.New(*logbufLines) logger := log.New(circularBuffer, "", log.LstdFlags) var configuration scanner.Configuration var err error configuration.ScanFilter, err = filter.NewFilter(constants.ScanExcludeList) if err != nil { fmt.Fprintf(os.Stderr, "Unable to set default scan exclusions\t%s\n", err) os.Exit(1) } configuration.FsScanContext = fsrateio.NewReaderContext(bytesPerSecond, blocksPerSecond, 0) defaultSpeed := configuration.FsScanContext.GetContext().SpeedPercent() if firstScan { configuration.FsScanContext.GetContext().SetSpeedPercent(100) } if *showStats { fmt.Println(configuration.FsScanContext) } var fsh scanner.FileSystemHistory fsChannel, disableScanner := scanner.StartScannerDaemon(workingRootDir, objectsDir, &configuration, logger) networkReaderContext := rateio.NewReaderContext( getCachedNetworkSpeed(netbenchFilename), constants.DefaultNetworkSpeedPercent, &rateio.ReadMeasurer{}) configuration.NetworkReaderContext = networkReaderContext rescanObjectCacheChannel := rpcd.Setup(&configuration, &fsh, objectsDir, networkReaderContext, netbenchFilename, oldTriggersFilename, disableScanner, logger) configMetricsDir, err := tricorder.RegisterDirectory("/config") if err != nil { fmt.Fprintf(os.Stderr, "Unable to create /config metrics directory\t%s\n", err) os.Exit(1) } configuration.RegisterMetrics(configMetricsDir) if err != nil { fmt.Fprintf(os.Stderr, "Unable to create config metrics\t%s\n", err) os.Exit(1) } httpd.AddHtmlWriter(&fsh) httpd.AddHtmlWriter(&configuration) httpd.AddHtmlWriter(circularBuffer) html.RegisterHtmlWriterForPattern("/dumpFileSystem", "Scanned File System", &DumpableFileSystemHistory{&fsh}) if err = httpd.StartServer(*portNum); err != nil { fmt.Fprintf(os.Stderr, "Unable to create http server\t%s\n", err) os.Exit(1) } fsh.Update(nil) invalidateNextScanObjectCache := false sighupChannel := make(chan os.Signal) signal.Notify(sighupChannel, syscall.SIGHUP) sigtermChannel := make(chan os.Signal) signal.Notify(sigtermChannel, syscall.SIGTERM, syscall.SIGINT) writePidfile() for iter := 0; true; { select { case <-sighupChannel: err = syscall.Exec(os.Args[0], os.Args, os.Environ()) if err != nil { logger.Printf("Unable to Exec:%s\t%s\n", os.Args[0], err) } case <-sigtermChannel: gracefulCleanup() case fs := <-fsChannel: if *showStats { fmt.Printf("Completed cycle: %d\n", iter) } if invalidateNextScanObjectCache { fs.ScanObjectCache() invalidateNextScanObjectCache = false } fsh.Update(fs) iter++ runtime.GC() // An opportune time to take out the garbage. if *showStats { fmt.Print(fsh) fmt.Print(fsh.FileSystem()) memstats.WriteMemoryStats(os.Stdout) fmt.Println() } if firstScan { configuration.FsScanContext.GetContext().SetSpeedPercent( defaultSpeed) firstScan = false if *showStats { fmt.Println(configuration.FsScanContext) } } case <-rescanObjectCacheChannel: invalidateNextScanObjectCache = true fsh.UpdateObjectCacheOnly() } } }
func main() { flag.Parse() tricorder.RegisterFlags() setupTls(*caFile, pathJoin(*certDir, *certFile), pathJoin(*certDir, *keyFile)) rlim := syscall.Rlimit{*fdLimit, *fdLimit} if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlim); err != nil { fmt.Fprintf(os.Stderr, "Cannot set FD limit\t%s\n", err) os.Exit(1) } if os.Geteuid() == 0 { if err := setUser(*username); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } } fi, err := os.Lstat(*stateDir) if err != nil { fmt.Fprintf(os.Stderr, "Cannot stat: %s\t%s\n", *stateDir, err) os.Exit(1) } if !fi.IsDir() { fmt.Fprintf(os.Stderr, "%s is not a directory\n", *stateDir) os.Exit(1) } interval := time.Duration(*minInterval) * time.Second circularBuffer := logbuf.New(*logbufLines) logger := log.New(circularBuffer, "", log.LstdFlags) mdbChannel := mdbd.StartMdbDaemon(path.Join(*stateDir, *mdbFile), logger) herd := herd.NewHerd(fmt.Sprintf("%s:%d", *imageServerHostname, *imageServerPortNum), logger) herd.AddHtmlWriter(circularBuffer) if err = herd.StartServer(*portNum, true); err != nil { fmt.Fprintf(os.Stderr, "Unable to create http server\t%s\n", err) os.Exit(1) } nextCycleStopTime := time.Now().Add(interval) for { select { case mdb := <-mdbChannel: herd.MdbUpdate(mdb) if *debug { showMdb(mdb) } runtime.GC() // An opportune time to take out the garbage. default: // Do work. if herd.PollNextSub() { if *debug { fmt.Print(".") } sleepTime := nextCycleStopTime.Sub(time.Now()) time.Sleep(sleepTime) nextCycleStopTime = time.Now().Add(interval) if sleepTime < 0 { // There was no time to rest. runtime.GC() // An opportune time to take out the garbage. } } } } }