func main() {
flag.Parse()
tricorder.RegisterFlags()
if os.Geteuid() == 0 {
fmt.Fprintln(os.Stderr, "Do not run the Image Server as root")
os.Exit(1)
}
setupTls(*caFile, *certFile, *keyFile)
circularBuffer := logbuf.New(*logbufLines)
logger := log.New(circularBuffer, "", log.LstdFlags)
objSrv, err := filesystem.NewObjectServer(*objectDir, logger)
if err != nil {
fmt.Fprintf(os.Stderr, "Cannot create ObjectServer\t%s\n", err)
os.Exit(1)
}
imdb, err := scanner.LoadImageDataBase(*imageDir, objSrv, logger)
if err != nil {
fmt.Fprintf(os.Stderr, "Cannot load image database\t%s\n", err)
os.Exit(1)
}
tricorder.RegisterMetric("/image-count",
func() uint { return imdb.CountImages() },
units.None, "number of images")
imageserverRpcd.Setup(imdb, logger)
rpcHtmlWriter := objectserverRpcd.Setup(objSrv, logger)
rpc.HandleHTTP()
httpd.AddHtmlWriter(imdb)
httpd.AddHtmlWriter(rpcHtmlWriter)
httpd.AddHtmlWriter(circularBuffer)
if err = httpd.StartServer(*portNum, imdb, false); err != nil {
fmt.Fprintf(os.Stderr, "Unable to create http server\t%s\n", err)
os.Exit(1)
}
}
func main() {
flag.Parse()
tricorder.RegisterFlags()
if os.Geteuid() == 0 {
fmt.Fprintln(os.Stderr, "Do not run the Image Server as root")
os.Exit(1)
}
if *archiveMode && *imageServerHostname == "" {
fmt.Fprintln(os.Stderr, "-imageServerHostname required in archive mode")
os.Exit(1)
}
circularBuffer := logbuf.New()
logger := log.New(circularBuffer, "", log.LstdFlags)
if err := setupserver.SetupTls(); err != nil {
logger.Println(err)
circularBuffer.Flush()
if !*permitInsecureMode {
os.Exit(1)
}
}
objSrv, err := filesystem.NewObjectServer(*objectDir, logger)
if err != nil {
fmt.Fprintf(os.Stderr, "Cannot create ObjectServer: %s\n", err)
os.Exit(1)
}
cleanupUnreferencedObjects := true
if *imageServerHostname != "" {
cleanupUnreferencedObjects = false
}
imdb, err := scanner.LoadImageDataBase(*imageDir, objSrv,
cleanupUnreferencedObjects, logger)
if err != nil {
fmt.Fprintf(os.Stderr, "Cannot load image database: %s\n", err)
os.Exit(1)
}
tricorder.RegisterMetric("/image-count",
func() uint { return imdb.CountImages() },
units.None, "number of images")
imgSrvRpcHtmlWriter := imageserverRpcd.Setup(imdb, *imageServerHostname,
logger)
objSrvRpcHtmlWriter := objectserverRpcd.Setup(objSrv, logger)
httpd.AddHtmlWriter(imdb)
httpd.AddHtmlWriter(&imageObjectServersType{imdb, objSrv})
httpd.AddHtmlWriter(imgSrvRpcHtmlWriter)
httpd.AddHtmlWriter(objSrvRpcHtmlWriter)
httpd.AddHtmlWriter(circularBuffer)
if *imageServerHostname != "" {
go replicator(fmt.Sprintf("%s:%d", *imageServerHostname,
*imageServerPortNum), imdb, objSrv, *archiveMode, logger)
}
if err = httpd.StartServer(*portNum, imdb, objSrv, false); err != nil {
fmt.Fprintf(os.Stderr, "Unable to create http server: %s\n", err)
os.Exit(1)
}
}
func main() {
flag.Usage = printUsage
flag.Parse()
tricorder.RegisterFlags()
circularBuffer := logbuf.New()
logger := log.New(circularBuffer, "", log.LstdFlags)
// We have to have inputs.
if *sourcesFile == "" {
printUsage()
os.Exit(2)
}
handleSignals(logger)
var generators []generator
readerChannel := fsutil.WatchFile(*sourcesFile, logger)
file, err := os.Open(*sourcesFile)
if err != nil {
showErrorAndDie(err)
}
(<-readerChannel).Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
fields := strings.Fields(scanner.Text())
if len(fields) == 0 || len(fields[0]) == 0 || fields[0][0] == '#' {
continue
}
gen, err := getSource(fields)
if err != nil {
showErrorAndDie(err)
}
generators = append(generators, gen)
}
if err := scanner.Err(); err != nil {
showErrorAndDie(err)
}
file.Close()
httpSrv, err := startHttpServer(*portNum)
if err != nil {
showErrorAndDie(err)
}
httpSrv.AddHtmlWriter(circularBuffer)
updateFunc := startRpcd(logger)
go runDaemon(generators, *mdbFile, *hostnameRegex, *datacentre,
*fetchInterval, updateFunc, logger, *debug)
<-readerChannel
fsutil.WatchFileStop()
if err := syscall.Exec(os.Args[0], os.Args, os.Environ()); err != nil {
logger.Printf("Unable to Exec:%s: %s\n", os.Args[0], err)
}
}
func main() {
flag.Usage = printUsage
flag.Parse()
tricorder.RegisterFlags()
if os.Geteuid() == 0 {
fmt.Fprintln(os.Stderr, "Do not run the filegen server as root")
os.Exit(1)
}
circularBuffer := logbuf.New()
logger := log.New(circularBuffer, "", log.LstdFlags)
if err := setupserver.SetupTls(); err != nil {
logger.Println(err)
circularBuffer.Flush()
if !*permitInsecureMode {
os.Exit(1)
}
}
manager := filegen.New(logger)
if *configFile != "" {
if err := util.LoadConfiguration(manager, *configFile); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
ch := fsutil.WatchFile(*configFile, nil)
(<-ch).Close() // Drain the first event.
go func() {
<-ch
err := syscall.Exec(os.Args[0], os.Args, os.Environ())
if err != nil {
logger.Printf("Unable to Exec:%s: %s\n", os.Args[0], err)
}
}()
}
httpd.AddHtmlWriter(manager)
httpd.AddHtmlWriter(circularBuffer)
for _, pathname := range flag.Args() {
if err := registerSourceDirectory(manager, pathname, "/"); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
if err := httpd.StartServer(*portNum, manager, false); err != nil {
fmt.Fprintf(os.Stderr, "Unable to create http server: %s\n", err)
os.Exit(1)
}
}
func main() {
tricorder.RegisterFlags()
flag.Parse()
circularBuffer := logbuf.New()
logger := log.New(circularBuffer, "", log.LstdFlags)
handleSignals(logger)
// Read configs early so that we will fail fast.
maybeNilMemoryManager := maybeCreateMemoryManager(logger)
consumerBuilders, err := newPStoreConsumers(maybeNilMemoryManager)
if err != nil {
log.Println("Pstore config file error:", err)
logger.Println("Pstore config file error:", err)
}
metricNameEngine := suggest.NewEngine()
metricNameAdder := newTsdbAdder(metricNameEngine)
tagkEngine := suggest.NewSuggester("appname", "HostName")
tagvEngine := suggest.NewEngine()
tagvAdder := newTsdbAdder(tagvEngine)
appList := createApplicationList()
for _, app := range appList.All() {
tagvAdder.Add(app.Name())
}
applicationStats := createApplicationStats(
appList, logger, tagvAdder, maybeNilMemoryManager)
rpc.RegisterName(
"Scotty",
&rpcType{AS: applicationStats},
)
rpc.HandleHTTP()
connectionErrors := newConnectionErrorsType()
if consumerBuilders == nil {
startCollector(
applicationStats,
connectionErrors,
totalCountCollectionType(nil),
metricNameAdder,
&maybeNilMemoryManagerWrapperType{maybeNilMemoryManager})
} else {
var coord coordinatorBuilderType
if *fCoord != "" {
var err error
coord, err = consul.NewCoordinator(logger)
if err != nil {
logger.Println(err)
coord = &blockingCoordinatorType{}
}
}
totalCounts := startPStoreLoops(
applicationStats,
consumerBuilders,
logger,
coord)
startCollector(
applicationStats,
connectionErrors,
totalCountCollectionType(totalCounts),
metricNameAdder,
&maybeNilMemoryManagerWrapperType{maybeNilMemoryManager})
}
http.Handle(
"/",
gzipHandler{&splash.Handler{
AS: applicationStats,
Log: circularBuffer,
}})
http.Handle(
"/showAllApps",
gzipHandler{&showallapps.Handler{
AS: applicationStats,
CollectionFreq: *fCollectionFrequency,
}})
http.Handle(
"/api/hosts/",
http.StripPrefix(
"/api/hosts/",
gzipHandler{&byEndpointHandler{
AS: applicationStats,
}}))
http.Handle(
"/api/latest/",
http.StripPrefix(
"/api/latest/",
gzipHandler{&latestHandler{
AS: applicationStats,
}}))
http.Handle(
"/api/errors/",
gzipHandler{&errorHandler{
ConnectionErrors: connectionErrors,
}},
)
tsdbServeMux := http.NewServeMux()
tsdbServeMux.Handle(
"/api/query",
tsdbexec.NewHandler(
func(r *tsdbjson.QueryRequest) ([]tsdbjson.TimeSeries, error) {
return tsdbexec.Query(
r, applicationStats, *fCollectionFrequency)
}))
tsdbServeMux.Handle(
"/api/suggest",
tsdbexec.NewHandler(
func(req url.Values) ([]string, error) {
return tsdbexec.Suggest(
req,
map[string]suggest.Suggester{
"metrics": metricNameEngine,
"tagk": tagkEngine,
"tagv": tagvEngine,
})
},
))
tsdbServeMux.Handle(
"/api/aggregators",
tsdbexec.NewHandler(
func(req url.Values) ([]string, error) {
return aggregators.Names(), nil
},
))
tsdbServeMux.Handle(
"/api/version",
tsdbexec.NewHandler(
func(req url.Values) (map[string]string, error) {
return map[string]string{
"version": "1.0",
}, nil
},
))
tsdbServeMux.Handle(
"/api/config",
tsdbexec.NewHandler(
func(req url.Values) (map[string]string, error) {
return map[string]string{
"tsd.ore.auto_create_metrics": "true",
"tsd.ore.auto_create_tagks": "true",
"tsd.ore.auto_create_tagvs": "true",
}, nil
},
))
tsdbServeMux.Handle(
"/api/config/filters",
tsdbexec.NewHandler(
func(req url.Values) (interface{}, error) {
return tsdbjson.AllFilterDescriptions(), nil
},
))
tsdbServeMux.Handle(
"/api/dropcaches",
tsdbexec.NewHandler(
func(req url.Values) (map[string]string, error) {
return map[string]string{
"message": "Caches dropped",
"status": "200",
}, nil
},
))
tsdbServeMux.Handle(
"/api",
tsdbexec.NotFoundHandler,
)
go func() {
if err := http.ListenAndServe(fmt.Sprintf(":%d", *fTsdbPort), tsdbServeMux); err != nil {
log.Fatal(err)
}
}()
if err := http.ListenAndServe(fmt.Sprintf(":%d", *fPort), nil); err != nil {
log.Fatal(err)
}
}
func main() {
flag.Parse()
tricorder.RegisterFlags()
circularBuffer := logbuf.New()
logger := log.New(circularBuffer, "", log.LstdFlags)
if err := setupserver.SetupTls(); err != nil {
logger.Println(err)
circularBuffer.Flush()
if !*permitInsecureMode {
os.Exit(1)
}
}
rlim := syscall.Rlimit{*fdLimit, *fdLimit}
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlim); err != nil {
fmt.Fprintf(os.Stderr, "Cannot set FD limit\t%s\n", err)
os.Exit(1)
}
if os.Geteuid() == 0 {
if err := setUser(*username); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
fi, err := os.Lstat(*stateDir)
if err != nil {
fmt.Fprintf(os.Stderr, "Cannot stat: %s\t%s\n", *stateDir, err)
os.Exit(1)
}
if !fi.IsDir() {
fmt.Fprintf(os.Stderr, "%s is not a directory\n", *stateDir)
os.Exit(1)
}
interval := time.Duration(*minInterval) * time.Second
mdbChannel := mdbd.StartMdbDaemon(path.Join(*stateDir, *mdbFile), logger)
objectServer, err := newObjectServer(path.Join(*stateDir, *objectsDir),
logger)
if err != nil {
fmt.Fprintf(os.Stderr, "Cannot load objectcache: %s\n", err)
os.Exit(1)
}
herd := herd.NewHerd(fmt.Sprintf("%s:%d", *imageServerHostname,
*imageServerPortNum), objectServer, logger)
herd.AddHtmlWriter(circularBuffer)
rpcd.Setup(herd, logger)
if err = herd.StartServer(*portNum, true); err != nil {
fmt.Fprintf(os.Stderr, "Unable to create http server\t%s\n", err)
os.Exit(1)
}
scanTokenChannel := make(chan bool, 1)
scanTokenChannel <- true
nextCycleStopTime := time.Now().Add(interval)
for {
select {
case mdb := <-mdbChannel:
herd.MdbUpdate(mdb)
if *debug {
showMdb(mdb)
}
case <-scanTokenChannel:
// Scan one sub.
if herd.PollNextSub() { // We've reached the end of a scan cycle.
if *debug {
fmt.Print(".")
}
go func(sleepDuration time.Duration) {
time.Sleep(sleepDuration)
nextCycleStopTime = time.Now().Add(interval)
scanTokenChannel <- true
}(nextCycleStopTime.Sub(time.Now()))
} else {
scanTokenChannel <- true
}
}
}
}
func main() {
flag.Parse()
tricorder.RegisterFlags()
setupTls(*caFile, *certFile, *keyFile)
subdDirPathname := path.Join(*rootDir, *subdDir)
workingRootDir := path.Join(subdDirPathname, "root")
objectsDir := path.Join(workingRootDir, *subdDir, "objects")
tmpDir := path.Join(subdDirPathname, "tmp")
netbenchFilename := path.Join(subdDirPathname, "netbench")
oldTriggersFilename := path.Join(subdDirPathname, "triggers.previous")
if !createDirectory(workingRootDir) {
os.Exit(1)
}
if !sanityCheck() {
os.Exit(1)
}
if !createDirectory(tmpDir) {
os.Exit(1)
}
if !mountTmpfs(tmpDir) {
os.Exit(1)
}
if !unshareAndBind(workingRootDir) {
os.Exit(1)
}
if !createDirectory(objectsDir) {
os.Exit(1)
}
runtime.GOMAXPROCS(int(*maxThreads))
bytesPerSecond, blocksPerSecond, firstScan, ok := getCachedFsSpeed(
workingRootDir, tmpDir)
if !ok {
os.Exit(1)
}
circularBuffer := logbuf.New(*logbufLines)
logger := log.New(circularBuffer, "", log.LstdFlags)
var configuration scanner.Configuration
var err error
configuration.ScanFilter, err = filter.NewFilter(constants.ScanExcludeList)
if err != nil {
fmt.Fprintf(os.Stderr, "Unable to set default scan exclusions\t%s\n",
err)
os.Exit(1)
}
configuration.FsScanContext = fsrateio.NewReaderContext(bytesPerSecond,
blocksPerSecond, 0)
defaultSpeed := configuration.FsScanContext.GetContext().SpeedPercent()
if firstScan {
configuration.FsScanContext.GetContext().SetSpeedPercent(100)
}
if *showStats {
fmt.Println(configuration.FsScanContext)
}
var fsh scanner.FileSystemHistory
fsChannel, disableScanner := scanner.StartScannerDaemon(workingRootDir,
objectsDir, &configuration, logger)
networkReaderContext := rateio.NewReaderContext(
getCachedNetworkSpeed(netbenchFilename),
constants.DefaultNetworkSpeedPercent, &rateio.ReadMeasurer{})
configuration.NetworkReaderContext = networkReaderContext
rescanObjectCacheChannel := rpcd.Setup(&configuration, &fsh, objectsDir,
networkReaderContext, netbenchFilename, oldTriggersFilename,
disableScanner, logger)
configMetricsDir, err := tricorder.RegisterDirectory("/config")
if err != nil {
fmt.Fprintf(os.Stderr,
"Unable to create /config metrics directory\t%s\n",
err)
os.Exit(1)
}
configuration.RegisterMetrics(configMetricsDir)
if err != nil {
fmt.Fprintf(os.Stderr, "Unable to create config metrics\t%s\n", err)
os.Exit(1)
}
httpd.AddHtmlWriter(&fsh)
httpd.AddHtmlWriter(&configuration)
httpd.AddHtmlWriter(circularBuffer)
html.RegisterHtmlWriterForPattern("/dumpFileSystem", "Scanned File System",
&DumpableFileSystemHistory{&fsh})
if err = httpd.StartServer(*portNum); err != nil {
fmt.Fprintf(os.Stderr, "Unable to create http server\t%s\n", err)
os.Exit(1)
}
fsh.Update(nil)
invalidateNextScanObjectCache := false
sighupChannel := make(chan os.Signal)
signal.Notify(sighupChannel, syscall.SIGHUP)
sigtermChannel := make(chan os.Signal)
signal.Notify(sigtermChannel, syscall.SIGTERM, syscall.SIGINT)
writePidfile()
for iter := 0; true; {
select {
case <-sighupChannel:
err = syscall.Exec(os.Args[0], os.Args, os.Environ())
if err != nil {
logger.Printf("Unable to Exec:%s\t%s\n", os.Args[0], err)
}
case <-sigtermChannel:
gracefulCleanup()
case fs := <-fsChannel:
if *showStats {
fmt.Printf("Completed cycle: %d\n", iter)
}
if invalidateNextScanObjectCache {
fs.ScanObjectCache()
invalidateNextScanObjectCache = false
}
fsh.Update(fs)
iter++
runtime.GC() // An opportune time to take out the garbage.
if *showStats {
fmt.Print(fsh)
fmt.Print(fsh.FileSystem())
memstats.WriteMemoryStats(os.Stdout)
fmt.Println()
}
if firstScan {
configuration.FsScanContext.GetContext().SetSpeedPercent(
defaultSpeed)
firstScan = false
if *showStats {
fmt.Println(configuration.FsScanContext)
}
}
case <-rescanObjectCacheChannel:
invalidateNextScanObjectCache = true
fsh.UpdateObjectCacheOnly()
}
}
}
func main() {
flag.Parse()
tricorder.RegisterFlags()
setupTls(*caFile,
pathJoin(*certDir, *certFile), pathJoin(*certDir, *keyFile))
rlim := syscall.Rlimit{*fdLimit, *fdLimit}
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlim); err != nil {
fmt.Fprintf(os.Stderr, "Cannot set FD limit\t%s\n", err)
os.Exit(1)
}
if os.Geteuid() == 0 {
if err := setUser(*username); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
fi, err := os.Lstat(*stateDir)
if err != nil {
fmt.Fprintf(os.Stderr, "Cannot stat: %s\t%s\n", *stateDir, err)
os.Exit(1)
}
if !fi.IsDir() {
fmt.Fprintf(os.Stderr, "%s is not a directory\n", *stateDir)
os.Exit(1)
}
interval := time.Duration(*minInterval) * time.Second
circularBuffer := logbuf.New(*logbufLines)
logger := log.New(circularBuffer, "", log.LstdFlags)
mdbChannel := mdbd.StartMdbDaemon(path.Join(*stateDir, *mdbFile), logger)
herd := herd.NewHerd(fmt.Sprintf("%s:%d", *imageServerHostname,
*imageServerPortNum), logger)
herd.AddHtmlWriter(circularBuffer)
if err = herd.StartServer(*portNum, true); err != nil {
fmt.Fprintf(os.Stderr, "Unable to create http server\t%s\n", err)
os.Exit(1)
}
nextCycleStopTime := time.Now().Add(interval)
for {
select {
case mdb := <-mdbChannel:
herd.MdbUpdate(mdb)
if *debug {
showMdb(mdb)
}
runtime.GC() // An opportune time to take out the garbage.
default:
// Do work.
if herd.PollNextSub() {
if *debug {
fmt.Print(".")
}
sleepTime := nextCycleStopTime.Sub(time.Now())
time.Sleep(sleepTime)
nextCycleStopTime = time.Now().Add(interval)
if sleepTime < 0 { // There was no time to rest.
runtime.GC() // An opportune time to take out the garbage.
}
}
}
}
}
