func (pl *ProberList) addProber(genericProber prober.Prober, path string, probeInterval uint8) { newProber := &proberType{ prober: genericProber, probeInterval: time.Duration(probeInterval) * time.Second, probeTimeDistribution: latencyBucketer.NewCumulativeDistribution(), } if err := tricorder.RegisterMetric(pl.proberPath+path+"/probe-duration", newProber.probeTimeDistribution, units.Millisecond, "duration of last probe"); err != nil { panic(err) } if probeInterval > 0 { if err := tricorder.RegisterMetric(pl.proberPath+path+"/probe-interval", &newProber.probeInterval, units.Second, "probe interval"); err != nil { panic(err) } if err := tricorder.RegisterMetric( pl.proberPath+path+"/probe-start-time", &newProber.probeStartTime, units.None, "start time of last probe"); err != nil { panic(err) } } pl.probers = append(pl.probers, newProber) }
func init() { latencyBucketer = tricorder.NewGeometricBucketer(0.1, 100e3) mdbCompareTimeDistribution = latencyBucketer.NewDistribution() tricorder.RegisterMetric("/mdb-compare-time", mdbCompareTimeDistribution, units.Millisecond, "time to compare new MDB with old MDB") mdbDecodeTimeDistribution = latencyBucketer.NewDistribution() tricorder.RegisterMetric("/mdb-decode-time", mdbDecodeTimeDistribution, units.Millisecond, "time to decode MDB data") mdbSortTimeDistribution = latencyBucketer.NewDistribution() tricorder.RegisterMetric("/mdb-sort-time", mdbSortTimeDistribution, units.Millisecond, "time to sort MDB data") }
func init() { loadCpuTimeDistribution = latencyBucketer.NewCumulativeDistribution() if err := tricorder.RegisterMetric("/load-cpu-time", loadCpuTimeDistribution, units.Millisecond, "load CPU time durations"); err != nil { panic(err) } loadTimeDistribution = latencyBucketer.NewCumulativeDistribution() if err := tricorder.RegisterMetric("/load-time", loadTimeDistribution, units.Millisecond, "load durations"); err != nil { panic(err) } }
func init() { latencyBucketer = tricorder.NewGeometricBucketer(0.1, 100e3) computeCpuTimeDistribution = latencyBucketer.NewDistribution() tricorder.RegisterMetric("/compute-cputime", computeCpuTimeDistribution, units.Millisecond, "compute CPU time") fullPollDistribution = latencyBucketer.NewDistribution() tricorder.RegisterMetric("/poll-full-latency", fullPollDistribution, units.Millisecond, "full poll duration") mdbUpdateTimeDistribution = latencyBucketer.NewDistribution() tricorder.RegisterMetric("/mdb-update-time", mdbUpdateTimeDistribution, units.Millisecond, "time to update Herd MDB data") shortPollDistribution = latencyBucketer.NewDistribution() tricorder.RegisterMetric("/poll-short-latency", shortPollDistribution, units.Millisecond, "short poll duration") }
func main() { flag.Parse() tricorder.RegisterFlags() if os.Geteuid() == 0 { fmt.Fprintln(os.Stderr, "Do not run the Image Server as root") os.Exit(1) } setupTls(*caFile, *certFile, *keyFile) circularBuffer := logbuf.New(*logbufLines) logger := log.New(circularBuffer, "", log.LstdFlags) objSrv, err := filesystem.NewObjectServer(*objectDir, logger) if err != nil { fmt.Fprintf(os.Stderr, "Cannot create ObjectServer\t%s\n", err) os.Exit(1) } imdb, err := scanner.LoadImageDataBase(*imageDir, objSrv, logger) if err != nil { fmt.Fprintf(os.Stderr, "Cannot load image database\t%s\n", err) os.Exit(1) } tricorder.RegisterMetric("/image-count", func() uint { return imdb.CountImages() }, units.None, "number of images") imageserverRpcd.Setup(imdb, logger) rpcHtmlWriter := objectserverRpcd.Setup(objSrv, logger) rpc.HandleHTTP() httpd.AddHtmlWriter(imdb) httpd.AddHtmlWriter(rpcHtmlWriter) httpd.AddHtmlWriter(circularBuffer) if err = httpd.StartServer(*portNum, imdb, false); err != nil { fmt.Fprintf(os.Stderr, "Unable to create http server\t%s\n", err) os.Exit(1) } }
func Setup(configuration *scanner.Configuration, fsh *scanner.FileSystemHistory, objectsDirname string, rootDirname string, netReaderContext *rateio.ReaderContext, netbenchFname string, oldTriggersFname string, disableScannerFunction func(disableScanner bool), rescanObjectCacheFunction func(), logger *log.Logger) *HtmlWriter { rpcObj := &rpcType{ scannerConfiguration: configuration, fileSystemHistory: fsh, objectsDir: objectsDirname, rootDir: rootDirname, networkReaderContext: netReaderContext, netbenchFilename: netbenchFname, oldTriggersFilename: oldTriggersFname, rescanObjectCacheFunction: rescanObjectCacheFunction, disableScannerFunc: disableScannerFunction, logger: logger} srpc.RegisterName("Subd", rpcObj) addObjectsHandler := &addObjectsHandlerType{ objectsDir: objectsDirname, logger: logger} srpc.RegisterName("ObjectServer", addObjectsHandler) tricorder.RegisterMetric("/image-name", &rpcObj.lastSuccessfulImageName, units.None, "name of the image for the last successful update") return &HtmlWriter{&rpcObj.lastSuccessfulImageName} }
func newProberList(proberPath string) *ProberList { pl := &ProberList{ proberPath: proberPath, probeTimeDistribution: latencyBucketer.NewCumulativeDistribution(), } if err := tricorder.RegisterMetric(proberPath+"/probe-duration", pl.probeTimeDistribution, units.Millisecond, "duration of last probe"); err != nil { panic(err) } if err := tricorder.RegisterMetric(proberPath+"/probe-start-time", &pl.probeStartTime, units.None, "start time of last probe"); err != nil { panic(err) } return pl }
func init() { statisticsComputeBucketer = tricorder.NewGeometricBucketer(0.1, 100e3) statisticsComputeCpuTimeDistribution = statisticsComputeBucketer.NewCumulativeDistribution() tricorder.RegisterMetric("/statistics-compute-cputime", statisticsComputeCpuTimeDistribution, units.Millisecond, "statistics compute CPU time") }
func main() { flag.Parse() tricorder.RegisterFlags() if os.Geteuid() == 0 { fmt.Fprintln(os.Stderr, "Do not run the Image Server as root") os.Exit(1) } if *archiveMode && *imageServerHostname == "" { fmt.Fprintln(os.Stderr, "-imageServerHostname required in archive mode") os.Exit(1) } circularBuffer := logbuf.New() logger := log.New(circularBuffer, "", log.LstdFlags) if err := setupserver.SetupTls(); err != nil { logger.Println(err) circularBuffer.Flush() if !*permitInsecureMode { os.Exit(1) } } objSrv, err := filesystem.NewObjectServer(*objectDir, logger) if err != nil { fmt.Fprintf(os.Stderr, "Cannot create ObjectServer: %s\n", err) os.Exit(1) } cleanupUnreferencedObjects := true if *imageServerHostname != "" { cleanupUnreferencedObjects = false } imdb, err := scanner.LoadImageDataBase(*imageDir, objSrv, cleanupUnreferencedObjects, logger) if err != nil { fmt.Fprintf(os.Stderr, "Cannot load image database: %s\n", err) os.Exit(1) } tricorder.RegisterMetric("/image-count", func() uint { return imdb.CountImages() }, units.None, "number of images") imgSrvRpcHtmlWriter := imageserverRpcd.Setup(imdb, *imageServerHostname, logger) objSrvRpcHtmlWriter := objectserverRpcd.Setup(objSrv, logger) httpd.AddHtmlWriter(imdb) httpd.AddHtmlWriter(&imageObjectServersType{imdb, objSrv}) httpd.AddHtmlWriter(imgSrvRpcHtmlWriter) httpd.AddHtmlWriter(objSrvRpcHtmlWriter) httpd.AddHtmlWriter(circularBuffer) if *imageServerHostname != "" { go replicator(fmt.Sprintf("%s:%d", *imageServerHostname, *imageServerPortNum), imdb, objSrv, *archiveMode, logger) } if err = httpd.StartServer(*portNum, imdb, objSrv, false); err != nil { fmt.Fprintf(os.Stderr, "Unable to create http server: %s\n", err) os.Exit(1) } }
func Setup(objSrv objectserver.ObjectServer, logger *log.Logger) *htmlWriter { getSemaphore := make(chan bool, 100) rpcObj := &objectServer{objSrv} srpcObj := &srpcType{objSrv, getSemaphore, logger} rpc.RegisterName("ObjectServer", rpcObj) srpc.RegisterName("ObjectServer", srpcObj) tricorder.RegisterMetric("/get-requests", func() uint { return uint(len(getSemaphore)) }, units.None, "number of GetObjects() requests in progress") return &htmlWriter{getSemaphore} }
func doMain() error { flag.Parse() runtime.GOMAXPROCS(int(*maxThreads)) runtime.LockOSThread() circularBuffer := logbuf.New(*logbufLines) logger := log.New(circularBuffer, "", log.LstdFlags) probers, err := setupProbers() if err != nil { return err } latencyBucketer := tricorder.NewGeometricBucketer(0.1, 100e3) scanTimeDistribution := latencyBucketer.NewDistribution() var scanStartTime time.Time if err := tricorder.RegisterMetric("scan-duration", scanTimeDistribution, units.Millisecond, "duration of last probe"); err != nil { return err } if err := tricorder.RegisterMetric("scan-start-time", &scanStartTime, units.None, "start time of last probe"); err != nil { return err } rpc.HandleHTTP() httpd.AddHtmlWriter(circularBuffer) if err := httpd.StartServer(*portNum); err != nil { return err } for { scanStartTime = time.Now() for _, p := range probers { if err := p.Probe(); err != nil { logger.Println(err) } } scanDuration := time.Since(scanStartTime) scanTimeDistribution.Add(scanDuration) time.Sleep(time.Second*time.Duration(*probeInterval) - scanDuration) } _ = logger return nil }
func init() { tricorder.RegisterMetric("/get-requests", func() uint { return uint(len(getSemaphore)) }, units.None, "number of GetObjects() requests in progress") }
func makeMetric(bucketer *tricorder.Bucketer, name string, comment string) *tricorder.CumulativeDistribution { distribution := latencyBucketer.NewCumulativeDistribution() tricorder.RegisterMetric(name, distribution, units.Millisecond, comment) return distribution }
func registerMetrics() { var temperature float64 var someBool bool var someDuration time.Duration rpcDistribution := tricorder.PowersOfTen.NewCumulativeDistribution() var sample []int kAnIntList = tricorder.NewList(sample, tricorder.MutableSlice) if err := tricorder.RegisterMetric( "/list/squares", kAnIntList, units.None, "Squares"); err != nil { log.Fatalf("Got error %v registering metric", err) } if err := tricorder.RegisterMetric( "/proc/rpc-latency", rpcDistribution, units.Millisecond, "RPC latency"); err != nil { log.Fatalf("Got error %v registering metric", err) } if err := tricorder.RegisterMetric( "/proc/rpc-count", rpcCountCallback, units.Unknown, "RPC count"); err != nil { log.Fatalf("Got error %v registering metric", err) } if err := tricorder.RegisterMetric("/proc/temperature", &temperature, units.Celsius, "Temperature"); err != nil { log.Fatalf("Got error %v registering metric", err) } if err := tricorder.RegisterMetric("/proc/duration", &someDuration, units.Second, "Duration"); err != nil { log.Fatalf("Got error %v registering metric", err) } fooDir, err := tricorder.RegisterDirectory("proc/foo") if err != nil { log.Fatalf("Got error %v registering directory", err) } barDir, err := fooDir.RegisterDirectory("bar") if err != nil { log.Fatalf("Got error %v registering directory", err) } err = barDir.RegisterMetric( "baz", bazCallback, units.None, "Another float value") if err != nil { log.Fatalf("Got error %v registering metric", err) } err = barDir.RegisterMetric("abool", &someBool, units.None, "A boolean value") if err != nil { log.Fatalf("Got error %v registering metric", err) } temperature = 22.5 someBool = true someDuration = 2*time.Minute + 5*time.Second // Add data points to the distribution // < 10: 10 // 10 - 25: 15 // 25 - 62.5: 38 // 62.5 - 156.25: 94 // 156.25 - 390.625: 234 for i := 0; i < 500; i++ { rpcDistribution.Add(float64(i)) } go updateList() }
func init() { latencyBucketer = tricorder.NewGeometricBucketer(1, 100e3) scanTimeDistribution = latencyBucketer.NewDistribution() tricorder.RegisterMetric("/scan-time", scanTimeDistribution, units.Second, "scan time") }
func startCollector( appStats *datastructs.ApplicationStatuses, connectionErrors *connectionErrorsType, totalCounts totalCountUpdaterType, metricNameAdder suggest.Adder, memoryChecker memoryCheckerType) { collector.SetConcurrentPolls(*fPollCount) collector.SetConcurrentConnects(*fConnectionCount) sweepDurationDist := tricorder.NewGeometricBucketer(1, 100000.0).NewCumulativeDistribution() collectionBucketer := tricorder.NewGeometricBucketer(1e-4, 100.0) collectionTimesDist := collectionBucketer.NewCumulativeDistribution() tricorderCollectionTimesDist := collectionBucketer.NewCumulativeDistribution() snmpCollectionTimesDist := collectionBucketer.NewCumulativeDistribution() jsonCollectionTimesDist := collectionBucketer.NewCumulativeDistribution() changedMetricsPerEndpointDist := tricorder.NewGeometricBucketer(1.0, 10000.0).NewCumulativeDistribution() if err := tricorder.RegisterMetric( "collector/collectionTimes", collectionTimesDist, units.Second, "Collection Times"); err != nil { log.Fatal(err) } if err := tricorder.RegisterMetric( "collector/collectionTimes_tricorder", tricorderCollectionTimesDist, units.Second, "Tricorder Collection Times"); err != nil { log.Fatal(err) } if err := tricorder.RegisterMetric( "collector/collectionTimes_snmp", snmpCollectionTimesDist, units.Second, "SNMP Collection Times"); err != nil { log.Fatal(err) } if err := tricorder.RegisterMetric( "collector/collectionTimes_json", jsonCollectionTimesDist, units.Second, "JSON Collection Times"); err != nil { log.Fatal(err) } if err := tricorder.RegisterMetric( "collector/changedMetricsPerEndpoint", changedMetricsPerEndpointDist, units.None, "Changed metrics per sweep"); err != nil { log.Fatal(err) } if err := tricorder.RegisterMetric( "collector/sweepDuration", sweepDurationDist, units.Millisecond, "Sweep duration"); err != nil { log.Fatal(err) } programStartTime := time.Now() if err := tricorder.RegisterMetric( "collector/elapsedTime", func() time.Duration { return time.Now().Sub(programStartTime) }, units.Second, "elapsed time"); err != nil { log.Fatal(err) } byProtocolDist := map[string]*tricorder.CumulativeDistribution{ "tricorder": tricorderCollectionTimesDist, "snmp": snmpCollectionTimesDist, "json": jsonCollectionTimesDist, } // Metric collection goroutine. Collect metrics periodically. go func() { // We assign each endpoint its very own nameSetType instance // to store metric names already sent to suggest. // Only that endpoint's fetch goroutine reads and modifies // the contents of its nameSetType instance. Although // this goroutine creates nameSetType instances and manages // the references to them, it never reads or modifies the // contents of any nameSetType instance after creating it. endpointToNamesSentToSuggest := make( map[*collector.Endpoint]nameSetType) for { endpoints, metricStore := appStats.ActiveEndpointIds() sweepTime := time.Now() for _, endpoint := range endpoints { namesSentToSuggest := endpointToNamesSentToSuggest[endpoint] if namesSentToSuggest == nil { namesSentToSuggest = make(nameSetType) endpointToNamesSentToSuggest[endpoint] = namesSentToSuggest } logger := &loggerType{ Store: metricStore, AppStats: appStats, ConnectionErrors: connectionErrors, CollectionTimesDist: collectionTimesDist, ByProtocolDist: byProtocolDist, ChangedMetricsDist: changedMetricsPerEndpointDist, NamesSentToSuggest: namesSentToSuggest, MetricNameAdder: metricNameAdder, TotalCounts: totalCounts, } endpoint.Poll(sweepTime, logger) } sweepDuration := time.Now().Sub(sweepTime) sweepDurationDist.Add(sweepDuration) memoryChecker.Check() if sweepDuration < *fCollectionFrequency { time.Sleep((*fCollectionFrequency) - sweepDuration) } } }() }
// RegisterMetrics registers the memory manager metrics func (m *memoryManagerType) RegisterMetrics() (err error) { var data memoryManagerMetricsType group := tricorder.NewGroup() group.RegisterUpdateFunc( func() time.Time { m.Metrics(&data) return time.Now() }) if err = tricorder.RegisterMetricInGroup( "/proc/memory-manager/alloc-bytes-needed-for-gc", &data.AllocBytesNeededForGC, group, units.Byte, "Number of allocated bytes needed to trigger GC"); err != nil { return } if err = tricorder.RegisterMetricInGroup( "/proc/memory-manager/largest-alloc-bytes-needed-for-gc", &data.LargestAllocBytesNeededForGC, group, units.Byte, "Number of allocated bytes needed to trigger GC"); err != nil { return } if err = tricorder.RegisterMetricInGroup( "/proc/memory-manager/inspected-cycle-count", &data.InspectedCycleCount, group, units.None, "Number of gc cycles inspected"); err != nil { return } if err = tricorder.RegisterMetricInGroup( "/proc/memory-manager/total-cycle-count", &data.TotalCycleCount, group, units.None, "Number of total gc cycles"); err != nil { return } if err = tricorder.RegisterMetricInGroup( "/proc/memory-manager/stw-gc-count", &data.STWGCCount, group, units.None, "Number of stop the world GCs"); err != nil { return } if err = tricorder.RegisterMetricInGroup( "/proc/memory-manager/no-expand-count", &data.NoExpandCount, group, units.None, "Inspected cycle counts where we disabled expanding"); err != nil { return } if err = tricorder.RegisterMetricInGroup( "/proc/memory-manager/page-free-bytes", &data.PageFreeInBytes, group, units.Byte, "Total number of pages freed in bytes"); err != nil { return } if err = tricorder.RegisterMetricInGroup( "/proc/memory-manager/page-free-count", &data.PageFreeCount, group, units.None, "Number of times pages freed"); err != nil { return } if err = tricorder.RegisterMetric( "proc/memory-manager/hard-limit", &m.hardLimit, units.Byte, "Target memory usage"); err != nil { return } if err = tricorder.RegisterMetric( "proc/memory-manager/high-water-mark", &m.highWaterMark, units.Byte, "Memory usage that triggers a GC"); err != nil { return } if err = tricorder.RegisterMetric( "proc/memory-manager/low-water-mark", &m.lowWaterMark, units.Byte, "Desired memory usage after GC happens"); err != nil { return } if err = tricorder.RegisterMetric( "proc/memory-manager/gc-percent", &m.gcPercent, units.None, "GC percentage"); err != nil { return } return }
func (p *pstoreHandlerType) RegisterMetrics() (err error) { var attributes pstore.ConsumerAttributes p.consumer.Attributes(&attributes) var data pstore.ConsumerMetrics var visitorData visitorMetricsType metricsStore := p.ConsumerMetricsStore() visitorMetricsStore := p.visitorMetricsStore group := tricorder.NewGroup() group.RegisterUpdateFunc( func() time.Time { metricsStore.Metrics(&data) visitorMetricsStore.Metrics(&visitorData) return time.Now() }) if err = tricorder.RegisterMetric( fmt.Sprintf("writer/%s/totalTimeSpent", p.Name()), p.totalTimeSpentDist, units.Second, "total time spent per sweep"); err != nil { return } if err = tricorder.RegisterMetric( fmt.Sprintf("writer/%s/writeTimePerMetric", p.Name()), p.perMetricWriteTimes, units.Millisecond, "Time spent writing each metric"); err != nil { return } if err = tricorder.RegisterMetric( fmt.Sprintf("writer/%s/maxRecordsPerSecond", p.Name()), attributes.TotalRecordsPerSecond, units.None, "Max records per second to write. 0 means unlimited"); err != nil { return } if err = tricorder.RegisterMetric( fmt.Sprintf("writer/%s/concurrency", p.Name()), &attributes.Concurrency, units.None, "Number of writing goroutines"); err != nil { return } if err = tricorder.RegisterMetric( fmt.Sprintf("writer/%s/batchSize", p.Name()), &attributes.BatchSize, units.None, "This many records written each time"); err != nil { return } if err = tricorder.RegisterMetric( fmt.Sprintf("writer/%s/rollUpSpan", p.Name()), &attributes.RollUpSpan, units.None, "Time period length for rolled up values. 0 means no roll up."); err != nil { return } if err = tricorder.RegisterMetricInGroup( fmt.Sprintf("writer/%s/valuesWritten", p.Name()), &data.ValuesWritten, group, units.None, "Number of values written to persistent storage"); err != nil { return } if err = tricorder.RegisterMetricInGroup( fmt.Sprintf("writer/%s/valuesNotWritten", p.Name()), &data.ValuesNotWritten, group, units.None, "Number of values not written to persistent storage"); err != nil { return } if err = tricorder.RegisterMetricInGroup( fmt.Sprintf("writer/%s/timeLeft", p.Name()), &visitorData.TimeLeft, group, units.None, "approximate time writer is behind"); err != nil { return } if err = tricorder.RegisterMetricInGroup( fmt.Sprintf("writer/%s/blocked", p.Name()), &visitorData.Blocked, group, units.None, "true if writer is awaiting a lease"); err != nil { return } if err = tricorder.RegisterMetricInGroup( fmt.Sprintf("writer/%s/writeAttempts", p.Name()), &data.WriteAttempts, group, units.None, "Number of attempts to write to persistent storage"); err != nil { return } if err = tricorder.RegisterMetricInGroup( fmt.Sprintf("writer/%s/successfulWrites", p.Name()), &data.SuccessfulWrites, group, units.None, "Number of successful writes to persistent storage"); err != nil { return } if err = tricorder.RegisterMetricInGroup( fmt.Sprintf("writer/%s/successfulWriteRatio", p.Name()), data.SuccessfulWriteRatio, group, units.None, "Ratio of successful writes to write attempts"); err != nil { return } if err = tricorder.RegisterMetricInGroup( fmt.Sprintf("writer/%s/lastWriteError", p.Name()), &data.LastWriteError, group, units.None, "Last write error"); err != nil { return } if err = tricorder.RegisterMetricInGroup( fmt.Sprintf("writer/%s/lastWriteErrorTime", p.Name()), &data.LastWriteErrorTS, group, units.None, "Time of last write error"); err != nil { return } if err = tricorder.RegisterMetricInGroup( fmt.Sprintf("writer/%s/lastSuccessfulWrite", p.Name()), &data.LastSuccessfulWriteTS, group, units.None, "Time of last successful write"); err != nil { return } return }
func publishFsSpeed(bytesPerSecond, blocksPerSecond uint64) { tricorder.RegisterMetric("/root-read-speed", &bytesPerSecond, units.BytePerSecond, "read speed of root file-system media") tricorder.RegisterMetric("/root-block-read-speed", &blocksPerSecond, units.None, "read speed of root file-system media in blocks/second") }