func register(dir *tricorder.DirectorySpec) *prober { p := new(prober) if err := p.findGateway(); err != nil { panic(err) } if err := dir.RegisterMetric("gateway-address", &p.gatewayAddress, units.None, "gateway address"); err != nil { panic(err) } if err := dir.RegisterMetric("gateway-interface", &p.gatewayInterfaceName, units.None, "gateway interface"); err != nil { panic(err) } latencyBucketer := tricorder.NewGeometricBucketer(0.1, 10e3) p.gatewayPingTimeDistribution = latencyBucketer.NewDistribution() if err := dir.RegisterMetric("gateway-ping-time", p.gatewayPingTimeDistribution, units.Millisecond, "ping time to gateway"); err != nil { panic(err) } p.gatewayRttDistribution = latencyBucketer.NewDistribution() if err := dir.RegisterMetric("gateway-rtt", p.gatewayRttDistribution, units.Millisecond, "round-trip time to gateway"); err != nil { panic(err) } return p }
func init() { statisticsComputeBucketer = tricorder.NewGeometricBucketer(0.1, 100e3) statisticsComputeCpuTimeDistribution = statisticsComputeBucketer.NewCumulativeDistribution() tricorder.RegisterMetric("/statistics-compute-cputime", statisticsComputeCpuTimeDistribution, units.Millisecond, "statistics compute CPU time") }
func init() { latencyBucketer = tricorder.NewGeometricBucketer(0.1, 100e3) mdbCompareTimeDistribution = latencyBucketer.NewDistribution() tricorder.RegisterMetric("/mdb-compare-time", mdbCompareTimeDistribution, units.Millisecond, "time to compare new MDB with old MDB") mdbDecodeTimeDistribution = latencyBucketer.NewDistribution() tricorder.RegisterMetric("/mdb-decode-time", mdbDecodeTimeDistribution, units.Millisecond, "time to decode MDB data") mdbSortTimeDistribution = latencyBucketer.NewDistribution() tricorder.RegisterMetric("/mdb-sort-time", mdbSortTimeDistribution, units.Millisecond, "time to sort MDB data") }
func (p *ldapconfig) register(dir *tricorder.DirectorySpec) error { if err := dir.RegisterMetric("healthy", &p.healthy, units.None, "Is LDAP reachable and configured?"); err != nil { return err } latencyBucketer := tricorder.NewGeometricBucketer(0.1, 10e3) p.ldapLatencyDistribution = latencyBucketer.NewCumulativeDistribution() if err := dir.RegisterMetric("latency", p.ldapLatencyDistribution, units.Millisecond, "LDAP latency"); err != nil { return err } return nil }
func init() { latencyBucketer = tricorder.NewGeometricBucketer(0.1, 100e3) computeCpuTimeDistribution = latencyBucketer.NewDistribution() tricorder.RegisterMetric("/compute-cputime", computeCpuTimeDistribution, units.Millisecond, "compute CPU time") fullPollDistribution = latencyBucketer.NewDistribution() tricorder.RegisterMetric("/poll-full-latency", fullPollDistribution, units.Millisecond, "full poll duration") mdbUpdateTimeDistribution = latencyBucketer.NewDistribution() tricorder.RegisterMetric("/mdb-update-time", mdbUpdateTimeDistribution, units.Millisecond, "time to update Herd MDB data") shortPollDistribution = latencyBucketer.NewDistribution() tricorder.RegisterMetric("/poll-short-latency", shortPollDistribution, units.Millisecond, "short poll duration") }
func init() { latencyBucketer = tricorder.NewGeometricBucketer(0.1, 100e3) computeCpuTimeDistribution = makeMetric(latencyBucketer, "/compute-cputime", "compute CPU time") computeSlotWaitTimeDistribution = makeMetric(latencyBucketer, "/compute-slot-wait-time", "compute slot wait time") computeTimeDistribution = makeMetric(latencyBucketer, "/compute-time", "compute time") connectDistribution = makeMetric(latencyBucketer, "/connect-latency", "connect duration") mdbUpdateTimeDistribution = makeMetric(latencyBucketer, "/mdb-update-time", "time to update Herd MDB data") fullPollDistribution = makeMetric(latencyBucketer, "/poll-full-latency", "full poll duration") shortPollDistribution = makeMetric(latencyBucketer, "/poll-short-latency", "short poll duration") pollWaitTimeDistribution = makeMetric(latencyBucketer, "/poll-wait-time", "poll wait time") }
func doMain() error { flag.Parse() runtime.GOMAXPROCS(int(*maxThreads)) runtime.LockOSThread() circularBuffer := logbuf.New(*logbufLines) logger := log.New(circularBuffer, "", log.LstdFlags) probers, err := setupProbers() if err != nil { return err } latencyBucketer := tricorder.NewGeometricBucketer(0.1, 100e3) scanTimeDistribution := latencyBucketer.NewDistribution() var scanStartTime time.Time if err := tricorder.RegisterMetric("scan-duration", scanTimeDistribution, units.Millisecond, "duration of last probe"); err != nil { return err } if err := tricorder.RegisterMetric("scan-start-time", &scanStartTime, units.None, "start time of last probe"); err != nil { return err } rpc.HandleHTTP() httpd.AddHtmlWriter(circularBuffer) if err := httpd.StartServer(*portNum); err != nil { return err } for { scanStartTime = time.Now() for _, p := range probers { if err := p.Probe(); err != nil { logger.Println(err) } } scanDuration := time.Since(scanStartTime) scanTimeDistribution.Add(scanDuration) time.Sleep(time.Second*time.Duration(*probeInterval) - scanDuration) } _ = logger return nil }
func newPStoreHandler( appList *datastructs.ApplicationList, consumer *pstore.ConsumerWithMetricsBuilder, maybeNilCoordBuilder coordinatorBuilderType) *pstoreHandlerType { bucketer := tricorder.NewGeometricBucketer(1e-4, 1000.0) perMetricWriteTimes := bucketer.NewCumulativeDistribution() consumer.SetPerMetricWriteTimeDist(perMetricWriteTimes) visitorMetricsStore := &visitorMetricsStoreType{} var maybeNilCoord store.Coordinator if maybeNilCoordBuilder != nil { maybeNilCoord = maybeNilCoordBuilder.WithStateListener( visitorMetricsStore.SetBlocked) } return &pstoreHandlerType{ consumer: consumer.Build(), appList: appList, totalTimeSpentDist: bucketer.NewCumulativeDistribution(), perMetricWriteTimes: perMetricWriteTimes, visitorMetricsStore: visitorMetricsStore, maybeNilCoord: maybeNilCoord, } }
func registerServerMetrics() { var err error serverMetricsDir, err = tricorder.RegisterDirectory("srpc/server") if err != nil { panic(err) } err = serverMetricsDir.RegisterMetric("num-connections", &numServerConnections, units.None, "number of connection attempts") if err != nil { panic(err) } err = serverMetricsDir.RegisterMetric("num-open-connections", &numOpenServerConnections, units.None, "number of open connections") if err != nil { panic(err) } err = serverMetricsDir.RegisterMetric("num-rejected-connections", &numRejectedServerConnections, units.None, "number of rejected connections") if err != nil { panic(err) } bucketer = tricorder.NewGeometricBucketer(0.1, 1e5) }
func init() { latencyBucketer = tricorder.NewGeometricBucketer(1, 100e3) scanTimeDistribution = latencyBucketer.NewDistribution() tricorder.RegisterMetric("/scan-time", scanTimeDistribution, units.Second, "scan time") }
"github.com/Symantec/Dominator/lib/mdb" "github.com/Symantec/tricorder/go/tricorder" "github.com/Symantec/tricorder/go/tricorder/units" "log" "os" "path" "reflect" "regexp" "runtime" "sort" "syscall" "time" ) var ( latencyBucketer = tricorder.NewGeometricBucketer(0.1, 100e3) loadCpuTimeDistribution *tricorder.CumulativeDistribution loadTimeDistribution *tricorder.CumulativeDistribution ) type genericEncoder interface { Encode(v interface{}) error } func init() { loadCpuTimeDistribution = latencyBucketer.NewCumulativeDistribution() if err := tricorder.RegisterMetric("/load-cpu-time", loadCpuTimeDistribution, units.Millisecond, "load CPU time durations"); err != nil { panic(err) } loadTimeDistribution = latencyBucketer.NewCumulativeDistribution()
package store import ( "github.com/Symantec/tricorder/go/tricorder" "github.com/Symantec/tricorder/go/tricorder/duration" "sync" "time" ) // This file contains all the code related to tracking scotty store metrics. var ( // 1 up to 1,000,000 bucketer kBucketer = tricorder.NewGeometricBucketer(1.0, 1e6) ) type storePrimitiveMetricsType struct { UniqueMetricValueCount int64 TimeStampPageCount int64 LatestEvictedTimeStamp float64 } func (s *storePrimitiveMetricsType) TimeSpan() time.Duration { return time.Since(duration.FloatToTime(s.LatestEvictedTimeStamp)) } type storeMetricsType struct { PagesPerMetricDist *tricorder.NonCumulativeDistribution lock sync.Mutex values storePrimitiveMetricsType }
func startCollector( appStats *datastructs.ApplicationStatuses, connectionErrors *connectionErrorsType, totalCounts totalCountUpdaterType, metricNameAdder suggest.Adder, memoryChecker memoryCheckerType) { collector.SetConcurrentPolls(*fPollCount) collector.SetConcurrentConnects(*fConnectionCount) sweepDurationDist := tricorder.NewGeometricBucketer(1, 100000.0).NewCumulativeDistribution() collectionBucketer := tricorder.NewGeometricBucketer(1e-4, 100.0) collectionTimesDist := collectionBucketer.NewCumulativeDistribution() tricorderCollectionTimesDist := collectionBucketer.NewCumulativeDistribution() snmpCollectionTimesDist := collectionBucketer.NewCumulativeDistribution() jsonCollectionTimesDist := collectionBucketer.NewCumulativeDistribution() changedMetricsPerEndpointDist := tricorder.NewGeometricBucketer(1.0, 10000.0).NewCumulativeDistribution() if err := tricorder.RegisterMetric( "collector/collectionTimes", collectionTimesDist, units.Second, "Collection Times"); err != nil { log.Fatal(err) } if err := tricorder.RegisterMetric( "collector/collectionTimes_tricorder", tricorderCollectionTimesDist, units.Second, "Tricorder Collection Times"); err != nil { log.Fatal(err) } if err := tricorder.RegisterMetric( "collector/collectionTimes_snmp", snmpCollectionTimesDist, units.Second, "SNMP Collection Times"); err != nil { log.Fatal(err) } if err := tricorder.RegisterMetric( "collector/collectionTimes_json", jsonCollectionTimesDist, units.Second, "JSON Collection Times"); err != nil { log.Fatal(err) } if err := tricorder.RegisterMetric( "collector/changedMetricsPerEndpoint", changedMetricsPerEndpointDist, units.None, "Changed metrics per sweep"); err != nil { log.Fatal(err) } if err := tricorder.RegisterMetric( "collector/sweepDuration", sweepDurationDist, units.Millisecond, "Sweep duration"); err != nil { log.Fatal(err) } programStartTime := time.Now() if err := tricorder.RegisterMetric( "collector/elapsedTime", func() time.Duration { return time.Now().Sub(programStartTime) }, units.Second, "elapsed time"); err != nil { log.Fatal(err) } byProtocolDist := map[string]*tricorder.CumulativeDistribution{ "tricorder": tricorderCollectionTimesDist, "snmp": snmpCollectionTimesDist, "json": jsonCollectionTimesDist, } // Metric collection goroutine. Collect metrics periodically. go func() { // We assign each endpoint its very own nameSetType instance // to store metric names already sent to suggest. // Only that endpoint's fetch goroutine reads and modifies // the contents of its nameSetType instance. Although // this goroutine creates nameSetType instances and manages // the references to them, it never reads or modifies the // contents of any nameSetType instance after creating it. endpointToNamesSentToSuggest := make( map[*collector.Endpoint]nameSetType) for { endpoints, metricStore := appStats.ActiveEndpointIds() sweepTime := time.Now() for _, endpoint := range endpoints { namesSentToSuggest := endpointToNamesSentToSuggest[endpoint] if namesSentToSuggest == nil { namesSentToSuggest = make(nameSetType) endpointToNamesSentToSuggest[endpoint] = namesSentToSuggest } logger := &loggerType{ Store: metricStore, AppStats: appStats, ConnectionErrors: connectionErrors, CollectionTimesDist: collectionTimesDist, ByProtocolDist: byProtocolDist, ChangedMetricsDist: changedMetricsPerEndpointDist, NamesSentToSuggest: namesSentToSuggest, MetricNameAdder: metricNameAdder, TotalCounts: totalCounts, } endpoint.Poll(sweepTime, logger) } sweepDuration := time.Now().Sub(sweepTime) sweepDurationDist.Add(sweepDuration) memoryChecker.Check() if sweepDuration < *fCollectionFrequency { time.Sleep((*fCollectionFrequency) - sweepDuration) } } }() }
func init() { sleepBucketer = tricorder.NewGeometricBucketer(1e-3, 1e3) }