func register(dir *tricorder.DirectorySpec) *prober {
p := new(prober)
if err := p.findGateway(); err != nil {
panic(err)
}
if err := dir.RegisterMetric("gateway-address", &p.gatewayAddress,
units.None, "gateway address"); err != nil {
panic(err)
}
if err := dir.RegisterMetric("gateway-interface", &p.gatewayInterfaceName,
units.None, "gateway interface"); err != nil {
panic(err)
}
latencyBucketer := tricorder.NewGeometricBucketer(0.1, 10e3)
p.gatewayPingTimeDistribution = latencyBucketer.NewDistribution()
if err := dir.RegisterMetric("gateway-ping-time",
p.gatewayPingTimeDistribution,
units.Millisecond, "ping time to gateway"); err != nil {
panic(err)
}
p.gatewayRttDistribution = latencyBucketer.NewDistribution()
if err := dir.RegisterMetric("gateway-rtt", p.gatewayRttDistribution,
units.Millisecond, "round-trip time to gateway"); err != nil {
panic(err)
}
return p
}
func init() {
statisticsComputeBucketer = tricorder.NewGeometricBucketer(0.1, 100e3)
statisticsComputeCpuTimeDistribution =
statisticsComputeBucketer.NewCumulativeDistribution()
tricorder.RegisterMetric("/statistics-compute-cputime",
statisticsComputeCpuTimeDistribution,
units.Millisecond, "statistics compute CPU time")
}
func init() {
latencyBucketer = tricorder.NewGeometricBucketer(0.1, 100e3)
mdbCompareTimeDistribution = latencyBucketer.NewDistribution()
tricorder.RegisterMetric("/mdb-compare-time", mdbCompareTimeDistribution,
units.Millisecond, "time to compare new MDB with old MDB")
mdbDecodeTimeDistribution = latencyBucketer.NewDistribution()
tricorder.RegisterMetric("/mdb-decode-time", mdbDecodeTimeDistribution,
units.Millisecond, "time to decode MDB data")
mdbSortTimeDistribution = latencyBucketer.NewDistribution()
tricorder.RegisterMetric("/mdb-sort-time", mdbSortTimeDistribution,
units.Millisecond, "time to sort MDB data")
}
func (p *ldapconfig) register(dir *tricorder.DirectorySpec) error {
if err := dir.RegisterMetric("healthy", &p.healthy, units.None,
"Is LDAP reachable and configured?"); err != nil {
return err
}
latencyBucketer := tricorder.NewGeometricBucketer(0.1, 10e3)
p.ldapLatencyDistribution = latencyBucketer.NewCumulativeDistribution()
if err := dir.RegisterMetric("latency", p.ldapLatencyDistribution,
units.Millisecond, "LDAP latency"); err != nil {
return err
}
return nil
}
func init() {
latencyBucketer = tricorder.NewGeometricBucketer(0.1, 100e3)
computeCpuTimeDistribution = latencyBucketer.NewDistribution()
tricorder.RegisterMetric("/compute-cputime", computeCpuTimeDistribution,
units.Millisecond, "compute CPU time")
fullPollDistribution = latencyBucketer.NewDistribution()
tricorder.RegisterMetric("/poll-full-latency", fullPollDistribution,
units.Millisecond, "full poll duration")
mdbUpdateTimeDistribution = latencyBucketer.NewDistribution()
tricorder.RegisterMetric("/mdb-update-time", mdbUpdateTimeDistribution,
units.Millisecond, "time to update Herd MDB data")
shortPollDistribution = latencyBucketer.NewDistribution()
tricorder.RegisterMetric("/poll-short-latency", shortPollDistribution,
units.Millisecond, "short poll duration")
}
func init() {
latencyBucketer = tricorder.NewGeometricBucketer(0.1, 100e3)
computeCpuTimeDistribution = makeMetric(latencyBucketer,
"/compute-cputime", "compute CPU time")
computeSlotWaitTimeDistribution = makeMetric(latencyBucketer,
"/compute-slot-wait-time", "compute slot wait time")
computeTimeDistribution = makeMetric(latencyBucketer,
"/compute-time", "compute time")
connectDistribution = makeMetric(latencyBucketer,
"/connect-latency", "connect duration")
mdbUpdateTimeDistribution = makeMetric(latencyBucketer,
"/mdb-update-time", "time to update Herd MDB data")
fullPollDistribution = makeMetric(latencyBucketer,
"/poll-full-latency", "full poll duration")
shortPollDistribution = makeMetric(latencyBucketer,
"/poll-short-latency", "short poll duration")
pollWaitTimeDistribution = makeMetric(latencyBucketer,
"/poll-wait-time", "poll wait time")
}
func doMain() error {
flag.Parse()
runtime.GOMAXPROCS(int(*maxThreads))
runtime.LockOSThread()
circularBuffer := logbuf.New(*logbufLines)
logger := log.New(circularBuffer, "", log.LstdFlags)
probers, err := setupProbers()
if err != nil {
return err
}
latencyBucketer := tricorder.NewGeometricBucketer(0.1, 100e3)
scanTimeDistribution := latencyBucketer.NewDistribution()
var scanStartTime time.Time
if err := tricorder.RegisterMetric("scan-duration",
scanTimeDistribution, units.Millisecond,
"duration of last probe"); err != nil {
return err
}
if err := tricorder.RegisterMetric("scan-start-time", &scanStartTime,
units.None, "start time of last probe"); err != nil {
return err
}
rpc.HandleHTTP()
httpd.AddHtmlWriter(circularBuffer)
if err := httpd.StartServer(*portNum); err != nil {
return err
}
for {
scanStartTime = time.Now()
for _, p := range probers {
if err := p.Probe(); err != nil {
logger.Println(err)
}
}
scanDuration := time.Since(scanStartTime)
scanTimeDistribution.Add(scanDuration)
time.Sleep(time.Second*time.Duration(*probeInterval) - scanDuration)
}
_ = logger
return nil
}
func newPStoreHandler(
appList *datastructs.ApplicationList,
consumer *pstore.ConsumerWithMetricsBuilder,
maybeNilCoordBuilder coordinatorBuilderType) *pstoreHandlerType {
bucketer := tricorder.NewGeometricBucketer(1e-4, 1000.0)
perMetricWriteTimes := bucketer.NewCumulativeDistribution()
consumer.SetPerMetricWriteTimeDist(perMetricWriteTimes)
visitorMetricsStore := &visitorMetricsStoreType{}
var maybeNilCoord store.Coordinator
if maybeNilCoordBuilder != nil {
maybeNilCoord = maybeNilCoordBuilder.WithStateListener(
visitorMetricsStore.SetBlocked)
}
return &pstoreHandlerType{
consumer: consumer.Build(),
appList: appList,
totalTimeSpentDist: bucketer.NewCumulativeDistribution(),
perMetricWriteTimes: perMetricWriteTimes,
visitorMetricsStore: visitorMetricsStore,
maybeNilCoord: maybeNilCoord,
}
}
func registerServerMetrics() {
var err error
serverMetricsDir, err = tricorder.RegisterDirectory("srpc/server")
if err != nil {
panic(err)
}
err = serverMetricsDir.RegisterMetric("num-connections",
&numServerConnections, units.None, "number of connection attempts")
if err != nil {
panic(err)
}
err = serverMetricsDir.RegisterMetric("num-open-connections",
&numOpenServerConnections, units.None, "number of open connections")
if err != nil {
panic(err)
}
err = serverMetricsDir.RegisterMetric("num-rejected-connections",
&numRejectedServerConnections, units.None,
"number of rejected connections")
if err != nil {
panic(err)
}
bucketer = tricorder.NewGeometricBucketer(0.1, 1e5)
}
func init() {
latencyBucketer = tricorder.NewGeometricBucketer(1, 100e3)
scanTimeDistribution = latencyBucketer.NewDistribution()
tricorder.RegisterMetric("/scan-time", scanTimeDistribution,
units.Second, "scan time")
}
"github.com/Symantec/Dominator/lib/mdb"
"github.com/Symantec/tricorder/go/tricorder"
"github.com/Symantec/tricorder/go/tricorder/units"
"log"
"os"
"path"
"reflect"
"regexp"
"runtime"
"sort"
"syscall"
"time"
)
var (
latencyBucketer = tricorder.NewGeometricBucketer(0.1, 100e3)
loadCpuTimeDistribution *tricorder.CumulativeDistribution
loadTimeDistribution *tricorder.CumulativeDistribution
)
type genericEncoder interface {
Encode(v interface{}) error
}
func init() {
loadCpuTimeDistribution = latencyBucketer.NewCumulativeDistribution()
if err := tricorder.RegisterMetric("/load-cpu-time", loadCpuTimeDistribution,
units.Millisecond, "load CPU time durations"); err != nil {
panic(err)
}
loadTimeDistribution = latencyBucketer.NewCumulativeDistribution()
package store
import (
"github.com/Symantec/tricorder/go/tricorder"
"github.com/Symantec/tricorder/go/tricorder/duration"
"sync"
"time"
)
// This file contains all the code related to tracking scotty store metrics.
var (
// 1 up to 1,000,000 bucketer
kBucketer = tricorder.NewGeometricBucketer(1.0, 1e6)
)
type storePrimitiveMetricsType struct {
UniqueMetricValueCount int64
TimeStampPageCount int64
LatestEvictedTimeStamp float64
}
func (s *storePrimitiveMetricsType) TimeSpan() time.Duration {
return time.Since(duration.FloatToTime(s.LatestEvictedTimeStamp))
}
type storeMetricsType struct {
PagesPerMetricDist *tricorder.NonCumulativeDistribution
lock sync.Mutex
values storePrimitiveMetricsType
}
func startCollector(
appStats *datastructs.ApplicationStatuses,
connectionErrors *connectionErrorsType,
totalCounts totalCountUpdaterType,
metricNameAdder suggest.Adder,
memoryChecker memoryCheckerType) {
collector.SetConcurrentPolls(*fPollCount)
collector.SetConcurrentConnects(*fConnectionCount)
sweepDurationDist := tricorder.NewGeometricBucketer(1, 100000.0).NewCumulativeDistribution()
collectionBucketer := tricorder.NewGeometricBucketer(1e-4, 100.0)
collectionTimesDist := collectionBucketer.NewCumulativeDistribution()
tricorderCollectionTimesDist := collectionBucketer.NewCumulativeDistribution()
snmpCollectionTimesDist := collectionBucketer.NewCumulativeDistribution()
jsonCollectionTimesDist := collectionBucketer.NewCumulativeDistribution()
changedMetricsPerEndpointDist := tricorder.NewGeometricBucketer(1.0, 10000.0).NewCumulativeDistribution()
if err := tricorder.RegisterMetric(
"collector/collectionTimes",
collectionTimesDist,
units.Second,
"Collection Times"); err != nil {
log.Fatal(err)
}
if err := tricorder.RegisterMetric(
"collector/collectionTimes_tricorder",
tricorderCollectionTimesDist,
units.Second,
"Tricorder Collection Times"); err != nil {
log.Fatal(err)
}
if err := tricorder.RegisterMetric(
"collector/collectionTimes_snmp",
snmpCollectionTimesDist,
units.Second,
"SNMP Collection Times"); err != nil {
log.Fatal(err)
}
if err := tricorder.RegisterMetric(
"collector/collectionTimes_json",
jsonCollectionTimesDist,
units.Second,
"JSON Collection Times"); err != nil {
log.Fatal(err)
}
if err := tricorder.RegisterMetric(
"collector/changedMetricsPerEndpoint",
changedMetricsPerEndpointDist,
units.None,
"Changed metrics per sweep"); err != nil {
log.Fatal(err)
}
if err := tricorder.RegisterMetric(
"collector/sweepDuration",
sweepDurationDist,
units.Millisecond,
"Sweep duration"); err != nil {
log.Fatal(err)
}
programStartTime := time.Now()
if err := tricorder.RegisterMetric(
"collector/elapsedTime",
func() time.Duration {
return time.Now().Sub(programStartTime)
},
units.Second,
"elapsed time"); err != nil {
log.Fatal(err)
}
byProtocolDist := map[string]*tricorder.CumulativeDistribution{
"tricorder": tricorderCollectionTimesDist,
"snmp": snmpCollectionTimesDist,
"json": jsonCollectionTimesDist,
}
// Metric collection goroutine. Collect metrics periodically.
go func() {
// We assign each endpoint its very own nameSetType instance
// to store metric names already sent to suggest.
// Only that endpoint's fetch goroutine reads and modifies
// the contents of its nameSetType instance. Although
// this goroutine creates nameSetType instances and manages
// the references to them, it never reads or modifies the
// contents of any nameSetType instance after creating it.
endpointToNamesSentToSuggest := make(
map[*collector.Endpoint]nameSetType)
for {
endpoints, metricStore := appStats.ActiveEndpointIds()
sweepTime := time.Now()
for _, endpoint := range endpoints {
namesSentToSuggest := endpointToNamesSentToSuggest[endpoint]
if namesSentToSuggest == nil {
namesSentToSuggest = make(nameSetType)
endpointToNamesSentToSuggest[endpoint] = namesSentToSuggest
}
logger := &loggerType{
Store: metricStore,
AppStats: appStats,
ConnectionErrors: connectionErrors,
CollectionTimesDist: collectionTimesDist,
ByProtocolDist: byProtocolDist,
ChangedMetricsDist: changedMetricsPerEndpointDist,
NamesSentToSuggest: namesSentToSuggest,
MetricNameAdder: metricNameAdder,
TotalCounts: totalCounts,
}
endpoint.Poll(sweepTime, logger)
}
sweepDuration := time.Now().Sub(sweepTime)
sweepDurationDist.Add(sweepDuration)
memoryChecker.Check()
if sweepDuration < *fCollectionFrequency {
time.Sleep((*fCollectionFrequency) - sweepDuration)
}
}
}()
}
func init() {
sleepBucketer = tricorder.NewGeometricBucketer(1e-3, 1e3)
}