func startPStoreLoops(
stats *datastructs.ApplicationStatuses,
consumerBuilders []*pstore.ConsumerWithMetricsBuilder,
logger *log.Logger,
maybeNilCoordBuilder coordinatorBuilderType) []*totalCountType {
result := make([]*totalCountType, len(consumerBuilders))
for i := range result {
pstoreHandler := newPStoreHandler(
stats.ApplicationList(),
consumerBuilders[i],
maybeNilCoordBuilder)
result[i] = pstoreHandler.TotalCount()
var attributes pstore.ConsumerAttributes
pstoreHandler.Attributes(&attributes)
refreshRate := *fPStoreUpdateFrequency
if attributes.RollUpSpan > 0 {
refreshRate = attributes.RollUpSpan
}
if err := pstoreHandler.RegisterMetrics(); err != nil {
log.Fatal(err)
}
go func(handler *pstoreHandlerType, refreshRate time.Duration) {
// persistent storage writing goroutine. Write every 30s by default.
// Notice that this single goroutine handles all the persistent
// storage writing as multiple goroutines must not access the
// pstoreHandler instance. accessing pstoreHandler metrics is the
// one exception to this rule.
for {
metricStore := stats.Store()
writeTime := time.Now()
handler.StartVisit()
metricStore.VisitAllEndpoints(handler)
handler.EndVisit(metricStore)
writeDuration := time.Now().Sub(writeTime)
if writeDuration < refreshRate {
time.Sleep(refreshRate - writeDuration)
}
}
}(pstoreHandler, refreshRate)
}
return result
}
func startCollector(
appStats *datastructs.ApplicationStatuses,
connectionErrors *connectionErrorsType,
totalCounts totalCountUpdaterType,
metricNameAdder suggest.Adder,
memoryChecker memoryCheckerType) {
collector.SetConcurrentPolls(*fPollCount)
collector.SetConcurrentConnects(*fConnectionCount)
sweepDurationDist := tricorder.NewGeometricBucketer(1, 100000.0).NewCumulativeDistribution()
collectionBucketer := tricorder.NewGeometricBucketer(1e-4, 100.0)
collectionTimesDist := collectionBucketer.NewCumulativeDistribution()
tricorderCollectionTimesDist := collectionBucketer.NewCumulativeDistribution()
snmpCollectionTimesDist := collectionBucketer.NewCumulativeDistribution()
jsonCollectionTimesDist := collectionBucketer.NewCumulativeDistribution()
changedMetricsPerEndpointDist := tricorder.NewGeometricBucketer(1.0, 10000.0).NewCumulativeDistribution()
if err := tricorder.RegisterMetric(
"collector/collectionTimes",
collectionTimesDist,
units.Second,
"Collection Times"); err != nil {
log.Fatal(err)
}
if err := tricorder.RegisterMetric(
"collector/collectionTimes_tricorder",
tricorderCollectionTimesDist,
units.Second,
"Tricorder Collection Times"); err != nil {
log.Fatal(err)
}
if err := tricorder.RegisterMetric(
"collector/collectionTimes_snmp",
snmpCollectionTimesDist,
units.Second,
"SNMP Collection Times"); err != nil {
log.Fatal(err)
}
if err := tricorder.RegisterMetric(
"collector/collectionTimes_json",
jsonCollectionTimesDist,
units.Second,
"JSON Collection Times"); err != nil {
log.Fatal(err)
}
if err := tricorder.RegisterMetric(
"collector/changedMetricsPerEndpoint",
changedMetricsPerEndpointDist,
units.None,
"Changed metrics per sweep"); err != nil {
log.Fatal(err)
}
if err := tricorder.RegisterMetric(
"collector/sweepDuration",
sweepDurationDist,
units.Millisecond,
"Sweep duration"); err != nil {
log.Fatal(err)
}
programStartTime := time.Now()
if err := tricorder.RegisterMetric(
"collector/elapsedTime",
func() time.Duration {
return time.Now().Sub(programStartTime)
},
units.Second,
"elapsed time"); err != nil {
log.Fatal(err)
}
byProtocolDist := map[string]*tricorder.CumulativeDistribution{
"tricorder": tricorderCollectionTimesDist,
"snmp": snmpCollectionTimesDist,
"json": jsonCollectionTimesDist,
}
// Metric collection goroutine. Collect metrics periodically.
go func() {
// We assign each endpoint its very own nameSetType instance
// to store metric names already sent to suggest.
// Only that endpoint's fetch goroutine reads and modifies
// the contents of its nameSetType instance. Although
// this goroutine creates nameSetType instances and manages
// the references to them, it never reads or modifies the
// contents of any nameSetType instance after creating it.
endpointToNamesSentToSuggest := make(
map[*collector.Endpoint]nameSetType)
for {
endpoints, metricStore := appStats.ActiveEndpointIds()
sweepTime := time.Now()
for _, endpoint := range endpoints {
namesSentToSuggest := endpointToNamesSentToSuggest[endpoint]
if namesSentToSuggest == nil {
namesSentToSuggest = make(nameSetType)
endpointToNamesSentToSuggest[endpoint] = namesSentToSuggest
}
logger := &loggerType{
Store: metricStore,
AppStats: appStats,
ConnectionErrors: connectionErrors,
CollectionTimesDist: collectionTimesDist,
ByProtocolDist: byProtocolDist,
ChangedMetricsDist: changedMetricsPerEndpointDist,
NamesSentToSuggest: namesSentToSuggest,
MetricNameAdder: metricNameAdder,
TotalCounts: totalCounts,
}
endpoint.Poll(sweepTime, logger)
}
sweepDuration := time.Now().Sub(sweepTime)
sweepDurationDist.Add(sweepDuration)
memoryChecker.Check()
if sweepDuration < *fCollectionFrequency {
time.Sleep((*fCollectionFrequency) - sweepDuration)
}
}
}()
}
func query(
endpoints *datastructs.ApplicationStatuses,
metricName string,
aggregatorGen tsdb.AggregatorGenerator,
start, end float64,
options *QueryOptions) (result *tsdb.TaggedTimeSeriesSet, err error) {
if options == nil {
options = &QueryOptions{}
}
apps, store := endpoints.AllWithStore()
var taggedTimeSeriesSlice []tsdb.TaggedTimeSeries
var metricNameFound bool
if options.GroupByHostName && options.GroupByAppName {
for i := range apps {
if options.isIncluded(apps[i].EndpointId.HostName(), apps[i].Name) {
timeSeries, ok := store.TsdbTimeSeries(
metricName,
apps[i].EndpointId,
start,
end)
if ok {
metricNameFound = true
var aggregator tsdb.Aggregator
aggregator, err = aggregatorGen(start, end)
if err != nil {
return
}
aggregator.Add(timeSeries)
aggregatedTimeSeries := aggregator.Aggregate()
if len(aggregatedTimeSeries) != 0 {
taggedTimeSeriesSlice = append(
taggedTimeSeriesSlice, tsdb.TaggedTimeSeries{
Tags: tsdb.TagSet{
HostName: apps[i].EndpointId.HostName(),
AppName: apps[i].Name,
},
Values: aggregatedTimeSeries,
})
}
}
}
}
} else {
aggregatorMap := make(map[tsdb.TagSet]tsdb.Aggregator)
for i := range apps {
if options.isIncluded(apps[i].EndpointId.HostName(), apps[i].Name) {
timeSeries, ok := store.TsdbTimeSeries(
metricName,
apps[i].EndpointId,
start,
end)
if ok {
metricNameFound = true
var tagSet tsdb.TagSet
if options.GroupByHostName {
tagSet.HostName = apps[i].EndpointId.HostName()
}
if options.GroupByAppName {
tagSet.AppName = apps[i].Name
}
aggregator := aggregatorMap[tagSet]
if aggregator == nil {
aggregator, err = aggregatorGen(start, end)
if err != nil {
return
}
aggregatorMap[tagSet] = aggregator
}
aggregator.Add(timeSeries)
}
}
}
if metricNameFound {
for k, v := range aggregatorMap {
aggregatedTimeSeries := v.Aggregate()
if len(aggregatedTimeSeries) != 0 {
taggedTimeSeriesSlice = append(
taggedTimeSeriesSlice, tsdb.TaggedTimeSeries{
Tags: k,
Values: aggregatedTimeSeries,
})
}
}
}
}
if metricNameFound {
return &tsdb.TaggedTimeSeriesSet{
MetricName: metricName,
Data: taggedTimeSeriesSlice,
GroupedByHostName: options.GroupByHostName,
GroupedByAppName: options.GroupByAppName,
}, nil
}
return nil, ErrNoSuchMetric
}