func newConsumerMetrics(consumerName, prefix string) *ConsumerMetrics {
kafkaMetrics := &ConsumerMetrics{
registry: metrics.DefaultRegistry,
}
// Ensure prefix ends with a dot (.) so it plays nice with statsd/graphite
prefix = strings.Trim(prefix, " ")
if prefix != "" && prefix[len(prefix)-1:] != "." {
prefix += "."
}
kafkaMetrics.consumerName = consumerName
kafkaMetrics.prefix = prefix
kafkaMetrics.fetchersIdleTimer = metrics.NewRegisteredTimer(fmt.Sprintf("%sFetchersIdleTime-%s", prefix, consumerName), kafkaMetrics.registry)
kafkaMetrics.fetchDurationTimer = metrics.NewRegisteredTimer(fmt.Sprintf("%sFetchDuration-%s", prefix, consumerName), kafkaMetrics.registry)
kafkaMetrics.numWorkerManagersGauge = metrics.NewRegisteredGauge(fmt.Sprintf("%sNumWorkerManagers-%s", prefix, consumerName), kafkaMetrics.registry)
kafkaMetrics.activeWorkersCounter = metrics.NewRegisteredCounter(fmt.Sprintf("%sWMsActiveWorkers-%s", prefix, consumerName), kafkaMetrics.registry)
kafkaMetrics.pendingWMsTasksCounter = metrics.NewRegisteredCounter(fmt.Sprintf("%sWMsPendingTasks-%s", prefix, consumerName), kafkaMetrics.registry)
kafkaMetrics.taskTimeoutCounter = metrics.NewRegisteredCounter(fmt.Sprintf("%sTaskTimeouts-%s", prefix, consumerName), kafkaMetrics.registry)
kafkaMetrics.wmsBatchDurationTimer = metrics.NewRegisteredTimer(fmt.Sprintf("%sWMsBatchDuration-%s", prefix, consumerName), kafkaMetrics.registry)
kafkaMetrics.wmsIdleTimer = metrics.NewRegisteredTimer(fmt.Sprintf("%sWMsIdleTime-%s", prefix, consumerName), kafkaMetrics.registry)
kafkaMetrics.numFetchedMessagesCounter = metrics.NewRegisteredCounter(fmt.Sprintf("%sFetchedMessages-%s", prefix, consumerName), kafkaMetrics.registry)
kafkaMetrics.numConsumedMessagesCounter = metrics.NewRegisteredCounter(fmt.Sprintf("%sConsumedMessages-%s", prefix, consumerName), kafkaMetrics.registry)
kafkaMetrics.numAcksCounter = metrics.NewRegisteredCounter(fmt.Sprintf("%sAcks-%s", prefix, consumerName), kafkaMetrics.registry)
kafkaMetrics.topicPartitionLag = make(map[TopicAndPartition]metrics.Gauge)
kafkaMetrics.reportingStopChannels = make([]chan struct{}, 0)
return kafkaMetrics
}
// Start calculating and reporting statistics on the repo and tiles.
//
// We presume the git.Update(true) is called somewhere else, usually this is done
// in the trace/db.Builder, so the repo is always as good as the loaded tiles.
func Start(nanoTileStore *db.Builder, git *gitinfo.GitInfo) {
coverage := metrics.NewRegisteredGaugeFloat64("stats.tests.bench_runs_per_changelist", metrics.DefaultRegistry)
skpLatency := metrics.NewRegisteredTimer("stats.skp.update_latency", metrics.DefaultRegistry)
commits := metrics.NewRegisteredGauge("stats.commits.total", metrics.DefaultRegistry)
go func() {
for _ = range time.Tick(2 * time.Minute) {
tile := nanoTileStore.GetTile()
numCommits := tile.LastCommitIndex() + 1
numTraces := len(tile.Traces)
total := 0
for _, tr := range tile.Traces {
for i := 0; i < numCommits; i++ {
if !tr.IsMissing(i) {
total += 1
}
}
}
cov := float64(total) / float64(numCommits*numTraces)
glog.Info("Coverage: ", cov)
coverage.Update(cov)
last, err := git.LastSkpCommit()
if err != nil {
glog.Warning("Failed to read last SKP commit: %s", err)
continue
}
skpLatency.Update(time.Since(last))
commits.Update(int64(git.NumCommits()))
}
}()
}
func newConsumerMetrics(consumerName string) *consumerMetrics {
kafkaMetrics := &consumerMetrics{
registry: metrics.NewRegistry(),
}
kafkaMetrics.fetchersIdleTimer = metrics.NewRegisteredTimer(fmt.Sprintf("FetchersIdleTime-%s", consumerName), kafkaMetrics.registry)
kafkaMetrics.fetchDurationTimer = metrics.NewRegisteredTimer(fmt.Sprintf("FetchDuration-%s", consumerName), kafkaMetrics.registry)
kafkaMetrics.numWorkerManagersGauge = metrics.NewRegisteredGauge(fmt.Sprintf("NumWorkerManagers-%s", consumerName), kafkaMetrics.registry)
kafkaMetrics.activeWorkersCounter = metrics.NewRegisteredCounter(fmt.Sprintf("WMsActiveWorkers-%s", consumerName), kafkaMetrics.registry)
kafkaMetrics.pendingWMsTasksCounter = metrics.NewRegisteredCounter(fmt.Sprintf("WMsPendingTasks-%s", consumerName), kafkaMetrics.registry)
kafkaMetrics.wmsBatchDurationTimer = metrics.NewRegisteredTimer(fmt.Sprintf("WMsBatchDuration-%s", consumerName), kafkaMetrics.registry)
kafkaMetrics.wmsIdleTimer = metrics.NewRegisteredTimer(fmt.Sprintf("WMsIdleTime-%s", consumerName), kafkaMetrics.registry)
return kafkaMetrics
}
func newConsumerMetrics(consumerName, prefix string) *ConsumerMetrics {
kafkaMetrics := &ConsumerMetrics{
registry: metrics.DefaultRegistry,
}
// Ensure prefix ends with a dot (.) so it plays nice with statsd/graphite
prefix = strings.Trim(prefix, " ")
if prefix != "" && prefix[len(prefix)-1:] != "." {
prefix += "."
}
kafkaMetrics.fetchersIdleTimer = metrics.NewRegisteredTimer(fmt.Sprintf("%sFetchersIdleTime-%s", prefix, consumerName), kafkaMetrics.registry)
kafkaMetrics.fetchDurationTimer = metrics.NewRegisteredTimer(fmt.Sprintf("%sFetchDuration-%s", prefix, consumerName), kafkaMetrics.registry)
kafkaMetrics.numWorkerManagersGauge = metrics.NewRegisteredGauge(fmt.Sprintf("%sNumWorkerManagers-%s", prefix, consumerName), kafkaMetrics.registry)
kafkaMetrics.activeWorkersCounter = metrics.NewRegisteredCounter(fmt.Sprintf("%sWMsActiveWorkers-%s", prefix, consumerName), kafkaMetrics.registry)
kafkaMetrics.pendingWMsTasksCounter = metrics.NewRegisteredCounter(fmt.Sprintf("%sWMsPendingTasks-%s", prefix, consumerName), kafkaMetrics.registry)
kafkaMetrics.taskTimeoutCounter = metrics.NewRegisteredCounter(fmt.Sprintf("%sTaskTimeouts-%s", prefix, consumerName), kafkaMetrics.registry)
kafkaMetrics.wmsBatchDurationTimer = metrics.NewRegisteredTimer(fmt.Sprintf("%sWMsBatchDuration-%s", prefix, consumerName), kafkaMetrics.registry)
kafkaMetrics.wmsIdleTimer = metrics.NewRegisteredTimer(fmt.Sprintf("%sWMsIdleTime-%s", prefix, consumerName), kafkaMetrics.registry)
return kafkaMetrics
}
// TRACKED_ITEM_URL_TEMPLATE is used to generate the URL that is
// embedded in an issue. It is also used to search for issues linked to a
// specific item (cluster). The format verb is to be replaced with the ID
// of the tracked item.
TRACKED_ITEM_URL_TEMPLATE = "https://perf.skia.org/cl/%d"
)
var (
// The number of clusters with a status of "New".
newClustersGauge = metrics.NewRegisteredGauge("alerting.new", metrics.DefaultRegistry)
// The number of times we've successfully done alert clustering.
runsCounter = metrics.NewRegisteredCounter("alerting.runs", metrics.DefaultRegistry)
// How long it takes to do a clustering run.
alertingLatency = metrics.NewRegisteredTimer("alerting.latency", metrics.DefaultRegistry)
// tileBuilder is the tracedb.Builder where we load Tiles from.
tileBuilder *tracedb.Builder
)
// CombineClusters combines freshly found clusters with existing clusters.
//
// Algorithm:
// Run clustering and pick out the "Interesting" clusters.
// Compare all the Interesting clusters to all the existing relevant clusters,
// where "relevant" clusters are ones whose Hash/timestamp of the step
// exists in the current tile.
// Start with an empty "list".
// For each cluster:
// For each relevant existing cluster:
