func timeDelivery(d amqp.Delivery, stats statsd.Statter, deliveryTimings map[string]time.Time) {
// If d is a call add to deliveryTimings and increment openCalls, if it is a
// response then get time.Since original call from deliveryTiming, send timing metric, and
// decrement openCalls, in both cases send the gauges RpcCallsOpen and RpcBodySize
if d.ReplyTo != "" {
openCalls++
deliveryTimings[fmt.Sprintf("%s:%s", d.CorrelationId, d.ReplyTo)] = time.Now()
} else {
openCalls--
rpcSent := deliveryTimings[fmt.Sprintf("%s:%s", d.CorrelationId, d.RoutingKey)]
if rpcSent != *new(time.Time) {
respTime := time.Since(rpcSent)
delete(deliveryTimings, fmt.Sprintf("%s:%s", d.CorrelationId, d.RoutingKey))
stats.TimingDuration(fmt.Sprintf("RpcCallTime.%s", d.Type), respTime, 1.0)
}
}
stats.Gauge("RpcCallsOpen", openCalls, 1.0)
stats.Gauge("RpcBodySize", int64(len(d.Body)), 1.0)
}
// HandlerTimer monitors HTTP performance and sends the details to StatsD.
func HandlerTimer(handler http.Handler, stats statsd.Statter) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
cStart := time.Now()
openConnections++
stats.Gauge("HttpConnectionsOpen", openConnections, 1.0)
handler.ServeHTTP(w, r)
openConnections--
stats.Gauge("HttpConnectionsOpen", openConnections, 1.0)
// (FIX: this doesn't seem to really work at catching errors...)
state := "Success"
for _, h := range w.Header()["Content-Type"] {
if h == "application/problem+json" {
state = "Error"
break
}
}
// set resp timing key based on success / failure
stats.TimingDuration(fmt.Sprintf("HttpResponseTime.%s.%s", r.URL, state), time.Since(cStart), 1.0)
})
}
// ProfileCmd runs forever, sending Go statistics to StatsD.
func ProfileCmd(profileName string, stats statsd.Statter) {
for {
var memoryStats runtime.MemStats
runtime.ReadMemStats(&memoryStats)
stats.Gauge(fmt.Sprintf("Gostats.%s.Goroutines", profileName), int64(runtime.NumGoroutine()), 1.0)
stats.Gauge(fmt.Sprintf("Gostats.%s.Heap.Objects", profileName), int64(memoryStats.HeapObjects), 1.0)
stats.Gauge(fmt.Sprintf("Gostats.%s.Heap.Idle", profileName), int64(memoryStats.HeapIdle), 1.0)
stats.Gauge(fmt.Sprintf("Gostats.%s.Heap.InUse", profileName), int64(memoryStats.HeapInuse), 1.0)
stats.Gauge(fmt.Sprintf("Gostats.%s.Heap.Released", profileName), int64(memoryStats.HeapReleased), 1.0)
gcPauseAvg := int64(memoryStats.PauseTotalNs) / int64(len(memoryStats.PauseNs))
stats.Timing(fmt.Sprintf("Gostats.%s.Gc.PauseAvg", profileName), gcPauseAvg, 1.0)
stats.Gauge(fmt.Sprintf("Gostats.%s.Gc.NextAt", profileName), int64(memoryStats.NextGC), 1.0)
time.Sleep(time.Second)
}
}
// ProfileCmd runs forever, sending Go runtime statistics to StatsD.
func ProfileCmd(profileName string, stats statsd.Statter) {
var memoryStats runtime.MemStats
prevNumGC := int64(0)
c := time.Tick(1 * time.Second)
for range c {
runtime.ReadMemStats(&memoryStats)
// Gather goroutine count
stats.Gauge(fmt.Sprintf("%s.Gostats.Goroutines", profileName), int64(runtime.NumGoroutine()), 1.0)
// Gather various heap metrics
stats.Gauge(fmt.Sprintf("%s.Gostats.Heap.Alloc", profileName), int64(memoryStats.HeapAlloc), 1.0)
stats.Gauge(fmt.Sprintf("%s.Gostats.Heap.Objects", profileName), int64(memoryStats.HeapObjects), 1.0)
stats.Gauge(fmt.Sprintf("%s.Gostats.Heap.Idle", profileName), int64(memoryStats.HeapIdle), 1.0)
stats.Gauge(fmt.Sprintf("%s.Gostats.Heap.InUse", profileName), int64(memoryStats.HeapInuse), 1.0)
stats.Gauge(fmt.Sprintf("%s.Gostats.Heap.Released", profileName), int64(memoryStats.HeapReleased), 1.0)
// Gather various GC related metrics
if memoryStats.NumGC > 0 {
totalRecentGC := uint64(0)
realBufSize := uint32(256)
if memoryStats.NumGC < 256 {
realBufSize = memoryStats.NumGC
}
for _, pause := range memoryStats.PauseNs {
totalRecentGC += pause
}
gcPauseAvg := totalRecentGC / uint64(realBufSize)
lastGC := memoryStats.PauseNs[(memoryStats.NumGC+255)%256]
stats.Timing(fmt.Sprintf("%s.Gostats.Gc.PauseAvg", profileName), int64(gcPauseAvg), 1.0)
stats.Gauge(fmt.Sprintf("%s.Gostats.Gc.LastPause", profileName), int64(lastGC), 1.0)
}
stats.Gauge(fmt.Sprintf("%s.Gostats.Gc.NextAt", profileName), int64(memoryStats.NextGC), 1.0)
// Send both a counter and a gauge here we can much more easily observe
// the GC rate (versus the raw number of GCs) in graphing tools that don't
// like deltas
stats.Gauge(fmt.Sprintf("%s.Gostats.Gc.Count", profileName), int64(memoryStats.NumGC), 1.0)
gcInc := int64(memoryStats.NumGC) - prevNumGC
stats.Inc(fmt.Sprintf("%s.Gostats.Gc.Rate", profileName), gcInc, 1.0)
prevNumGC += gcInc
}
}
func main() {
// command line flags
var opts struct {
HostPort string `long:"host" default:"127.0.0.1:8125" description:"host:port of statsd server"`
Prefix string `long:"prefix" default:"test-client" description:"Statsd prefix"`
StatType string `long:"type" default:"count" description:"stat type to send. Can be timing, count, guage"`
StatValue int64 `long:"value" default:"1" description:"Value to send"`
Name string `short:"n" long:"name" default:"counter" description:"stat name"`
Rate float32 `short:"r" long:"rate" default:"1.0" description:"sample rate"`
Volume int `short:"c" long:"count" default:"1000" description:"Number of stats to send. Volume."`
Nil bool `long:"nil" default:"false" description:"Use nil client"`
Buffered bool `long:"buffered" default:"false" description:"Use a buffered client"`
Duration time.Duration `short:"d" long:"duration" default:"10s" description:"How long to spread the volume across. Each second of duration volume/seconds events will be sent."`
}
// parse said flags
_, err := flags.Parse(&opts)
if err != nil {
if e, ok := err.(*flags.Error); ok {
if e.Type == flags.ErrHelp {
os.Exit(0)
}
}
fmt.Printf("Error: %+v\n", err)
os.Exit(1)
}
if opts.Nil && opts.Buffered {
fmt.Printf("Specifying both nil and buffered together is invalid.")
os.Exit(1)
}
var client statsd.Statter
if !opts.Nil {
if !opts.Buffered {
client, err = statsd.NewClient(opts.HostPort, opts.Prefix)
} else {
client, err = statsd.NewBufferedClient(opts.HostPort, opts.Prefix, opts.Duration/time.Duration(4), 0)
}
if err != nil {
log.Fatal(err)
}
defer client.Close()
}
var stat func(stat string, value int64, rate float32) error
switch opts.StatType {
case "count":
stat = func(stat string, value int64, rate float32) error {
return client.Inc(stat, value, rate)
}
case "gauge":
stat = func(stat string, value int64, rate float32) error {
return client.Gauge(stat, value, rate)
}
case "timing":
stat = func(stat string, value int64, rate float32) error {
return client.Timing(stat, value, rate)
}
default:
log.Fatal("Unsupported state type")
}
pertick := opts.Volume / int(opts.Duration.Seconds()) / 10
// add some extra tiem, because the first tick takes a while
ender := time.After(opts.Duration + 100*time.Millisecond)
c := time.Tick(time.Second / 10)
count := 0
for {
select {
case <-c:
for x := 0; x < pertick; x++ {
err := stat(opts.Name, opts.StatValue, opts.Rate)
if err != nil {
log.Printf("Got Error: %+v", err)
break
}
count += 1
}
case <-ender:
log.Printf("%d events called", count)
os.Exit(0)
return
}
}
}
// ProfileCmd runs forever, sending Go runtime statistics to StatsD.
func ProfileCmd(profileName string, stats statsd.Statter) {
c := time.Tick(1 * time.Second)
for range c {
var memoryStats runtime.MemStats
runtime.ReadMemStats(&memoryStats)
stats.Gauge(fmt.Sprintf("%s.Gostats.Goroutines", profileName), int64(runtime.NumGoroutine()), 1.0)
stats.Gauge(fmt.Sprintf("%s.Gostats.Heap.Alloc", profileName), int64(memoryStats.HeapAlloc), 1.0)
stats.Gauge(fmt.Sprintf("%s.Gostats.Heap.Objects", profileName), int64(memoryStats.HeapObjects), 1.0)
stats.Gauge(fmt.Sprintf("%s.Gostats.Heap.Idle", profileName), int64(memoryStats.HeapIdle), 1.0)
stats.Gauge(fmt.Sprintf("%s.Gostats.Heap.InUse", profileName), int64(memoryStats.HeapInuse), 1.0)
stats.Gauge(fmt.Sprintf("%s.Gostats.Heap.Released", profileName), int64(memoryStats.HeapReleased), 1.0)
// Calculate average and last and convert from nanoseconds to milliseconds
gcPauseAvg := (int64(memoryStats.PauseTotalNs) / int64(len(memoryStats.PauseNs))) / 1000000
lastGC := int64(memoryStats.PauseNs[(memoryStats.NumGC+255)%256]) / 1000000
stats.Timing(fmt.Sprintf("%s.Gostats.Gc.PauseAvg", profileName), gcPauseAvg, 1.0)
stats.Gauge(fmt.Sprintf("%s.Gostats.Gc.LastPauseLatency", profileName), lastGC, 1.0)
stats.Gauge(fmt.Sprintf("%s.Gostats.Gc.NextAt", profileName), int64(memoryStats.NextGC), 1.0)
}
}