func (eb *Winlogbeat) Cleanup(b *beat.Beat) error {
logp.Debug("winlogbeat", "Dumping runtime metrics...")
expvar.Do(func(kv expvar.KeyValue) {
logp.Debug("winlogbeat", "%s=%s", kv.Key, kv.Value.String())
})
return nil
}
func TestSchemaInfoQueryCache(t *testing.T) {
db := fakesqldb.Register()
for query, result := range getSchemaInfoTestSupportedQueries() {
db.AddQuery(query, result)
}
firstQuery := "select * from test_table_01"
secondQuery := "select * from test_table_02"
db.AddQuery("select * from test_table_01 where 1 != 1", &sqltypes.Result{})
db.AddQuery("select * from test_table_02 where 1 != 1", &sqltypes.Result{})
schemaInfo := newTestSchemaInfo(10, 10*time.Second, 10*time.Second, true)
dbaParams := sqldb.ConnParams{Engine: db.Name}
schemaInfo.Open(&dbaParams, true)
defer schemaInfo.Close()
ctx := context.Background()
logStats := newLogStats("GetPlanStats", ctx)
schemaInfo.SetQueryCacheCap(1)
firstPlan := schemaInfo.GetPlan(ctx, logStats, firstQuery)
if firstPlan == nil {
t.Fatalf("plan should not be nil")
}
secondPlan := schemaInfo.GetPlan(ctx, logStats, secondQuery)
if secondPlan == nil {
t.Fatalf("plan should not be nil")
}
expvar.Do(func(kv expvar.KeyValue) {
_ = kv.Value.String()
})
schemaInfo.ClearQueryPlanCache()
}
// Export expvars to Cube right now. Use the provided timestamp for the
// submitted event. This function sends variables once and returns.
//
// You shouldn't need this function under normal circumstances. Use Run()
// instead.
func ExportVariablesWithTimestamp(collectionType string, putUrl string, timestamp time.Time) error {
variables := make([]string, 0)
expvar.Do(func(entry expvar.KeyValue) {
variables = append(variables, fmt.Sprintf("%q: %s", entry.Key, entry.Value))
})
request := fmt.Sprintf(
`[
{
"type": "%s",
"time": "%s",
"data": { %s }
}
]`,
collectionType,
timestamp.Format(time.ANSIC),
strings.Join(variables, ","))
response, err := http.Post(putUrl, "application/json", bytes.NewBufferString(request))
if err != nil {
log.Printf("Error POSTing events to Cube collector: %v", err)
log.Printf("The request we tried to post: %v", request)
return err
}
defer response.Body.Close()
return nil
}
// By using StatsLog, you can print stats on stdout every second, which is sometimes handy to check the state
// of the server. The stats themselves are declared using the "expvar" package
// to use this function, just before starting your listeners, create a goroutine like this
// go logging.StatsLog()
func StatsLogInterval(seconds int) {
// If we are running in debug mode, do not clog the screen
if IsDebug() {
log.Println("disabling logger in debug mode")
return
}
log.Println("starting logger")
info := log.New(os.Stdout, "s:", log.Ldate|log.Ltime)
sleepTime := time.Duration(seconds) * time.Second
for _ = range time.Tick(sleepTime) {
var buffer bytes.Buffer
expvar.Do(func(k expvar.KeyValue) {
if strings.HasPrefix(k.Key, StatsPrefix) {
buffer.WriteString(fmt.Sprintf("[%s %s] ", strings.TrimLeft(k.Key, StatsPrefix), k.Value))
// reset stats every nseconds
if v, ok := (k.Value).(*expvar.Int); ok {
v.Set(0)
}
}
})
info.Println(buffer.String())
}
}
// RegisterDebug adds handlers for /debug/vars (expvar) and /debug/pprof (net/http/pprof) support
func (this *HttpWeb) RegisterDebug(m *martini.ClassicMartini) {
m.Get("/debug/vars", func(w http.ResponseWriter, r *http.Request) {
// from expvar.go, since the expvarHandler isn't exported :(
w.Header().Set("Content-Type", "application/json; charset=utf-8")
fmt.Fprintf(w, "{\n")
first := true
expvar.Do(func(kv expvar.KeyValue) {
if !first {
fmt.Fprintf(w, ",\n")
}
first = false
fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
})
fmt.Fprintf(w, "\n}\n")
})
// list all the /debug/ endpoints we want
m.Get("/debug/pprof", pprof.Index)
m.Get("/debug/pprof/cmdline", pprof.Cmdline)
m.Get("/debug/pprof/profile", pprof.Profile)
m.Get("/debug/pprof/symbol", pprof.Symbol)
m.Post("/debug/pprof/symbol", pprof.Symbol)
m.Get("/debug/pprof/block", pprof.Handler("block").ServeHTTP)
m.Get("/debug/pprof/heap", pprof.Handler("heap").ServeHTTP)
m.Get("/debug/pprof/goroutine", pprof.Handler("goroutine").ServeHTTP)
m.Get("/debug/pprof/threadcreate", pprof.Handler("threadcreate").ServeHTTP)
}
// ExpvarHandler dumps json representation of expvars to http response.
//
// Expvars may be filtered by regexp provided via 'r' query argument.
//
// See https://golang.org/pkg/expvar/ for details.
func ExpvarHandler(ctx *fasthttp.RequestCtx) {
expvarHandlerCalls.Add(1)
ctx.Response.Reset()
r, err := getExpvarRegexp(ctx)
if err != nil {
expvarRegexpErrors.Add(1)
fmt.Fprintf(ctx, "Error when obtaining expvar regexp: %s", err)
ctx.SetStatusCode(fasthttp.StatusBadRequest)
return
}
fmt.Fprintf(ctx, "{\n")
first := true
expvar.Do(func(kv expvar.KeyValue) {
if r.MatchString(kv.Key) {
if !first {
fmt.Fprintf(ctx, ",\n")
}
first = false
fmt.Fprintf(ctx, "\t%q: %s", kv.Key, kv.Value)
}
})
fmt.Fprintf(ctx, "\n}\n")
ctx.SetContentType("application/json; charset=utf-8")
}
func (daemon *SSEDaemon) Status(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, "{\"status\":\"OK\"")
expvar.Do(func(kv expvar.KeyValue) {
fmt.Fprintf(w, ",%q:%s", kv.Key, kv.Value)
})
fmt.Fprintf(w, "}")
}
// dumpMetrics is used to log metrics on shutdown.
func dumpMetrics() {
logp.Info("Dumping runtime metrics...")
expvar.Do(func(kv expvar.KeyValue) {
if kv.Key != "memstats" {
logp.Info("%s=%s", kv.Key, kv.Value.String())
}
})
}
// LogOnDebugLevel logs all the current metrics, if logging is on Debug level.
func LogOnDebugLevel() {
if log.GetLevel() == log.DebugLevel {
fields := log.Fields{}
expvar.Do(func(kv expvar.KeyValue) {
fields[kv.Key] = kv.Value
})
logger.WithFields(fields).Debug("current values of metrics")
}
}
// Cleanup performs clean-up after Run completes.
func (bt *Metricbeat) Cleanup(b *beat.Beat) error {
logp.Info("Dumping runtime metrics...")
expvar.Do(func(kv expvar.KeyValue) {
if kv.Key != "memstats" {
logp.Info("%s=%s", kv.Key, kv.Value.String())
}
})
return nil
}
// snapshotExpvars iterates through all the defined expvars, and for the vars
// that are integers it snapshots the name and value in a separate (flat) map.
func snapshotExpvars(varsMap map[string]int64) {
expvar.Do(func(kv expvar.KeyValue) {
switch kv.Value.(type) {
case *expvar.Int:
varsMap[kv.Key], _ = strconv.ParseInt(kv.Value.String(), 10, 64)
case *expvar.Map:
snapshotMap(varsMap, kv.Key, kv.Value.(*expvar.Map))
}
})
}
func printVars() {
log.Println("expvars:")
expvar.Do(func(kv expvar.KeyValue) {
switch kv.Key {
case "memstats":
// Do nothing, this is a big output.
default:
log.Printf("\t%s -> %s", kv.Key, kv.Value)
}
})
}
// cleanup attempts to remove any files or data it may have created which should
// not be persisted.
func (eb *Winlogbeat) cleanup(b *beat.Beat) {
logp.Info("Dumping runtime metrics...")
expvar.Do(func(kv expvar.KeyValue) {
logf := logp.Info
if kv.Key == "memstats" {
logf = memstatsf
}
logf("%s=%s", kv.Key, kv.Value.String())
})
}
// serveExpvar serves registered expvar information over HTTP.
func serveExpvar(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "{\n")
first := true
expvar.Do(func(kv expvar.KeyValue) {
if !first {
fmt.Fprintf(w, ",\n")
}
first = false
fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
})
fmt.Fprintf(w, "\n}\n")
}
func writeMetrics(w io.Writer) {
fmt.Fprintf(w, "{\n")
first := true
expvar.Do(func(kv expvar.KeyValue) {
if !first {
fmt.Fprintf(w, ",\n")
}
first = false
fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
})
fmt.Fprintf(w, "\n}\n")
}
func do(f func(expvar.KeyValue)) {
expvar.Do(func(kv expvar.KeyValue) {
switch kv.Key {
case "cmdline",
"memstats":
return
}
if _, ok := kv.Value.(*expvar.Int); !ok {
return
}
f(kv)
})
}
func writeMetrics(w io.Writer) {
numGoroutines.Set(int64(runtime.NumGoroutine()))
fmt.Fprint(w, "{\n")
first := true
expvar.Do(func(kv expvar.KeyValue) {
if !first {
fmt.Fprint(w, ",\n")
}
first = false
fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
})
fmt.Fprint(w, "\n}\n")
}
// LogOnDebugLevel logs all the current metrics, if logging is on Debug level.
func LogOnDebugLevel() {
if !*config.Metrics.Enabled {
log.Debug("metrics: not enabled")
return
}
if log.GetLevel() == log.DebugLevel {
fields := log.Fields{}
expvar.Do(func(kv expvar.KeyValue) {
fields[kv.Key] = kv.Value
})
log.WithFields(fields).Debug("metrics: current values")
}
}
// serveExpvar serves registered expvar information over HTTP.
func serveExpvar(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
fmt.Fprintf(w, "{\n")
first := true
expvar.Do(func(kv expvar.KeyValue) {
if !first {
fmt.Fprintf(w, ",\n")
}
first = false
fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
})
fmt.Fprintf(w, "\n}\n")
}
func TestSchemaInfoExportVars(t *testing.T) {
db := fakesqldb.Register()
for query, result := range getSchemaInfoTestSupportedQueries() {
db.AddQuery(query, result)
}
schemaInfo := newTestSchemaInfo(10, 1*time.Second, 1*time.Second, true)
dbaParams := sqldb.ConnParams{Engine: db.Name}
schemaInfo.Open(&dbaParams, true)
defer schemaInfo.Close()
expvar.Do(func(kv expvar.KeyValue) {
_ = kv.Value.String()
})
}
// GetByDo takes expvar map using Do method from expvar package
func GetByDo() (string, error) {
result := "{"
first := true
expvar.Do(func(kv expvar.KeyValue) {
if !first {
result += ",\n"
}
first = false
result += fmt.Sprintf("%q: %s", kv.Key, kv.Value)
})
result += "}"
return result, nil
}
// expvarJSON returns expvars as a JSON-formatted byte slice.
func expvarJSON() []byte {
var buf bytes.Buffer
fmt.Fprintf(&buf, "{\n")
first := true
expvar.Do(func(kv expvar.KeyValue) {
if !first {
fmt.Fprintf(&buf, ",\n")
}
first = false
fmt.Fprintf(&buf, "%q: %s", kv.Key, kv.Value)
})
fmt.Fprintf(&buf, "\n}\n")
return buf.Bytes()
}
func StatsLog() {
for _ = range time.Tick(time.Second) {
var buffer bytes.Buffer
expvar.Do(func(k expvar.KeyValue) {
if strings.HasPrefix(k.Key, "rps") {
buffer.WriteString(fmt.Sprintf("[%s %s] ", k.Key, k.Value))
// reset stats every second
if v, ok := (k.Value).(*expvar.Int); ok {
v.Set(0)
}
}
})
log.Println(buffer.String())
}
}
// Pretty prints the latency histograms
func latencyHandler(w http.ResponseWriter, r *http.Request) {
expvar.Do(func(kv expvar.KeyValue) {
if strings.HasSuffix(kv.Key, "Histogram") {
template.Must(template.New("latency").Parse(
`<html>
<head>
<title>/debug/latency</title>
</head>
<body>
<pre>{{.}}</pre>
</body>
</html>
`)).Execute(w, template.HTML(strings.Replace(kv.Value.String(), "\\n", "<br />", -1)))
}
})
}
// handle expvar request.
func (s *httpServer) expvarHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
fmt.Fprintf(w, "{\n")
first := true
expvar.Do(func(kv expvar.KeyValue) {
if !first {
fmt.Fprintf(w, ",\n")
}
first = false
data, err := json.Marshal(kv.Value)
if err != nil {
logging.Errorf("%v encoding statistics: %v\n", s.logPrefix, err)
}
fmt.Fprintf(w, "%q: %s", kv.Key, data)
})
fmt.Fprintf(w, "\n}\n")
}
func TestSchemaInfoExportVars(t *testing.T) {
fakecacheservice.Register()
db := fakesqldb.Register()
for query, result := range getSchemaInfoTestSupportedQueries() {
db.AddQuery(query, result)
}
schemaInfo := newTestSchemaInfo(10, 1*time.Second, 1*time.Second, true)
appParams := sqldb.ConnParams{}
dbaParams := sqldb.ConnParams{}
schemaInfo.cachePool.Open()
defer schemaInfo.cachePool.Close()
schemaInfo.Open(&appParams, &dbaParams, []SchemaOverride{}, true)
defer schemaInfo.Close()
expvar.Do(func(kv expvar.KeyValue) {
_ = kv.Value.String()
})
}
func Handler() gin.HandlerFunc {
return func(c *gin.Context) {
w := c.Writer
c.Header("Content-Type", "application/json; charset=utf-8")
w.Write([]byte("{\n"))
first := true
expvar.Do(func(kv expvar.KeyValue) {
if !first {
w.Write([]byte(",\n"))
}
first = false
fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
})
w.Write([]byte("\n}\n"))
c.AbortWithStatus(200)
}
}
// PushAll pushes all expvar stats to InfluxDB
func (backend *InfluxDBBackend) PushAll() error {
series := []*influxClient.Series{}
expvar.Do(func(kv expvar.KeyValue) {
series = append(series, &influxClient.Series{
Name: "stats",
// TODO(aaijazi): This would be much better suited to InfluxDB v0.90's tags.
// Ideally, we'd use some of the expvars as tags, and some as values.
// However, as of 03/11/2015, InfluxDB v0.90 hasn't proven quite stable enough to use.
Columns: []string{"key", "value"},
Points: [][]interface{}{
{kv.Key, statToValue(kv.Value)},
},
})
})
err := backend.client.WriteSeries(series)
return err
}
func (exp *exp) expHandler(w http.ResponseWriter, r *http.Request) {
// load our variables into expvar
exp.syncToExpvar()
// now just run the official expvar handler code (which is not publicly callable, so pasted inline)
w.Header().Set("Content-Type", "application/json; charset=utf-8")
fmt.Fprintf(w, "{\n")
first := true
expvar.Do(func(kv expvar.KeyValue) {
if !first {
fmt.Fprintf(w, ",\n")
}
first = false
fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
})
fmt.Fprintf(w, "\n}\n")
}
func TestSchemaInfoQueryCache(t *testing.T) {
fakecacheservice.Register()
db := fakesqldb.Register()
for query, result := range getSchemaInfoTestSupportedQueries() {
db.AddQuery(query, result)
}
firstQuery := "select * from test_table_01"
secondQuery := "select * from test_table_02"
db.AddQuery("select * from test_table_01 where 1 != 1", &mproto.QueryResult{})
db.AddQuery("select * from test_table_02 where 1 != 1", &mproto.QueryResult{})
schemaInfo := newTestSchemaInfo(10, 10*time.Second, 10*time.Second, true)
appParams := sqldb.ConnParams{}
dbaParams := sqldb.ConnParams{}
cachePool := newTestSchemaInfoCachePool(true, schemaInfo.queryServiceStats)
cachePool.Open()
defer cachePool.Close()
schemaOverrides := getSchemaInfoTestSchemaOverride()
// test cache type RW
schemaInfo.Open(&appParams, &dbaParams, schemaOverrides, cachePool, true)
defer schemaInfo.Close()
ctx := context.Background()
logStats := newLogStats("GetPlanStats", ctx)
schemaInfo.SetQueryCacheSize(1)
firstPlan := schemaInfo.GetPlan(ctx, logStats, firstQuery)
if firstPlan == nil {
t.Fatalf("plan should not be nil")
}
secondPlan := schemaInfo.GetPlan(ctx, logStats, secondQuery)
if secondPlan == nil {
t.Fatalf("plan should not be nil")
}
expvar.Do(func(kv expvar.KeyValue) {
_ = kv.Value.String()
})
schemaInfo.ClearQueryPlanCache()
}