func TestSensorRecordSummary(t *testing.T) {
testServer := httptest.NewServer(prometheus.UninstrumentedHandler())
defer testServer.Close()
sensor := &Sensor{
Type: "summary",
collector: prometheus.NewSummary(prometheus.SummaryOpts{
Namespace: "telemetry",
Subsystem: "sensors",
Name: "TestSensorRecordSummary",
Help: "help",
})}
prometheus.MustRegister(sensor.collector)
patt := `telemetry_sensors_TestSensorRecordSummary{quantile="([\.0-9]*)"} ([0-9\.]*)`
// need a bunch of metrics to make quantiles make any sense
for i := 1; i <= 10; i++ {
sensor.record(fmt.Sprintf("%v", i))
}
resp := getFromTestServer(t, testServer)
expected := [][]string{{"0.5", "5"}, {"0.9", "9"}, {"0.99", "9"}}
if !checkBuckets(resp, patt, expected) {
t.Fatalf("Failed to get match for sensor in response")
}
for i := 1; i <= 5; i++ {
// add a new record for each one in the bottom half
sensor.record(fmt.Sprintf("%v", i))
}
resp = getFromTestServer(t, testServer)
expected = [][]string{{"0.5", "4"}, {"0.9", "8"}, {"0.99", "9"}}
if !checkBuckets(resp, patt, expected) {
t.Fatalf("Failed to get match for sensor in response")
}
}
func TestSensorRecordHistogram(t *testing.T) {
testServer := httptest.NewServer(prometheus.UninstrumentedHandler())
defer testServer.Close()
sensor := &Sensor{
Type: "histogram",
collector: prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: "telemetry",
Subsystem: "sensors",
Name: "TestSensorRecordHistogram",
Help: "help",
})}
prometheus.MustRegister(sensor.collector)
patt := `telemetry_sensors_TestSensorRecordHistogram_bucket{le="([\.0-9|\+Inf]*)"} ([1-9])`
sensor.record("1.2")
resp := getFromTestServer(t, testServer)
expected := [][]string{{"2.5", "1"}, {"5", "1"}, {"10", "1"}, {"+Inf", "1"}}
if !checkBuckets(resp, patt, expected) {
t.Fatalf("Failed to get match for sensor in response")
}
sensor.record("1.2") // same value should add
resp = getFromTestServer(t, testServer)
expected = [][]string{{"2.5", "2"}, {"5", "2"}, {"10", "2"}, {"+Inf", "2"}}
if !checkBuckets(resp, patt, expected) {
t.Fatalf("Failed to get match for sensor in response")
}
sensor.record("4.5") // overlapping should overlap
resp = getFromTestServer(t, testServer)
expected = [][]string{{"2.5", "2"}, {"5", "3"}, {"10", "3"}, {"+Inf", "3"}}
if !checkBuckets(resp, patt, expected) {
t.Fatalf("Failed to get match for sensor in response")
}
}
func TestCounter(t *testing.T) {
s := httptest.NewServer(stdprometheus.UninstrumentedHandler())
defer s.Close()
scrape := func() string {
resp, _ := http.Get(s.URL)
buf, _ := ioutil.ReadAll(resp.Body)
return string(buf)
}
namespace, subsystem, name := "ns", "ss", "foo"
re := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `{alpha="alpha-value",beta="beta-value"} ([0-9\.]+)`)
counter := NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: name,
Help: "This is the help string.",
}, []string{"alpha", "beta"}).With("beta", "beta-value", "alpha", "alpha-value") // order shouldn't matter
value := func() float64 {
matches := re.FindStringSubmatch(scrape())
f, _ := strconv.ParseFloat(matches[1], 64)
return f
}
if err := teststat.TestCounter(counter, value); err != nil {
t.Fatal(err)
}
}
func (s *Server) route() {
s.r = mux.NewRouter().StrictSlash(true)
s.r.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
s.serveErrorPage("page not found", http.StatusNotFound, w, r)
})
s.r.Path("/").Methods("OPTIONS").HandlerFunc(s.handleProbe)
s.r.Path("/robots.txt").HandlerFunc(s.handleRobotsTxt)
s.r.Path("/metrics").Handler(
prometheus.InstrumentHandler("metrics", prometheus.UninstrumentedHandler()))
s.r.PathPrefix("/static/").Handler(
prometheus.InstrumentHandler("static", http.HandlerFunc(s.handleStatic)))
s.r.Handle("/", prometheus.InstrumentHandlerFunc("home", s.handleHomeStatic))
s.r.PathPrefix("/about").Handler(
prometheus.InstrumentHandler("about", http.HandlerFunc(s.handleAboutStatic)))
s.r.HandleFunc("/room/{prefix:(pm:)?}{room:[a-z0-9]+}/ws", instrumentSocketHandlerFunc("ws", s.handleRoom))
s.r.Handle(
"/room/{prefix:(pm:)?}{room:[a-z0-9]+}/", prometheus.InstrumentHandlerFunc("room_static", s.handleRoomStatic))
s.r.Handle(
"/prefs/reset-password",
prometheus.InstrumentHandlerFunc("prefsResetPassword", s.handlePrefsResetPassword))
s.r.Handle(
"/prefs/verify", prometheus.InstrumentHandlerFunc("prefsVerify", s.handlePrefsVerify))
}
func Start(config *Config) error {
if err := clone(config); err != nil {
return err
}
handler := handler(config)
ops := http.NewServeMux()
if config.AllowHooks {
ops.Handle("/hooks/", prometheus.InstrumentHandler("hooks", http.StripPrefix("/hooks", hooksHandler(config))))
}
/*ops.Handle("/reflect/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
fmt.Fprintf(os.Stdout, "%s %s\n", r.Method, r.URL)
io.Copy(os.Stdout, r.Body)
}))*/
ops.Handle("/metrics", prometheus.UninstrumentedHandler())
healthz.InstallHandler(ops)
mux := http.NewServeMux()
mux.Handle("/", prometheus.InstrumentHandler("git", handler))
mux.Handle("/_/", http.StripPrefix("/_", ops))
log.Printf("Serving %s on %s", config.Home, config.Listen)
return http.ListenAndServe(config.Listen, mux)
}
func (s *Server) route() {
s.r = mux.NewRouter().StrictSlash(true)
s.r.Path("/").Methods("OPTIONS").HandlerFunc(s.handleProbe)
s.r.Path("/robots.txt").HandlerFunc(s.handleRobotsTxt)
s.r.Path("/metrics").Handler(
prometheus.InstrumentHandler("metrics", prometheus.UninstrumentedHandler()))
s.r.PathPrefix("/static/").Handler(
prometheus.InstrumentHandler("static", http.StripPrefix("/static", http.HandlerFunc(s.handleStatic))))
s.r.Handle("/", prometheus.InstrumentHandlerFunc("home", s.handleHomeStatic))
s.r.PathPrefix("/about").Handler(
prometheus.InstrumentHandler("about", http.HandlerFunc(s.handleAboutStatic)))
s.r.HandleFunc("/room/{room:[a-z0-9]+}/ws", instrumentSocketHandlerFunc("ws", s.handleRoom))
s.r.Handle(
"/room/{room:[a-z0-9]+}/", prometheus.InstrumentHandlerFunc("room_static", s.handleRoomStatic))
s.r.Handle(
"/prefs/reset-password",
prometheus.InstrumentHandlerFunc("prefsResetPassword", s.handleResetPassword))
s.r.Handle(
"/prefs/verify", prometheus.InstrumentHandlerFunc("prefsVerify", s.handlePrefsVerify))
}
func TestGauge(t *testing.T) {
s := httptest.NewServer(stdprometheus.UninstrumentedHandler())
defer s.Close()
scrape := func() string {
resp, _ := http.Get(s.URL)
buf, _ := ioutil.ReadAll(resp.Body)
return string(buf)
}
namespace, subsystem, name := "aaa", "bbb", "ccc"
re := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `{foo="bar"} ([0-9\.]+)`)
gauge := NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: name,
Help: "This is a different help string.",
}, []string{"foo"}).With("foo", "bar")
value := func() float64 {
matches := re.FindStringSubmatch(scrape())
f, _ := strconv.ParseFloat(matches[1], 64)
return f
}
if err := teststat.TestGauge(gauge, value); err != nil {
t.Fatal(err)
}
}
func TestSensorRecordGauge(t *testing.T) {
testServer := httptest.NewServer(prometheus.UninstrumentedHandler())
defer testServer.Close()
sensor := &Sensor{
Type: "gauge",
collector: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "telemetry",
Subsystem: "sensors",
Name: "TestSensorRecordGauge",
Help: "help",
})}
prometheus.MustRegister(sensor.collector)
sensor.record("1.2")
resp := getFromTestServer(t, testServer)
if strings.Count(resp, "telemetry_sensors_TestSensorRecordGauge 1.2") != 1 {
t.Fatalf("Failed to get match for sensor in response: %s", resp)
}
sensor.record("2.3")
resp = getFromTestServer(t, testServer)
if strings.Count(resp, "telemetry_sensors_TestSensorRecordGauge 2.3") != 1 {
t.Fatalf("Failed to get match for sensor in response: %s", resp)
}
}
func (o *ObserveOptions) dumpMetrics() {
if !o.printMetricsOnExit {
return
}
w := httptest.NewRecorder()
prometheus.UninstrumentedHandler().ServeHTTP(w, &http.Request{})
if w.Code == http.StatusOK {
fmt.Fprintf(o.out, w.Body.String())
}
}
func scrapePrometheus(t *testing.T) string {
server := httptest.NewServer(stdprometheus.UninstrumentedHandler())
defer server.Close()
resp, err := http.Get(server.URL)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
buf, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
return strings.TrimSpace(string(buf))
}
func ExampleInstrumentHandler() {
// Handle the "/doc" endpoint with the standard http.FileServer handler.
// By wrapping the handler with InstrumentHandler, request count,
// request and response sizes, and request latency are automatically
// exported to Prometheus, partitioned by HTTP status code and method
// and by the handler name (here "fileserver").
http.Handle("/doc", prometheus.InstrumentHandler(
"fileserver", http.FileServer(http.Dir("/usr/share/doc")),
))
// The Prometheus handler still has to be registered to handle the
// "/metrics" endpoint. The handler returned by prometheus.Handler() is
// already instrumented - with "prometheus" as the handler name. In this
// example, we want the handler name to be "metrics", so we instrument
// the uninstrumented Prometheus handler ourselves.
http.Handle("/metrics", prometheus.InstrumentHandler(
"metrics", prometheus.UninstrumentedHandler(),
))
}
// partial metric name parses ok and write out as expected
func TestSensorPartialName(t *testing.T) {
testServer := httptest.NewServer(prometheus.UninstrumentedHandler())
defer testServer.Close()
jsonFragment := []byte(`[{
"name": "telemetry_sensors_partial_name",
"help": "help text",
"type": "counter"}]`)
sensor := parseSensors(t, jsonFragment)[0]
if _, ok := sensor.collector.(prometheus.Counter); !ok {
t.Fatalf("Incorrect collector; expected Counter but got %v", sensor.collector)
}
sensor.record("1")
resp := getFromTestServer(t, testServer)
if strings.Count(resp, "telemetry_sensors_partial_name 1") != 1 {
t.Fatalf("Failed to get match for sensor in response: %s", resp)
}
}
func TestSensorPollAction(t *testing.T) {
testServer := httptest.NewServer(prometheus.UninstrumentedHandler())
defer testServer.Close()
cmd, _ := commands.NewCommand("./testdata/test.sh measureStuff", "0")
sensor := &Sensor{
Type: "counter",
checkCmd: cmd,
collector: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: "telemetry",
Subsystem: "sensors",
Name: "TestSensorPollAction",
Help: "help",
})}
prometheus.MustRegister(sensor.collector)
sensor.PollAction()
resp := getFromTestServer(t, testServer)
if strings.Count(resp, "telemetry_sensors_TestSensorPollAction 42") != 1 {
t.Fatalf("Failed to get match for sensor in response: %s", resp)
}
}
func export(json string) ([]byte, error) {
exporter := NewExporter(&testScraper{json})
prometheus.MustRegister(exporter)
defer prometheus.Unregister(exporter)
server := httptest.NewServer(prometheus.UninstrumentedHandler())
defer server.Close()
response, err := http.Get(server.URL)
if err != nil {
return nil, err
}
defer response.Body.Close()
body, err := ioutil.ReadAll(response.Body)
if err != nil {
return nil, err
}
return body, nil
}
func TestSummary(t *testing.T) {
s := httptest.NewServer(stdprometheus.UninstrumentedHandler())
defer s.Close()
scrape := func() string {
resp, _ := http.Get(s.URL)
buf, _ := ioutil.ReadAll(resp.Body)
return string(buf)
}
namespace, subsystem, name := "test", "prometheus", "summary"
re50 := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `{a="a",b="b",quantile="0.5"} ([0-9\.]+)`)
re90 := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `{a="a",b="b",quantile="0.9"} ([0-9\.]+)`)
re99 := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `{a="a",b="b",quantile="0.99"} ([0-9\.]+)`)
summary := NewSummaryFrom(stdprometheus.SummaryOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: name,
Help: "This is the help string for the summary.",
}, []string{"a", "b"}).With("b", "b").With("a", "a")
quantiles := func() (float64, float64, float64, float64) {
buf := scrape()
match50 := re50.FindStringSubmatch(buf)
p50, _ := strconv.ParseFloat(match50[1], 64)
match90 := re90.FindStringSubmatch(buf)
p90, _ := strconv.ParseFloat(match90[1], 64)
match99 := re99.FindStringSubmatch(buf)
p99, _ := strconv.ParseFloat(match99[1], 64)
p95 := p90 + ((p99 - p90) / 2) // Prometheus, y u no p95??? :< #yolo
return p50, p90, p95, p99
}
if err := teststat.TestHistogram(summary, quantiles, 0.01); err != nil {
t.Fatal(err)
}
}
// 1. parse args
// 2. start the prometheus listener if configured
// 3. start the loghisto metric system
// 4. start the processing and printing goroutines
// 5. open the pcap handler
// 6. hand off packets from the handler to the decoder
func main() {
portsArg := flag.String("ports", "4001,2379", "etcd listening ports")
iface := flag.String("iface", "eth0", "interface for sniffing traffic on")
promisc := flag.Bool("promiscuous", false, "promiscuous mode")
period := flag.Uint("period", 60, "seconds between submissions")
topK := flag.Uint("topk", 10, "submit stats for the top <K> sniffed paths")
prometheusPort := flag.Uint("prometheus-port", 0, "port for prometheus exporter to listen on")
flag.Parse()
if *prometheusPort != 0 {
http.Handle("/metrics", prometheus.UninstrumentedHandler())
go http.ListenAndServe(":"+strconv.Itoa(int(*prometheusPort)), nil)
}
numCPU := runtime.NumCPU()
runtime.GOMAXPROCS(numCPU)
ms := loghisto.NewMetricSystem(time.Duration(*period)*time.Second, false)
ms.Start()
metricStream := make(chan *loghisto.ProcessedMetricSet, 2)
ms.SubscribeToProcessedMetrics(metricStream)
defer ms.UnsubscribeFromProcessedMetrics(metricStream)
go statPrinter(metricStream, *topK, *period)
ports := []uint16{}
for _, p := range strings.Split(*portsArg, ",") {
p, err := strconv.Atoi(p)
if err == nil {
ports = append(ports, uint16(p))
}
}
h, err := pcap.Openlive(*iface, 1518, *promisc, 1000)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
defer h.Close()
portArray := strings.Split(*portsArg, ",")
dst := strings.Join(portArray, " or dst port ")
src := strings.Join(portArray, " or src port ")
filter := fmt.Sprintf("tcp and (dst port %s or src port %s)", dst, src)
fmt.Println("using bpf filter: ", filter)
if err := h.Setfilter(filter); err != nil {
fmt.Println(err)
os.Exit(1)
}
unparsedPackets := make(chan *pcap.Packet, 10240)
parsedPackets := make(chan *pcap.Packet, 10240)
for i := 0; i < 5; i++ {
go packetDecoder(unparsedPackets, parsedPackets)
}
processors := []chan *pcap.Packet{}
for i := 0; i < 50; i++ {
p := make(chan *pcap.Packet, 10240)
processors = append(processors, p)
go processor(ms, p)
}
go streamRouter(ports, parsedPackets, processors)
for {
pkt := h.Next()
if pkt != nil {
select {
case unparsedPackets <- pkt:
default:
fmt.Println("SHEDDING IN MAIN")
}
}
}
}
func prometheusHandler() gin.HandlerFunc {
h := prometheus.UninstrumentedHandler()
return func(c *gin.Context) {
h.ServeHTTP(c.Writer, c.Request)
}
}
func TestHistogram(t *testing.T) {
// Prometheus reports histograms as a count of observations that fell into
// each predefined bucket, with the bucket value representing a global upper
// limit. That is, the count monotonically increases over the buckets. This
// requires a different strategy to test.
s := httptest.NewServer(stdprometheus.UninstrumentedHandler())
defer s.Close()
scrape := func() string {
resp, _ := http.Get(s.URL)
buf, _ := ioutil.ReadAll(resp.Body)
return string(buf)
}
namespace, subsystem, name := "test", "prometheus", "histogram"
re := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `_bucket{x="1",le="([0-9]+|\+Inf)"} ([0-9\.]+)`)
numStdev := 3
bucketMin := (teststat.Mean - (numStdev * teststat.Stdev))
bucketMax := (teststat.Mean + (numStdev * teststat.Stdev))
if bucketMin < 0 {
bucketMin = 0
}
bucketCount := 10
bucketDelta := (bucketMax - bucketMin) / bucketCount
buckets := []float64{}
for i := bucketMin; i <= bucketMax; i += bucketDelta {
buckets = append(buckets, float64(i))
}
histogram := NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: name,
Help: "This is the help string for the histogram.",
Buckets: buckets,
}, []string{"x"}).With("x", "1")
// Can't TestHistogram, because Prometheus Histograms don't dynamically
// compute quantiles. Instead, they fill up buckets. So, let's populate the
// histogram kind of manually.
teststat.PopulateNormalHistogram(histogram, rand.Int())
// Then, we use ExpectedObservationsLessThan to validate.
for _, line := range strings.Split(scrape(), "\n") {
match := re.FindStringSubmatch(line)
if match == nil {
continue
}
bucket, _ := strconv.ParseInt(match[1], 10, 64)
have, _ := strconv.ParseInt(match[2], 10, 64)
want := teststat.ExpectedObservationsLessThan(bucket)
if match[1] == "+Inf" {
want = int64(teststat.Count) // special case
}
// Unfortunately, we observe experimentally that Prometheus is quite
// imprecise at the extremes. I'm setting a very high tolerance for now.
// It would be great to dig in and figure out whether that's a problem
// with my Expected calculation, or in Prometheus.
tolerance := 0.25
if delta := math.Abs(float64(want) - float64(have)); (delta / float64(want)) > tolerance {
t.Errorf("Bucket %d: want %d, have %d (%.1f%%)", bucket, want, have, (100.0 * delta / float64(want)))
}
}
}