func TestSensorRecordSummary(t *testing.T) { testServer := httptest.NewServer(prometheus.UninstrumentedHandler()) defer testServer.Close() sensor := &Sensor{ Type: "summary", collector: prometheus.NewSummary(prometheus.SummaryOpts{ Namespace: "telemetry", Subsystem: "sensors", Name: "TestSensorRecordSummary", Help: "help", })} prometheus.MustRegister(sensor.collector) patt := `telemetry_sensors_TestSensorRecordSummary{quantile="([\.0-9]*)"} ([0-9\.]*)` // need a bunch of metrics to make quantiles make any sense for i := 1; i <= 10; i++ { sensor.record(fmt.Sprintf("%v", i)) } resp := getFromTestServer(t, testServer) expected := [][]string{{"0.5", "5"}, {"0.9", "9"}, {"0.99", "9"}} if !checkBuckets(resp, patt, expected) { t.Fatalf("Failed to get match for sensor in response") } for i := 1; i <= 5; i++ { // add a new record for each one in the bottom half sensor.record(fmt.Sprintf("%v", i)) } resp = getFromTestServer(t, testServer) expected = [][]string{{"0.5", "4"}, {"0.9", "8"}, {"0.99", "9"}} if !checkBuckets(resp, patt, expected) { t.Fatalf("Failed to get match for sensor in response") } }
func TestSensorRecordHistogram(t *testing.T) { testServer := httptest.NewServer(prometheus.UninstrumentedHandler()) defer testServer.Close() sensor := &Sensor{ Type: "histogram", collector: prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "telemetry", Subsystem: "sensors", Name: "TestSensorRecordHistogram", Help: "help", })} prometheus.MustRegister(sensor.collector) patt := `telemetry_sensors_TestSensorRecordHistogram_bucket{le="([\.0-9|\+Inf]*)"} ([1-9])` sensor.record("1.2") resp := getFromTestServer(t, testServer) expected := [][]string{{"2.5", "1"}, {"5", "1"}, {"10", "1"}, {"+Inf", "1"}} if !checkBuckets(resp, patt, expected) { t.Fatalf("Failed to get match for sensor in response") } sensor.record("1.2") // same value should add resp = getFromTestServer(t, testServer) expected = [][]string{{"2.5", "2"}, {"5", "2"}, {"10", "2"}, {"+Inf", "2"}} if !checkBuckets(resp, patt, expected) { t.Fatalf("Failed to get match for sensor in response") } sensor.record("4.5") // overlapping should overlap resp = getFromTestServer(t, testServer) expected = [][]string{{"2.5", "2"}, {"5", "3"}, {"10", "3"}, {"+Inf", "3"}} if !checkBuckets(resp, patt, expected) { t.Fatalf("Failed to get match for sensor in response") } }
func TestCounter(t *testing.T) { s := httptest.NewServer(stdprometheus.UninstrumentedHandler()) defer s.Close() scrape := func() string { resp, _ := http.Get(s.URL) buf, _ := ioutil.ReadAll(resp.Body) return string(buf) } namespace, subsystem, name := "ns", "ss", "foo" re := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `{alpha="alpha-value",beta="beta-value"} ([0-9\.]+)`) counter := NewCounterFrom(stdprometheus.CounterOpts{ Namespace: namespace, Subsystem: subsystem, Name: name, Help: "This is the help string.", }, []string{"alpha", "beta"}).With("beta", "beta-value", "alpha", "alpha-value") // order shouldn't matter value := func() float64 { matches := re.FindStringSubmatch(scrape()) f, _ := strconv.ParseFloat(matches[1], 64) return f } if err := teststat.TestCounter(counter, value); err != nil { t.Fatal(err) } }
func (s *Server) route() { s.r = mux.NewRouter().StrictSlash(true) s.r.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { s.serveErrorPage("page not found", http.StatusNotFound, w, r) }) s.r.Path("/").Methods("OPTIONS").HandlerFunc(s.handleProbe) s.r.Path("/robots.txt").HandlerFunc(s.handleRobotsTxt) s.r.Path("/metrics").Handler( prometheus.InstrumentHandler("metrics", prometheus.UninstrumentedHandler())) s.r.PathPrefix("/static/").Handler( prometheus.InstrumentHandler("static", http.HandlerFunc(s.handleStatic))) s.r.Handle("/", prometheus.InstrumentHandlerFunc("home", s.handleHomeStatic)) s.r.PathPrefix("/about").Handler( prometheus.InstrumentHandler("about", http.HandlerFunc(s.handleAboutStatic))) s.r.HandleFunc("/room/{prefix:(pm:)?}{room:[a-z0-9]+}/ws", instrumentSocketHandlerFunc("ws", s.handleRoom)) s.r.Handle( "/room/{prefix:(pm:)?}{room:[a-z0-9]+}/", prometheus.InstrumentHandlerFunc("room_static", s.handleRoomStatic)) s.r.Handle( "/prefs/reset-password", prometheus.InstrumentHandlerFunc("prefsResetPassword", s.handlePrefsResetPassword)) s.r.Handle( "/prefs/verify", prometheus.InstrumentHandlerFunc("prefsVerify", s.handlePrefsVerify)) }
func Start(config *Config) error { if err := clone(config); err != nil { return err } handler := handler(config) ops := http.NewServeMux() if config.AllowHooks { ops.Handle("/hooks/", prometheus.InstrumentHandler("hooks", http.StripPrefix("/hooks", hooksHandler(config)))) } /*ops.Handle("/reflect/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() fmt.Fprintf(os.Stdout, "%s %s\n", r.Method, r.URL) io.Copy(os.Stdout, r.Body) }))*/ ops.Handle("/metrics", prometheus.UninstrumentedHandler()) healthz.InstallHandler(ops) mux := http.NewServeMux() mux.Handle("/", prometheus.InstrumentHandler("git", handler)) mux.Handle("/_/", http.StripPrefix("/_", ops)) log.Printf("Serving %s on %s", config.Home, config.Listen) return http.ListenAndServe(config.Listen, mux) }
func (s *Server) route() { s.r = mux.NewRouter().StrictSlash(true) s.r.Path("/").Methods("OPTIONS").HandlerFunc(s.handleProbe) s.r.Path("/robots.txt").HandlerFunc(s.handleRobotsTxt) s.r.Path("/metrics").Handler( prometheus.InstrumentHandler("metrics", prometheus.UninstrumentedHandler())) s.r.PathPrefix("/static/").Handler( prometheus.InstrumentHandler("static", http.StripPrefix("/static", http.HandlerFunc(s.handleStatic)))) s.r.Handle("/", prometheus.InstrumentHandlerFunc("home", s.handleHomeStatic)) s.r.PathPrefix("/about").Handler( prometheus.InstrumentHandler("about", http.HandlerFunc(s.handleAboutStatic))) s.r.HandleFunc("/room/{room:[a-z0-9]+}/ws", instrumentSocketHandlerFunc("ws", s.handleRoom)) s.r.Handle( "/room/{room:[a-z0-9]+}/", prometheus.InstrumentHandlerFunc("room_static", s.handleRoomStatic)) s.r.Handle( "/prefs/reset-password", prometheus.InstrumentHandlerFunc("prefsResetPassword", s.handleResetPassword)) s.r.Handle( "/prefs/verify", prometheus.InstrumentHandlerFunc("prefsVerify", s.handlePrefsVerify)) }
func TestGauge(t *testing.T) { s := httptest.NewServer(stdprometheus.UninstrumentedHandler()) defer s.Close() scrape := func() string { resp, _ := http.Get(s.URL) buf, _ := ioutil.ReadAll(resp.Body) return string(buf) } namespace, subsystem, name := "aaa", "bbb", "ccc" re := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `{foo="bar"} ([0-9\.]+)`) gauge := NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: subsystem, Name: name, Help: "This is a different help string.", }, []string{"foo"}).With("foo", "bar") value := func() float64 { matches := re.FindStringSubmatch(scrape()) f, _ := strconv.ParseFloat(matches[1], 64) return f } if err := teststat.TestGauge(gauge, value); err != nil { t.Fatal(err) } }
func TestSensorRecordGauge(t *testing.T) { testServer := httptest.NewServer(prometheus.UninstrumentedHandler()) defer testServer.Close() sensor := &Sensor{ Type: "gauge", collector: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "telemetry", Subsystem: "sensors", Name: "TestSensorRecordGauge", Help: "help", })} prometheus.MustRegister(sensor.collector) sensor.record("1.2") resp := getFromTestServer(t, testServer) if strings.Count(resp, "telemetry_sensors_TestSensorRecordGauge 1.2") != 1 { t.Fatalf("Failed to get match for sensor in response: %s", resp) } sensor.record("2.3") resp = getFromTestServer(t, testServer) if strings.Count(resp, "telemetry_sensors_TestSensorRecordGauge 2.3") != 1 { t.Fatalf("Failed to get match for sensor in response: %s", resp) } }
func (o *ObserveOptions) dumpMetrics() { if !o.printMetricsOnExit { return } w := httptest.NewRecorder() prometheus.UninstrumentedHandler().ServeHTTP(w, &http.Request{}) if w.Code == http.StatusOK { fmt.Fprintf(o.out, w.Body.String()) } }
func scrapePrometheus(t *testing.T) string { server := httptest.NewServer(stdprometheus.UninstrumentedHandler()) defer server.Close() resp, err := http.Get(server.URL) if err != nil { t.Fatal(err) } defer resp.Body.Close() buf, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatal(err) } return strings.TrimSpace(string(buf)) }
func ExampleInstrumentHandler() { // Handle the "/doc" endpoint with the standard http.FileServer handler. // By wrapping the handler with InstrumentHandler, request count, // request and response sizes, and request latency are automatically // exported to Prometheus, partitioned by HTTP status code and method // and by the handler name (here "fileserver"). http.Handle("/doc", prometheus.InstrumentHandler( "fileserver", http.FileServer(http.Dir("/usr/share/doc")), )) // The Prometheus handler still has to be registered to handle the // "/metrics" endpoint. The handler returned by prometheus.Handler() is // already instrumented - with "prometheus" as the handler name. In this // example, we want the handler name to be "metrics", so we instrument // the uninstrumented Prometheus handler ourselves. http.Handle("/metrics", prometheus.InstrumentHandler( "metrics", prometheus.UninstrumentedHandler(), )) }
// partial metric name parses ok and write out as expected func TestSensorPartialName(t *testing.T) { testServer := httptest.NewServer(prometheus.UninstrumentedHandler()) defer testServer.Close() jsonFragment := []byte(`[{ "name": "telemetry_sensors_partial_name", "help": "help text", "type": "counter"}]`) sensor := parseSensors(t, jsonFragment)[0] if _, ok := sensor.collector.(prometheus.Counter); !ok { t.Fatalf("Incorrect collector; expected Counter but got %v", sensor.collector) } sensor.record("1") resp := getFromTestServer(t, testServer) if strings.Count(resp, "telemetry_sensors_partial_name 1") != 1 { t.Fatalf("Failed to get match for sensor in response: %s", resp) } }
func TestSensorPollAction(t *testing.T) { testServer := httptest.NewServer(prometheus.UninstrumentedHandler()) defer testServer.Close() cmd, _ := commands.NewCommand("./testdata/test.sh measureStuff", "0") sensor := &Sensor{ Type: "counter", checkCmd: cmd, collector: prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "telemetry", Subsystem: "sensors", Name: "TestSensorPollAction", Help: "help", })} prometheus.MustRegister(sensor.collector) sensor.PollAction() resp := getFromTestServer(t, testServer) if strings.Count(resp, "telemetry_sensors_TestSensorPollAction 42") != 1 { t.Fatalf("Failed to get match for sensor in response: %s", resp) } }
func export(json string) ([]byte, error) { exporter := NewExporter(&testScraper{json}) prometheus.MustRegister(exporter) defer prometheus.Unregister(exporter) server := httptest.NewServer(prometheus.UninstrumentedHandler()) defer server.Close() response, err := http.Get(server.URL) if err != nil { return nil, err } defer response.Body.Close() body, err := ioutil.ReadAll(response.Body) if err != nil { return nil, err } return body, nil }
func TestSummary(t *testing.T) { s := httptest.NewServer(stdprometheus.UninstrumentedHandler()) defer s.Close() scrape := func() string { resp, _ := http.Get(s.URL) buf, _ := ioutil.ReadAll(resp.Body) return string(buf) } namespace, subsystem, name := "test", "prometheus", "summary" re50 := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `{a="a",b="b",quantile="0.5"} ([0-9\.]+)`) re90 := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `{a="a",b="b",quantile="0.9"} ([0-9\.]+)`) re99 := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `{a="a",b="b",quantile="0.99"} ([0-9\.]+)`) summary := NewSummaryFrom(stdprometheus.SummaryOpts{ Namespace: namespace, Subsystem: subsystem, Name: name, Help: "This is the help string for the summary.", }, []string{"a", "b"}).With("b", "b").With("a", "a") quantiles := func() (float64, float64, float64, float64) { buf := scrape() match50 := re50.FindStringSubmatch(buf) p50, _ := strconv.ParseFloat(match50[1], 64) match90 := re90.FindStringSubmatch(buf) p90, _ := strconv.ParseFloat(match90[1], 64) match99 := re99.FindStringSubmatch(buf) p99, _ := strconv.ParseFloat(match99[1], 64) p95 := p90 + ((p99 - p90) / 2) // Prometheus, y u no p95??? :< #yolo return p50, p90, p95, p99 } if err := teststat.TestHistogram(summary, quantiles, 0.01); err != nil { t.Fatal(err) } }
// 1. parse args // 2. start the prometheus listener if configured // 3. start the loghisto metric system // 4. start the processing and printing goroutines // 5. open the pcap handler // 6. hand off packets from the handler to the decoder func main() { portsArg := flag.String("ports", "4001,2379", "etcd listening ports") iface := flag.String("iface", "eth0", "interface for sniffing traffic on") promisc := flag.Bool("promiscuous", false, "promiscuous mode") period := flag.Uint("period", 60, "seconds between submissions") topK := flag.Uint("topk", 10, "submit stats for the top <K> sniffed paths") prometheusPort := flag.Uint("prometheus-port", 0, "port for prometheus exporter to listen on") flag.Parse() if *prometheusPort != 0 { http.Handle("/metrics", prometheus.UninstrumentedHandler()) go http.ListenAndServe(":"+strconv.Itoa(int(*prometheusPort)), nil) } numCPU := runtime.NumCPU() runtime.GOMAXPROCS(numCPU) ms := loghisto.NewMetricSystem(time.Duration(*period)*time.Second, false) ms.Start() metricStream := make(chan *loghisto.ProcessedMetricSet, 2) ms.SubscribeToProcessedMetrics(metricStream) defer ms.UnsubscribeFromProcessedMetrics(metricStream) go statPrinter(metricStream, *topK, *period) ports := []uint16{} for _, p := range strings.Split(*portsArg, ",") { p, err := strconv.Atoi(p) if err == nil { ports = append(ports, uint16(p)) } } h, err := pcap.Openlive(*iface, 1518, *promisc, 1000) if err != nil { fmt.Println(err) os.Exit(1) } defer h.Close() portArray := strings.Split(*portsArg, ",") dst := strings.Join(portArray, " or dst port ") src := strings.Join(portArray, " or src port ") filter := fmt.Sprintf("tcp and (dst port %s or src port %s)", dst, src) fmt.Println("using bpf filter: ", filter) if err := h.Setfilter(filter); err != nil { fmt.Println(err) os.Exit(1) } unparsedPackets := make(chan *pcap.Packet, 10240) parsedPackets := make(chan *pcap.Packet, 10240) for i := 0; i < 5; i++ { go packetDecoder(unparsedPackets, parsedPackets) } processors := []chan *pcap.Packet{} for i := 0; i < 50; i++ { p := make(chan *pcap.Packet, 10240) processors = append(processors, p) go processor(ms, p) } go streamRouter(ports, parsedPackets, processors) for { pkt := h.Next() if pkt != nil { select { case unparsedPackets <- pkt: default: fmt.Println("SHEDDING IN MAIN") } } } }
func prometheusHandler() gin.HandlerFunc { h := prometheus.UninstrumentedHandler() return func(c *gin.Context) { h.ServeHTTP(c.Writer, c.Request) } }
func TestHistogram(t *testing.T) { // Prometheus reports histograms as a count of observations that fell into // each predefined bucket, with the bucket value representing a global upper // limit. That is, the count monotonically increases over the buckets. This // requires a different strategy to test. s := httptest.NewServer(stdprometheus.UninstrumentedHandler()) defer s.Close() scrape := func() string { resp, _ := http.Get(s.URL) buf, _ := ioutil.ReadAll(resp.Body) return string(buf) } namespace, subsystem, name := "test", "prometheus", "histogram" re := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + `_bucket{x="1",le="([0-9]+|\+Inf)"} ([0-9\.]+)`) numStdev := 3 bucketMin := (teststat.Mean - (numStdev * teststat.Stdev)) bucketMax := (teststat.Mean + (numStdev * teststat.Stdev)) if bucketMin < 0 { bucketMin = 0 } bucketCount := 10 bucketDelta := (bucketMax - bucketMin) / bucketCount buckets := []float64{} for i := bucketMin; i <= bucketMax; i += bucketDelta { buckets = append(buckets, float64(i)) } histogram := NewHistogramFrom(stdprometheus.HistogramOpts{ Namespace: namespace, Subsystem: subsystem, Name: name, Help: "This is the help string for the histogram.", Buckets: buckets, }, []string{"x"}).With("x", "1") // Can't TestHistogram, because Prometheus Histograms don't dynamically // compute quantiles. Instead, they fill up buckets. So, let's populate the // histogram kind of manually. teststat.PopulateNormalHistogram(histogram, rand.Int()) // Then, we use ExpectedObservationsLessThan to validate. for _, line := range strings.Split(scrape(), "\n") { match := re.FindStringSubmatch(line) if match == nil { continue } bucket, _ := strconv.ParseInt(match[1], 10, 64) have, _ := strconv.ParseInt(match[2], 10, 64) want := teststat.ExpectedObservationsLessThan(bucket) if match[1] == "+Inf" { want = int64(teststat.Count) // special case } // Unfortunately, we observe experimentally that Prometheus is quite // imprecise at the extremes. I'm setting a very high tolerance for now. // It would be great to dig in and figure out whether that's a problem // with my Expected calculation, or in Prometheus. tolerance := 0.25 if delta := math.Abs(float64(want) - float64(have)); (delta / float64(want)) > tolerance { t.Errorf("Bucket %d: want %d, have %d (%.1f%%)", bucket, want, have, (100.0 * delta / float64(want))) } } }