func (e *Exporter) scrape(csvRows chan<- []string) { defer close(csvRows) e.totalScrapes.Inc() resp, err := e.client.Get(e.URI) if err != nil { e.up.Set(0) log.Printf("Error while scraping HAProxy: %v", err) return } defer resp.Body.Close() e.up.Set(1) reader := csv.NewReader(resp.Body) reader.TrailingComma = true reader.Comment = '#' for { row, err := reader.Read() if err == io.EOF { break } if err != nil { log.Printf("Error while reading CSV: %v", err) e.csvParseFailures.Inc() break } if len(row) == 0 { continue } csvRows <- row } }
// Collect implements prometheus.Collector. func (c collectdCollector) Collect(ch chan<- prometheus.Metric) { ch <- lastPush c.mu.Lock() valueLists := make([]api.ValueList, 0, len(c.valueLists)) for _, vl := range c.valueLists { valueLists = append(valueLists, vl) } c.mu.Unlock() now := time.Now() for _, vl := range valueLists { validUntil := vl.Time.Add(timeout * vl.Interval) if validUntil.Before(now) { continue } for i := range vl.Values { m, err := newMetric(vl, i) if err != nil { log.Printf("newMetric: %v", err) continue } ch <- m } } }
// Collect fetches the stats from configured nginx location and delivers them // as Prometheus metrics. It implements prometheus.Collector. func (e *Exporter) Collect(ch chan<- prometheus.Metric) { e.mutex.Lock() // To protect metrics from concurrent collects. defer e.mutex.Unlock() if err := e.scrape(ch); err != nil { log.Printf("Error scraping apache2: %s", err) e.scrapeFailures.Inc() e.scrapeFailures.Collect(ch) } e.version.Collect(ch) e.totalRequests.Collect(ch) e.uptimeDays.Collect(ch) e.uptimeHour.Collect(ch) e.totalAccesses.Collect(ch) e.cpuUsageUser.Collect(ch) e.cpuUsageSystem.Collect(ch) e.cpuLoad.Collect(ch) e.totalTraffic.Collect(ch) e.requestBeingProcessed.Collect(ch) e.idleWorkers.Collect(ch) e.requestsPerSecond.Collect(ch) e.dataPerSecond.Collect(ch) e.dataPerRequest.Collect(ch) e.clientRequests.Collect(ch) e.virtualHosts.Collect(ch) return }
func (e *Exporter) exportCsvFields(metrics map[int]*prometheus.GaugeVec, csvRow []string, labels ...string) { for fieldIdx, metric := range metrics { valueStr := csvRow[fieldIdx] if valueStr == "" { continue } var value int64 var err error switch valueStr { // UP or UP going down case "UP", "UP 1/3", "UP 2/3": value = 1 // DOWN or DOWN going up case "DOWN", "DOWN 1/2": value = 0 case "OPEN": value = 0 case "no check": continue default: value, err = strconv.ParseInt(valueStr, 10, 64) if err != nil { log.Printf("Error while parsing CSV field value %s: %v", valueStr, err) e.csvParseFailures.Inc() continue } } metric.WithLabelValues(labels...).Set(float64(value)) } }
func (e *Exporter) setMetrics(csvRows <-chan []string) { for csvRow := range csvRows { if len(csvRow) < expectedCsvFieldCount { log.Printf("Wrong CSV field count: %d vs. %d", len(csvRow), expectedCsvFieldCount) e.csvParseFailures.Inc() continue } pxname, svname, type_ := csvRow[0], csvRow[1], csvRow[32] const ( frontend = "0" backend = "1" server = "2" listener = "3" ) switch type_ { case frontend: e.exportCsvFields(e.frontendMetrics, csvRow, pxname) case backend: e.exportCsvFields(e.backendMetrics, csvRow, pxname) case server: e.exportCsvFields(e.serverMetrics, csvRow, pxname, svname) } } }
func watchConfig(fileName string, mapper *metricMapper) { watcher, err := fsnotify.NewWatcher() if err != nil { log.Fatal(err) } err = watcher.WatchFlags(fileName, fsnotify.FSN_MODIFY) if err != nil { log.Fatal(err) } for { select { case ev := <-watcher.Event: log.Printf("Config file changed (%s), attempting reload", ev) err = mapper.initFromFile(fileName) if err != nil { log.Println("Error reloading config:", err) configLoads.WithLabelValues("failure").Inc() } else { log.Println("Config reloaded successfully") configLoads.WithLabelValues("success").Inc() } // Re-add the file watcher since it can get lost on some changes. E.g. // saving a file with vim results in a RENAME-MODIFY-DELETE event // sequence, after which the newly written file is no longer watched. err = watcher.WatchFlags(fileName, fsnotify.FSN_MODIFY) case err := <-watcher.Error: log.Println("Error watching config:", err) } } }
func main() { flag.Parse() exporter := NewExporter(*nginxScrapeURI) prometheus.MustRegister(exporter) log.Printf("Starting Server: %s", *listeningAddress) http.Handle(*metricsEndpoint, prometheus.Handler()) log.Fatal(http.ListenAndServe(*listeningAddress, nil)) }
func (e *Exporter) Collect(ch chan<- prometheus.Metric) { e.mutex.Lock() // To protect metrics from concurrent collects. defer e.mutex.Unlock() if err := e.collect(ch); err != nil { log.Printf("Error scraping apache: %s", err) e.scrapeFailures.Inc() e.scrapeFailures.Collect(ch) } return }
// Collect fetches the stats of a user and delivers them // as Prometheus metrics. It implements prometheus.Collector. func (e *Exporter) Collect(ch chan<- prometheus.Metric) { e.mutex.Lock() // To protect metrics from concurrent collects. defer e.mutex.Unlock() if err := e.scrape(ch); err != nil { log.Printf("Error scraping cron: %s", err) } e.totalCronJobs.Collect(ch) e.cronRunningStatus.Collect(ch) return }
func (dms *DiskMetricStore) loop(persistenceInterval time.Duration) { lastPersist := time.Now() persistScheduled := false lastWrite := time.Time{} persistDone := make(chan time.Time) var persistTimer *time.Timer checkPersist := func() { if !persistScheduled && lastWrite.After(lastPersist) { persistTimer = time.AfterFunc( persistenceInterval-lastWrite.Sub(lastPersist), func() { persistStarted := time.Now() if err := dms.persist(); err != nil { log.Print("Error persisting metrics: ", err) } else { log.Printf( "Metrics persisted to '%s'.", dms.persistenceFile, ) } persistDone <- persistStarted }, ) persistScheduled = true } } for { select { case wr := <-dms.writeQueue: dms.processWriteRequest(wr) lastWrite = time.Now() checkPersist() case lastPersist = <-persistDone: persistScheduled = false checkPersist() // In case something has been written in the meantime. case <-dms.drain: // Prevent a scheduled persist from firing later. if persistTimer != nil { persistTimer.Stop() } // Now draining... for { select { case wr := <-dms.writeQueue: dms.processWriteRequest(wr) default: dms.done <- dms.persist() return } } } } }
// Collect fetches the stats of a user and delivers them // as Prometheus metrics. It implements prometheus.Collector. func (e *Exporter) Collect(ch chan<- prometheus.Metric) { e.mutex.Lock() // To protect metrics from concurrent collects. defer e.mutex.Unlock() if err := e.scrape(ch); err != nil { log.Printf("Error scraping postfix: %s", err) } e.totalQ.Collect(ch) e.incomingQ.Collect(ch) e.activeQ.Collect(ch) e.maildropQ.Collect(ch) e.deferredQ.Collect(ch) e.holdQ.Collect(ch) e.bounceQ.Collect(ch) return }
func (c *jenkinsCollector) fetchJenkinsMetrics() { ticker := time.NewTicker(jenkinsPollInterval).C auth := &gojenkins.Auth{ ApiToken: *jenkinsToken, Username: *jenkinsUser, } // Jenkins client object jenkins := gojenkins.NewJenkins(auth, *jenkinsUrl) for { select { case <-ticker: // TODO: // - Fetch Jenkins Jobs list --> jenkins.GetJobs() // - Foreach Job 'job' : // - Fetch last 5 builds --> job.LastCompletetedBuild.Number - 5 ... job.LastCompletetedBuild.Number // - Foreach Build 'build' : // - Create prometheus.Metrics from build.duration etc // with prometheus.NewConstMetric(desc, valueType, value) // - Create metricsStoreEntry from current timestamp & metrics // - Append metricsStoreEntry to metricsStore jobs, err := jenkins.GetJobs() log.Printf("Jobs: %v, jobs: %v", jobs, err) // Garbage collect expired value lists. now := time.Now() c.mu.Lock() // TODO: Clean up entries older than a certain timestamp // Maybe change metricStore to a queue data structure to easily find out how far to cleanup log.Printf("GC at %v", now) c.mu.Unlock() } } }
func main() { var ( listenAddress = flag.String("web.listen-address", ":9101", "Address to listen on for web interface and telemetry.") metricsPath = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.") haProxyScrapeUri = flag.String("haproxy.scrape-uri", "http://localhost/;csv", "URI on which to scrape HAProxy.") haProxyServerMetricFields = flag.String("haproxy.server-metric-fields", "", "If specified, only export the given csv fields. Comma-seperated list of numbers. See http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#9.1") haProxyTimeout = flag.Duration("haproxy.timeout", 5*time.Second, "Timeout for trying to get stats from HAProxy.") haProxyPidFile = flag.String("haproxy.pid-file", "", "Path to haproxy's pid file.") ) flag.Parse() exporter := NewExporter(*haProxyScrapeUri, *haProxyServerMetricFields, *haProxyTimeout) prometheus.MustRegister(exporter) if *haProxyPidFile != "" { procExporter := prometheus.NewProcessCollectorPIDFn( func() (int, error) { content, err := ioutil.ReadFile(*haProxyPidFile) if err != nil { return 0, fmt.Errorf("error reading pid file: %s", err) } value, err := strconv.Atoi(strings.TrimSpace(string(content))) if err != nil { return 0, fmt.Errorf("error parsing pid file: %s", err) } return value, nil }, namespace) prometheus.MustRegister(procExporter) } log.Printf("Starting Server: %s", *listenAddress) http.Handle(*metricsPath, prometheus.Handler()) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(`<html> <head><title>Haproxy Exporter</title></head> <body> <h1>Haproxy Exporter</h1> <p><a href='` + *metricsPath + `'>Metrics</a></p> </body> </html>`)) }) log.Fatal(http.ListenAndServe(*listenAddress, nil)) }
// GetMetricFamilies implements the MetricStore interface. func (dms *DiskMetricStore) GetMetricFamilies() []*dto.MetricFamily { result := []*dto.MetricFamily{} mfStatByName := map[string]mfStat{} dms.lock.RLock() defer dms.lock.RUnlock() for _, group := range dms.metricGroups { for name, tmf := range group.Metrics { mf := tmf.MetricFamily stat, exists := mfStatByName[name] if exists { existingMF := result[stat.pos] if !stat.copied { mfStatByName[name] = mfStat{ pos: stat.pos, copied: true, } existingMF = copyMetricFamily(existingMF) result[stat.pos] = existingMF } if mf.GetHelp() != existingMF.GetHelp() || mf.GetType() != existingMF.GetType() { log.Printf( "Metric families '%s' and '%s' are inconsistent, help and type of the latter will have priority. This is bad. Fix your pushed metrics!", mf, existingMF, ) } for _, metric := range mf.Metric { existingMF.Metric = append(existingMF.Metric, metric) } } else { mfStatByName[name] = mfStat{ pos: len(result), copied: false, } result = append(result, mf) } } } return result }
func main() { var server, apikey, listenAddress, metricPath string var period int flag.StringVar(&apikey, "api.key", "", "NewRelic API key") flag.StringVar(&server, "api.server", "https://api.newrelic.com", "NewRelic API URL") flag.IntVar(&period, "api.period", 60, "Period of data to extract in seconds") flag.StringVar(&listenAddress, "web.listen-address", ":9126", "Address to listen on for web interface and telemetry.") flag.StringVar(&metricPath, "web.telemetry-path", "/metrics", "Path under which to expose metrics.") flag.Parse() api := NewNewRelicApi(server, apikey) api.period = period exporter := NewExporter() exporter.api = *api prometheus.MustRegister(exporter) http.Handle(metricPath, prometheus.Handler()) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(`<html> <head><title>NewRelic exporter</title></head> <body> <h1>NewRelic exporter</h1> <p><a href='` + metricPath + `'>Metrics</a></p> </body> </html> `)) }) log.Printf("Listening on %s.", listenAddress) err := http.ListenAndServe(listenAddress, nil) if err != nil { log.Fatal(err) } log.Print("HTTP server stopped.") }
func (e *Exporter) exportCsvFields(metrics map[int]*prometheus.GaugeVec, csvRow []string, labels ...string) { for fieldIdx, metric := range metrics { valueStr := csvRow[fieldIdx] if valueStr == "" { continue } var value int64 switch fieldIdx { case statusField: value = parseStatusField(valueStr) default: var err error value, err = strconv.ParseInt(valueStr, 10, 64) if err != nil { log.Printf("Error while parsing CSV field value %s: %v", valueStr, err) e.csvParseFailures.Inc() continue } } metric.WithLabelValues(labels...).Set(float64(value)) } }
func (e *periodicExporter) updateSlaves() { log.Debug("discovering slaves...") // This will redirect us to the elected mesos master redirectURL := fmt.Sprintf("%s/master/redirect", e.opts.masterURL) rReq, err := http.NewRequest("GET", redirectURL, nil) if err != nil { panic(err) } tr := http.Transport{ DisableKeepAlives: true, } rresp, err := tr.RoundTrip(rReq) if err != nil { log.Warn(err) return } defer rresp.Body.Close() // This will/should return http://master.ip:5050 masterLoc := rresp.Header.Get("Location") if masterLoc == "" { log.Warnf("%d response missing Location header", rresp.StatusCode) return } log.Debugf("current elected master at: %s", masterLoc) // Find all active slaves stateURL := fmt.Sprintf("%s/master/state.json", masterLoc) resp, err := http.Get(stateURL) if err != nil { log.Warn(err) return } defer resp.Body.Close() type slave struct { Active bool `json:"active"` Hostname string `json:"hostname"` Pid string `json:"pid"` } var req struct { Slaves []*slave `json:"slaves"` } if err := json.NewDecoder(resp.Body).Decode(&req); err != nil { log.Warnf("failed to deserialize request: %s", err) return } var slaveURLs []string for _, slave := range req.Slaves { if slave.Active { var host, port string pid, err := upid.Parse(slave.Pid) if err != nil { log.Printf("failed to parse slave pid %q: %s", slave.Pid, err) host, port = slave.Hostname, "5051" } else { host, port = pid.Host, pid.Port } slaveURLs = append(slaveURLs, fmt.Sprintf("http://%s:%s", host, port)) } } log.Debugf("%d slaves discovered", len(slaveURLs)) e.slaves.Lock() e.slaves.urls = slaveURLs e.slaves.Unlock() }
func main() { flag.Parse() versionInfoTmpl.Execute(os.Stdout, BuildInfo) flags := map[string]string{} flag.VisitAll(func(f *flag.Flag) { flags[f.Name] = f.Value.String() }) ms := storage.NewDiskMetricStore(*persistenceFile, *persistenceInterval) prometheus.SetMetricFamilyInjectionHook(ms.GetMetricFamilies) // Enable collect checks for debugging. // prometheus.EnableCollectChecks(true) r := httprouter.New() r.Handler("GET", *metricsPath, prometheus.Handler()) // Handlers for pushing and deleting metrics. r.PUT("/metrics/job/:job/*labels", handler.Push(ms, true)) r.POST("/metrics/job/:job/*labels", handler.Push(ms, false)) r.DELETE("/metrics/job/:job/*labels", handler.Delete(ms)) r.PUT("/metrics/job/:job", handler.Push(ms, true)) r.POST("/metrics/job/:job", handler.Push(ms, false)) r.DELETE("/metrics/job/:job", handler.Delete(ms)) // Handlers for the deprecated API. r.PUT("/metrics/jobs/:job/instances/:instance", handler.LegacyPush(ms, true)) r.POST("/metrics/jobs/:job/instances/:instance", handler.LegacyPush(ms, false)) r.DELETE("/metrics/jobs/:job/instances/:instance", handler.LegacyDelete(ms)) r.PUT("/metrics/jobs/:job", handler.LegacyPush(ms, true)) r.POST("/metrics/jobs/:job", handler.LegacyPush(ms, false)) r.DELETE("/metrics/jobs/:job", handler.LegacyDelete(ms)) r.Handler("GET", "/static/*filepath", prometheus.InstrumentHandler( "static", http.FileServer( &assetfs.AssetFS{Asset: Asset, AssetDir: AssetDir}, ), )) statusHandler := prometheus.InstrumentHandlerFunc("status", handler.Status(ms, Asset, flags, BuildInfo)) r.Handler("GET", "/status", statusHandler) r.Handler("GET", "/", statusHandler) // Re-enable pprof. r.GET("/debug/pprof/*pprof", handlePprof) log.Printf("Listening on %s.", *listenAddress) l, err := net.Listen("tcp", *listenAddress) if err != nil { log.Fatal(err) } go interruptHandler(l) err = (&http.Server{Addr: *listenAddress, Handler: r}).Serve(l) log.Print("HTTP server stopped: ", err) // To give running connections a chance to submit their payload, we wait // for 1sec, but we don't want to wait long (e.g. until all connections // are done) to not delay the shutdown. time.Sleep(time.Second) if err := ms.Shutdown(); err != nil { log.Print("Problem shutting down metric storage: ", err) } }
func (l *StatsDListener) handlePacket(packet []byte, e chan<- Events) { lines := strings.Split(string(packet), "\n") events := Events{} for _, line := range lines { if line == "" { continue } elements := strings.Split(line, ":") if len(elements) < 2 { networkStats.WithLabelValues("malformed_line").Inc() log.Println("Bad line from StatsD:", line) continue } metric := elements[0] samples := elements[1:] for _, sample := range samples { components := strings.Split(sample, "|") samplingFactor := 1.0 if len(components) < 2 || len(components) > 3 { networkStats.WithLabelValues("malformed_component").Inc() log.Println("Bad component on line:", line) continue } valueStr, statType := components[0], components[1] value, err := strconv.ParseFloat(valueStr, 64) if err != nil { log.Printf("Bad value %s on line: %s", valueStr, line) networkStats.WithLabelValues("malformed_value").Inc() continue } if len(components) == 3 { if statType != "c" { log.Println("Illegal sampling factor for non-counter metric on line", line) networkStats.WithLabelValues("illegal_sample_factor").Inc() } samplingStr := components[2] if samplingStr[0] != '@' { log.Printf("Invalid sampling factor %s on line %s", samplingStr, line) networkStats.WithLabelValues("invalid_sample_factor").Inc() continue } samplingFactor, err = strconv.ParseFloat(samplingStr[1:], 64) if err != nil { log.Printf("Invalid sampling factor %s on line %s", samplingStr, line) networkStats.WithLabelValues("invalid_sample_factor").Inc() continue } if samplingFactor == 0 { // This should never happen, but avoid division by zero if it does. log.Printf("Invalid zero sampling factor %s on line %s, setting to 1", samplingStr, line) samplingFactor = 1 } value /= samplingFactor } event, err := buildEvent(statType, metric, value) if err != nil { log.Printf("Error building event on line %s: %s", line, err) networkStats.WithLabelValues("illegal_event").Inc() continue } events = append(events, event) networkStats.WithLabelValues("legal").Inc() } } e <- events }