func (e *Exporter) scrape(csvRows chan<- []string) {
defer close(csvRows)
e.totalScrapes.Inc()
resp, err := e.client.Get(e.URI)
if err != nil {
e.up.Set(0)
log.Printf("Error while scraping HAProxy: %v", err)
return
}
defer resp.Body.Close()
e.up.Set(1)
reader := csv.NewReader(resp.Body)
reader.TrailingComma = true
reader.Comment = '#'
for {
row, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
log.Printf("Error while reading CSV: %v", err)
e.csvParseFailures.Inc()
break
}
if len(row) == 0 {
continue
}
csvRows <- row
}
}
// Collect implements prometheus.Collector.
func (c collectdCollector) Collect(ch chan<- prometheus.Metric) {
ch <- lastPush
c.mu.Lock()
valueLists := make([]api.ValueList, 0, len(c.valueLists))
for _, vl := range c.valueLists {
valueLists = append(valueLists, vl)
}
c.mu.Unlock()
now := time.Now()
for _, vl := range valueLists {
validUntil := vl.Time.Add(timeout * vl.Interval)
if validUntil.Before(now) {
continue
}
for i := range vl.Values {
m, err := newMetric(vl, i)
if err != nil {
log.Printf("newMetric: %v", err)
continue
}
ch <- m
}
}
}
// Collect fetches the stats from configured nginx location and delivers them
// as Prometheus metrics. It implements prometheus.Collector.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
e.mutex.Lock() // To protect metrics from concurrent collects.
defer e.mutex.Unlock()
if err := e.scrape(ch); err != nil {
log.Printf("Error scraping apache2: %s", err)
e.scrapeFailures.Inc()
e.scrapeFailures.Collect(ch)
}
e.version.Collect(ch)
e.totalRequests.Collect(ch)
e.uptimeDays.Collect(ch)
e.uptimeHour.Collect(ch)
e.totalAccesses.Collect(ch)
e.cpuUsageUser.Collect(ch)
e.cpuUsageSystem.Collect(ch)
e.cpuLoad.Collect(ch)
e.totalTraffic.Collect(ch)
e.requestBeingProcessed.Collect(ch)
e.idleWorkers.Collect(ch)
e.requestsPerSecond.Collect(ch)
e.dataPerSecond.Collect(ch)
e.dataPerRequest.Collect(ch)
e.clientRequests.Collect(ch)
e.virtualHosts.Collect(ch)
return
}
func (e *Exporter) exportCsvFields(metrics map[int]*prometheus.GaugeVec, csvRow []string, labels ...string) {
for fieldIdx, metric := range metrics {
valueStr := csvRow[fieldIdx]
if valueStr == "" {
continue
}
var value int64
var err error
switch valueStr {
// UP or UP going down
case "UP", "UP 1/3", "UP 2/3":
value = 1
// DOWN or DOWN going up
case "DOWN", "DOWN 1/2":
value = 0
case "OPEN":
value = 0
case "no check":
continue
default:
value, err = strconv.ParseInt(valueStr, 10, 64)
if err != nil {
log.Printf("Error while parsing CSV field value %s: %v", valueStr, err)
e.csvParseFailures.Inc()
continue
}
}
metric.WithLabelValues(labels...).Set(float64(value))
}
}
func (e *Exporter) setMetrics(csvRows <-chan []string) {
for csvRow := range csvRows {
if len(csvRow) < expectedCsvFieldCount {
log.Printf("Wrong CSV field count: %d vs. %d", len(csvRow), expectedCsvFieldCount)
e.csvParseFailures.Inc()
continue
}
pxname, svname, type_ := csvRow[0], csvRow[1], csvRow[32]
const (
frontend = "0"
backend = "1"
server = "2"
listener = "3"
)
switch type_ {
case frontend:
e.exportCsvFields(e.frontendMetrics, csvRow, pxname)
case backend:
e.exportCsvFields(e.backendMetrics, csvRow, pxname)
case server:
e.exportCsvFields(e.serverMetrics, csvRow, pxname, svname)
}
}
}
func watchConfig(fileName string, mapper *metricMapper) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Fatal(err)
}
err = watcher.WatchFlags(fileName, fsnotify.FSN_MODIFY)
if err != nil {
log.Fatal(err)
}
for {
select {
case ev := <-watcher.Event:
log.Printf("Config file changed (%s), attempting reload", ev)
err = mapper.initFromFile(fileName)
if err != nil {
log.Println("Error reloading config:", err)
configLoads.WithLabelValues("failure").Inc()
} else {
log.Println("Config reloaded successfully")
configLoads.WithLabelValues("success").Inc()
}
// Re-add the file watcher since it can get lost on some changes. E.g.
// saving a file with vim results in a RENAME-MODIFY-DELETE event
// sequence, after which the newly written file is no longer watched.
err = watcher.WatchFlags(fileName, fsnotify.FSN_MODIFY)
case err := <-watcher.Error:
log.Println("Error watching config:", err)
}
}
}
func main() {
flag.Parse()
exporter := NewExporter(*nginxScrapeURI)
prometheus.MustRegister(exporter)
log.Printf("Starting Server: %s", *listeningAddress)
http.Handle(*metricsEndpoint, prometheus.Handler())
log.Fatal(http.ListenAndServe(*listeningAddress, nil))
}
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
e.mutex.Lock() // To protect metrics from concurrent collects.
defer e.mutex.Unlock()
if err := e.collect(ch); err != nil {
log.Printf("Error scraping apache: %s", err)
e.scrapeFailures.Inc()
e.scrapeFailures.Collect(ch)
}
return
}
// Collect fetches the stats of a user and delivers them
// as Prometheus metrics. It implements prometheus.Collector.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
e.mutex.Lock() // To protect metrics from concurrent collects.
defer e.mutex.Unlock()
if err := e.scrape(ch); err != nil {
log.Printf("Error scraping cron: %s", err)
}
e.totalCronJobs.Collect(ch)
e.cronRunningStatus.Collect(ch)
return
}
func (dms *DiskMetricStore) loop(persistenceInterval time.Duration) {
lastPersist := time.Now()
persistScheduled := false
lastWrite := time.Time{}
persistDone := make(chan time.Time)
var persistTimer *time.Timer
checkPersist := func() {
if !persistScheduled && lastWrite.After(lastPersist) {
persistTimer = time.AfterFunc(
persistenceInterval-lastWrite.Sub(lastPersist),
func() {
persistStarted := time.Now()
if err := dms.persist(); err != nil {
log.Print("Error persisting metrics: ", err)
} else {
log.Printf(
"Metrics persisted to '%s'.",
dms.persistenceFile,
)
}
persistDone <- persistStarted
},
)
persistScheduled = true
}
}
for {
select {
case wr := <-dms.writeQueue:
dms.processWriteRequest(wr)
lastWrite = time.Now()
checkPersist()
case lastPersist = <-persistDone:
persistScheduled = false
checkPersist() // In case something has been written in the meantime.
case <-dms.drain:
// Prevent a scheduled persist from firing later.
if persistTimer != nil {
persistTimer.Stop()
}
// Now draining...
for {
select {
case wr := <-dms.writeQueue:
dms.processWriteRequest(wr)
default:
dms.done <- dms.persist()
return
}
}
}
}
}
// Collect fetches the stats of a user and delivers them
// as Prometheus metrics. It implements prometheus.Collector.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
e.mutex.Lock() // To protect metrics from concurrent collects.
defer e.mutex.Unlock()
if err := e.scrape(ch); err != nil {
log.Printf("Error scraping postfix: %s", err)
}
e.totalQ.Collect(ch)
e.incomingQ.Collect(ch)
e.activeQ.Collect(ch)
e.maildropQ.Collect(ch)
e.deferredQ.Collect(ch)
e.holdQ.Collect(ch)
e.bounceQ.Collect(ch)
return
}
func (c *jenkinsCollector) fetchJenkinsMetrics() {
ticker := time.NewTicker(jenkinsPollInterval).C
auth := &gojenkins.Auth{
ApiToken: *jenkinsToken,
Username: *jenkinsUser,
}
// Jenkins client object
jenkins := gojenkins.NewJenkins(auth, *jenkinsUrl)
for {
select {
case <-ticker:
// TODO:
// - Fetch Jenkins Jobs list --> jenkins.GetJobs()
// - Foreach Job 'job' :
// - Fetch last 5 builds --> job.LastCompletetedBuild.Number - 5 ... job.LastCompletetedBuild.Number
// - Foreach Build 'build' :
// - Create prometheus.Metrics from build.duration etc
// with prometheus.NewConstMetric(desc, valueType, value)
// - Create metricsStoreEntry from current timestamp & metrics
// - Append metricsStoreEntry to metricsStore
jobs, err := jenkins.GetJobs()
log.Printf("Jobs: %v, jobs: %v", jobs, err)
// Garbage collect expired value lists.
now := time.Now()
c.mu.Lock()
// TODO: Clean up entries older than a certain timestamp
// Maybe change metricStore to a queue data structure to easily find out how far to cleanup
log.Printf("GC at %v", now)
c.mu.Unlock()
}
}
}
func main() {
var (
listenAddress = flag.String("web.listen-address", ":9101", "Address to listen on for web interface and telemetry.")
metricsPath = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
haProxyScrapeUri = flag.String("haproxy.scrape-uri", "http://localhost/;csv", "URI on which to scrape HAProxy.")
haProxyServerMetricFields = flag.String("haproxy.server-metric-fields", "", "If specified, only export the given csv fields. Comma-seperated list of numbers. See http://cbonte.github.io/haproxy-dconv/configuration-1.5.html#9.1")
haProxyTimeout = flag.Duration("haproxy.timeout", 5*time.Second, "Timeout for trying to get stats from HAProxy.")
haProxyPidFile = flag.String("haproxy.pid-file", "", "Path to haproxy's pid file.")
)
flag.Parse()
exporter := NewExporter(*haProxyScrapeUri, *haProxyServerMetricFields, *haProxyTimeout)
prometheus.MustRegister(exporter)
if *haProxyPidFile != "" {
procExporter := prometheus.NewProcessCollectorPIDFn(
func() (int, error) {
content, err := ioutil.ReadFile(*haProxyPidFile)
if err != nil {
return 0, fmt.Errorf("error reading pid file: %s", err)
}
value, err := strconv.Atoi(strings.TrimSpace(string(content)))
if err != nil {
return 0, fmt.Errorf("error parsing pid file: %s", err)
}
return value, nil
}, namespace)
prometheus.MustRegister(procExporter)
}
log.Printf("Starting Server: %s", *listenAddress)
http.Handle(*metricsPath, prometheus.Handler())
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`<html>
<head><title>Haproxy Exporter</title></head>
<body>
<h1>Haproxy Exporter</h1>
<p><a href='` + *metricsPath + `'>Metrics</a></p>
</body>
</html>`))
})
log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
// GetMetricFamilies implements the MetricStore interface.
func (dms *DiskMetricStore) GetMetricFamilies() []*dto.MetricFamily {
result := []*dto.MetricFamily{}
mfStatByName := map[string]mfStat{}
dms.lock.RLock()
defer dms.lock.RUnlock()
for _, group := range dms.metricGroups {
for name, tmf := range group.Metrics {
mf := tmf.MetricFamily
stat, exists := mfStatByName[name]
if exists {
existingMF := result[stat.pos]
if !stat.copied {
mfStatByName[name] = mfStat{
pos: stat.pos,
copied: true,
}
existingMF = copyMetricFamily(existingMF)
result[stat.pos] = existingMF
}
if mf.GetHelp() != existingMF.GetHelp() || mf.GetType() != existingMF.GetType() {
log.Printf(
"Metric families '%s' and '%s' are inconsistent, help and type of the latter will have priority. This is bad. Fix your pushed metrics!",
mf, existingMF,
)
}
for _, metric := range mf.Metric {
existingMF.Metric = append(existingMF.Metric, metric)
}
} else {
mfStatByName[name] = mfStat{
pos: len(result),
copied: false,
}
result = append(result, mf)
}
}
}
return result
}
func main() {
var server, apikey, listenAddress, metricPath string
var period int
flag.StringVar(&apikey, "api.key", "", "NewRelic API key")
flag.StringVar(&server, "api.server", "https://api.newrelic.com", "NewRelic API URL")
flag.IntVar(&period, "api.period", 60, "Period of data to extract in seconds")
flag.StringVar(&listenAddress, "web.listen-address", ":9126", "Address to listen on for web interface and telemetry.")
flag.StringVar(&metricPath, "web.telemetry-path", "/metrics", "Path under which to expose metrics.")
flag.Parse()
api := NewNewRelicApi(server, apikey)
api.period = period
exporter := NewExporter()
exporter.api = *api
prometheus.MustRegister(exporter)
http.Handle(metricPath, prometheus.Handler())
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`<html>
<head><title>NewRelic exporter</title></head>
<body>
<h1>NewRelic exporter</h1>
<p><a href='` + metricPath + `'>Metrics</a></p>
</body>
</html>
`))
})
log.Printf("Listening on %s.", listenAddress)
err := http.ListenAndServe(listenAddress, nil)
if err != nil {
log.Fatal(err)
}
log.Print("HTTP server stopped.")
}
func (e *Exporter) exportCsvFields(metrics map[int]*prometheus.GaugeVec, csvRow []string, labels ...string) {
for fieldIdx, metric := range metrics {
valueStr := csvRow[fieldIdx]
if valueStr == "" {
continue
}
var value int64
switch fieldIdx {
case statusField:
value = parseStatusField(valueStr)
default:
var err error
value, err = strconv.ParseInt(valueStr, 10, 64)
if err != nil {
log.Printf("Error while parsing CSV field value %s: %v", valueStr, err)
e.csvParseFailures.Inc()
continue
}
}
metric.WithLabelValues(labels...).Set(float64(value))
}
}
func (e *periodicExporter) updateSlaves() {
log.Debug("discovering slaves...")
// This will redirect us to the elected mesos master
redirectURL := fmt.Sprintf("%s/master/redirect", e.opts.masterURL)
rReq, err := http.NewRequest("GET", redirectURL, nil)
if err != nil {
panic(err)
}
tr := http.Transport{
DisableKeepAlives: true,
}
rresp, err := tr.RoundTrip(rReq)
if err != nil {
log.Warn(err)
return
}
defer rresp.Body.Close()
// This will/should return http://master.ip:5050
masterLoc := rresp.Header.Get("Location")
if masterLoc == "" {
log.Warnf("%d response missing Location header", rresp.StatusCode)
return
}
log.Debugf("current elected master at: %s", masterLoc)
// Find all active slaves
stateURL := fmt.Sprintf("%s/master/state.json", masterLoc)
resp, err := http.Get(stateURL)
if err != nil {
log.Warn(err)
return
}
defer resp.Body.Close()
type slave struct {
Active bool `json:"active"`
Hostname string `json:"hostname"`
Pid string `json:"pid"`
}
var req struct {
Slaves []*slave `json:"slaves"`
}
if err := json.NewDecoder(resp.Body).Decode(&req); err != nil {
log.Warnf("failed to deserialize request: %s", err)
return
}
var slaveURLs []string
for _, slave := range req.Slaves {
if slave.Active {
var host, port string
pid, err := upid.Parse(slave.Pid)
if err != nil {
log.Printf("failed to parse slave pid %q: %s", slave.Pid, err)
host, port = slave.Hostname, "5051"
} else {
host, port = pid.Host, pid.Port
}
slaveURLs = append(slaveURLs, fmt.Sprintf("http://%s:%s", host, port))
}
}
log.Debugf("%d slaves discovered", len(slaveURLs))
e.slaves.Lock()
e.slaves.urls = slaveURLs
e.slaves.Unlock()
}
func main() {
flag.Parse()
versionInfoTmpl.Execute(os.Stdout, BuildInfo)
flags := map[string]string{}
flag.VisitAll(func(f *flag.Flag) {
flags[f.Name] = f.Value.String()
})
ms := storage.NewDiskMetricStore(*persistenceFile, *persistenceInterval)
prometheus.SetMetricFamilyInjectionHook(ms.GetMetricFamilies)
// Enable collect checks for debugging.
// prometheus.EnableCollectChecks(true)
r := httprouter.New()
r.Handler("GET", *metricsPath, prometheus.Handler())
// Handlers for pushing and deleting metrics.
r.PUT("/metrics/job/:job/*labels", handler.Push(ms, true))
r.POST("/metrics/job/:job/*labels", handler.Push(ms, false))
r.DELETE("/metrics/job/:job/*labels", handler.Delete(ms))
r.PUT("/metrics/job/:job", handler.Push(ms, true))
r.POST("/metrics/job/:job", handler.Push(ms, false))
r.DELETE("/metrics/job/:job", handler.Delete(ms))
// Handlers for the deprecated API.
r.PUT("/metrics/jobs/:job/instances/:instance", handler.LegacyPush(ms, true))
r.POST("/metrics/jobs/:job/instances/:instance", handler.LegacyPush(ms, false))
r.DELETE("/metrics/jobs/:job/instances/:instance", handler.LegacyDelete(ms))
r.PUT("/metrics/jobs/:job", handler.LegacyPush(ms, true))
r.POST("/metrics/jobs/:job", handler.LegacyPush(ms, false))
r.DELETE("/metrics/jobs/:job", handler.LegacyDelete(ms))
r.Handler("GET", "/static/*filepath", prometheus.InstrumentHandler(
"static",
http.FileServer(
&assetfs.AssetFS{Asset: Asset, AssetDir: AssetDir},
),
))
statusHandler := prometheus.InstrumentHandlerFunc("status", handler.Status(ms, Asset, flags, BuildInfo))
r.Handler("GET", "/status", statusHandler)
r.Handler("GET", "/", statusHandler)
// Re-enable pprof.
r.GET("/debug/pprof/*pprof", handlePprof)
log.Printf("Listening on %s.", *listenAddress)
l, err := net.Listen("tcp", *listenAddress)
if err != nil {
log.Fatal(err)
}
go interruptHandler(l)
err = (&http.Server{Addr: *listenAddress, Handler: r}).Serve(l)
log.Print("HTTP server stopped: ", err)
// To give running connections a chance to submit their payload, we wait
// for 1sec, but we don't want to wait long (e.g. until all connections
// are done) to not delay the shutdown.
time.Sleep(time.Second)
if err := ms.Shutdown(); err != nil {
log.Print("Problem shutting down metric storage: ", err)
}
}
func (l *StatsDListener) handlePacket(packet []byte, e chan<- Events) {
lines := strings.Split(string(packet), "\n")
events := Events{}
for _, line := range lines {
if line == "" {
continue
}
elements := strings.Split(line, ":")
if len(elements) < 2 {
networkStats.WithLabelValues("malformed_line").Inc()
log.Println("Bad line from StatsD:", line)
continue
}
metric := elements[0]
samples := elements[1:]
for _, sample := range samples {
components := strings.Split(sample, "|")
samplingFactor := 1.0
if len(components) < 2 || len(components) > 3 {
networkStats.WithLabelValues("malformed_component").Inc()
log.Println("Bad component on line:", line)
continue
}
valueStr, statType := components[0], components[1]
value, err := strconv.ParseFloat(valueStr, 64)
if err != nil {
log.Printf("Bad value %s on line: %s", valueStr, line)
networkStats.WithLabelValues("malformed_value").Inc()
continue
}
if len(components) == 3 {
if statType != "c" {
log.Println("Illegal sampling factor for non-counter metric on line", line)
networkStats.WithLabelValues("illegal_sample_factor").Inc()
}
samplingStr := components[2]
if samplingStr[0] != '@' {
log.Printf("Invalid sampling factor %s on line %s", samplingStr, line)
networkStats.WithLabelValues("invalid_sample_factor").Inc()
continue
}
samplingFactor, err = strconv.ParseFloat(samplingStr[1:], 64)
if err != nil {
log.Printf("Invalid sampling factor %s on line %s", samplingStr, line)
networkStats.WithLabelValues("invalid_sample_factor").Inc()
continue
}
if samplingFactor == 0 {
// This should never happen, but avoid division by zero if it does.
log.Printf("Invalid zero sampling factor %s on line %s, setting to 1", samplingStr, line)
samplingFactor = 1
}
value /= samplingFactor
}
event, err := buildEvent(statType, metric, value)
if err != nil {
log.Printf("Error building event on line %s: %s", line, err)
networkStats.WithLabelValues("illegal_event").Inc()
continue
}
events = append(events, event)
networkStats.WithLabelValues("legal").Inc()
}
}
e <- events
}