func watchConfig(fileName string, mapper *metricMapper) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Fatal(err)
}
err = watcher.WatchFlags(fileName, fsnotify.FSN_MODIFY)
if err != nil {
log.Fatal(err)
}
for {
select {
case ev := <-watcher.Event:
log.Printf("Config file changed (%s), attempting reload", ev)
err = mapper.initFromFile(fileName)
if err != nil {
log.Println("Error reloading config:", err)
configLoads.WithLabelValues("failure").Inc()
} else {
log.Println("Config reloaded successfully")
configLoads.WithLabelValues("success").Inc()
}
// Re-add the file watcher since it can get lost on some changes. E.g.
// saving a file with vim results in a RENAME-MODIFY-DELETE event
// sequence, after which the newly written file is no longer watched.
err = watcher.WatchFlags(fileName, fsnotify.FSN_MODIFY)
case err := <-watcher.Error:
log.Println("Error watching config:", err)
}
}
}
func main() {
flag.Parse()
log.Println("Starting StatsD -> Prometheus Bridge...")
log.Println("Accepting StatsD Traffic on", *statsdListenAddress)
log.Println("Accepting Prometheus Requests on", *listenAddress)
go serveHTTP()
events := make(chan Events, 1024)
defer close(events)
listenAddr := udpAddrFromString(*statsdListenAddress)
conn, err := net.ListenUDP("udp", listenAddr)
if err != nil {
log.Fatal(err)
}
l := &StatsDListener{conn: conn}
go l.Listen(events)
mapper := &metricMapper{}
if *mappingConfig != "" {
err := mapper.initFromFile(*mappingConfig)
if err != nil {
log.Fatal("Error loading config:", err)
}
go watchConfig(*mappingConfig, mapper)
}
bridge := NewBridge(mapper)
bridge.Listen(events)
}
func (b *Bridge) Listen(e <-chan Events) {
for {
events := <-e
for _, event := range events {
metricName := ""
prometheusLabels := prometheus.Labels{}
labels, present := b.mapper.getMapping(event.MetricName())
if present {
metricName = labels["name"]
for label, value := range labels {
if label != "name" {
prometheusLabels[label] = value
}
}
} else {
metricName = escapeMetricName(event.MetricName())
}
switch event.(type) {
case *CounterEvent:
counter := b.Counters.Get(
metricName+"_counter",
prometheusLabels,
)
counter.Add(event.Value())
eventStats.WithLabelValues("counter").Inc()
case *GaugeEvent:
gauge := b.Gauges.Get(
metricName+"_gauge",
prometheusLabels,
)
gauge.Set(event.Value())
eventStats.WithLabelValues("gauge").Inc()
case *TimerEvent:
summary := b.Summaries.Get(
metricName+"_timer",
prometheusLabels,
)
summary.Observe(event.Value())
eventStats.WithLabelValues("timer").Inc()
default:
log.Println("Unsupported event type")
eventStats.WithLabelValues("illegal").Inc()
}
}
}
}
func (e *Exporter) scrape() {
now := time.Now().UnixNano()
e.totalScrapes.Inc()
for _, m := range e.metrics {
err := m.Scrape(db)
if err != nil {
log.Println(err)
e.errors.Set(1)
e.duration.Set(float64(time.Now().UnixNano()-now) / 1000000000)
return
}
}
e.errors.Set(0)
e.duration.Set(float64(time.Now().UnixNano()-now) / 1000000000)
}
func (e *Exporter) scrape(scrapes chan<- []string) {
defer close(scrapes)
now := time.Now().UnixNano()
e.totalScrapes.Inc()
db, err := sql.Open("mysql", e.dsn)
if err != nil {
log.Println("error opening connection to database:", err)
e.error.Set(1)
e.duration.Set(float64(time.Now().UnixNano()-now) / 1000000000)
return
}
defer db.Close()
// fetch database status
rows, err := db.Query("SHOW GLOBAL STATUS")
if err != nil {
log.Println("error running status query on database:", err)
e.error.Set(1)
e.duration.Set(float64(time.Now().UnixNano()-now) / 1000000000)
return
}
defer rows.Close()
var key, val []byte
for rows.Next() {
// get RawBytes from data
err = rows.Scan(&key, &val)
if err != nil {
log.Println("error getting result set:", err)
return
}
var res []string = make([]string, 2)
res[0] = string(key)
res[1] = string(val)
scrapes <- res
}
// fetch slave status
rows, err = db.Query("SHOW SLAVE STATUS")
if err != nil {
log.Println("error running show slave query on database:", err)
e.error.Set(1)
e.duration.Set(float64(time.Now().UnixNano()-now) / 1000000000)
return
}
defer rows.Close()
var slaveCols []string
slaveCols, err = rows.Columns()
if err != nil {
log.Println("error retrieving column list:", err)
e.error.Set(1)
e.duration.Set(float64(time.Now().UnixNano()-now) / 1000000000)
return
}
var slaveData = make([]sql.RawBytes, len(slaveCols))
// As the number of columns varies with mysqld versions,
// and sql.Scan requires []interface{}, we need to create a
// slice of pointers to the elements of slaveData.
scanArgs := make([]interface{}, len(slaveCols))
for i := range slaveData {
scanArgs[i] = &slaveData[i]
}
for rows.Next() {
err = rows.Scan(scanArgs...)
if err != nil {
log.Println("error retrieving result set:", err)
e.error.Set(1)
e.duration.Set(float64(time.Now().UnixNano()-now) / 1000000000)
return
}
}
for i, col := range slaveCols {
var res []string = make([]string, 2)
res[0] = col
res[1] = parseStatus(slaveData[i])
scrapes <- res
}
e.error.Set(0)
e.duration.Set(float64(time.Now().UnixNano()-now) / 1000000000)
}
func (l *StatsDListener) handlePacket(packet []byte, e chan<- Events) {
lines := strings.Split(string(packet), "\n")
events := Events{}
for _, line := range lines {
if line == "" {
continue
}
elements := strings.Split(line, ":")
if len(elements) < 2 {
networkStats.WithLabelValues("malformed_line").Inc()
log.Println("Bad line from StatsD:", line)
continue
}
metric := elements[0]
samples := elements[1:]
for _, sample := range samples {
components := strings.Split(sample, "|")
samplingFactor := 1.0
if len(components) < 2 || len(components) > 3 {
networkStats.WithLabelValues("malformed_component").Inc()
log.Println("Bad component on line:", line)
continue
}
valueStr, statType := components[0], components[1]
value, err := strconv.ParseFloat(valueStr, 64)
if err != nil {
log.Printf("Bad value %s on line: %s", valueStr, line)
networkStats.WithLabelValues("malformed_value").Inc()
continue
}
if len(components) == 3 {
if statType != "c" {
log.Println("Illegal sampling factor for non-counter metric on line", line)
networkStats.WithLabelValues("illegal_sample_factor").Inc()
}
samplingStr := components[2]
if samplingStr[0] != '@' {
log.Printf("Invalid sampling factor %s on line %s", samplingStr, line)
networkStats.WithLabelValues("invalid_sample_factor").Inc()
continue
}
samplingFactor, err = strconv.ParseFloat(samplingStr[1:], 64)
if err != nil {
log.Printf("Invalid sampling factor %s on line %s", samplingStr, line)
networkStats.WithLabelValues("invalid_sample_factor").Inc()
continue
}
if samplingFactor == 0 {
// This should never happen, but avoid division by zero if it does.
log.Printf("Invalid zero sampling factor %s on line %s, setting to 1", samplingStr, line)
samplingFactor = 1
}
value /= samplingFactor
}
event, err := buildEvent(statType, metric, value)
if err != nil {
log.Printf("Error building event on line %s: %s", line, err)
networkStats.WithLabelValues("illegal_event").Inc()
continue
}
events = append(events, event)
networkStats.WithLabelValues("legal").Inc()
}
}
e <- events
}
func (e *Exporter) scrape(ch chan<- prometheus.Metric) {
e.totalScrapes.Inc()
var err error
defer func(begun time.Time) {
e.duration.Set(time.Since(begun).Seconds())
if err == nil {
e.error.Set(0)
} else {
e.error.Set(1)
}
}(time.Now())
db, err := sql.Open("mysql", e.dsn)
if err != nil {
log.Println("Error opening connection to database:", err)
return
}
defer db.Close()
if err = scrapeGlobalStatus(db, ch); err != nil {
log.Println("Error scraping global state:", err)
return
}
if err = scrapeGlobalVariables(db, ch); err != nil {
log.Println("Error scraping global variables:", err)
return
}
if err = scrapeSlaveStatus(db, ch); err != nil {
log.Println("Error scraping slave state:", err)
return
}
if *autoIncrementColumns {
if err = scrapeInformationSchema(db, ch); err != nil {
log.Println("Error scraping information schema:", err)
return
}
}
if *binlogSize {
if err = scrapeBinlogSize(db, ch); err != nil {
log.Println("Error scraping binlog size:", err)
return
}
}
if *perfTableIOWaits {
if err = scrapePerfTableIOWaits(db, ch); err != nil {
log.Println("Error scraping performance schema:", err)
return
}
}
if *perfTableIOWaitsTime {
if err = scrapePerfTableIOWaitsTime(db, ch); err != nil {
log.Println("Error scraping performance schema:", err)
return
}
}
if *perfIndexIOWaits {
if err = scrapePerfIndexIOWaits(db, ch); err != nil {
log.Println("Error scraping performance schema:", err)
return
}
}
if *perfIndexIOWaitsTime {
if err = scrapePerfIndexIOWaitsTime(db, ch); err != nil {
log.Println("Error scraping performance schema:", err)
return
}
}
if *perfTableLockWaits {
if err = scrapePerfTableLockWaits(db, ch); err != nil {
log.Println("Error scraping performance schema:", err)
return
}
}
if *perfTableLockWaitsTime {
if err = scrapePerfTableLockWaitsTime(db, ch); err != nil {
log.Println("Error scraping performance schema:", err)
return
}
}
if *perfEventsStatements {
if err = scrapePerfEventsStatements(db, ch); err != nil {
log.Println("Error scraping performance schema:", err)
return
}
}
if *userStat {
if err = scrapeUserStat(db, ch); err != nil {
log.Println("Error scraping user stat:", err)
return
}
}
}
func (e *Exporter) scrape(ch chan<- prometheus.Metric) {
defer func(begun time.Time) {
e.duration.Set(time.Since(begun).Seconds())
}(time.Now())
e.error.Set(0)
e.totalScrapes.Inc()
db, err := sql.Open("postgres", e.dsn)
if err != nil {
log.Println("Error opening connection to database:", err)
e.error.Set(1)
return
}
defer db.Close()
for namespaceAndQuery, mapping := range e.metricMap {
namespace := namespaceAndQuery.namespace
log.Debugln("Querying namespace: ", namespace, " query: ", namespaceAndQuery.query)
func() { // Don't fail on a bad scrape of one metric
rows, err := db.Query(namespaceAndQuery.query)
if err != nil {
log.Println("Error running query on database: ", namespace, err)
e.error.Set(1)
return
}
defer rows.Close()
var columnNames []string
columnNames, err = rows.Columns()
if err != nil {
log.Println("Error retrieving column list for: ", namespace, err)
e.error.Set(1)
return
}
// Make a lookup map for the column indices
var columnIdx = make(map[string]int, len(columnNames))
for i, n := range columnNames {
columnIdx[n] = i
}
var columnData = make([]interface{}, len(columnNames))
var scanArgs = make([]interface{}, len(columnNames))
for i := range columnData {
scanArgs[i] = &columnData[i]
}
for rows.Next() {
err = rows.Scan(scanArgs...)
if err != nil {
log.Println("Error retrieving rows:", namespace, err)
e.error.Set(1)
return
}
// Get the label values for this row
var labels = make([]string, len(mapping.labels))
for idx, columnName := range mapping.labels {
labels[idx], _ = dbToString(columnData[columnIdx[columnName]])
}
// Loop over column names, and match to scan data. Unknown columns
// will be filled with an untyped metric number *if* they can be
// converted to float64s. NULLs are allowed and treated as NaN.
for idx, columnName := range columnNames {
if metricMapping, ok := mapping.columnMappings[columnName]; ok {
// Is this a metricy metric?
if metricMapping.discard {
continue
}
value, ok := dbToFloat64(columnData[idx])
if !ok {
e.error.Set(1)
log.Errorln("Unexpected error parsing column: ", namespace, columnName, columnData[idx])
continue
}
// Generate the metric
ch <- prometheus.MustNewConstMetric(metricMapping.desc, metricMapping.vtype, value, labels...)
} else {
// Unknown metric. Report as untyped if scan to float64 works, else note an error too.
desc := prometheus.NewDesc(fmt.Sprintf("%s_%s", namespace, columnName), fmt.Sprintf("Unknown metric from %s", namespace), nil, nil)
// Its not an error to fail here, since the values are
// unexpected anyway.
value, ok := dbToFloat64(columnData[idx])
if !ok {
log.Warnln("Unparseable column type - discarding: ", namespace, columnName, err)
continue
}
ch <- prometheus.MustNewConstMetric(desc, prometheus.UntypedValue, value, labels...)
}
}
}
}()
}
}
func (e *Exporter) scrape(ch chan<- prometheus.Metric) {
defer func(begun time.Time) {
e.duration.Set(time.Since(begun).Seconds())
}(time.Now())
e.error.Set(0)
e.totalScrapes.Inc()
db, err := sql.Open("mysql", e.dsn)
if err != nil {
log.Println("Error opening connection to database:", err)
e.error.Set(1)
return
}
defer db.Close()
globalStatusRows, err := db.Query("SHOW GLOBAL STATUS")
if err != nil {
log.Println("Error running status query on database:", err)
e.error.Set(1)
return
}
defer globalStatusRows.Close()
var key string
var val sql.RawBytes
for globalStatusRows.Next() {
if err := globalStatusRows.Scan(&key, &val); err != nil {
log.Println("Error getting result set:", err)
e.error.Set(1)
return
}
if floatVal, ok := parseStatus(val); ok {
match := globalStatusRE.FindStringSubmatch(key)
if match == nil {
ch <- prometheus.MustNewConstMetric(
newDesc(globalStatus, strings.ToLower(key), "Generic metric from SHOW GLOBAL STATUS."),
prometheus.UntypedValue,
floatVal,
)
continue
}
switch match[1] {
case "com":
ch <- prometheus.MustNewConstMetric(
globalCommandsDesc, prometheus.CounterValue, floatVal, match[2],
)
case "connection_errors":
ch <- prometheus.MustNewConstMetric(
globalConnectionErrorsDesc, prometheus.CounterValue, floatVal, match[2],
)
case "innodb_rows":
ch <- prometheus.MustNewConstMetric(
globalInnoDBRowOpsDesc, prometheus.CounterValue, floatVal, match[2],
)
case "performance_schema":
ch <- prometheus.MustNewConstMetric(
globalPerformanceSchemaLostDesc, prometheus.CounterValue, floatVal, match[2],
)
}
}
}
slaveStatusRows, err := db.Query("SHOW SLAVE STATUS")
if err != nil {
log.Println("Error running show slave query on database:", err)
e.error.Set(1)
return
}
defer slaveStatusRows.Close()
if slaveStatusRows.Next() {
// There is either no row in SHOW SLAVE STATUS (if this is not a
// slave server), or exactly one. In case of multi-source
// replication, things work very much differently. This code
// cannot deal with that case.
slaveCols, err := slaveStatusRows.Columns()
if err != nil {
log.Println("Error retrieving column list:", err)
e.error.Set(1)
return
}
// As the number of columns varies with mysqld versions,
// and sql.Scan requires []interface{}, we need to create a
// slice of pointers to the elements of slaveData.
scanArgs := make([]interface{}, len(slaveCols))
for i := range scanArgs {
scanArgs[i] = &sql.RawBytes{}
}
if err := slaveStatusRows.Scan(scanArgs...); err != nil {
log.Println("Error retrieving result set:", err)
e.error.Set(1)
return
}
for i, col := range slaveCols {
if value, ok := parseStatus(*scanArgs[i].(*sql.RawBytes)); ok {
ch <- prometheus.MustNewConstMetric(
newDesc(slaveStatus, strings.ToLower(col), "Generic metric from SHOW SLAVE STATUS."),
prometheus.UntypedValue,
value,
)
}
}
}
if *perfTableIOWaits {
perfSchemaTableWaitsRows, err := db.Query("SELECT OBJECT_SCHEMA, OBJECT_NAME, COUNT_READ, COUNT_WRITE, COUNT_FETCH, COUNT_INSERT, COUNT_UPDATE, COUNT_DELETE FROM performance_schema.table_io_waits_summary_by_table WHERE OBJECT_SCHEMA NOT IN ('mysql', 'performance_schema')")
if err != nil {
log.Println("Error running performance schema query on database:", err)
e.error.Set(1)
return
}
defer perfSchemaTableWaitsRows.Close()
var (
objectSchema string
objectName string
countRead int64
countWrite int64
countFetch int64
countInsert int64
countUpdate int64
countDelete int64
)
for perfSchemaTableWaitsRows.Next() {
if err := perfSchemaTableWaitsRows.Scan(
&objectSchema, &objectName, &countRead, &countWrite, &countFetch, &countInsert, &countUpdate, &countDelete,
); err != nil {
log.Println("error getting result set:", err)
return
}
ch <- prometheus.MustNewConstMetric(
performanceSchemaTableWaitsDesc, prometheus.CounterValue, float64(countRead),
objectSchema, objectName, "read",
)
ch <- prometheus.MustNewConstMetric(
performanceSchemaTableWaitsDesc, prometheus.CounterValue, float64(countWrite),
objectSchema, objectName, "write",
)
ch <- prometheus.MustNewConstMetric(
performanceSchemaTableWaitsDesc, prometheus.CounterValue, float64(countFetch),
objectSchema, objectName, "fetch",
)
ch <- prometheus.MustNewConstMetric(
performanceSchemaTableWaitsDesc, prometheus.CounterValue, float64(countInsert),
objectSchema, objectName, "insert",
)
ch <- prometheus.MustNewConstMetric(
performanceSchemaTableWaitsDesc, prometheus.CounterValue, float64(countUpdate),
objectSchema, objectName, "update",
)
ch <- prometheus.MustNewConstMetric(
performanceSchemaTableWaitsDesc, prometheus.CounterValue, float64(countDelete),
objectSchema, objectName, "delete",
)
}
}
}
