// Generate XXX
func (g *InterfaceGenerator) Generate() (metrics.Values, error) {
prevValues, err := g.collectIntarfacesValues()
if err != nil {
return nil, err
}
time.Sleep(g.Interval)
currValues, err := g.collectIntarfacesValues()
if err != nil {
return nil, err
}
ret := make(map[string]float64)
for name, value := range prevValues {
if !postInterfaceMetricsRegexp.MatchString(name) {
continue
}
currValue, ok := currValues[name]
if ok {
ret[name+".delta"] = (currValue - value) / g.Interval.Seconds()
}
}
return metrics.Values(ret), nil
}
// Generate XXX
func (g *FilesystemGenerator) Generate() (metrics.Values, error) {
filesystems, err := util.CollectDfValues(dfColumnSpecs)
if err != nil {
return nil, err
}
ret := make(map[string]float64)
for name, values := range filesystems {
// https://github.com/docker/docker/blob/v1.5.0/daemon/graphdriver/devmapper/deviceset.go#L981
if regexp.MustCompile(`^/dev/mapper/docker-`).FindStringSubmatch(name) != nil {
continue
}
if matches := regexp.MustCompile(`^/dev/(.*)$`).FindStringSubmatch(name); matches != nil {
device := regexp.MustCompile(`[^A-Za-z0-9_-]`).ReplaceAllString(matches[1], "_")
for key, value := range values {
intValue, valueTypeOk := value.(int64)
if valueTypeOk {
// kilo bytes -> bytes
ret["filesystem."+device+"."+key] = float64(intValue) * 1024
}
}
}
}
return metrics.Values(ret), nil
}
// Generate XXX
func (g *CPUUsageGenerator) Generate() (metrics.Values, error) {
prevValues, prevTotal, _, err := g.collectProcStatValues()
if err != nil {
return nil, err
}
time.Sleep(g.Interval)
currValues, currTotal, cpuCount, err := g.collectProcStatValues()
if err != nil {
return nil, err
}
ret := make(map[string]float64)
for i, name := range cpuUsageMetricNames {
// Values in /proc/stat differ in Linux kernel versions.
// Not all metrics in cpuUsageMetricNames can be retrieved.
// ref: `man 5 proc`
if i >= len(currValues) || i >= len(prevValues) {
break
}
// percentage of increased amount of CPU time
ret[name+".percentage"] = (currValues[i] - prevValues[i]) * 100.0 * float64(cpuCount) / (currTotal - prevTotal)
}
return metrics.Values(ret), nil
}
// Generate XXX
func (g *UptimeGenerator) Generate() (metrics.Values, error) {
if g == nil {
return nil, errors.New("UptimeGenerator is not initialized")
}
r, _, err := windows.GetTickCount.Call()
if r == 0 {
return nil, err
}
return metrics.Values(map[string]float64{"uptime": float64(r) / 1000}), nil
}
func (g *InterfaceGenerator) collectIntarfacesValues() (metrics.Values, error) {
out, err := exec.Command("netstat", "-bni").Output()
if err != nil {
interfaceLogger.Errorf("Failed to invoke netstat: %s", err)
return nil, err
}
lineScanner := bufio.NewScanner(bytes.NewReader(out))
results := make(map[string]float64)
hasNonZeroValue := false
for lineScanner.Scan() {
line := lineScanner.Text()
fields := strings.Fields(line)
name := regexp.MustCompile(`[^A-Za-z0-9_-]`).ReplaceAllString(regexp.MustCompile(`\*`).ReplaceAllString(fields[0], ""), "_")
if match, _ := regexp.MatchString(`^lo\d+$`, name); match {
continue
}
if match, _ := regexp.MatchString(`^<Link#\d+>$`, fields[2]); match {
rxIndex, txIndex := getFieldIndex(fields)
rxKey := fmt.Sprintf("interface.%s.rxBytes", name)
rxValue, rxErr := strconv.ParseFloat(fields[rxIndex], 64)
if rxErr != nil {
interfaceLogger.Warningf("Failed to parse host interfaces: %s", err)
break
}
results[rxKey] = rxValue
if rxValue != 0 {
hasNonZeroValue = true
}
txKey := fmt.Sprintf("interface.%s.txBytes", name)
txValue, txErr := strconv.ParseFloat(fields[txIndex], 64)
if txErr != nil {
interfaceLogger.Warningf("Failed to parse host interfaces: %s", err)
break
}
results[txKey] = txValue
if txValue != 0 {
hasNonZeroValue = true
}
}
}
// results (eth0) sample
/** [%!s(*metrics.Value=&{interface.eth0.rxBytes 6.6074069281e+10})
%!s(*metrics.Value=&{interface.eth0.txBytes 9.180531994e+09})
]
**/
if hasNonZeroValue {
return metrics.Values(results), nil
}
return nil, nil
}
// Generate generate metrics values
func (g *MemoryGenerator) Generate() (metrics.Values, error) {
file, err := os.Open("/proc/meminfo")
if err != nil {
memoryLogger.Errorf("Failed (skip these metrics): %s", err)
return nil, err
}
scanner := bufio.NewScanner(file)
ret := make(map[string]float64)
used := float64(0)
usedCnt := 0
for scanner.Scan() {
line := scanner.Text()
for k, regexp := range memItems {
if matches := regexp.FindStringSubmatch(line); matches != nil {
// ex.) MemTotal: 3916792 kB
// matches[1] = 3916792, matches[2] = kB
if matches[2] != "kB" {
memoryLogger.Warningf("/proc/meminfo contains an invalid unit: %s", k)
break
}
value, err := strconv.ParseFloat(matches[1], 64)
if err != nil {
memoryLogger.Warningf("Failed to parse memory metrics: %s", err)
break
}
ret["memory."+k] = value * 1024
if k == "free" || k == "buffers" || k == "cached" {
used -= value
usedCnt++
}
if k == "total" {
used += value
usedCnt++
}
break
}
}
}
if err := scanner.Err(); err != nil {
memoryLogger.Errorf("Failed (skip these metrics): %s", err)
return nil, err
}
if usedCnt == 4 { // 4 is free, buffers, cached and total
ret["memory.used"] = used * 1024
}
return metrics.Values(ret), nil
}
// Generate XXX
func (g *Loadavg5Generator) Generate() (metrics.Values, error) {
contentbytes, err := ioutil.ReadFile("/proc/loadavg")
if err != nil {
loadavg5Logger.Errorf("Failed (skip these metrics): %s", err)
return nil, err
}
content := string(contentbytes)
cols := strings.Split(content, " ")
f, err := strconv.ParseFloat(cols[1], 64)
if err != nil {
loadavg5Logger.Errorf("Failed to parse loadavg5 metrics (skip these metrics): %s", err)
return nil, err
}
return metrics.Values(map[string]float64{"loadavg5": f}), nil
}
// Generate generate metrics values
func (g *UptimeGenerator) Generate() (metrics.Values, error) {
contentbytes, err := ioutil.ReadFile("/proc/uptime")
if err != nil {
uptimeLogger.Errorf("Failed (skip these metrics): %s", err)
return nil, err
}
content := string(contentbytes)
cols := strings.Split(content, " ")
f, err := strconv.ParseFloat(cols[0], 64)
if err != nil {
uptimeLogger.Errorf("Failed to parse values (skip these metrics): %s", err)
return nil, err
}
return metrics.Values(map[string]float64{"uptime": f / 86400}), nil
}
// Generate XXX
func (g *FilesystemGenerator) Generate() (metrics.Values, error) {
filesystems, err := windows.CollectFilesystemValues()
if err != nil {
return nil, err
}
ret := make(map[string]float64)
for name, values := range filesystems {
if matches := regexp.MustCompile(`^(.*):`).FindStringSubmatch(name); matches != nil {
device := regexp.MustCompile(`[^A-Za-z0-9_-]`).ReplaceAllString(matches[1], "_")
ret["filesystem."+device+".size"] = values.KbSize * 1024
ret["filesystem."+device+".used"] = values.KbUsed * 1024
}
}
logger.Debugf("%q", ret)
return metrics.Values(ret), nil
}
// Generate returns current CPU usage of the host.
// Keys below are expected:
// - cpu.user.percentage
// - cpu.system.percentage
// - cpu.idle.percentage
func (g *CPUUsageGenerator) Generate() (metrics.Values, error) {
// % iostat -c2 -C
// CPU
// us ni sy in id
// 0 0 0 0 100
// 0 0 0 0 100
iostatBytes, err := exec.Command("iostat", "-c2", "-d", "-C").Output()
if err != nil {
cpuUsageLogger.Errorf("Failed to invoke iostat: %s", err)
return nil, err
}
iostat := string(iostatBytes)
lines := strings.Split(iostat, "\n")
if len(lines) != 5 {
return nil, fmt.Errorf("iostat result malformed: [%q]", iostat)
}
fields := strings.Fields(lines[3])
if len(fields) < len(iostatFieldToMetricName) {
return nil, fmt.Errorf("iostat result malformed: [%q]", iostat)
}
cpuUsage := make(map[string]float64, len(iostatFieldToMetricName))
for i, n := range iostatFieldToMetricName {
if i == 3 {
continue
}
value, err := strconv.ParseFloat(fields[i], 64)
if err != nil {
return nil, err
}
cpuUsage["cpu."+n+".percentage"] = value
}
return metrics.Values(cpuUsage), nil
}
// Generate XXX
func (g *MemoryGenerator) Generate() (metrics.Values, error) {
ret := make(map[string]float64)
var memoryStatusEx windows.MEMORY_STATUS_EX
memoryStatusEx.Length = uint32(unsafe.Sizeof(memoryStatusEx))
r, _, err := windows.GlobalMemoryStatusEx.Call(uintptr(unsafe.Pointer(&memoryStatusEx)))
if r == 0 {
return nil, err
}
free := float64(memoryStatusEx.AvailPhys)
total := float64(memoryStatusEx.TotalPhys)
ret["memory.free"] = free
ret["memory.total"] = total
ret["memory.used"] = total - free
ret["memory.pagefile_total"] = float64(memoryStatusEx.TotalPageFile) / 1024
ret["memory.pagefile_free"] = float64(memoryStatusEx.AvailPageFile) / 1024
memoryLogger.Debugf("memory : %s", ret)
return metrics.Values(ret), nil
}
func parseNetdev(out []byte) (metrics.Values, error) {
lineScanner := bufio.NewScanner(bytes.NewReader(out))
results := make(map[string]float64)
for lineScanner.Scan() {
line := lineScanner.Text()
if kv := strings.SplitN(line, ":", 2); len(kv) == 2 {
name := sanitizerReg.ReplaceAllString(strings.TrimSpace(kv[0]), "_")
if name == "lo" {
continue
}
cols := strings.Fields(kv[1])
if len(cols) < len(interfaceMetrics) {
continue
}
interfaceResult := make(map[string]float64)
hasNonZeroValue := false
for i, metricName := range interfaceMetrics {
key := fmt.Sprintf("interface.%s.%s", name, metricName)
value, err := strconv.ParseFloat(cols[i], 64)
if err != nil {
interfaceLogger.Warningf("Failed to parse host interfaces: %s", err)
break
}
if value != 0 {
hasNonZeroValue = true
}
interfaceResult[key] = value
}
if hasNonZeroValue {
for k, v := range interfaceResult {
results[k] = v
}
}
}
}
return metrics.Values(results), nil
}
// Generate generate metrics values
func (g *MemoryGenerator) Generate() (metrics.Values, error) {
outBytes, err := exec.Command("vm_stat").Output()
if err != nil {
memoryLogger.Errorf("Failed (skip these metrics): %s", err)
return nil, err
}
out := string(outBytes)
lines := strings.Split(out, "\n")
stats := map[string]int64{}
for _, line := range lines {
if matches := statReg.FindStringSubmatch(line); matches != nil {
v, _ := strconv.ParseInt(matches[2], 0, 64)
// `vm_stat` returns the page sise of 4096 bytes
stats[matches[1]] = v * 4096
}
}
wired := stats["Pages wired down"]
compressed := stats["Pages occupied by compressor"]
cached := stats["Pages purgeable"] + stats["File-backed pages"]
active := stats["Pages active"]
inactive := stats["Pages inactive"]
used := wired + compressed + active + inactive + stats["Pages speculative"] - cached
free := stats["Pages free"]
total := used + cached + free
ret := map[string]float64{
"memory.total": float64(total),
"memory.used": float64(used),
"memory.cached": float64(cached),
"memory.free": float64(free),
"memory.active": float64(active),
"memory.inactive": float64(inactive),
}
return metrics.Values(ret), nil
}
func generateValues(generators []metrics.Generator) chan metrics.Values {
processed := make(chan metrics.Values)
finish := make(chan bool)
result := make(chan metrics.Values)
go func() {
allValues := metrics.Values(make(map[string]float64))
for {
select {
case values := <-processed:
allValues.Merge(values)
case <-finish:
result <- allValues
return
}
}
}()
go func() {
var wg sync.WaitGroup
for _, g := range generators {
wg.Add(1)
go func(g metrics.Generator) {
defer wg.Done()
values, err := g.Generate()
if err != nil {
logger.Errorf("Failed to generate value in %T (skip this metric): %s", g, err.Error())
return
}
processed <- values
}(g)
}
wg.Wait()
finish <- true // processed all jobs
}()
return result
}
// Generate XXX
func (g *FilesystemGenerator) Generate() (metrics.Values, error) {
filesystems, err := util.CollectDfValues(dfColumnSpecs)
if err != nil {
return nil, err
}
ret := make(map[string]float64)
for name, values := range filesystems {
if matches := regexp.MustCompile(`^/dev/(.*)$`).FindStringSubmatch(name); matches != nil {
device := regexp.MustCompile(`[^A-Za-z0-9_-]`).ReplaceAllString(matches[1], "_")
for key, value := range values {
intValue, valueTypeOk := value.(int64)
if valueTypeOk {
// kilo bytes -> bytes
ret["filesystem."+device+"."+key] = float64(intValue) * 1024
}
}
}
}
return metrics.Values(ret), nil
}
// Generate generate swap values
func (g *SwapGenerator) Generate() (metrics.Values, error) {
outBytes, err := exec.Command("sysctl", "vm.swapusage").Output()
if err != nil {
swapLogger.Errorf("Failed (skip these metrics): %s", err)
return nil, err
}
out := string(outBytes)
matches := swapReg.FindStringSubmatch(out)
if matches == nil || len(matches) != 4 {
return nil, fmt.Errorf("faild to parse vm.swapusage result: [%q]", out)
}
t, _ := strconv.ParseFloat(matches[1], 64)
// swap_used are calculated at server, so don't send it
// u, _ := strconv.ParseFloat(matches[2], 64)
f, _ := strconv.ParseFloat(matches[3], 64)
const mb = 1024.0 * 1024.0
return metrics.Values(map[string]float64{
"memory.swap_total": t * mb,
"memory.swap_free": f * mb,
}), nil
}
// Generate returns current CPU usage of the host.
// Keys below are expected:
// - cpu.user.percentage
// - cpu.system.percentage
// - cpu.idle.percentage
func (g *CPUUsageGenerator) Generate() (metrics.Values, error) {
// $ iostat -n0 -c2
// cpu load average
// us sy id 1m 5m 15m
// 13 7 81 1.93 2.23 2.65
// 13 7 81 1.93 2.23 2.65
iostatBytes, err := exec.Command("iostat", "-n0", "-c2").Output()
if err != nil {
cpuUsageLogger.Errorf("Failed to invoke iostat: %s", err)
return nil, err
}
iostat := string(iostatBytes)
lines := strings.Split(iostat, "\n")
if len(lines) != 5 {
return nil, fmt.Errorf("iostat result malformed: [%q]", iostat)
}
fields := strings.Fields(lines[3])
if len(fields) < len(iostatFieldToMetricName) {
return nil, fmt.Errorf("iostat result malformed: [%q]", iostat)
}
cpuUsage := make(map[string]float64, len(iostatFieldToMetricName))
for i, n := range iostatFieldToMetricName {
value, err := strconv.ParseFloat(fields[i], 64)
if err != nil {
return nil, err
}
cpuUsage["cpu."+n+".percentage"] = value
}
return metrics.Values(cpuUsage), nil
}
// Generate generate metrics values
func (g *MemoryGenerator) Generate() (metrics.Values, error) {
var errRet error
outBytes, err := exec.Command("top", "-bn", "1").Output()
if err != nil {
logger.Warningf("'top -bn 1' command exited with a non-zero status: '%s'", err)
errRet = err
}
ret := make(map[string]float64)
out := string(outBytes)
lines := strings.Split(out, "\n")
for _, line := range lines {
fields := strings.Fields(line)
if len(fields) < 1 {
continue
}
if fields[0] == "Memory:" {
for i := 1; i < len(fields); i += 2 {
v, err := getValue(fields[i])
if err == nil {
k := fields[i+1]
if strings.HasSuffix(k, ",") {
k = k[0 : len(k)-1]
}
switch k {
case "Act":
ret["memory.act"] = v
case "Wired":
ret["memory.wired"] = v
case "Exec":
ret["memory.exec"] = v
case "File":
ret["memory.file"] = v
case "Free":
ret["memory.free"] = v
}
} else {
errRet = err
}
}
}
if fields[0] == "Swap:" {
if v, err := getValue(fields[1]); err == nil {
ret["memory.swap_total"] = v
} else {
errRet = err
}
swapFreeIndex := 5
if len(fields) == 5 {
swapFreeIndex = 3
}
if v, err := getValue(fields[swapFreeIndex]); err == nil {
ret["memory.swap_free"] = v
} else {
errRet = err
}
}
}
v, err := getTotalMem()
if err != nil {
return nil, err
}
ret["memory.total"] = v
if errRet == nil {
return metrics.Values(ret), nil
}
return nil, errRet
}
func (g *InterfaceGenerator) collectIntarfacesValues() (metrics.Values, error) {
file, err := os.Open("/proc/net/dev")
if err != nil {
interfaceLogger.Errorf("Failed (skip these metrics): %s", err)
return nil, err
}
lineScanner := bufio.NewScanner(bufio.NewReader(file))
results := make(map[string]float64)
for lineScanner.Scan() {
line := lineScanner.Text()
if matches := regexp.MustCompile(`^\s*([^:]+):\s*(.*)$`).FindStringSubmatch(line); matches != nil {
name := regexp.MustCompile(`[^A-Za-z0-9_-]`).ReplaceAllString(matches[1], "_")
if name == "lo" {
continue
}
cols := regexp.MustCompile(`\s+`).Split(matches[2], len(interfaceMetrics))
if len(cols) < len(interfaceMetrics) {
continue
}
interfaceResult := make(map[string]float64)
hasNonZeroValue := false
for i := range interfaceMetrics {
key := fmt.Sprintf("interface.%s.%s", name, interfaceMetrics[i])
value, err := strconv.ParseFloat(cols[i], 64)
if err != nil {
interfaceLogger.Warningf("Failed to parse host interfaces: %s", err)
break
}
if value != 0 {
hasNonZeroValue = true
}
interfaceResult[key] = value
}
if hasNonZeroValue {
for k, v := range interfaceResult {
results[k] = v
}
}
}
}
// results (eth0) sample
/** [%!s(*metrics.Value=&{interface.eth0.rxBytes 6.6074069281e+10})
%!s(*metrics.Value=&{interface.eth0.rxPackets 1.0483646e+08})
%!s(*metrics.Value=&{interface.eth0.rxErrors 0})
%!s(*metrics.Value=&{interface.eth0.rxDrops 1})
%!s(*metrics.Value=&{interface.eth0.rxFifo 0})
%!s(*metrics.Value=&{interface.eth0.rxFrame 0})
%!s(*metrics.Value=&{interface.eth0.rxCompressed 0})
%!s(*metrics.Value=&{interface.eth0.rxMulticast 0})
%!s(*metrics.Value=&{interface.eth0.txBytes 9.180531994e+09})
%!s(*metrics.Value=&{interface.eth0.txPackets 5.3107958e+07})
%!s(*metrics.Value=&{interface.eth0.txErrors 0})
%!s(*metrics.Value=&{interface.eth0.txDrops 0})
%!s(*metrics.Value=&{interface.eth0.txFifo 0})
%!s(*metrics.Value=&{interface.eth0.txColls 0})
%!s(*metrics.Value=&{interface.eth0.txCarrier 0})
%!s(*metrics.Value=&{interface.eth0.txCompressed 0})
]
**/
return metrics.Values(results), nil
}