func (l *Launchd) Restart(serviceName string) error {
path := l.resolvePlist(serviceName)
if path == "" {
return &ServiceError{l.Name(), serviceName, ErrServiceNotFound}
}
cmd := exec.Command("launchctl", "unload", path)
sout, err := util.SafeRun(cmd)
if err != nil {
return &ServiceError{l.Name(), serviceName, err}
}
lines, err := util.ReadLines(sout)
if len(lines) != 0 {
return &ServiceError{l.Name(), serviceName, errors.New("Unexpected output: " + strings.Join(lines, "\n"))}
}
cmd = exec.Command("launchctl", "load", path)
sout, err = util.SafeRun(cmd)
if err != nil {
return &ServiceError{l.Name(), serviceName, err}
}
lines, err = util.ReadLines(sout)
if len(lines) != 0 {
return &ServiceError{l.Name(), serviceName, errors.New("Unexpected output: " + strings.Join(lines, "\n"))}
}
return nil
}
func (hs *hostStorage) collectDisk(path string) error {
var lines []string
if path == "" {
cmd := exec.Command("df", "-P")
sout, err := util.SafeRun(cmd)
if err != nil {
return err
}
lines, err = util.ReadLines(sout)
if err != nil {
return err
}
} else {
data, err := ioutil.ReadFile(path)
if err != nil {
return err
}
lines, err = util.ReadLines(data)
if err != nil {
return err
}
}
usage := map[string]float64{}
for _, line := range lines {
if line[0] == '/' {
items := strings.Fields(line)
if len(items) < 5 {
util.Debug("Cannot parse df output: %v", items)
continue
}
pct := items[4]
if pct[len(pct)-1] == '%' {
val, err := strconv.ParseInt(pct[0:len(pct)-1], 10, 32)
if err != nil {
util.Debug("Cannot parse df output: " + line)
}
usage[items[len(items)-1]] = float64(val)
}
}
}
for name, used := range usage {
hs.saveType("disk", name, used, Gauge)
}
return nil
}
func (rs *redisSource) runCli(funk executor) (metrics.Map, error) {
sout, err := funk("redis-cli", rs.buildArgs(), nil)
lines, err := util.ReadLines(sout)
if err != nil {
return nil, err
}
values := map[string]float64{}
for _, line := range lines {
if line == "" || line[0] == '#' {
continue
}
parts := strings.Split(line, ":")
if rs.metrics[parts[0]] {
val, err := strconv.ParseInt(parts[1], 10, 64)
if err != nil {
return nil, errors.New("Invalid metric input for '" + line + "': " + err.Error())
}
values[parts[0]] = float64(val)
}
}
if len(rs.metrics) > len(values) {
for k := range rs.metrics {
if _, ok := values[k]; !ok {
util.Warn("Could not find metric redis(%s), did you spell it right?", k)
}
}
}
return values, nil
}
func (rs *mysqlSource) runRepl(values metrics.Map, funk executor) (metrics.Map, error) {
args := rs.buildArgs()
args = append(args, "-e")
args = append(args, "show slave status\\G")
sout, err := funk("mysql", args, nil)
lines, err := util.ReadLines(sout)
if err != nil {
return nil, err
}
for _, line := range lines {
if line == "" || line[0] == '*' {
continue
}
parts := strings.Fields(line)
if parts[0] == "Seconds_Behind_Master:" {
if parts[0] == "NULL" {
values["Seconds_Behind_Master"] = 999999
} else {
val, err := strconv.ParseInt(parts[1], 10, 64)
if err != nil {
return nil, errors.New("Invalid metric input for '" + line + "': " + err.Error())
}
values["Seconds_Behind_Master"] = float64(val)
}
}
}
return values, nil
}
/*
Collecting total RSS for a process is actually rather involved on Linux.
Because of this, we only collect the value if the user defines a rule
for it. This is a "dynamicCollector".
*/
func totalRssCollector(mypid int, ps *processStorage) error {
matches, err := filepath.Glob(fmt.Sprintf("%s/[1-9][0-9]*/status", ps.path))
if err != nil {
return err
}
var live []processEntry
for _, file := range matches {
pe := processEntry{}
data, err := ioutil.ReadFile(file)
if err != nil {
// race condition between globbing and reading, process
// can disappear at any moment
continue
}
lines, err := util.ReadLines(data)
if err != nil {
return err
}
for _, line := range lines {
if line[0] == 'V' || line[0] == 'P' {
items := strings.Split(line, ":")
switch items[0] {
case "Pid":
pid, err := strconv.Atoi(strings.TrimSpace(items[1]))
if err != nil {
return err
}
pe.pid = pid
case "PPid":
ppid, err := strconv.Atoi(strings.TrimSpace(items[1]))
if err != nil {
return err
}
pe.ppid = ppid
case "VmRSS":
vals := strings.Fields(items[1])
val, err := strconv.ParseInt(vals[0], 10, 64)
if err != nil {
return err
}
pe.rss = val * 1024
}
}
}
if pe.rss != 0 {
live = append(live, pe)
}
}
util.DebugDebug("Calculating %d processes", len(live))
rss := memoryFor(live, mypid)
util.DebugDebug("Total RSS for %d: %d", mypid, rss)
ps.Save("memory", "total_rss", float64(rss))
return nil
}
func runSql(pg *pgSource, stmt string) ([][]string, error) {
args := pg.buildArgs()
args = append(args, "-c")
args = append(args, stmt)
sout, err := pg.execFunk("psql", args, nil)
if err != nil {
return nil, errors.New(string(sout))
}
lines, err := util.ReadLines(sout)
if err != nil {
return nil, err
}
var table [][]string
for _, line := range lines {
if line == "" {
continue
}
var row []string
cells := strings.Split(line, "|")
for _, cell := range cells {
row = append(row, strings.TrimSpace(cell))
}
table = append(table, row)
}
return table, nil
}
func (ps *processStorage) captureVM(pid int) error {
dir := ps.path + "/" + strconv.Itoa(int(pid))
data, err := ioutil.ReadFile(dir + "/status")
if err != nil {
return err
}
lines, err := util.ReadLines(data)
if err != nil {
return err
}
for _, line := range lines {
if line[0] == 'V' {
items := strings.Fields(line)
switch items[0] {
case "VmRSS:":
val, err := strconv.ParseInt(items[1], 10, 64)
if err != nil {
return err
}
ps.Save("memory", "rss", float64(1024*val))
}
}
}
return nil
}
func (l *Launchd) LookupService(serviceName string) (*ProcessStatus, error) {
cmd := exec.Command("launchctl", "list")
sout, err := util.SafeRun(cmd)
if err != nil {
return nil, &ServiceError{l.Name(), serviceName, err}
}
lines, err := util.ReadLines(sout)
if err != nil {
return nil, &ServiceError{l.Name(), serviceName, err}
}
for _, line := range lines {
if strings.Contains(line, serviceName) {
util.Debug("launchctl found " + serviceName)
parts := strings.SplitN(line, "\t", 3)
pid, err := strconv.ParseInt(parts[0], 10, 32)
if err != nil {
return nil, &ServiceError{l.Name(), serviceName, err}
}
return &ProcessStatus{int(pid), Up}, nil
}
}
path := l.resolvePlist(serviceName)
if path != "" {
return &ProcessStatus{0, Down}, nil
}
return nil, &ServiceError{l.Name(), serviceName, ErrServiceNotFound}
}
func (u *Upstart) LookupService(serviceName string) (*ProcessStatus, error) {
matches, err := filepath.Glob(u.path + "/" + serviceName + ".conf")
if err != nil {
return nil, &ServiceError{u.Name(), serviceName, err}
}
if len(matches) == 0 {
return nil, &ServiceError{u.Name(), serviceName, ErrServiceNotFound}
}
var sout []byte
if u.dummyOutput != nil {
sout = []byte(*u.dummyOutput)
} else {
cmd := exec.Command("initctl", "status", serviceName)
sout, err = util.SafeRun(cmd)
if err != nil {
return nil, &ServiceError{u.Name(), serviceName, err}
}
}
lines, err := util.ReadLines(sout)
if len(lines) != 1 {
return nil, &ServiceError{u.Name(), serviceName, errors.New("Unexpected output: " + strings.Join(lines, "\n"))}
}
// mysql start/running, process 14190
// sshdgenkeys stop/waiting
line := lines[0]
if strings.Contains(line, "Unknown job") {
return nil, &ServiceError{u.Name(), serviceName, ErrServiceNotFound}
}
results := pidScanner.FindStringSubmatch(line)
if len(results) == 4 && len(results[3]) > 0 {
pid, err := strconv.ParseInt(results[3], 10, 32)
if err != nil {
return nil, &ServiceError{u.Name(), serviceName, err}
}
return &ProcessStatus{int(pid), Up}, nil
}
if len(results) == 4 {
switch {
case results[1] == "start":
return &ProcessStatus{0, Starting}, nil
case results[1] == "stop":
return &ProcessStatus{0, Down}, nil
}
}
return nil, &ServiceError{u.Name(), serviceName, errors.New("Unknown upstart output: " + line)}
}
/*
* So many hacks in this. OSX support can be seen as "bad" at best.
*/
func (ps *processStorage) capturePs(pid int) error {
cmd := exec.Command("ps", "So", "rss,time,utime", "-p", strconv.Itoa(pid))
sout, err := util.SafeRun(cmd)
if err != nil {
return err
}
lines, err := util.ReadLines(sout)
if err != nil {
return err
}
if len(lines) < 2 {
return errors.New("Insufficient output from ps")
}
fields := strings.Fields(lines[1])
val, err := strconv.ParseInt(fields[0], 10, 64)
if err != nil {
return err
}
ps.Save("memory", "rss", float64(1024*val))
times := timeRegexp.FindStringSubmatch(fields[1])
if times == nil {
util.Debug("Unable to parse CPU time in " + lines[1])
return nil
}
min, _ := strconv.ParseUint(times[1], 10, 32)
sec, _ := strconv.ParseUint(times[2], 10, 32)
cs, _ := strconv.ParseUint(times[3], 10, 32)
ticks := min*60*100 + sec*100 + cs
times = timeRegexp.FindStringSubmatch(fields[2])
if times == nil {
util.Debug("Unable to parse User time in " + lines[1])
return nil
}
min, _ = strconv.ParseUint(times[1], 10, 32)
sec, _ = strconv.ParseUint(times[2], 10, 32)
cs, _ = strconv.ParseUint(times[3], 10, 32)
uticks := min*60*100 + sec*100 + cs
ps.Save("cpu", "user", float64(uticks))
ps.Save("cpu", "system", float64(ticks-uticks))
return nil
}
func (hs *hostStorage) collectLoadAverage() error {
// TODO make this a one-time check so we don't incur the overhead
// on every cycle.
ok, err := util.FileExists(hs.path + "/loadavg")
if err != nil {
return err
}
var loadavgString string
if ok {
contentBytes, err := ioutil.ReadFile(hs.path + "/loadavg")
if err != nil {
return err
}
loadavgString = string(contentBytes)
} else {
cmd := exec.Command("sysctl", "-n", "vm.loadavg")
cmd.Env = []string{"LANG=C"}
sout, err := util.SafeRun(cmd)
if err != nil {
return err
}
lines, err := util.ReadLines(sout)
if err != nil {
return err
}
loadavgString = lines[0][2 : len(lines[0])-2] // trim braces
}
slices := strings.Split(loadavgString, " ")
load1, err := strconv.ParseFloat(slices[0], 64)
if err != nil {
return err
}
load5, err := strconv.ParseFloat(slices[1], 64)
if err != nil {
return err
}
load15, err := strconv.ParseFloat(slices[2], 64)
if err != nil {
return err
}
hs.Save("load", "1", load1)
hs.Save("load", "5", load5)
hs.Save("load", "15", load15)
return nil
}
func (s *Systemd) LookupService(serviceName string) (*ProcessStatus, error) {
var sout []byte
var err error
if len(s.dummyOutput) != 0 {
sout = []byte(s.dummyOutput)
} else {
cmd := exec.Command("systemctl", "show", "-p", "MainPID", serviceName)
sout, err = util.SafeRun(cmd)
}
if err != nil {
return nil, &ServiceError{s.Name(), serviceName, ErrServiceNotFound}
}
lines, err := util.ReadLines(sout)
if len(lines) != 1 {
return nil, &ServiceError{s.Name(), serviceName, errors.New("Unexpected output: " + strings.Join(lines, "\n"))}
}
// Output will be "MainPID=1234" or
// "MainPID=0" if service does not exist.
line := lines[0]
fields := strings.Split(line, "=")
if fields[1] != "0" {
pid, err := strconv.ParseInt(fields[1], 10, 32)
if err != nil {
return nil, &ServiceError{s.Name(), serviceName, err}
}
return &ProcessStatus{int(pid), Up}, nil
}
if len(s.dummyOutput2) != 0 {
sout = []byte(s.dummyOutput2)
} else {
cmd := exec.Command("systemctl", "is-enabled", serviceName)
sout, err = util.SafeRun(cmd)
}
if err != nil || string(sout) != "enabled\n" {
return nil, &ServiceError{s.Name(), serviceName, ErrServiceNotFound}
}
return &ProcessStatus{0, Down}, nil
}
func (u *Upstart) serviceCommand(serviceName string, command string, timeout time.Duration, expectedLines int) error {
var err error
var sout []byte
if u.dummyOutput != nil {
sout = []byte(*u.dummyOutput)
} else {
cmd := exec.Command("initctl", command, serviceName)
sout, err = util.SafeRun(cmd, timeout)
if err != nil {
return &ServiceError{u.Name(), serviceName, err}
}
}
lines, err := util.ReadLines(sout)
if len(lines) != expectedLines {
return &ServiceError{u.Name(), serviceName, errors.New("Unexpected output: " + strings.Join(lines, "\n"))}
}
return nil
}
func (ps *processStorage) captureCPU(pid int) error {
dir := ps.path + "/" + strconv.Itoa(int(pid))
data, err := ioutil.ReadFile(dir + "/stat")
if err != nil {
return err
}
lines, err := util.ReadLines(data)
if err != nil {
return err
}
for _, line := range lines {
fields := strings.Fields(line)
utime, err := strconv.ParseInt(fields[13], 10, 64)
if err != nil {
return err
}
stime, err := strconv.ParseInt(fields[14], 10, 64)
if err != nil {
return err
}
cutime, err := strconv.ParseInt(fields[15], 10, 64)
if err != nil {
return err
}
cstime, err := strconv.ParseInt(fields[16], 10, 64)
if err != nil {
return err
}
ps.Save("cpu", "user", float64(utime))
ps.Save("cpu", "system", float64(stime))
ps.Save("cpu", "total_user", float64(cutime))
ps.Save("cpu", "total_system", float64(cstime))
}
return nil
}
func (rs *memcachedSource) runCli(funk executor) (metrics.Map, error) {
sout, err := funk("nc", []string{rs.Hostname, rs.Port}, []byte("stats\n"))
if err != nil {
return nil, err
}
lines, err := util.ReadLines(sout)
if err != nil {
return nil, err
}
values := map[string]float64{}
for _, line := range lines {
if line == "" || line[0] != 'S' {
continue
}
parts := strings.Fields(line)
if rs.metrics[parts[1]] {
val, err := strconv.ParseFloat(parts[2], 64)
if err != nil {
return nil, errors.New("Invalid metric input for '" + line + "': " + err.Error())
}
values[parts[1]] = val
}
}
if len(rs.metrics) > len(values) {
for k := range rs.metrics {
if _, ok := values[k]; !ok {
util.Info("Could not find metric %s(%s), did you spell it right?", rs.Name(), k)
}
}
}
return values, nil
}
func (rs *mysqlSource) runStatus(funk executor) (metrics.Map, error) {
args := rs.buildArgs()
args = append(args, "-e")
args = append(args, "show global status")
sout, err := funk("mysql", args, nil)
lines, err := util.ReadLines(sout)
if err != nil {
return nil, err
}
values := map[string]float64{}
for _, line := range lines {
if line == "" || line[0] == '#' {
continue
}
parts := strings.Fields(line)
if rs.metrics[parts[0]] {
val, err := strconv.ParseInt(parts[1], 10, 64)
if err != nil {
return nil, errors.New("Invalid metric input for '" + line + "': " + err.Error())
}
values[parts[0]] = float64(val)
}
}
if len(rs.metrics) > len(values) {
for k := range rs.metrics {
if _, ok := values[k]; !ok {
util.Warn("Could not find metric mysql(%s), did you spell it right?", k)
}
}
}
return values, nil
}
func (rs *mysqlSource) Prepare() error {
if !rs.metrics["Seconds_Behind_Master"] {
return nil
}
args := rs.buildArgs()
args = append(args, "-e")
args = append(args, "show status like 'Slave_running'")
sout, err := rs.exec("mysql", args, nil)
if err != nil {
return err
}
lines, err := util.ReadLines(sout)
if err != nil {
return err
}
parts := strings.Fields(lines[1])
if parts[1] != "ON" {
return errors.New("Cannot monitor mysql replication, slave not running")
}
delete(rs.metrics, "Seconds_Behind_Master")
rs.captureRepl = true
return nil
}
func (hs *hostStorage) collectMemory() error {
ok, err := util.FileExists(hs.path + "/meminfo")
if err != nil {
return err
}
if ok {
contentBytes, err := ioutil.ReadFile(hs.path + "/meminfo")
if err != nil {
return err
}
lines := strings.Split(string(contentBytes), "\n")
memMetrics := make(map[string]float64)
for _, line := range lines {
if line == "" {
continue
}
results := meminfoParser.FindStringSubmatch(line)
if results == nil {
util.Warn("Unknown input: " + line)
continue
}
val, err := strconv.ParseInt(results[2], 10, 64)
if err != nil {
util.Warn("Unexpected input: " + results[2] + " in " + line)
return err
}
memMetrics[results[1]] = float64(val)
}
free := memMetrics["SwapFree"]
total := memMetrics["SwapTotal"]
if free == 0 {
hs.Save("swap", "", 100)
} else if free == total {
hs.Save("swap", "", 0)
} else {
hs.Save("swap", "", float64(100-int8(100*(float64(free)/float64(total)))))
}
} else {
cmd := exec.Command("sysctl", "-n", "vm.swapusage")
cmd.Env = []string{"LANG=C"}
sout, err := util.SafeRun(cmd)
if err != nil {
return err
}
lines, err := util.ReadLines(sout)
if err != nil {
return err
}
rest := lines[0]
matches := swapRegexp.FindStringSubmatch(rest)
total := matches[1]
rest = matches[2]
matches = swapRegexp.FindStringSubmatch(rest)
used := matches[1]
tot, err := strconv.ParseFloat(total[0:len(total)-1], 64)
if err != nil {
return err
}
usd, err := strconv.ParseFloat(used[0:len(used)-1], 64)
if err != nil {
return err
}
t := normalizeSwap(tot, rune(total[len(total)-1]))
u := normalizeSwap(usd, rune(used[len(used)-1]))
if t == 0 {
hs.Save("swap", "", 100)
} else {
hs.Save("swap", "", float64(100*(u/t)))
}
}
return nil
}