func (_ *SystemStats) Gather(acc telegraf.Accumulator) error {
loadavg, err := load.Avg()
if err != nil {
return err
}
hostinfo, err := host.Info()
if err != nil {
return err
}
users, err := host.Users()
if err != nil {
return err
}
fields := map[string]interface{}{
"load1": loadavg.Load1,
"load5": loadavg.Load5,
"load15": loadavg.Load15,
"uptime": hostinfo.Uptime,
"n_users": len(users),
"uptime_format": format_uptime(hostinfo.Uptime),
"n_cpus": runtime.NumCPU(),
}
acc.AddFields("system", fields, nil)
return nil
}
func loadMetrics(cycle int) []*types.TimeSeriesData {
cnt, err := load.Avg()
if err != nil {
return nil
}
ts := time.Now().Unix()
metrics := make([]*types.TimeSeriesData, 3)
metrics[0] = &types.TimeSeriesData{
Metric: "sys.load1",
Value: cnt.Load1,
Cycle: cycle,
Timestamp: ts,
DataType: "GAUGE",
}
metrics[1] = &types.TimeSeriesData{
Metric: "sys.load5",
Value: cnt.Load5,
Cycle: cycle,
Timestamp: ts,
DataType: "GAUGE",
}
metrics[2] = &types.TimeSeriesData{
Metric: "sys.load15",
Value: cnt.Load15,
Cycle: cycle,
Timestamp: ts,
DataType: "GAUGE",
}
return metrics
}
func gaugesUpdate(results runner.BenchmarkResults) {
if s, err := load.Avg(); err == nil {
gauges["cpu_load1"].Set(s.Load1)
gauges["cpu_load5"].Set(s.Load5)
gauges["cpu_load15"].Set(s.Load15)
}
if s, err := mem.VirtualMemory(); err == nil {
gauges["mem_total"].Set(float64(s.Total))
gauges["mem_used"].Set(float64(s.Used))
}
if s, err := disk.Usage("/"); err == nil {
gauges["disk_total"].Set(float64(s.Total))
gauges["disk_used"].Set(float64(s.Used))
}
if results != nil {
gauges["io_time"].Set(results["IO"].Time)
gauges["io_wall_time"].Set(results["IO"].WallTime)
gauges["io_memory"].Set(float64(results["IO"].Memory))
gauges["cpu_time"].Set(results["CPU"].Time)
gauges["cpu_wall_time"].Set(results["CPU"].WallTime)
gauges["cpu_memory"].Set(float64(results["CPU"].Memory))
gauges["memory_time"].Set(results["Memory"].Time)
gauges["memory_wall_time"].Set(results["Memory"].WallTime)
gauges["memory_memory"].Set(float64(results["Memory"].Memory))
}
}
// GetSystem gets statistics about the system
func GetSystem() *api.SystemStats {
status := new(api.SystemStats)
if load, err := load.Avg(); err == nil {
status.Load = &api.SystemStats_Loadstats{
Load1: float32(load.Load1),
Load5: float32(load.Load5),
Load15: float32(load.Load15),
}
}
if cpu, err := cpu.Times(false); err == nil && len(cpu) == 1 {
status.Cpu = &api.SystemStats_CPUStats{
User: float32(cpu[0].User),
System: float32(cpu[0].System),
Idle: float32(cpu[0].Idle),
}
}
if mem, err := mem.VirtualMemory(); err == nil {
status.Memory = &api.SystemStats_MemoryStats{
Total: mem.Total,
Available: mem.Available,
Used: mem.Used,
}
}
return status
}
func GetRenderContext() *load.AvgStat {
if avg, err := load.Avg(); err == nil {
return avg
} else {
log.Fatal(err)
}
return nil
}
func GetLoadStats() string {
l, err := load.Avg()
format := "last1=%f last5=%f last15=%f"
if err == nil {
return fmt.Sprintf(format, l.Load1, l.Load5, l.Load15)
} else {
return fmt.Sprintf(format, 0, 0, 0)
}
}
func getCPULoad() float32 {
c, err := load.Avg()
if err != nil {
log.Printf("ERROR: Unable to get CPU load for the scout.")
log.Print(err)
}
return float32(c.Load5)
}
func (c *Cpu) populateValues() error {
load, err := load.Avg()
if err != nil {
return err
}
c.oneLoad = load.Load1
c.fiveLoad = load.Load5
c.fifteenLoad = load.Load15
return nil
}
// LoadAverage - returns load avg
func LoadAverage() LoadStruct {
cores, _ := cpu.Counts(true)
load, _ := load.Avg()
l := LoadStruct{
Minute: load.Load1,
FiveMinutes: load.Load5,
FifteenMinutes: load.Load15,
Cores: cores,
}
return l
}
func gaugesUpdate() {
if s, err := load.Avg(); err == nil {
gauges["cpu_load1"].Set(s.Load1)
gauges["cpu_load5"].Set(s.Load5)
gauges["cpu_load15"].Set(s.Load15)
}
if s, err := mem.VirtualMemory(); err == nil {
gauges["mem_total"].Set(float64(s.Total))
gauges["mem_used"].Set(float64(s.Used))
}
if s, err := disk.Usage("/"); err == nil {
gauges["disk_total"].Set(float64(s.Total))
gauges["disk_used"].Set(float64(s.Used))
}
}
func loadAvg(nss []core.Namespace) ([]plugin.MetricType, error) {
load, err := load.Avg()
if err != nil {
return nil, err
}
results := make([]plugin.MetricType, len(nss))
for i, ns := range nss {
switch ns.Element(len(ns) - 1).Value {
case "load1":
results[i] = plugin.MetricType{
Namespace_: ns,
Data_: load.Load1,
Unit_: "Load/1M",
Timestamp_: time.Now(),
}
case "load5":
results[i] = plugin.MetricType{
Namespace_: ns,
Data_: load.Load5,
Unit_: "Load/5M",
Timestamp_: time.Now(),
}
case "load15":
results[i] = plugin.MetricType{
Namespace_: ns,
Data_: load.Load15,
Unit_: "Load/15M",
Timestamp_: time.Now(),
}
default:
return nil, fmt.Errorf("Requested load statistic %s is not found", ns.Element(len(ns)-1).Value)
}
}
return results, nil
}
func runRktMonitor(cmd *cobra.Command, args []string) {
if len(args) != 1 {
cmd.Usage()
os.Exit(1)
}
d, err := time.ParseDuration(flagDuration)
if err != nil {
fmt.Printf("%v\n", err)
os.Exit(1)
}
if os.Getuid() != 0 {
fmt.Printf("need to be root to run rkt images\n")
os.Exit(1)
}
f, err := os.Open(args[0])
if err != nil {
fmt.Printf("%v\n", err)
os.Exit(1)
}
decoder := json.NewDecoder(f)
podManifest := false
man := schema.PodManifest{}
err = decoder.Decode(&man)
if err == nil {
podManifest = true
}
var flavorType string
if flagStage1Path == "" {
flavorType = "stage1-coreos.aci"
} else {
_, flavorType = filepath.Split(flagStage1Path)
}
var execCmd *exec.Cmd
var loadAvg *load.AvgStat
var containerStarting, containerStarted, containerStopping, containerStopped time.Time
records := [][]string{{"Time", "PID name", "PID number", "RSS", "CPU"}} // csv headers
summaryRecords := [][]string{{"Load1", "Load5", "Load15", "StartTime", "StopTime"}} // csv summary headers
var rktBinary string
if flagRktDir != "" {
rktBinary = flagRktDir + "/rkt"
} else {
rktBinary = "rkt"
}
for i := 0; i < flagRepetitionNumber; i++ {
containerStarting = time.Now()
// build argument list for execCmd
argv := []string{"run"}
if flagStage1Path != "" {
argv = append(argv, fmt.Sprintf("--stage1-path=%v", flagStage1Path))
}
if podManifest {
argv = append(argv, "--pod-manifest", args[0])
} else {
argv = append(argv, args[0], "--insecure-options=image")
}
argv = append(argv, "--net=default-restricted")
execCmd = exec.Command(rktBinary, argv...)
if flagShowOutput {
execCmd.Stdout = os.Stdout
execCmd.Stderr = os.Stderr
}
err = execCmd.Start()
containerStarted = time.Now()
if err != nil {
fmt.Printf("%v\n", err)
os.Exit(1)
}
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
go func() {
for range c {
err := killAllChildren(int32(execCmd.Process.Pid))
if err != nil {
fmt.Fprintf(os.Stderr, "cleanup failed: %v\n", err)
}
os.Exit(1)
}
}()
usages := make(map[int32][]*ProcessStatus)
timeToStop := time.Now().Add(d)
for time.Now().Before(timeToStop) {
usage, err := getUsage(int32(execCmd.Process.Pid))
if err != nil {
panic(err)
}
if flagVerbose {
printUsage(usage)
}
if flagSaveToCsv {
records = addRecords(usage, records)
}
for _, ps := range usage {
usages[ps.Pid] = append(usages[ps.Pid], ps)
}
_, err = process.NewProcess(int32(execCmd.Process.Pid))
if err != nil {
// process.Process.IsRunning is not implemented yet
fmt.Fprintf(os.Stderr, "rkt exited prematurely\n")
break
}
time.Sleep(time.Second)
}
loadAvg, err = load.Avg()
if err != nil {
fmt.Fprintf(os.Stderr, "measure load avg failed: %v\n", err)
}
containerStopping = time.Now()
err = killAllChildren(int32(execCmd.Process.Pid))
containerStopped = time.Now()
if err != nil {
fmt.Fprintf(os.Stderr, "cleanup failed: %v\n", err)
}
for _, processHistory := range usages {
var avgCPU float64
var avgMem uint64
var peakMem uint64
for _, p := range processHistory {
avgCPU += p.CPU
avgMem += p.RSS
if peakMem < p.RSS {
peakMem = p.RSS
}
}
avgCPU = avgCPU / float64(len(processHistory))
avgMem = avgMem / uint64(len(processHistory))
if !flagSaveToCsv {
fmt.Printf("%s(%d): seconds alive: %d avg CPU: %f%% avg Mem: %s peak Mem: %s\n", processHistory[0].Name, processHistory[0].Pid, len(processHistory), avgCPU, formatSize(avgMem), formatSize(peakMem))
}
}
if flagSaveToCsv {
summaryRecords = append(summaryRecords, []string{
strconv.FormatFloat(loadAvg.Load1, 'g', 3, 64),
strconv.FormatFloat(loadAvg.Load5, 'g', 3, 64),
strconv.FormatFloat(loadAvg.Load15, 'g', 3, 64),
strconv.FormatInt(containerStarted.Sub(containerStarting).Nanoseconds(), 10),
strconv.FormatInt(containerStopped.Sub(containerStopping).Nanoseconds(), 10)})
}
fmt.Printf("load average: Load1: %f Load5: %f Load15: %f\n", loadAvg.Load1, loadAvg.Load5, loadAvg.Load15)
fmt.Printf("container start time: %dns\n", containerStarted.Sub(containerStarting).Nanoseconds())
fmt.Printf("container stop time: %dns\n", containerStopped.Sub(containerStopping).Nanoseconds())
}
t := time.Now()
prefix := fmt.Sprintf("%d-%02d-%02d_%02d-%02d_%s_", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), flavorType)
if flagSaveToCsv {
err = saveRecords(records, flagCsvDir, prefix+"rkt_benchmark_interval.csv")
if err != nil {
fmt.Fprintf(os.Stderr, "Can't write to a file: %v\n", err)
}
err = saveRecords(summaryRecords, flagCsvDir, prefix+"rkt_benchmark_summary.csv")
if err != nil {
fmt.Fprintf(os.Stderr, "Can't write to a summary file: %v\n", err)
}
}
}
func runRktMonitor(cmd *cobra.Command, args []string) {
if len(args) != 1 {
cmd.Usage()
os.Exit(254)
}
d, err := time.ParseDuration(flagDuration)
if err != nil {
fmt.Printf("%v\n", err)
os.Exit(254)
}
if os.Getuid() != 0 {
fmt.Printf("need to be root to run rkt images\n")
os.Exit(254)
}
f, err := os.Open(args[0])
if err != nil {
fmt.Printf("%v\n", err)
os.Exit(254)
}
decoder := json.NewDecoder(f)
podManifest := false
man := schema.PodManifest{}
err = decoder.Decode(&man)
if err == nil {
podManifest = true
}
var flavorType, testImage string
if flagStage1Path == "" {
flavorType = "stage1-coreos.aci"
} else {
if !fileExist(flagStage1Path) {
fmt.Fprintln(os.Stderr, "Given stage1 file path doesn't exist")
os.Exit(254)
}
_, flavorType = filepath.Split(flagStage1Path)
}
var containerId string
var stopCmd, execCmd, gcCmd *exec.Cmd
var loadAvg *load.AvgStat
var containerStarting, containerStarted, containerStopping, containerStopped time.Time
records := [][]string{{"Time", "PID name", "PID number", "RSS", "CPU"}} // csv headers
summaryRecords := [][]string{{"Load1", "Load5", "Load15", "StartTime", "StopTime"}} // csv summary headers
var rktBinary string
if flagRktDir != "" {
rktBinary = filepath.Join(flagRktDir, "rkt")
if !fileExist(rktBinary) {
fmt.Fprintln(os.Stderr, "rkt binary not found!")
os.Exit(1)
}
} else {
rktBinary = "rkt"
}
for i := 0; i < flagRepetitionNumber; i++ {
// build argument list for execCmd
argv := []string{"run", "--debug"}
if flagStage1Path != "" {
argv = append(argv, fmt.Sprintf("--stage1-path=%v", flagStage1Path))
}
if podManifest {
argv = append(argv, "--pod-manifest", args[0])
} else {
argv = append(argv, args[0], "--insecure-options=image")
}
argv = append(argv, "--net=default-restricted")
execCmd = exec.Command(rktBinary, argv...)
if flagShowOutput {
execCmd.Stderr = os.Stderr
}
cmdReader, err := execCmd.StdoutPipe()
if err != nil {
fmt.Fprintln(os.Stderr, "Error creating StdoutPipe for execCmd", err)
os.Exit(254)
}
execCmdScanner := bufio.NewScanner(cmdReader)
startConfirmation := make(chan string, 1)
go func() {
var containerId string
for execCmdScanner.Scan() {
if flagShowOutput {
fmt.Println(execCmdScanner.Text())
}
if strings.Contains(execCmdScanner.Text(), "APP-STARTED!") {
startConfirmation <- containerId
} else if strings.Contains(execCmdScanner.Text(), "Set hostname to") {
sl := strings.SplitAfter(execCmdScanner.Text(), "<rkt-")
containerId = sl[len(sl)-1]
containerId = containerId[:len(containerId)-2]
}
}
}()
containerStarting = time.Now()
err = execCmd.Start()
containerId = <-startConfirmation
containerStarted = time.Now() //here we are sure - container is running (issue: #3019)
close(startConfirmation)
if flagShowOutput {
execCmd.Stdout = os.Stdout
}
if err != nil {
fmt.Printf("%v\n", err)
os.Exit(254)
}
usages := make(map[int32][]*ProcessStatus)
timeToStop := time.Now().Add(d)
for time.Now().Before(timeToStop) {
usage, err := getUsage(int32(execCmd.Process.Pid))
if err != nil {
panic(err)
}
if flagVerbose {
printUsage(usage)
}
if flagSaveToCsv {
records = addRecords(usage, records)
}
for _, ps := range usage {
usages[ps.Pid] = append(usages[ps.Pid], ps)
}
_, err = process.NewProcess(int32(execCmd.Process.Pid))
if err != nil {
// process.Process.IsRunning is not implemented yet
fmt.Fprintf(os.Stderr, "rkt exited prematurely\n")
break
}
time.Sleep(time.Second)
}
loadAvg, err = load.Avg()
if err != nil {
fmt.Fprintf(os.Stderr, "measure load avg failed: %v\n", err)
}
stopCmd = exec.Command(rktBinary, "stop", containerId)
cmdStopReader, err := stopCmd.StdoutPipe()
if err != nil {
fmt.Fprintln(os.Stderr, "Error creating StdoutPipe for stopCmd", err)
os.Exit(254)
}
cmdStopScanner := bufio.NewScanner(cmdStopReader)
containerStopping = time.Now()
stopConfirmation := make(chan bool, 1)
go func() {
for cmdStopScanner.Scan() {
if strings.Contains(cmdStopScanner.Text(), containerId) {
stopConfirmation <- true
return
}
}
stopConfirmation <- false
}()
err = stopCmd.Start()
if !<-stopConfirmation {
fmt.Println("WARNING: There was a problem stopping the container! (Container already stopped?)")
}
containerStopped = time.Now()
close(stopConfirmation)
if err != nil {
fmt.Printf("%v\n", err)
os.Exit(254)
}
gcCmd = exec.Command(rktBinary, "gc", "--grace-period=0")
gcCmd.Start()
for _, processHistory := range usages {
var avgCPU float64
var avgMem uint64
var peakMem uint64
for _, p := range processHistory {
avgCPU += p.CPU
avgMem += p.RSS
if peakMem < p.RSS {
peakMem = p.RSS
}
}
avgCPU = avgCPU / float64(len(processHistory))
avgMem = avgMem / uint64(len(processHistory))
if !flagSaveToCsv {
fmt.Printf("%s(%d): seconds alive: %d avg CPU: %f%% avg Mem: %s peak Mem: %s\n", processHistory[0].Name, processHistory[0].Pid, len(processHistory), avgCPU, formatSize(avgMem), formatSize(peakMem))
}
}
if flagSaveToCsv {
summaryRecords = append(summaryRecords, []string{
strconv.FormatFloat(loadAvg.Load1, 'g', 3, 64),
strconv.FormatFloat(loadAvg.Load5, 'g', 3, 64),
strconv.FormatFloat(loadAvg.Load15, 'g', 3, 64),
strconv.FormatFloat(float64(containerStarted.Sub(containerStarting).Nanoseconds())/float64(time.Millisecond), 'g', -1, 64),
strconv.FormatFloat(float64(containerStopped.Sub(containerStopping).Nanoseconds())/float64(time.Millisecond), 'g', -1, 64)})
}
fmt.Printf("load average: Load1: %f Load5: %f Load15: %f\n", loadAvg.Load1, loadAvg.Load5, loadAvg.Load15)
fmt.Printf("container start time: %sms\n", strconv.FormatFloat(float64(containerStarted.Sub(containerStarting).Nanoseconds())/float64(time.Millisecond), 'g', -1, 64))
fmt.Printf("container stop time: %sms\n", strconv.FormatFloat(float64(containerStopped.Sub(containerStopping).Nanoseconds())/float64(time.Millisecond), 'g', -1, 64))
}
t := time.Now()
_, testImage = filepath.Split(args[0])
prefix := fmt.Sprintf("%d-%02d-%02d_%02d-%02d_%s_%s", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), flavorType, testImage)
if flagSaveToCsv {
err = saveRecords(records, flagCsvDir, prefix+"_rkt_benchmark_interval.csv")
if err != nil {
fmt.Fprintf(os.Stderr, "Can't write to a file: %v\n", err)
}
err = saveRecords(summaryRecords, flagCsvDir, prefix+"_rkt_benchmark_summary.csv")
if err != nil {
fmt.Fprintf(os.Stderr, "Can't write to a summary file: %v\n", err)
}
}
}