func (p *Procstat) createProcesses() error {
var errstring string
var outerr error
pids, err := p.getAllPids()
if err != nil {
errstring += err.Error() + " "
}
for _, pid := range pids {
_, ok := p.pidmap[pid]
if !ok {
proc, err := process.NewProcess(pid)
if err == nil {
p.pidmap[pid] = proc
} else {
errstring += err.Error() + " "
}
}
}
if errstring != "" {
outerr = fmt.Errorf("%s", errstring)
}
return outerr
}
func (sc *ServiceCommand) waitForAnyPort(cancel <-chan struct{}, command *exec.Cmd) error {
for true {
time.Sleep(100 * time.Millisecond)
select {
case <-cancel:
return nil
default:
}
connections, err := net.Connections("all")
if err != nil {
return errors.WithStack(err)
}
proc, err := process.NewProcess(int32(command.Process.Pid))
if err != nil {
return errors.WithStack(err)
}
if hasPort(proc, connections) {
return nil
}
}
return errors.New("exited check loop unexpectedly")
}
func memoryUtil() (value int64, rate float32) {
pids := apachePID()
total := make([]int64, len(pids))
var sum int64
for i, pid := range pids {
k, _ := process.NewProcess(int32(pid))
rss, _ := k.MemoryInfo()
val := rss.RSS / 1024
total[i] = int64(val)
}
for k := 0; k < len(total); k++ {
sum = sum + total[k]
}
rat := (sum / 1024) * 100
fmt.Println(sum, rat)
return sum / 1024, float32(rat)
}
// pidStats returns the resource usage stats per pid
func (e *UniversalExecutor) pidStats() (map[string]*cstructs.ResourceUsage, error) {
stats := make(map[string]*cstructs.ResourceUsage)
e.pidLock.RLock()
pids := make(map[int]*nomadPid, len(e.pids))
for k, v := range e.pids {
pids[k] = v
}
e.pidLock.RUnlock()
for pid, np := range pids {
p, err := process.NewProcess(int32(pid))
if err != nil {
e.logger.Printf("[DEBUG] executor: unable to create new process with pid: %v", pid)
continue
}
ms := &cstructs.MemoryStats{}
if memInfo, err := p.MemoryInfo(); err == nil {
ms.RSS = memInfo.RSS
ms.Swap = memInfo.Swap
ms.Measured = ExecutorBasicMeasuredMemStats
}
cs := &cstructs.CpuStats{}
if cpuStats, err := p.Times(); err == nil {
cs.SystemMode = np.cpuStatsSys.Percent(cpuStats.System * float64(time.Second))
cs.UserMode = np.cpuStatsUser.Percent(cpuStats.User * float64(time.Second))
cs.Measured = ExecutorBasicMeasuredCpuStats
// calculate cpu usage percent
cs.Percent = np.cpuStatsTotal.Percent(cpuStats.Total() * float64(time.Second))
}
stats[strconv.Itoa(pid)] = &cstructs.ResourceUsage{MemoryStats: ms, CpuStats: cs}
}
return stats, nil
}
func (spec *Specification) createProcesses() ([]*process.Process, error) {
var out []*process.Process
var errstring string
var outerr error
pids, err := spec.getAllPids()
if err != nil {
errstring += err.Error() + " "
}
for _, pid := range pids {
p, err := process.NewProcess(int32(pid))
if err == nil {
out = append(out, p)
} else {
errstring += err.Error() + " "
}
}
if errstring != "" {
outerr = fmt.Errorf("%s", errstring)
}
return out, outerr
}
// pidStats returns the resource usage stats per pid
func (e *UniversalExecutor) pidStats() (map[string]*cstructs.ResourceUsage, error) {
stats := make(map[string]*cstructs.ResourceUsage)
e.pidLock.RLock()
pids := make([]*nomadPid, len(e.pids))
copy(pids, e.pids)
e.pidLock.RUnlock()
for _, pid := range pids {
p, err := process.NewProcess(int32(pid.pid))
if err != nil {
e.logger.Printf("[DEBUG] executor: unable to create new process with pid: %v", pid.pid)
continue
}
ms := &cstructs.MemoryStats{}
if memInfo, err := p.MemoryInfo(); err == nil {
ms.RSS = memInfo.RSS
ms.Swap = memInfo.Swap
}
cs := &cstructs.CpuStats{}
if cpuStats, err := p.Times(); err == nil {
cs.SystemMode = cpuStats.System
cs.UserMode = cpuStats.User
// calculate cpu usage percent
cs.Percent = pid.cpuStats.Percent(cpuStats.Total())
}
stats[strconv.Itoa(pid.pid)] = &cstructs.ResourceUsage{MemoryStats: ms, CpuStats: cs}
}
return stats, nil
}
func main() {
flag.Parse()
pid, err := readIntFromFile("pid")
if err != nil {
fmt.Fprintf(os.Stderr, "error reading pid: %v\n", err)
os.Exit(254)
}
process, err := process.NewProcess(pid)
if err != nil {
fmt.Fprintf(os.Stderr, "unable to create process %d instance: %v\n", pid, err)
os.Exit(254)
}
if force {
if process.Kill() != nil {
fmt.Fprintf(os.Stderr, "unable to kill process %d: %v\n", pid, err)
os.Exit(254)
}
} else {
if process.Terminate() != nil {
fmt.Fprintf(os.Stderr, "unable to terminate process %d: %v\n", pid, err)
os.Exit(254)
}
}
}
// GetComponent gets statistics about this component
func GetComponent() *api.ComponentStats {
status := new(api.ComponentStats)
status.Uptime = uint64(time.Now().Sub(startTime).Seconds())
process, err := process.NewProcess(int32(os.Getpid()))
if err == nil {
if memory, err := process.MemoryInfo(); err == nil {
status.Memory = &api.ComponentStats_MemoryStats{
Memory: memory.RSS,
Swap: memory.Swap,
}
}
if cpu, err := process.Times(); err == nil {
status.Cpu = &api.ComponentStats_CPUStats{
User: float32(cpu.User),
System: float32(cpu.System),
Idle: float32(cpu.Idle),
}
}
}
status.Goroutines = uint64(runtime.NumGoroutine())
memstats := new(runtime.MemStats)
runtime.ReadMemStats(memstats)
status.GcCpuFraction = float32(memstats.GCCPUFraction)
return status
}
// dockerData gather date from docker daemon directly (using DOCKER_HOST)
// and from proc/DOCKER_PID/status and publish those to conn
func storeProc(pidfile string, interval time.Duration) {
pid, err := loadPid(pidfile)
if err != nil {
warn(err)
return
}
p, err := process.NewProcess(int32(pid))
warn(err)
if err != nil {
return
}
// threads
threads, err := p.NumThreads()
warn(err)
if err != nil {
return
}
mi, err := p.MemoryInfo()
warn(err)
if err != nil {
return
}
procData := map[string]interface{}{"threads": threads, "vmsize": int(mi.VMS), "rss": int(mi.RSS)}
log.Println("proc = ", procData)
store("process", map[string]string{"pid": strconv.Itoa(int(pid))}, procData)
}
func (sc *ServiceConfig) Status() ([]ServiceStatus, error) {
command, err := sc.GetCommand()
if err != nil {
return nil, errors.WithStack(err)
}
status := ServiceStatus{
Service: sc,
Status: "STOPPED",
}
if command.Pid != 0 {
status.Status = "RUNNING"
status.Pid = command.Pid
proc, err := process.NewProcess(int32(command.Pid))
if err != nil {
return nil, errors.WithStack(err)
}
epochStart, err := proc.CreateTime()
if err != nil {
return nil, errors.WithStack(err)
}
status.StartTime = time.Unix(epochStart/1000, 0)
status.Ports, err = sc.getPorts(proc)
if err != nil {
return nil, errors.WithStack(err)
}
}
return []ServiceStatus{
status,
}, nil
}
//Returns the command line slice for a given process name
func getProcCmdline(procname string) (cmd []string, err error) {
var proc *process.Process
pid := getProcPID(procname)
proc, err = process.NewProcess(int32(pid))
cmd, err = proc.CmdlineSlice()
return cmd, err
}
func killAllChildren(pid int32) error {
p, err := process.NewProcess(pid)
if err != nil {
return err
}
processes := []*process.Process{p}
for i := 0; i < len(processes); i++ {
children, err := processes[i].Children()
if err != nil && err != process.ErrorNoChildren {
return err
}
processes = append(processes, children...)
}
for _, p := range processes {
osProcess, err := os.FindProcess(int(p.Pid))
if err != nil {
if err.Error() == "os: process already finished" {
continue
}
return err
}
err = osProcess.Kill()
if err != nil {
return err
}
}
return nil
}
// Run gathers ps information from gosigar.
func (p *Ps) Run() error {
pids := sigar.ProcList{}
err := pids.Get()
if err != nil {
return err
}
for _, pid := range pids.List {
state := sigar.ProcState{}
mem := sigar.ProcMem{}
time := sigar.ProcTime{}
if err := state.Get(pid); err != nil {
continue
}
if err := mem.Get(pid); err != nil {
continue
}
if err := time.Get(pid); err != nil {
continue
}
procData := make(map[string]interface{})
procData["Name"] = state.Name
procData["Pid"] = pid
procData["ParentPid"] = state.Ppid
procData["StartTime"] = time.FormatStartTime()
procData["RunTime"] = time.FormatTotal()
procData["MemoryResident"] = mem.Resident / 1024
procData["State"] = string(state.State)
gopsutilProcess, err := gopsutil_process.NewProcess(int32(pid))
if err != nil {
continue
}
mmaps, err := gopsutilProcess.MemoryMaps(false)
if err == nil {
procData["MemoryMaps"] = mmaps
}
ios, err := gopsutilProcess.IOCounters()
if err == nil {
procData["IOCounters"] = ios
}
ctxSwitches, err := gopsutilProcess.NumCtxSwitches()
if err == nil {
procData["CtxSwitches"] = ctxSwitches
}
if len(procData) > 0 {
p.Data[strconv.Itoa(pid)] = procData
}
}
return nil
}
func runCpuMonitor(monitor *CpuLoadMonitor) {
pid := os.Getpid()
process, _ := process.NewProcess(int32(pid))
for monitor.running {
monitor.sample, _ = process.CPUPercent(0)
monitor.cpu = monitor.alpha*monitor.sample + (1-monitor.alpha)*monitor.cpu
time.Sleep(monitor.samplingInterval)
}
}
func (ds *DaemonStat) ProcessStats() {
ds.PointsRate = float64(ds.PointsCounter) / float64(ds.Interval)
p, err := process.NewProcess(int32(os.Getpid()))
if err == nil {
ds.MemGauge, _ = p.MemoryInfo()
} else {
ds.MemGauge = nil
log.Errorf("%v", err)
Stat.ErrorIncr()
}
}
func ProcessTable() map[int32]string {
res := make(map[int32]string)
pids, _ := process.Pids()
for i := 0; i < len(pids); i++ {
pid := pids[i]
proc, _ := process.NewProcess(pid)
cline, _ := proc.Cmdline()
res[pid] = cline
}
return res
}
func getCPU() {
stats, _ := process.NewProcess(int32(os.Getpid()))
for {
cpu, _ := stats.Percent(1 * time.Second)
if cpu > 30 {
log.Printf("CPU:%v\n", cpu)
go WatchCPU()
}
time.Sleep(1 * time.Second)
}
}
// dockerData gather date from docker daemon directly (using DOCKER_HOST)
// and from proc/DOCKER_PID/status and publish those to conn
func dockerData(tags string, conn net.Conn) {
if os.Getenv("DOCKER_HOST") == "" {
log.Fatal(`please provide eg. DOCKER_HOST="tcp://127.0.0.1:8080"`)
}
// pid
pidS := os.Getenv("DOCKER_PID")
if pidS == "" {
log.Fatal("cannot find docker deamon - please provide DOCKER_PID - try DOCKER_PID=`sudo lsof -t -sTCP:LISTEN -i :8080`")
}
pid, err := strconv.Atoi(pidS)
if err != nil {
log.Fatal(err)
}
log.Printf("docker pid=%d", pid)
// docker client
dockerClient, _ := docker.NewClientFromEnv()
for {
p, err := process.NewProcess(int32(pid))
// threads
threads, err := p.NumThreads()
if err != nil {
log.Fatal(err)
}
mi, err := p.MemoryInfo()
if err != nil {
log.Fatal(err)
}
rss := mi.RSS
vms := mi.VMS
// docker info
info, err := dockerClient.Info()
// log.Printf("info=%#v", info)
output := fmt.Sprintf("docker,driver=%s%s containers=%di,goroutines=%di,images=%di,threads=%di,vmsize=%di,rss=%di",
info.Get("Driver"), tags, info.GetInt("Containers"), info.GetInt("NGoroutines"), info.GetInt("Images"), threads, vms, rss)
n, err := conn.Write([]byte(output))
if err != nil {
log.Fatal(err)
}
log.Println(n, output)
time.Sleep(1 * time.Second)
}
}
func GetLoggingLevelByInspectingParent() (logrus.Level, error) {
ppid := os.Getppid()
process, err := process.NewProcess(int32(ppid))
if err != nil {
return logrus.WarnLevel, err
}
cmdline, err := process.Cmdline()
if err != nil {
return logrus.WarnLevel, err
}
if strings.Contains(cmdline, "-vv") {
return logrus.DebugLevel, nil
} else if strings.Contains(cmdline, "-v") {
return logrus.InfoLevel, nil
}
return logrus.WarnLevel, nil
}
func main() {
flag.Parse()
pid, err := readIntFromFile("ppid")
if err != nil {
fmt.Fprintf(os.Stderr, "error reading pid: %v\n", err)
os.Exit(1)
}
process, err := process.NewProcess(pid)
if err != nil {
fmt.Fprintf(os.Stderr, "unable to create process %d instance: %v\n", pid, err)
os.Exit(1)
}
if force {
children, err := process.Children()
if err != nil {
fmt.Fprintf(os.Stderr, "cannot get child processes from %d: %v\n", pid, err)
os.Exit(1)
}
for _, child := range children {
if err := child.Kill(); err != nil {
fmt.Fprintf(os.Stderr, "unable to kill process %d: %v\n", child.Pid, err)
os.Exit(1)
}
}
if err := process.Kill(); err != nil {
fmt.Fprintf(os.Stderr, "unable to kill process %d: %v\n", pid, err)
os.Exit(1)
}
} else {
if process.Terminate() != nil {
fmt.Fprintf(os.Stderr, "unable to terminate process %d: %v\n", pid, err)
os.Exit(1)
}
}
}
// Run runs the trigger. encountered errors go to the configured channel (if any).
// you probably want to run this in a new goroutine.
func (cpu Cpu) Run() {
tick := time.NewTicker(cpu.checkEvery)
pid := os.Getpid()
p, err := process.NewProcess(int32(pid))
if err != nil {
cpu.logError(err)
return
}
for ts := range tick.C {
percent, err := p.Percent(0)
if err != nil {
cpu.logError(err)
continue
}
//fmt.Println("percent is now", percent)
unix := ts.Unix()
// we discard the decimals of the percentage. an integer with percent resolution should be good enough.
if int(percent) >= cpu.threshold && unix >= cpu.lastUnix+int64(cpu.minTimeDiff) {
f, err := os.Create(fmt.Sprintf("%s/%d.profile-cpu", cpu.path, unix))
if err != nil {
cpu.logError(err)
continue
}
err = pprof.StartCPUProfile(f)
if err != nil {
cpu.logError(err)
}
time.Sleep(cpu.profDur)
pprof.StopCPUProfile()
cpu.lastUnix = unix
err = f.Close()
if err != nil {
cpu.logError(err)
}
}
}
}
func getUsage(pid int32) ([]*ProcessStatus, error) {
var statuses []*ProcessStatus
pids := []int32{pid}
for i := 0; i < len(pids); i++ {
proc, ok := pidMap[pids[i]]
if !ok {
var err error
proc, err = process.NewProcess(pids[i])
if err != nil {
return nil, err
}
pidMap[pids[i]] = proc
}
s, err := getProcStatus(proc)
if err != nil {
return nil, err
}
statuses = append(statuses, s)
children, err := proc.Children()
if err != nil && err != process.ErrorNoChildren {
return nil, err
}
childloop:
for _, child := range children {
for _, p := range pids {
if p == child.Pid {
fmt.Printf("%d is in %#v\n", p, pids)
continue childloop
}
}
pids = append(pids, child.Pid)
}
}
return statuses, nil
}
func (s *ServiceConfig) GetCommand() (*ServiceCommand, error) {
s.printf("Building control command for: %v\n", s.Name)
dir := home.EdwardConfig.LogDir
logs := struct {
Build string
Run string
Stop string
}{
Build: path.Join(dir, s.Name+"-build.log"),
Run: path.Join(dir, s.Name+".log"),
Stop: path.Join(dir, s.Name+"-stop.log"),
}
path := ""
if s.Path != nil {
path = *s.Path
}
command := &ServiceCommand{
Service: s,
Scripts: struct {
Build Script
Launch Script
Stop Script
}{
Build: Script{
Path: path,
Command: s.Commands.Build,
Log: logs.Build,
},
Launch: Script{
Path: path,
Command: s.Commands.Launch,
Log: logs.Run,
},
Stop: Script{
Path: path,
Command: s.Commands.Stop,
Log: logs.Stop,
},
},
Logger: s.Logger,
}
// Retrieve the PID if available
pidFile := command.getPidPath()
s.printf("Checking pidfile for %v: %v\n", s.Name, pidFile)
if _, err := os.Stat(pidFile); err == nil {
dat, err := ioutil.ReadFile(pidFile)
if err != nil {
return nil, errors.WithStack(err)
}
pid, err := strconv.Atoi(string(dat))
if err != nil {
return nil, errors.WithStack(err)
}
command.Pid = pid
exists, err := process.PidExists(int32(command.Pid))
if err != nil {
return nil, errors.WithStack(err)
}
if !exists {
s.printf("Process for %v was not found, resetting.\n", s.Name)
command.clearState()
}
proc, err := process.NewProcess(int32(command.Pid))
if err != nil {
return nil, errors.WithStack(err)
}
cmdline, err := proc.Cmdline()
if err != nil {
return nil, errors.WithStack(err)
}
if !strings.Contains(cmdline, s.Name) {
s.printf("Process for %v was not as expected (found %v), resetting.\n", s.Name, cmdline)
command.clearState()
}
} else {
s.printf("No pidfile for %v", s.Name)
}
// TODO: Set status
return command, nil
}
// currentState returns the current state of system conforming
// to definitions
func currentState(d Def) State {
s, err := host.HostInfo()
if err != nil {
log.Printf("failed to load host information. (%v)", err.Error())
return State{}
}
state := State{
Host: s.Hostname,
Uptime: s.Uptime,
Ports: []Port{},
Sockets: []Socket{},
Processes: []Process{},
}
pids, err := process.Pids()
if err != nil {
log.Printf("failed to query running processes. (%v)", err.Error())
return State{}
}
for _, i := range pids {
p, err := process.NewProcess(i)
if err != nil {
log.Printf("failed to get handle of process with ID: %v. (%v)", i, err.Error())
continue
}
n, _ := p.Name()
cmd, err := p.Cmdline()
if err != nil {
log.Printf("failed to query the command which started the process with ID: %v. (%v)", i, err.Error())
continue
}
cnt := false
for k := range d.Processes {
if strings.Contains(n, k) {
cnt = true
break
}
}
conn, err := p.Connections()
if err != nil {
log.Printf("failed to query connections made by process with ID: %v. (%v)", i, err.Error())
continue
}
pr := Process{
Name: n,
Command: cmd,
Connections: []Conn{},
}
for _, c := range conn {
ncn := Conn{Local: c.Laddr, Remote: c.Raddr}
pr.Connections = append(pr.Connections, ncn)
if _, match := d.Ports[ncn.Local.Port]; match {
state.Ports = append(state.Ports, Port(ncn.Remote.Port))
}
if cnt {
state.Processes = append(state.Processes, pr)
}
}
}
for k := range d.Sockets {
if _, err := os.Stat(k); err != nil {
state.Sockets = append(state.Sockets, Socket(k))
}
}
return state
}
func (r region) communicate() {
//collect region statistics
ticker := time.NewTicker(5 * time.Second)
//object holding process reference
var exe *exec.Cmd
exe = nil
var start time.Time
var proc *process.Process
//process communication
terminated := make(chan bool)
for {
select {
case <-terminated:
//the process exited for some Reason
exe = nil
case cmd := <-r.cmds:
switch cmd.command {
case "start":
//if already running, exit
if exe != nil {
r.log.Error("Region is already running", r.UUID)
continue
}
//execute binaries
os.Chdir(r.dir)
cmdName := "/usr/bin/mono"
cmdArgs := []string{"OpenSim.exe", "-console", "rest"}
exe = exec.Command(cmdName, cmdArgs...)
err := exe.Start()
if err != nil {
errMsg := fmt.Sprintf("Error starting process: %s", err.Error())
r.log.Error(errMsg)
continue
}
r.log.Info("Started Successfully")
start = time.Now()
proc, _ = process.NewProcess(int32(exe.Process.Pid))
go func() {
//wait for process, ignoring process-specific errors
_ = exe.Wait()
r.log.Error("Terminated")
exe = nil
terminated <- true
}()
case "kill":
//if not running, exit
if exe == nil {
errMsg := fmt.Sprintf("Kill region %v failed, region is not running", r.UUID.String())
r.log.Error(errMsg)
continue
}
if err := exe.Process.Kill(); err != nil {
errMsg := fmt.Sprintf("Error killing process: %s", err.Error())
r.log.Error(errMsg)
}
default:
r.log.Info("Received unexpected command: %v", cmd.command)
}
case <-ticker.C:
stat := mgm.RegionStat{UUID: r.UUID}
if exe == nil {
//trivially halted if we never started
r.rStat <- stat
continue
}
stat.Running = true
cpuPercent, err := proc.CPUPercent(0)
if err != nil {
errMsg := fmt.Sprintf("Error getting cpu for pid: %s", err.Error())
r.log.Error(errMsg)
} else {
stat.CPUPercent = cpuPercent
}
//memory info from this module may not be correct....
memInfo, err := proc.MemoryInfo()
if err != nil {
errMsg := fmt.Sprintf("Error getting memory for pid: %s", err.Error())
r.log.Error(errMsg)
} else {
stat.MemKB = (float64(memInfo.RSS) / 1024.0)
}
elapsed := time.Since(start)
stat.Uptime = elapsed
r.rStat <- stat
}
}
}
func runRktMonitor(cmd *cobra.Command, args []string) {
if len(args) != 1 {
cmd.Usage()
os.Exit(254)
}
d, err := time.ParseDuration(flagDuration)
if err != nil {
fmt.Printf("%v\n", err)
os.Exit(254)
}
if os.Getuid() != 0 {
fmt.Printf("need to be root to run rkt images\n")
os.Exit(254)
}
f, err := os.Open(args[0])
if err != nil {
fmt.Printf("%v\n", err)
os.Exit(254)
}
decoder := json.NewDecoder(f)
podManifest := false
man := schema.PodManifest{}
err = decoder.Decode(&man)
if err == nil {
podManifest = true
}
var flavorType, testImage string
if flagStage1Path == "" {
flavorType = "stage1-coreos.aci"
} else {
if !fileExist(flagStage1Path) {
fmt.Fprintln(os.Stderr, "Given stage1 file path doesn't exist")
os.Exit(254)
}
_, flavorType = filepath.Split(flagStage1Path)
}
var containerId string
var stopCmd, execCmd, gcCmd *exec.Cmd
var loadAvg *load.AvgStat
var containerStarting, containerStarted, containerStopping, containerStopped time.Time
records := [][]string{{"Time", "PID name", "PID number", "RSS", "CPU"}} // csv headers
summaryRecords := [][]string{{"Load1", "Load5", "Load15", "StartTime", "StopTime"}} // csv summary headers
var rktBinary string
if flagRktDir != "" {
rktBinary = filepath.Join(flagRktDir, "rkt")
if !fileExist(rktBinary) {
fmt.Fprintln(os.Stderr, "rkt binary not found!")
os.Exit(1)
}
} else {
rktBinary = "rkt"
}
for i := 0; i < flagRepetitionNumber; i++ {
// build argument list for execCmd
argv := []string{"run", "--debug"}
if flagStage1Path != "" {
argv = append(argv, fmt.Sprintf("--stage1-path=%v", flagStage1Path))
}
if podManifest {
argv = append(argv, "--pod-manifest", args[0])
} else {
argv = append(argv, args[0], "--insecure-options=image")
}
argv = append(argv, "--net=default-restricted")
execCmd = exec.Command(rktBinary, argv...)
if flagShowOutput {
execCmd.Stderr = os.Stderr
}
cmdReader, err := execCmd.StdoutPipe()
if err != nil {
fmt.Fprintln(os.Stderr, "Error creating StdoutPipe for execCmd", err)
os.Exit(254)
}
execCmdScanner := bufio.NewScanner(cmdReader)
startConfirmation := make(chan string, 1)
go func() {
var containerId string
for execCmdScanner.Scan() {
if flagShowOutput {
fmt.Println(execCmdScanner.Text())
}
if strings.Contains(execCmdScanner.Text(), "APP-STARTED!") {
startConfirmation <- containerId
} else if strings.Contains(execCmdScanner.Text(), "Set hostname to") {
sl := strings.SplitAfter(execCmdScanner.Text(), "<rkt-")
containerId = sl[len(sl)-1]
containerId = containerId[:len(containerId)-2]
}
}
}()
containerStarting = time.Now()
err = execCmd.Start()
containerId = <-startConfirmation
containerStarted = time.Now() //here we are sure - container is running (issue: #3019)
close(startConfirmation)
if flagShowOutput {
execCmd.Stdout = os.Stdout
}
if err != nil {
fmt.Printf("%v\n", err)
os.Exit(254)
}
usages := make(map[int32][]*ProcessStatus)
timeToStop := time.Now().Add(d)
for time.Now().Before(timeToStop) {
usage, err := getUsage(int32(execCmd.Process.Pid))
if err != nil {
panic(err)
}
if flagVerbose {
printUsage(usage)
}
if flagSaveToCsv {
records = addRecords(usage, records)
}
for _, ps := range usage {
usages[ps.Pid] = append(usages[ps.Pid], ps)
}
_, err = process.NewProcess(int32(execCmd.Process.Pid))
if err != nil {
// process.Process.IsRunning is not implemented yet
fmt.Fprintf(os.Stderr, "rkt exited prematurely\n")
break
}
time.Sleep(time.Second)
}
loadAvg, err = load.Avg()
if err != nil {
fmt.Fprintf(os.Stderr, "measure load avg failed: %v\n", err)
}
stopCmd = exec.Command(rktBinary, "stop", containerId)
cmdStopReader, err := stopCmd.StdoutPipe()
if err != nil {
fmt.Fprintln(os.Stderr, "Error creating StdoutPipe for stopCmd", err)
os.Exit(254)
}
cmdStopScanner := bufio.NewScanner(cmdStopReader)
containerStopping = time.Now()
stopConfirmation := make(chan bool, 1)
go func() {
for cmdStopScanner.Scan() {
if strings.Contains(cmdStopScanner.Text(), containerId) {
stopConfirmation <- true
return
}
}
stopConfirmation <- false
}()
err = stopCmd.Start()
if !<-stopConfirmation {
fmt.Println("WARNING: There was a problem stopping the container! (Container already stopped?)")
}
containerStopped = time.Now()
close(stopConfirmation)
if err != nil {
fmt.Printf("%v\n", err)
os.Exit(254)
}
gcCmd = exec.Command(rktBinary, "gc", "--grace-period=0")
gcCmd.Start()
for _, processHistory := range usages {
var avgCPU float64
var avgMem uint64
var peakMem uint64
for _, p := range processHistory {
avgCPU += p.CPU
avgMem += p.RSS
if peakMem < p.RSS {
peakMem = p.RSS
}
}
avgCPU = avgCPU / float64(len(processHistory))
avgMem = avgMem / uint64(len(processHistory))
if !flagSaveToCsv {
fmt.Printf("%s(%d): seconds alive: %d avg CPU: %f%% avg Mem: %s peak Mem: %s\n", processHistory[0].Name, processHistory[0].Pid, len(processHistory), avgCPU, formatSize(avgMem), formatSize(peakMem))
}
}
if flagSaveToCsv {
summaryRecords = append(summaryRecords, []string{
strconv.FormatFloat(loadAvg.Load1, 'g', 3, 64),
strconv.FormatFloat(loadAvg.Load5, 'g', 3, 64),
strconv.FormatFloat(loadAvg.Load15, 'g', 3, 64),
strconv.FormatFloat(float64(containerStarted.Sub(containerStarting).Nanoseconds())/float64(time.Millisecond), 'g', -1, 64),
strconv.FormatFloat(float64(containerStopped.Sub(containerStopping).Nanoseconds())/float64(time.Millisecond), 'g', -1, 64)})
}
fmt.Printf("load average: Load1: %f Load5: %f Load15: %f\n", loadAvg.Load1, loadAvg.Load5, loadAvg.Load15)
fmt.Printf("container start time: %sms\n", strconv.FormatFloat(float64(containerStarted.Sub(containerStarting).Nanoseconds())/float64(time.Millisecond), 'g', -1, 64))
fmt.Printf("container stop time: %sms\n", strconv.FormatFloat(float64(containerStopped.Sub(containerStopping).Nanoseconds())/float64(time.Millisecond), 'g', -1, 64))
}
t := time.Now()
_, testImage = filepath.Split(args[0])
prefix := fmt.Sprintf("%d-%02d-%02d_%02d-%02d_%s_%s", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), flavorType, testImage)
if flagSaveToCsv {
err = saveRecords(records, flagCsvDir, prefix+"_rkt_benchmark_interval.csv")
if err != nil {
fmt.Fprintf(os.Stderr, "Can't write to a file: %v\n", err)
}
err = saveRecords(summaryRecords, flagCsvDir, prefix+"_rkt_benchmark_summary.csv")
if err != nil {
fmt.Fprintf(os.Stderr, "Can't write to a summary file: %v\n", err)
}
}
}
func runRktMonitor(cmd *cobra.Command, args []string) {
if len(args) != 1 {
cmd.Usage()
os.Exit(1)
}
d, err := time.ParseDuration(flagDuration)
if err != nil {
fmt.Printf("%v\n", err)
os.Exit(1)
}
if os.Getuid() != 0 {
fmt.Printf("need to be root to run rkt images\n")
os.Exit(1)
}
f, err := os.Open(args[0])
if err != nil {
fmt.Printf("%v\n", err)
os.Exit(1)
}
decoder := json.NewDecoder(f)
podManifest := false
man := schema.PodManifest{}
err = decoder.Decode(&man)
if err == nil {
podManifest = true
}
var flavorType string
if flagStage1Path == "" {
flavorType = "stage1-coreos.aci"
} else {
_, flavorType = filepath.Split(flagStage1Path)
}
var execCmd *exec.Cmd
var loadAvg *load.AvgStat
var containerStarting, containerStarted, containerStopping, containerStopped time.Time
records := [][]string{{"Time", "PID name", "PID number", "RSS", "CPU"}} // csv headers
summaryRecords := [][]string{{"Load1", "Load5", "Load15", "StartTime", "StopTime"}} // csv summary headers
var rktBinary string
if flagRktDir != "" {
rktBinary = flagRktDir + "/rkt"
} else {
rktBinary = "rkt"
}
for i := 0; i < flagRepetitionNumber; i++ {
containerStarting = time.Now()
// build argument list for execCmd
argv := []string{"run"}
if flagStage1Path != "" {
argv = append(argv, fmt.Sprintf("--stage1-path=%v", flagStage1Path))
}
if podManifest {
argv = append(argv, "--pod-manifest", args[0])
} else {
argv = append(argv, args[0], "--insecure-options=image")
}
argv = append(argv, "--net=default-restricted")
execCmd = exec.Command(rktBinary, argv...)
if flagShowOutput {
execCmd.Stdout = os.Stdout
execCmd.Stderr = os.Stderr
}
err = execCmd.Start()
containerStarted = time.Now()
if err != nil {
fmt.Printf("%v\n", err)
os.Exit(1)
}
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
go func() {
for range c {
err := killAllChildren(int32(execCmd.Process.Pid))
if err != nil {
fmt.Fprintf(os.Stderr, "cleanup failed: %v\n", err)
}
os.Exit(1)
}
}()
usages := make(map[int32][]*ProcessStatus)
timeToStop := time.Now().Add(d)
for time.Now().Before(timeToStop) {
usage, err := getUsage(int32(execCmd.Process.Pid))
if err != nil {
panic(err)
}
if flagVerbose {
printUsage(usage)
}
if flagSaveToCsv {
records = addRecords(usage, records)
}
for _, ps := range usage {
usages[ps.Pid] = append(usages[ps.Pid], ps)
}
_, err = process.NewProcess(int32(execCmd.Process.Pid))
if err != nil {
// process.Process.IsRunning is not implemented yet
fmt.Fprintf(os.Stderr, "rkt exited prematurely\n")
break
}
time.Sleep(time.Second)
}
loadAvg, err = load.Avg()
if err != nil {
fmt.Fprintf(os.Stderr, "measure load avg failed: %v\n", err)
}
containerStopping = time.Now()
err = killAllChildren(int32(execCmd.Process.Pid))
containerStopped = time.Now()
if err != nil {
fmt.Fprintf(os.Stderr, "cleanup failed: %v\n", err)
}
for _, processHistory := range usages {
var avgCPU float64
var avgMem uint64
var peakMem uint64
for _, p := range processHistory {
avgCPU += p.CPU
avgMem += p.RSS
if peakMem < p.RSS {
peakMem = p.RSS
}
}
avgCPU = avgCPU / float64(len(processHistory))
avgMem = avgMem / uint64(len(processHistory))
if !flagSaveToCsv {
fmt.Printf("%s(%d): seconds alive: %d avg CPU: %f%% avg Mem: %s peak Mem: %s\n", processHistory[0].Name, processHistory[0].Pid, len(processHistory), avgCPU, formatSize(avgMem), formatSize(peakMem))
}
}
if flagSaveToCsv {
summaryRecords = append(summaryRecords, []string{
strconv.FormatFloat(loadAvg.Load1, 'g', 3, 64),
strconv.FormatFloat(loadAvg.Load5, 'g', 3, 64),
strconv.FormatFloat(loadAvg.Load15, 'g', 3, 64),
strconv.FormatInt(containerStarted.Sub(containerStarting).Nanoseconds(), 10),
strconv.FormatInt(containerStopped.Sub(containerStopping).Nanoseconds(), 10)})
}
fmt.Printf("load average: Load1: %f Load5: %f Load15: %f\n", loadAvg.Load1, loadAvg.Load5, loadAvg.Load15)
fmt.Printf("container start time: %dns\n", containerStarted.Sub(containerStarting).Nanoseconds())
fmt.Printf("container stop time: %dns\n", containerStopped.Sub(containerStopping).Nanoseconds())
}
t := time.Now()
prefix := fmt.Sprintf("%d-%02d-%02d_%02d-%02d_%s_", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), flavorType)
if flagSaveToCsv {
err = saveRecords(records, flagCsvDir, prefix+"rkt_benchmark_interval.csv")
if err != nil {
fmt.Fprintf(os.Stderr, "Can't write to a file: %v\n", err)
}
err = saveRecords(summaryRecords, flagCsvDir, prefix+"rkt_benchmark_summary.csv")
if err != nil {
fmt.Fprintf(os.Stderr, "Can't write to a summary file: %v\n", err)
}
}
}
// Run gathers ps information from gosigar.
func (p *Ps) Run() error {
pids := sigar.ProcList{}
err := pids.Get()
if err != nil {
return err
}
for _, pid := range pids.List {
state := sigar.ProcState{}
mem := sigar.ProcMem{}
time := sigar.ProcTime{}
if err := state.Get(pid); err != nil {
continue
}
if err := mem.Get(pid); err != nil {
continue
}
if err := time.Get(pid); err != nil {
continue
}
// If NameFilter is defined, skip process if its name does not match filter.
if len(p.NameFilter) > 0 {
matched := false
for _, nameFilterInterface := range p.NameFilter {
nameFilter := nameFilterInterface.(string)
if strings.Contains(state.Name, nameFilter) {
matched = true
break
}
}
if matched == false {
continue
}
}
procData := make(map[string]interface{})
procData["Name"] = state.Name
procData["Pid"] = pid
procData["ParentPid"] = state.Ppid
procData["StartTime"] = time.FormatStartTime()
procData["RunTime"] = time.FormatTotal()
procData["MemoryResident"] = mem.Resident / 1024
procData["State"] = string(state.State)
gopsutilProcess, err := gopsutil_process.NewProcess(int32(pid))
if err != nil {
continue
}
mmaps, err := gopsutilProcess.MemoryMaps(false)
if err == nil {
procData["MemoryMaps"] = mmaps
}
ios, err := gopsutilProcess.IOCounters()
if err == nil {
procData["IOCounters"] = ios
}
ctxSwitches, err := gopsutilProcess.NumCtxSwitches()
if err == nil {
procData["CtxSwitches"] = ctxSwitches
}
if len(procData) > 0 {
p.Data[strconv.Itoa(pid)] = procData
}
}
return nil
}