func markTerminated(conn *zk.Conn, hss *zzk.HostServiceState) {
ssPath := zzk.ServiceStatePath(hss.ServiceId, hss.ServiceStateId)
_, stats, err := conn.Get(ssPath)
if err != nil {
glog.V(0).Infof("Unable to get service state %s for delete because: %v", ssPath, err)
return
}
err = conn.Delete(ssPath, stats.Version)
if err != nil {
glog.V(0).Infof("Unable to delete service state %s because: %v", ssPath, err)
return
}
hssPath := zzk.HostServiceStatePath(hss.HostId, hss.ServiceStateId)
_, stats, err = conn.Get(hssPath)
if err != nil {
glog.V(0).Infof("Unable to get host service state %s for delete becaus: %v", hssPath, err)
return
}
err = conn.Delete(hssPath, stats.Version)
if err != nil {
glog.V(0).Infof("Unable to delete host service state %s", hssPath)
}
}
func (a *HostAgent) waitForProcessToDie(conn *zk.Conn, cmd *exec.Cmd, procFinished chan<- int, serviceState *dao.ServiceState) {
a.dockerRemove(serviceState.Id)
defer func() {
procFinished <- 1
}()
// save the last circularBufferSize bytes from each container run
lastStdout := circular.NewBuffer(circularBufferSize)
lastStderr := circular.NewBuffer(circularBufferSize)
if stdout, err := cmd.StdoutPipe(); err != nil {
glog.Errorf("Unable to read standard out for service state %s: %v", serviceState.Id, err)
return
} else {
go io.Copy(lastStdout, stdout)
}
if stderr, err := cmd.StderrPipe(); err != nil {
glog.Errorf("Unable to read standard error for service state %s: %v", serviceState.Id, err)
return
} else {
go io.Copy(lastStderr, stderr)
}
if err := cmd.Start(); err != nil {
glog.Errorf("Problem starting command '%s %s': %v", cmd.Path, cmd.Args, err)
dumpOut(lastStdout, lastStderr, circularBufferSize)
return
}
// We are name the container the same as its service state ID, so use that as an alias
dockerId := serviceState.Id
serviceState.DockerId = dockerId
time.Sleep(1 * time.Second) // Sleep to give docker a chance to start
var containerState ContainerState
var err error
for i := 0; i < 30; i++ {
if containerState, err = getDockerState(dockerId); err != nil {
time.Sleep(3 * time.Second) // Sleep to give docker a chance to start
glog.V(2).Infof("Problem getting service state for %s :%v", serviceState.Id, err)
a.dockerTerminate(dockerId)
dumpOut(lastStdout, lastStderr, circularBufferSize)
} else {
break
}
}
if err != nil {
return
}
if err = zzk.LoadAndUpdateServiceState(conn, serviceState.ServiceId, serviceState.Id, func(ss *dao.ServiceState) {
ss.DockerId = containerState.ID
ss.Started = time.Now()
ss.Terminated = time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC)
ss.PrivateIp = containerState.NetworkSettings.IPAddress
ss.PortMapping = containerState.NetworkSettings.Ports
}); err != nil {
glog.Warningf("Unable to update service state %s: %v", serviceState.Id, err)
} else {
glog.V(1).Infof("SSPath: %s, PortMapping: %v", zzk.ServiceStatePath(serviceState.ServiceId, serviceState.Id), serviceState.PortMapping)
if err := cmd.Wait(); err != nil {
if exiterr, ok := err.(*exec.ExitError); ok {
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
statusCode := status.ExitStatus()
switch {
case statusCode == 137:
glog.V(1).Infof("Docker process killed: %s", serviceState.Id)
case statusCode == 2:
glog.V(1).Infof("Docker process stopped: %s", serviceState.Id)
default:
glog.V(0).Infof("Docker process %s exited with code %d", serviceState.Id, statusCode)
dumpOut(lastStdout, lastStderr, circularBufferSize)
}
}
} else {
glog.V(1).Info("Unable to determine exit code for %s", serviceState.Id)
}
} else {
glog.V(0).Infof("Process for service state %s finished", serviceState.Id)
}
if err = zzk.ResetServiceState(conn, serviceState.ServiceId, serviceState.Id); err != nil {
glog.Errorf("Caught error marking process termination time for %s: %v", serviceState.Id, err)
}
}
}