func statePreparing(ctx *VmContext, ev VmEvent) {
switch ev.Event() {
case EVENT_VM_EXIT, ERROR_INTERRUPTED:
glog.Info("VM exited before start...")
case COMMAND_SHUTDOWN, COMMAND_RELEASE:
glog.Info("got shutdown or release command, not started yet")
ctx.reportVmShutdown()
ctx.Become(nil, "NONE")
case COMMAND_EXEC:
ctx.execCmd(ev.(*ExecCommand))
case COMMAND_WINDOWSIZE:
cmd := ev.(*WindowSizeCommand)
ctx.setWindowSize(cmd.ClientTag, cmd.Size)
case COMMAND_RUN_POD, COMMAND_REPLACE_POD:
glog.Info("got spec, prepare devices")
if ok := ctx.lazyPrepareDevice(ev.(*RunPodCommand)); ok {
ctx.startSocks()
ctx.DCtx.(LazyDriverContext).LazyLaunch(ctx)
ctx.setTimeout(60)
ctx.Become(stateStarting, "STARTING")
} else {
glog.Warning("Fail to prepare devices, quit")
ctx.Become(nil, "None")
}
default:
glog.Warning("got event during pod initiating")
}
}
func (daemon *Daemon) CmdTty(job *engine.Job) (err error) {
if len(job.Args) < 3 {
return nil
}
var (
podID = job.Args[0]
tag = job.Args[1]
h = job.Args[2]
w = job.Args[3]
container string
vmid string
)
if strings.Contains(podID, "pod-") {
container = ""
vmid, err = daemon.GetPodVmByName(podID)
if err != nil {
return err
}
} else if strings.Contains(podID, "vm-") {
vmid = podID
} else {
container = podID
podID, err = daemon.GetPodByContainer(container)
if err != nil {
return err
}
vmid, err = daemon.GetPodVmByName(podID)
if err != nil {
return err
}
}
vm, ok := daemon.VmList[vmid]
if !ok {
return fmt.Errorf("vm %s doesn't exist!")
}
row, err := strconv.Atoi(h)
if err != nil {
glog.Warning("Window row %s incorrect!", h)
}
column, err := strconv.Atoi(w)
if err != nil {
glog.Warning("Window column %s incorrect!", h)
}
err = vm.Tty(tag, row, column)
if err != nil {
return err
}
glog.V(1).Infof("Success to resize the tty!")
return nil
}
func stateDestroying(ctx *VmContext, ev VmEvent) {
if processed, _ := deviceRemoveHandler(ctx, ev); processed {
if closed := ctx.tryClose(); closed {
glog.Info("resources reclaimed, quit...")
}
} else {
switch ev.Event() {
case EVENT_VM_EXIT:
glog.Info("Got VM shutdown event")
ctx.unsetTimeout()
if closed := ctx.onVmExit(false); closed {
glog.Info("VM Context closed.")
}
case EVENT_VM_KILL:
glog.Info("Got VM force killed message")
ctx.unsetTimeout()
if closed := ctx.onVmExit(true); closed {
glog.Info("VM Context closed.")
}
case ERROR_INTERRUPTED:
glog.V(1).Info("Connection interrupted while destroying")
case COMMAND_RELEASE:
glog.Info("vm destroying, got release")
ctx.reportVmShutdown()
case EVENT_VM_TIMEOUT:
glog.Info("Device removing timeout")
ctx.Close()
default:
glog.Warning("got event during vm cleaning up")
}
}
}
func (vm *Vm) handlePodEvent(mypod *Pod) {
glog.V(1).Infof("hyperHandlePodEvent pod %s, vm %s", mypod.Id, vm.Id)
_, ret2, ret3, err := vm.GetVmChan()
if err != nil {
return
}
glog.V(1).Infof("hyperHandlePodEvent pod %s, vm %s", mypod.Id, vm.Id)
Status := ret2.(chan *types.VmResponse)
subStatus := ret3.(chan *types.VmResponse)
for {
defer func() {
err := recover()
if err != nil {
glog.Warning("panic during send shutdown message to channel")
}
}()
Response := <-Status
subStatus <- Response
exit := mypod.Handler.Handle(Response, mypod.Handler.Data, mypod, vm)
if exit {
break
}
}
}
func watchDog(qc *QemuContext, hub chan hypervisor.VmEvent) {
wdt := qc.wdt
for {
msg, ok := <-wdt
if ok {
switch msg {
case "quit":
glog.V(1).Info("quit watch dog.")
return
case "kill":
success := false
if qc.process != nil {
glog.V(0).Infof("kill Qemu... %d", qc.process.Pid)
if err := qc.process.Kill(); err == nil {
success = true
}
} else {
glog.Warning("no process to be killed")
}
hub <- &hypervisor.VmKilledEvent{Success: success}
return
}
} else {
glog.V(1).Info("chan closed, quit watch dog.")
break
}
}
}
func stateCleaning(ctx *VmContext, ev VmEvent) {
if processed := commonStateHandler(ctx, ev, false); processed {
} else if processed, success := deviceRemoveHandler(ctx, ev); processed {
if !success {
glog.Warning("fail to unplug devices for stop")
ctx.poweroffVM(true, "fail to unplug devices")
ctx.Become(stateDestroying, "DESTROYING")
} else if ctx.deviceReady() {
// ctx.reset()
// ctx.unsetTimeout()
// ctx.reportPodStopped()
// glog.V(1).Info("device ready, could run pod.")
// ctx.Become(stateInit, "INIT")
ctx.vm <- &DecodedMessage{
code: INIT_READY,
message: []byte{},
}
glog.V(1).Info("device ready, could run pod.")
}
} else if processed := initFailureHandler(ctx, ev); processed {
ctx.poweroffVM(true, "fail to unplug devices")
ctx.Become(stateDestroying, "DESTROYING")
} else {
switch ev.Event() {
case COMMAND_RELEASE:
glog.Info("vm cleaning to idle, got release, quit")
ctx.reportVmShutdown()
ctx.Become(stateDestroying, "DESTROYING")
case EVENT_VM_TIMEOUT:
glog.Warning("VM did not exit in time, try to stop it")
ctx.poweroffVM(true, "pod stopp/unplug timeout")
ctx.Become(stateDestroying, "DESTROYING")
case COMMAND_ACK:
ack := ev.(*CommandAck)
glog.V(1).Infof("[cleaning] Got reply to %d: '%s'", ack.reply, string(ack.msg))
if ack.reply == INIT_READY {
ctx.reset()
ctx.unsetTimeout()
ctx.reportPodStopped()
glog.Info("init has been acknowledged, could run pod.")
ctx.Become(stateInit, "INIT")
}
default:
glog.V(1).Info("got event message while cleaning")
}
}
}
func (daemon *Daemon) StartVm(vmId string, cpu, mem int, lazy bool, keep int) (*hypervisor.Vm, error) {
b := &hypervisor.BootConfig{
CPU: cpu,
Memory: mem,
Kernel: daemon.Kernel,
Initrd: daemon.Initrd,
Bios: daemon.Bios,
Cbfs: daemon.Cbfs,
Vbox: daemon.VboxImage,
}
vm := daemon.NewVm(vmId, cpu, mem, lazy, keep)
err := vm.Launch(b)
if err != nil {
return nil, err
}
_, r1, r2, err1 := vm.GetVmChan()
if err1 != nil {
return nil, err1
}
vmStatus := r1.(chan *types.VmResponse)
subVmStatus := r2.(chan *types.VmResponse)
go func(interface{}) {
defer func() {
err := recover()
if err != nil {
glog.Warning("panic during send shutdown message to channel")
}
}()
for {
vmResponse := <-vmStatus
subVmStatus <- vmResponse
}
}(subVmStatus)
var vmResponse *types.VmResponse
for {
vmResponse = <-subVmStatus
glog.V(1).Infof("Get the response from VM, VM id is %s, response code is %d!", vmResponse.VmId, vmResponse.Code)
if vmResponse.VmId == vmId {
if vmResponse.Code == types.E_VM_RUNNING {
glog.Infof("Got E_VM_RUNNING code response")
break
} else {
break
}
}
}
if vmResponse.Code != types.E_VM_RUNNING {
return nil, fmt.Errorf("Vbox does not start successfully")
}
return vm, nil
}
func stateTerminating(ctx *VmContext, ev VmEvent) {
switch ev.Event() {
case EVENT_VM_EXIT:
glog.Info("Got VM shutdown event while terminating, go to cleaning up")
ctx.unsetTimeout()
if closed := ctx.onVmExit(true); !closed {
ctx.Become(stateDestroying, "DESTROYING")
}
case EVENT_VM_KILL:
glog.Info("Got VM force killed message, go to cleaning up")
ctx.unsetTimeout()
if closed := ctx.onVmExit(true); !closed {
ctx.Become(stateDestroying, "DESTROYING")
}
case COMMAND_RELEASE:
glog.Info("vm terminating, got release")
ctx.reportVmShutdown()
case COMMAND_ACK:
ack := ev.(*CommandAck)
glog.V(1).Infof("[Terminating] Got reply to %d: '%s'", ack.reply, string(ack.msg))
if ack.reply == INIT_DESTROYPOD {
glog.Info("POD destroyed ", string(ack.msg))
ctx.poweroffVM(false, "")
}
case ERROR_CMD_FAIL:
ack := ev.(*CommandError)
if ack.context.code == INIT_DESTROYPOD {
glog.Warning("Destroy pod failed")
ctx.poweroffVM(true, "Destroy pod failed")
}
case EVENT_VM_TIMEOUT:
glog.Warning("VM did not exit in time, try to stop it")
ctx.poweroffVM(true, "vm terminating timeout")
case ERROR_INTERRUPTED:
glog.V(1).Info("Connection interrupted while terminating")
default:
glog.V(1).Info("got event during terminating")
}
}
// reportVmShutdown() send report to daemon, notify about that:
// 1. Vm has been shutdown
func (ctx *VmContext) reportVmShutdown() {
defer func() {
err := recover()
if err != nil {
glog.Warning("panic during send shutdown message to channel")
}
}()
ctx.client <- &types.VmResponse{
VmId: ctx.Id,
Code: types.E_VM_SHUTDOWN,
Cause: "VM shut down",
}
}
func qmpCommander(handler chan QmpInteraction, conn *net.UnixConn, session *QmpSession, feedback chan QmpInteraction) {
glog.V(1).Info("Begin process command session")
for _, cmd := range session.commands {
msg, err := json.Marshal(*cmd)
if err != nil {
handler <- qmpFail("cannot marshal command", session.callback)
return
}
success := false
var qe *QmpError = nil
for repeat := 0; !success && repeat < 3; repeat++ {
if len(cmd.Scm) > 0 {
glog.V(1).Infof("send cmd with scm (%d bytes) (%d) %s", len(cmd.Scm), repeat+1, string(msg))
f, _ := conn.File()
fd := f.Fd()
syscall.Sendmsg(int(fd), msg, cmd.Scm, nil, 0)
} else {
glog.V(1).Infof("sending command (%d) %s", repeat+1, string(msg))
conn.Write(msg)
}
res, ok := <-feedback
if !ok {
glog.Info("QMP command result chan closed")
return
}
switch res.MessageType() {
case QMP_RESULT:
success = true
break
//success
case QMP_ERROR:
glog.Warning("got one qmp error")
qe = res.(*QmpError)
time.Sleep(1000 * time.Millisecond)
case QMP_INTERNAL_ERROR:
glog.Info("QMP quit... commander quit... ")
return
}
}
if !success {
handler <- qe.Finish(session.callback)
return
}
}
handler <- session.Finish()
return
}
func (daemon *Daemon) Restore() error {
if daemon.GetPodNum() == 0 {
return nil
}
podList := map[string]string{}
iter := daemon.db.NewIterator(util.BytesPrefix([]byte("pod-")), nil)
for iter.Next() {
key := iter.Key()
value := iter.Value()
if strings.Contains(string(key), "pod-container-") {
glog.V(1).Infof(string(value))
continue
}
glog.V(1).Infof("Get the pod item, pod is %s!", key)
err := daemon.db.Delete(key, nil)
if err != nil {
return err
}
podList[string(key)[4:]] = string(value)
}
iter.Release()
err := iter.Error()
if err != nil {
return err
}
daemon.PodsMutex.Lock()
glog.V(2).Infof("lock PodList")
defer glog.V(2).Infof("unlock PodList")
defer daemon.PodsMutex.Unlock()
for k, v := range podList {
err = daemon.CreatePod(k, v, nil, false)
if err != nil {
glog.Warning("Got a unexpected error, %s", err.Error())
continue
}
vmId, err := daemon.GetVmByPod(k)
if err != nil {
glog.V(1).Info(err.Error(), " for ", k)
continue
}
daemon.PodList[k].Vm = string(vmId)
}
// associate all VMs
daemon.AssociateAllVms()
return nil
}
func stateRunning(ctx *VmContext, ev VmEvent) {
if processed := commonStateHandler(ctx, ev, true); processed {
} else if processed := initFailureHandler(ctx, ev); processed {
ctx.shutdownVM(true, "Fail during reconnect to a running pod")
ctx.Become(stateTerminating, "TERMINATING")
} else {
switch ev.Event() {
case COMMAND_STOP_POD:
ctx.stopPod()
ctx.Become(statePodStopping, "STOPPING")
case COMMAND_RELEASE:
glog.Info("pod is running, got release command, let VM fly")
ctx.Become(nil, "NONE")
ctx.reportSuccess("", nil)
case COMMAND_EXEC:
ctx.execCmd(ev.(*ExecCommand))
case COMMAND_ATTACH:
ctx.attachCmd(ev.(*AttachCommand))
case COMMAND_WINDOWSIZE:
cmd := ev.(*WindowSizeCommand)
if ctx.userSpec.Tty {
ctx.setWindowSize(cmd.ClientTag, cmd.Size)
}
case EVENT_POD_FINISH:
result := ev.(*PodFinished)
ctx.reportPodFinished(result)
if ctx.Keep == types.VM_KEEP_NONE {
ctx.exitVM(false, "", true, false)
}
case COMMAND_ACK:
ack := ev.(*CommandAck)
glog.V(1).Infof("[running] got init ack to %d", ack.reply)
case ERROR_CMD_FAIL:
ack := ev.(*CommandError)
if ack.context.code == INIT_EXECCMD {
cmd := ExecCommand{}
json.Unmarshal(ack.context.message, &cmd)
ctx.ptys.Close(ctx, cmd.Sequence)
glog.V(0).Infof("Exec command %s on session %d failed", cmd.Command[0], cmd.Sequence)
}
case COMMAND_GET_POD_IP:
ctx.reportPodIP()
default:
glog.Warning("got unexpected event during pod running")
}
}
}
func (ctx *VmContext) ReleaseInterface(index int, ipAddr string, file *os.File,
maps []pod.UserContainerPort) {
var err error
success := true
if HDriver.BuildinNetwork() {
err = ctx.DCtx.ReleaseNetwork(ctx.Id, ipAddr, maps, file)
} else {
err = network.Release(ctx.Id, ipAddr, maps, file)
}
if err != nil {
glog.Warning("Unable to release network interface, address: ", ipAddr, err)
success = false
}
ctx.Hub <- &InterfaceReleased{Index: index, Success: success}
}
func stateInit(ctx *VmContext, ev VmEvent) {
if processed := commonStateHandler(ctx, ev, false); processed {
//processed by common
} else if processed := initFailureHandler(ctx, ev); processed {
ctx.shutdownVM(true, "Fail during init environment")
ctx.Become(stateDestroying, "DESTROYING")
} else {
switch ev.Event() {
case EVENT_VM_START_FAILED:
glog.Error("VM did not start up properly, go to cleaning up")
ctx.reportVmFault("VM did not start up properly, go to cleaning up")
ctx.Close()
case EVENT_INIT_CONNECTED:
glog.Info("begin to wait vm commands")
ctx.reportVmRun()
case COMMAND_RELEASE:
glog.Info("no pod on vm, got release, quit.")
ctx.shutdownVM(false, "")
ctx.Become(stateDestroying, "DESTRYING")
ctx.reportVmShutdown()
case COMMAND_EXEC:
ctx.execCmd(ev.(*ExecCommand))
case COMMAND_WINDOWSIZE:
cmd := ev.(*WindowSizeCommand)
ctx.setWindowSize(cmd.ClientTag, cmd.Size)
case COMMAND_RUN_POD, COMMAND_REPLACE_POD:
glog.Info("got spec, prepare devices")
if ok := ctx.prepareDevice(ev.(*RunPodCommand)); ok {
ctx.setTimeout(60)
ctx.Become(stateStarting, "STARTING")
}
case COMMAND_GET_POD_IP:
ctx.reportPodIP()
default:
glog.Warning("got event during pod initiating")
}
}
}
func statePodStopping(ctx *VmContext, ev VmEvent) {
if processed := commonStateHandler(ctx, ev, true); processed {
} else {
switch ev.Event() {
case COMMAND_RELEASE:
glog.Info("pod stopping, got release, quit.")
ctx.unsetTimeout()
ctx.shutdownVM(false, "got release, quit")
ctx.Become(stateTerminating, "TERMINATING")
ctx.reportVmShutdown()
case COMMAND_ACK, EVENT_POD_FINISH:
ack := ev.(*CommandAck)
glog.V(1).Infof("[Stopping] got init ack to %d", ack.reply)
if ack.reply == INIT_STOPPOD {
glog.Info("POD stopped ", string(ack.msg))
ctx.detachDevice()
ctx.Become(stateCleaning, "CLEANING")
}
case ERROR_CMD_FAIL:
ack := ev.(*CommandError)
if ack.context.code == INIT_STOPPOD {
ctx.unsetTimeout()
ctx.shutdownVM(true, "Stop pod failed as init report")
ctx.Become(stateTerminating, "TERMINATING")
glog.Error("Stop pod failed as init report")
}
case EVENT_VM_TIMEOUT:
reason := "stopping POD timeout"
ctx.shutdownVM(true, reason)
ctx.Become(stateTerminating, "TERMINATING")
glog.Error(reason)
default:
glog.Warning("got unexpected event during pod stopping")
}
}
}
func qmpHandler(ctx *hypervisor.VmContext) {
go qmpInitializer(ctx)
qc := ctx.DCtx.(*QemuContext)
timer := time.AfterFunc(10*time.Second, func() {
glog.Warning("Initializer Timeout.")
qc.qmp <- &QmpTimeout{}
})
type msgHandler func(QmpInteraction)
var handler msgHandler = nil
var conn *net.UnixConn = nil
buf := []*QmpSession{}
res := make(chan QmpInteraction, 128)
loop := func(msg QmpInteraction) {
switch msg.MessageType() {
case QMP_SESSION:
glog.Info("got new session")
buf = append(buf, msg.(*QmpSession))
if len(buf) == 1 {
go qmpCommander(qc.qmp, conn, msg.(*QmpSession), res)
}
case QMP_FINISH:
glog.Infof("session finished, buffer size %d", len(buf))
r := msg.(*QmpFinish)
if r.success {
glog.V(1).Info("success ")
if r.callback != nil {
ctx.Hub <- r.callback
}
} else {
reason := "unknown"
if c, ok := r.reason["error"]; ok {
reason = c.(string)
}
glog.Error("QMP command failed ", reason)
ctx.Hub <- &hypervisor.DeviceFailed{
Session: r.callback,
}
}
buf = buf[1:]
if len(buf) > 0 {
go qmpCommander(qc.qmp, conn, buf[0], res)
}
case QMP_RESULT, QMP_ERROR:
res <- msg
case QMP_EVENT:
ev := msg.(*QmpEvent)
glog.V(1).Info("got QMP event ", ev.Type)
if ev.Type == QMP_EVENT_SHUTDOWN {
glog.Info("got QMP shutdown event, quit...")
handler = nil
ctx.Hub <- &hypervisor.VmExit{}
}
case QMP_INTERNAL_ERROR:
res <- msg
handler = nil
glog.Info("QMP handler quit as received ", msg.(*QmpInternalError).cause)
ctx.Hub <- &hypervisor.Interrupted{Reason: msg.(*QmpInternalError).cause}
case QMP_QUIT:
handler = nil
}
}
initializing := func(msg QmpInteraction) {
switch msg.MessageType() {
case QMP_INIT:
timer.Stop()
init := msg.(*QmpInit)
conn = init.conn
handler = loop
glog.Info("QMP initialzed, go into main QMP loop")
//routine for get message
go qmpReceiver(qc.qmp, qc.waitQmp, init.decoder)
if len(buf) > 0 {
go qmpCommander(qc.qmp, conn, buf[0], res)
}
case QMP_FINISH:
finish := msg.(*QmpFinish)
if !finish.success {
timer.Stop()
ctx.Hub <- &hypervisor.InitFailedEvent{
Reason: finish.reason["error"].(string),
}
handler = nil
glog.Error("QMP initialize failed")
}
case QMP_TIMEOUT:
ctx.Hub <- &hypervisor.InitFailedEvent{
Reason: "QMP Init timeout",
}
handler = nil
glog.Error("QMP initialize timeout")
case QMP_SESSION:
glog.Info("got new session during initializing")
buf = append(buf, msg.(*QmpSession))
}
}
handler = initializing
for handler != nil {
msg, ok := <-qc.qmp
if !ok {
glog.Info("QMP channel closed, Quit qmp handler")
break
}
handler(msg)
}
}
func stateStarting(ctx *VmContext, ev VmEvent) {
if processed := commonStateHandler(ctx, ev, true); processed {
//processed by common
} else if processed := deviceInitHandler(ctx, ev); processed {
if ctx.deviceReady() {
glog.V(1).Info("device ready, could run pod.")
ctx.startPod()
}
} else if processed := initFailureHandler(ctx, ev); processed {
ctx.shutdownVM(true, "Fail during init pod running environment")
ctx.Become(stateTerminating, "TERMINATING")
} else {
switch ev.Event() {
case EVENT_VM_START_FAILED:
glog.Info("VM did not start up properly, go to cleaning up")
if closed := ctx.onVmExit(true); !closed {
ctx.Become(stateDestroying, "DESTROYING")
}
case EVENT_INIT_CONNECTED:
glog.Info("begin to wait vm commands")
ctx.reportVmRun()
case COMMAND_RELEASE:
glog.Info("pod starting, got release, please wait")
ctx.reportBusy("")
case COMMAND_ATTACH:
ctx.attachCmd(ev.(*AttachCommand))
case COMMAND_WINDOWSIZE:
cmd := ev.(*WindowSizeCommand)
if ctx.userSpec.Tty {
ctx.setWindowSize(cmd.ClientTag, cmd.Size)
}
case COMMAND_ACK:
ack := ev.(*CommandAck)
glog.V(1).Infof("[starting] got init ack to %d", ack.reply)
if ack.reply == INIT_STARTPOD {
ctx.unsetTimeout()
var pinfo []byte = []byte{}
persist, err := ctx.dump()
if err == nil {
buf, err := persist.serialize()
if err == nil {
pinfo = buf
}
}
ctx.reportSuccess("Start POD success", pinfo)
ctx.Become(stateRunning, "RUNNING")
glog.Info("pod start success ", string(ack.msg))
}
case ERROR_CMD_FAIL:
ack := ev.(*CommandError)
if ack.context.code == INIT_STARTPOD {
reason := "Start POD failed"
ctx.shutdownVM(true, reason)
ctx.Become(stateTerminating, "TERMINATING")
glog.Error(reason)
}
case EVENT_VM_TIMEOUT:
reason := "Start POD timeout"
ctx.shutdownVM(true, reason)
ctx.Become(stateTerminating, "TERMINATING")
glog.Error(reason)
default:
glog.Warning("got event during pod initiating")
}
}
}
