func statePreparing(ctx *VmContext, ev VmEvent) { switch ev.Event() { case EVENT_VM_EXIT, ERROR_INTERRUPTED: glog.Info("VM exited before start...") case COMMAND_SHUTDOWN, COMMAND_RELEASE: glog.Info("got shutdown or release command, not started yet") ctx.reportVmShutdown() ctx.Become(nil, "NONE") case COMMAND_EXEC: ctx.execCmd(ev.(*ExecCommand)) case COMMAND_WINDOWSIZE: cmd := ev.(*WindowSizeCommand) ctx.setWindowSize(cmd.ClientTag, cmd.Size) case COMMAND_RUN_POD, COMMAND_REPLACE_POD: glog.Info("got spec, prepare devices") if ok := ctx.lazyPrepareDevice(ev.(*RunPodCommand)); ok { ctx.startSocks() ctx.DCtx.(LazyDriverContext).LazyLaunch(ctx) ctx.setTimeout(60) ctx.Become(stateStarting, "STARTING") } else { glog.Warning("Fail to prepare devices, quit") ctx.Become(nil, "None") } default: glog.Warning("got event during pod initiating") } }
func waitInitReady(ctx *VmContext) { conn, err := UnixSocketConnect(ctx.HyperSockName) if err != nil { glog.Error("Cannot connect to hyper socket ", err.Error()) ctx.Hub <- &InitFailedEvent{ Reason: "Cannot connect to hyper socket " + err.Error(), } return } glog.Info("Wating for init messages...") msg, err := readVmMessage(conn.(*net.UnixConn)) if err != nil { glog.Error("read init message failed... ", err.Error()) ctx.Hub <- &InitFailedEvent{ Reason: "read init message failed... " + err.Error(), } conn.Close() } else if msg.code == INIT_READY { glog.Info("Get init ready message") ctx.Hub <- &InitConnectedEvent{conn: conn.(*net.UnixConn)} go waitCmdToInit(ctx, conn.(*net.UnixConn)) } else { glog.Warningf("Get init message %d", msg.code) ctx.Hub <- &InitFailedEvent{ Reason: fmt.Sprintf("Get init message %d", msg.code), } conn.Close() } }
func InitDriver() *XenDriver { if probeXend() { glog.Info("xend is running, can not start with xl.") return nil } ctx, res := HyperxlInitializeDriver() if res != 0 { glog.Info("failed to initialize xen context") return nil } else if ctx.Version < REQUIRED_VERSION { glog.Info("Xen version is not new enough (%d), need 4.5 or higher", ctx.Version) return nil } else { glog.V(1).Info("Xen capabilities: ", ctx.Capabilities) hvm := false caps := strings.Split(ctx.Capabilities, " ") for _, cap := range caps { if strings.HasPrefix(cap, "hvm-") { hvm = true break } } if !hvm { glog.Info("Xen installation does not support HVM, current capabilities: %s", ctx.Capabilities) return nil } } sigchan := make(chan os.Signal, 1) go func() { for { _, ok := <-sigchan if !ok { break } glog.V(1).Info("got SIGCHLD, send msg to libxl") HyperxlSigchldHandler(ctx.Ctx) } }() signal.Notify(sigchan, syscall.SIGCHLD) xd := &XenDriver{ Ctx: ctx.Ctx, Logger: ctx.Logger, Version: ctx.Version, Capabilities: ctx.Capabilities, } xd.domains = make(map[uint32]*hypervisor.VmContext) globalDriver = xd return globalDriver }
func qmpCommander(handler chan QmpInteraction, conn *net.UnixConn, session *QmpSession, feedback chan QmpInteraction) { glog.V(1).Info("Begin process command session") for _, cmd := range session.commands { msg, err := json.Marshal(*cmd) if err != nil { handler <- qmpFail("cannot marshal command", session.callback) return } success := false var qe *QmpError = nil for repeat := 0; !success && repeat < 3; repeat++ { if len(cmd.Scm) > 0 { glog.V(1).Infof("send cmd with scm (%d bytes) (%d) %s", len(cmd.Scm), repeat+1, string(msg)) f, _ := conn.File() fd := f.Fd() syscall.Sendmsg(int(fd), msg, cmd.Scm, nil, 0) } else { glog.V(1).Infof("sending command (%d) %s", repeat+1, string(msg)) conn.Write(msg) } res, ok := <-feedback if !ok { glog.Info("QMP command result chan closed") return } switch res.MessageType() { case QMP_RESULT: success = true break //success case QMP_ERROR: glog.Warning("got one qmp error") qe = res.(*QmpError) time.Sleep(1000 * time.Millisecond) case QMP_INTERNAL_ERROR: glog.Info("QMP quit... commander quit... ") return } } if !success { handler <- qe.Finish(session.callback) return } } handler <- session.Finish() return }
func waitConsoleOutput(ctx *VmContext) { conn, err := UnixSocketConnect(ctx.ConsoleSockName) if err != nil { glog.Error("failed to connected to ", ctx.ConsoleSockName, " ", err.Error()) return } glog.V(1).Info("connected to ", ctx.ConsoleSockName) tc, err := telnet.NewConn(conn) if err != nil { glog.Error("fail to init telnet connection to ", ctx.ConsoleSockName, ": ", err.Error()) return } glog.V(1).Infof("connected %s as telnet mode.", ctx.ConsoleSockName) cout := make(chan string, 128) go TtyLiner(tc, cout) for { line, ok := <-cout if ok { glog.V(1).Info("[console] ", line) } else { glog.Info("console output end") break } } }
func (daemon *Daemon) CmdPodRun(job *engine.Job) error { // we can only support 1024 Pods if daemon.GetRunningPodNum() >= 1024 { return fmt.Errorf("Pod full, the maximum Pod is 1024!") } var autoremove bool = false podArgs := job.Args[0] if job.Args[1] == "yes" { autoremove = true } podId := fmt.Sprintf("pod-%s", pod.RandStr(10, "alpha")) glog.Info(podArgs) var lazy bool = hypervisor.HDriver.SupportLazyMode() code, cause, err := daemon.StartPod(podId, podArgs, "", nil, lazy, autoremove, types.VM_KEEP_NONE) if err != nil { glog.Error(err.Error()) return err } // Prepare the VM status to client v := &engine.Env{} v.Set("ID", podId) v.SetInt("Code", code) v.Set("Cause", cause) if _, err := v.WriteTo(job.Stdout); err != nil { return err } return nil }
func stateCleaning(ctx *VmContext, ev VmEvent) { if processed := commonStateHandler(ctx, ev, false); processed { } else if processed, success := deviceRemoveHandler(ctx, ev); processed { if !success { glog.Warning("fail to unplug devices for stop") ctx.poweroffVM(true, "fail to unplug devices") ctx.Become(stateDestroying, "DESTROYING") } else if ctx.deviceReady() { // ctx.reset() // ctx.unsetTimeout() // ctx.reportPodStopped() // glog.V(1).Info("device ready, could run pod.") // ctx.Become(stateInit, "INIT") ctx.vm <- &DecodedMessage{ code: INIT_READY, message: []byte{}, } glog.V(1).Info("device ready, could run pod.") } } else if processed := initFailureHandler(ctx, ev); processed { ctx.poweroffVM(true, "fail to unplug devices") ctx.Become(stateDestroying, "DESTROYING") } else { switch ev.Event() { case COMMAND_RELEASE: glog.Info("vm cleaning to idle, got release, quit") ctx.reportVmShutdown() ctx.Become(stateDestroying, "DESTROYING") case EVENT_VM_TIMEOUT: glog.Warning("VM did not exit in time, try to stop it") ctx.poweroffVM(true, "pod stopp/unplug timeout") ctx.Become(stateDestroying, "DESTROYING") case COMMAND_ACK: ack := ev.(*CommandAck) glog.V(1).Infof("[cleaning] Got reply to %d: '%s'", ack.reply, string(ack.msg)) if ack.reply == INIT_READY { ctx.reset() ctx.unsetTimeout() ctx.reportPodStopped() glog.Info("init has been acknowledged, could run pod.") ctx.Become(stateInit, "INIT") } default: glog.V(1).Info("got event message while cleaning") } } }
// InitDeviceContext will init device info in context func (ctx *VmContext) InitDeviceContext(spec *pod.UserPod, wg *sync.WaitGroup, cInfo []*ContainerInfo, vInfo []*VolumeInfo) { ctx.lock.Lock() defer ctx.lock.Unlock() for i := 0; i < ctx.InterfaceCount; i++ { ctx.progress.adding.networks[i] = true } if cInfo == nil { cInfo = []*ContainerInfo{} } if vInfo == nil { vInfo = []*VolumeInfo{} } ctx.initVolumeMap(spec) if glog.V(3) { for i, c := range cInfo { glog.Infof("#%d Container Info:", i) b, err := json.MarshalIndent(c, "...|", " ") if err == nil { glog.Info("\n", string(b)) } } } containers := make([]VmContainer, len(spec.Containers)) for i, container := range spec.Containers { ctx.initContainerInfo(i, &containers[i], &container) ctx.setContainerInfo(i, &containers[i], cInfo[i]) if spec.Tty { containers[i].Tty = ctx.attachId ctx.attachId++ ctx.ptys.ttys[containers[i].Tty] = newAttachments(i, true) } } ctx.vmSpec = &VmPod{ Hostname: spec.Name, Containers: containers, Interfaces: nil, Routes: nil, ShareDir: ShareDirTag, } for _, vol := range vInfo { ctx.setVolumeInfo(vol) } ctx.userSpec = spec ctx.wg = wg }
func stateDestroying(ctx *VmContext, ev VmEvent) { if processed, _ := deviceRemoveHandler(ctx, ev); processed { if closed := ctx.tryClose(); closed { glog.Info("resources reclaimed, quit...") } } else { switch ev.Event() { case EVENT_VM_EXIT: glog.Info("Got VM shutdown event") ctx.unsetTimeout() if closed := ctx.onVmExit(false); closed { glog.Info("VM Context closed.") } case EVENT_VM_KILL: glog.Info("Got VM force killed message") ctx.unsetTimeout() if closed := ctx.onVmExit(true); closed { glog.Info("VM Context closed.") } case ERROR_INTERRUPTED: glog.V(1).Info("Connection interrupted while destroying") case COMMAND_RELEASE: glog.Info("vm destroying, got release") ctx.reportVmShutdown() case EVENT_VM_TIMEOUT: glog.Info("Device removing timeout") ctx.Close() default: glog.Warning("got event during vm cleaning up") } } }
func (qc *QemuContext) Kill(ctx *hypervisor.VmContext) { defer func() { err := recover() if glog.V(1) && err != nil { glog.Info("kill qemu, but channel has already been closed") } }() qc.wdt <- "kill" }
// state machine func commonStateHandler(ctx *VmContext, ev VmEvent, hasPod bool) bool { processed := true switch ev.Event() { case EVENT_VM_EXIT: glog.Info("Got VM shutdown event, go to cleaning up") ctx.unsetTimeout() if closed := ctx.onVmExit(hasPod); !closed { ctx.Become(stateDestroying, "DESTROYING") } case ERROR_INTERRUPTED: glog.Info("Connection interrupted, quit...") ctx.exitVM(true, "connection to VM broken", false, false) ctx.onVmExit(hasPod) case COMMAND_SHUTDOWN: glog.Info("got shutdown command, shutting down") ctx.exitVM(false, "", hasPod, ev.(*ShutdownCommand).Wait) default: processed = false } return processed }
func stateTerminating(ctx *VmContext, ev VmEvent) { switch ev.Event() { case EVENT_VM_EXIT: glog.Info("Got VM shutdown event while terminating, go to cleaning up") ctx.unsetTimeout() if closed := ctx.onVmExit(true); !closed { ctx.Become(stateDestroying, "DESTROYING") } case EVENT_VM_KILL: glog.Info("Got VM force killed message, go to cleaning up") ctx.unsetTimeout() if closed := ctx.onVmExit(true); !closed { ctx.Become(stateDestroying, "DESTROYING") } case COMMAND_RELEASE: glog.Info("vm terminating, got release") ctx.reportVmShutdown() case COMMAND_ACK: ack := ev.(*CommandAck) glog.V(1).Infof("[Terminating] Got reply to %d: '%s'", ack.reply, string(ack.msg)) if ack.reply == INIT_DESTROYPOD { glog.Info("POD destroyed ", string(ack.msg)) ctx.poweroffVM(false, "") } case ERROR_CMD_FAIL: ack := ev.(*CommandError) if ack.context.code == INIT_DESTROYPOD { glog.Warning("Destroy pod failed") ctx.poweroffVM(true, "Destroy pod failed") } case EVENT_VM_TIMEOUT: glog.Warning("VM did not exit in time, try to stop it") ctx.poweroffVM(true, "vm terminating timeout") case ERROR_INTERRUPTED: glog.V(1).Info("Connection interrupted while terminating") default: glog.V(1).Info("got event during terminating") } }
func (pts *pseudoTtys) ptyConnect(ctx *VmContext, container int, session uint64, tty *TtyIO) { pts.lock.Lock() if ta, ok := pts.ttys[session]; ok { ta.attach(tty) } else { pts.ttys[session] = newAttachmentsWithTty(container, false, tty) } pts.lock.Unlock() if tty.Stdin != nil { go func() { buf := make([]byte, 32) defer pts.Detach(ctx, session, tty) defer func() { recover() }() for { nr, err := tty.Stdin.Read(buf) if err != nil { glog.Info("a stdin closed, ", err.Error()) return } else if nr == 1 && buf[0] == ExitChar { glog.Info("got stdin detach char, exit term") return } glog.V(3).Infof("trying to input char: %d and %d chars", buf[0], nr) mbuf := make([]byte, nr) copy(mbuf, buf[:nr]) pts.channel <- &ttyMessage{ session: session, message: mbuf[:nr], } } }() } return }
func stateInit(ctx *VmContext, ev VmEvent) { if processed := commonStateHandler(ctx, ev, false); processed { //processed by common } else if processed := initFailureHandler(ctx, ev); processed { ctx.shutdownVM(true, "Fail during init environment") ctx.Become(stateDestroying, "DESTROYING") } else { switch ev.Event() { case EVENT_VM_START_FAILED: glog.Error("VM did not start up properly, go to cleaning up") ctx.reportVmFault("VM did not start up properly, go to cleaning up") ctx.Close() case EVENT_INIT_CONNECTED: glog.Info("begin to wait vm commands") ctx.reportVmRun() case COMMAND_RELEASE: glog.Info("no pod on vm, got release, quit.") ctx.shutdownVM(false, "") ctx.Become(stateDestroying, "DESTRYING") ctx.reportVmShutdown() case COMMAND_EXEC: ctx.execCmd(ev.(*ExecCommand)) case COMMAND_WINDOWSIZE: cmd := ev.(*WindowSizeCommand) ctx.setWindowSize(cmd.ClientTag, cmd.Size) case COMMAND_RUN_POD, COMMAND_REPLACE_POD: glog.Info("got spec, prepare devices") if ok := ctx.prepareDevice(ev.(*RunPodCommand)); ok { ctx.setTimeout(60) ctx.Become(stateStarting, "STARTING") } case COMMAND_GET_POD_IP: ctx.reportPodIP() default: glog.Warning("got event during pod initiating") } } }
// launchQemu run qemu and wait it's quit, includes func launchQemu(qc *QemuContext, ctx *hypervisor.VmContext) { qemu := qc.driver.executable if qemu == "" { ctx.Hub <- &hypervisor.VmStartFailEvent{Message: "can not find qemu executable"} return } args := qc.arguments(ctx) if glog.V(1) { glog.Info("cmdline arguments: ", strings.Join(args, " ")) } pipe := make([]int, 2) err := syscall.Pipe(pipe) if err != nil { glog.Error("fail to create pipe") ctx.Hub <- &hypervisor.VmStartFailEvent{Message: "fail to create pipe"} return } err = daemon(qemu, append([]string{"qemu-system-x86_64"}, args...), pipe[1]) if err != nil { //fail to daemonize glog.Error("try to start qemu failed") ctx.Hub <- &hypervisor.VmStartFailEvent{Message: "try to start qemu failed"} return } buf := make([]byte, 4) nr, err := syscall.Read(pipe[0], buf) if err != nil || nr != 4 { glog.Error("try to start qemu failed") ctx.Hub <- &hypervisor.VmStartFailEvent{Message: "try to start qemu failed"} return } syscall.Close(pipe[1]) syscall.Close(pipe[0]) pid := binary.BigEndian.Uint32(buf[:nr]) glog.V(1).Infof("starting daemon with pid: %d", pid) err = ctx.DCtx.(*QemuContext).watchPid(int(pid), ctx.Hub) if err != nil { glog.Error("watch qemu process failed") ctx.Hub <- &hypervisor.VmStartFailEvent{Message: "watch qemu process failed"} return } }
func Init() { if daemonCfg.LogConfig.Config == nil { daemonCfg.LogConfig.Config = make(map[string]string) } daemonCfg.InstallFlags() registryCfg.InstallFlags() hyperd.NewDockerImpl = func() (docker hyperd.DockerInterface, e error) { docker, e = NewDocker() if e != nil { return nil, fmt.Errorf("failed to create docker instance") } return docker, nil } glog.Info("success to create docker") }
func NewDocker() (*Docker, error) { registryService := registry.NewService(registryCfg) daemonCfg.TrustKeyPath = getDaemonConfDir() + "/" + defaultTrustKeyFile daemonCfg.Root = utils.HYPER_ROOT d, err := daemon.NewDaemon(daemonCfg, registryService) if err != nil { glog.Errorf("Error starting daemon: %v", err) return nil, err } glog.Info("Daemon has completed initialization") return &Docker{ daemon: d, }, nil }
func statePodStopping(ctx *VmContext, ev VmEvent) { if processed := commonStateHandler(ctx, ev, true); processed { } else { switch ev.Event() { case COMMAND_RELEASE: glog.Info("pod stopping, got release, quit.") ctx.unsetTimeout() ctx.shutdownVM(false, "got release, quit") ctx.Become(stateTerminating, "TERMINATING") ctx.reportVmShutdown() case COMMAND_ACK, EVENT_POD_FINISH: ack := ev.(*CommandAck) glog.V(1).Infof("[Stopping] got init ack to %d", ack.reply) if ack.reply == INIT_STOPPOD { glog.Info("POD stopped ", string(ack.msg)) ctx.detachDevice() ctx.Become(stateCleaning, "CLEANING") } case ERROR_CMD_FAIL: ack := ev.(*CommandError) if ack.context.code == INIT_STOPPOD { ctx.unsetTimeout() ctx.shutdownVM(true, "Stop pod failed as init report") ctx.Become(stateTerminating, "TERMINATING") glog.Error("Stop pod failed as init report") } case EVENT_VM_TIMEOUT: reason := "stopping POD timeout" ctx.shutdownVM(true, reason) ctx.Become(stateTerminating, "TERMINATING") glog.Error(reason) default: glog.Warning("got unexpected event during pod stopping") } } }
func stateRunning(ctx *VmContext, ev VmEvent) { if processed := commonStateHandler(ctx, ev, true); processed { } else if processed := initFailureHandler(ctx, ev); processed { ctx.shutdownVM(true, "Fail during reconnect to a running pod") ctx.Become(stateTerminating, "TERMINATING") } else { switch ev.Event() { case COMMAND_STOP_POD: ctx.stopPod() ctx.Become(statePodStopping, "STOPPING") case COMMAND_RELEASE: glog.Info("pod is running, got release command, let VM fly") ctx.Become(nil, "NONE") ctx.reportSuccess("", nil) case COMMAND_EXEC: ctx.execCmd(ev.(*ExecCommand)) case COMMAND_ATTACH: ctx.attachCmd(ev.(*AttachCommand)) case COMMAND_WINDOWSIZE: cmd := ev.(*WindowSizeCommand) if ctx.userSpec.Tty { ctx.setWindowSize(cmd.ClientTag, cmd.Size) } case EVENT_POD_FINISH: result := ev.(*PodFinished) ctx.reportPodFinished(result) if ctx.Keep == types.VM_KEEP_NONE { ctx.exitVM(false, "", true, false) } case COMMAND_ACK: ack := ev.(*CommandAck) glog.V(1).Infof("[running] got init ack to %d", ack.reply) case ERROR_CMD_FAIL: ack := ev.(*CommandError) if ack.context.code == INIT_EXECCMD { cmd := ExecCommand{} json.Unmarshal(ack.context.message, &cmd) ctx.ptys.Close(ctx, cmd.Sequence) glog.V(0).Infof("Exec command %s on session %d failed", cmd.Command[0], cmd.Sequence) } case COMMAND_GET_POD_IP: ctx.reportPodIP() default: glog.Warning("got unexpected event during pod running") } } }
func (ctx *VmContext) prepareDevice(cmd *RunPodCommand) bool { if len(cmd.Spec.Containers) != len(cmd.Containers) { ctx.reportBadRequest("Spec and Container Info mismatch") return false } ctx.InitDeviceContext(cmd.Spec, cmd.Wg, cmd.Containers, cmd.Volumes) if glog.V(2) { res, _ := json.MarshalIndent(*ctx.vmSpec, " ", " ") glog.Info("initial vm spec: ", string(res)) } ctx.allocateDevices() return true }
func qmpReceiver(qmp chan QmpInteraction, wait chan int, decoder *json.Decoder) { glog.V(0).Info("Begin receive QMP message") for { rsp := &QmpResponse{} if err := decoder.Decode(rsp); err != nil { glog.Info("QMP exit as got error:", err.Error()) qmp <- &QmpInternalError{cause: err.Error()} /* After Error report, send wait notification to close qmp channel */ wait <- 1 return } msg := rsp.msg qmp <- msg if msg.MessageType() == QMP_EVENT && msg.(*QmpEvent).Type == QMP_EVENT_SHUTDOWN { glog.V(0).Info("Shutdown, quit QMP receiver") return } } }
func (ctx *VmContext) lazyPrepareDevice(cmd *RunPodCommand) bool { if len(cmd.Spec.Containers) != len(cmd.Containers) { ctx.reportBadRequest("Spec and Container Info mismatch") return false } ctx.InitDeviceContext(cmd.Spec, cmd.Wg, cmd.Containers, cmd.Volumes) if glog.V(2) { res, _ := json.MarshalIndent(*ctx.vmSpec, " ", " ") glog.Info("initial vm spec: ", string(res)) } err := ctx.lazyAllocateNetworks() if err != nil { ctx.reportVmFault(err.Error()) return false } ctx.lazyAddBlockDevices() return true }
//export hyperxl_log_cgo func hyperxl_log_cgo(msg *C.char, len C.int) { if glog.V(1) { glog.Info("[libxl] ", C.GoStringN(msg, len)) } }
func (daemon *Daemon) CmdPodRun(job *engine.Job) error { // we can only support 1024 Pods if daemon.GetRunningPodNum() >= 1024 { return fmt.Errorf("Pod full, the maximum Pod is 1024!") } var ( autoremove bool = false tag string = "" ttys []*hypervisor.TtyIO = []*hypervisor.TtyIO{} ttyCallback chan *types.VmResponse ) podArgs := job.Args[0] if job.Args[1] == "yes" { autoremove = true } if len(job.Args) > 2 { tag = job.Args[2] } if tag != "" { glog.V(1).Info("Pod Run with client terminal tag: ", tag) ttyCallback = make(chan *types.VmResponse, 1) ttys = append(ttys, &hypervisor.TtyIO{ Stdin: job.Stdin, Stdout: job.Stdout, ClientTag: tag, Callback: ttyCallback, }) } podId := fmt.Sprintf("pod-%s", pod.RandStr(10, "alpha")) glog.Info(podArgs) var lazy bool = hypervisor.HDriver.SupportLazyMode() daemon.PodList.Lock() glog.V(2).Infof("lock PodList") defer glog.V(2).Infof("unlock PodList") defer daemon.PodList.Unlock() code, cause, err := daemon.StartPod(podId, podArgs, "", nil, lazy, autoremove, types.VM_KEEP_NONE, ttys) if err != nil { glog.Error(err.Error()) return err } if len(ttys) > 0 { <-ttyCallback return nil } // Prepare the VM status to client v := &engine.Env{} v.Set("ID", podId) v.SetInt("Code", code) v.Set("Cause", cause) if _, err := v.WriteTo(job.Stdout); err != nil { return err } return nil }
func daemon(cmd string, argv []string, pipe int) error { // create a subprocess pid, err := fork(false) if err != nil { return err } else if pid > 0 { go func() { wp, err := syscall.Wait4(int(pid), nil, 0, nil) if err == nil { glog.V(3).Infof("collect child %d", wp) } else { glog.Errorf("error during wait %d: %s", pid, err.Error()) } }() // return the parent return nil } // exit the created one, create the daemon _, err = fork(true) if err != nil { glog.Error("second fork failed: ", err.Error()) os.Exit(-1) } cur := os.Getpid() glog.V(1).Infof("qemu daemon pid %d.", cur) //Change the file mode mask _ = syscall.Umask(0) // create a new SID for the child process s_ret, err := syscall.Setsid() if err != nil { glog.Info("Error: syscall.Setsid errno: ", err.Error()) os.Exit(-1) } if s_ret < 0 { glog.Errorf("setsid return negative value: %d", s_ret) os.Exit(-1) } os.Chdir("/") f, e := os.OpenFile("/dev/null", os.O_RDWR, 0) if e == nil { fd := f.Fd() syscall.Dup2(int(fd), int(os.Stdin.Fd())) syscall.Dup2(int(fd), int(os.Stdout.Fd())) syscall.Dup2(int(fd), int(os.Stderr.Fd())) } buf := make([]byte, 4) binary.BigEndian.PutUint32(buf, uint32(cur)) syscall.Write(pipe, buf) syscall.Close(pipe) fds := listFd() for _, fd := range fds { if f, err := strconv.Atoi(fd); err == nil && f > 2 { glog.V(1).Infof("close fd %d", f) syscall.Close(f) } } err = syscall.Exec(cmd, argv, []string{}) if err != nil { glog.Error("fail to exec qemu process") os.Exit(-1) } return nil }
func qmpHandler(ctx *hypervisor.VmContext) { go qmpInitializer(ctx) qc := ctx.DCtx.(*QemuContext) timer := time.AfterFunc(10*time.Second, func() { glog.Warning("Initializer Timeout.") qc.qmp <- &QmpTimeout{} }) type msgHandler func(QmpInteraction) var handler msgHandler = nil var conn *net.UnixConn = nil buf := []*QmpSession{} res := make(chan QmpInteraction, 128) loop := func(msg QmpInteraction) { switch msg.MessageType() { case QMP_SESSION: glog.Info("got new session") buf = append(buf, msg.(*QmpSession)) if len(buf) == 1 { go qmpCommander(qc.qmp, conn, msg.(*QmpSession), res) } case QMP_FINISH: glog.Infof("session finished, buffer size %d", len(buf)) r := msg.(*QmpFinish) if r.success { glog.V(1).Info("success ") if r.callback != nil { ctx.Hub <- r.callback } } else { reason := "unknown" if c, ok := r.reason["error"]; ok { reason = c.(string) } glog.Error("QMP command failed ", reason) ctx.Hub <- &hypervisor.DeviceFailed{ Session: r.callback, } } buf = buf[1:] if len(buf) > 0 { go qmpCommander(qc.qmp, conn, buf[0], res) } case QMP_RESULT, QMP_ERROR: res <- msg case QMP_EVENT: ev := msg.(*QmpEvent) glog.V(1).Info("got QMP event ", ev.Type) if ev.Type == QMP_EVENT_SHUTDOWN { glog.Info("got QMP shutdown event, quit...") handler = nil ctx.Hub <- &hypervisor.VmExit{} } case QMP_INTERNAL_ERROR: res <- msg handler = nil glog.Info("QMP handler quit as received ", msg.(*QmpInternalError).cause) ctx.Hub <- &hypervisor.Interrupted{Reason: msg.(*QmpInternalError).cause} case QMP_QUIT: handler = nil } } initializing := func(msg QmpInteraction) { switch msg.MessageType() { case QMP_INIT: timer.Stop() init := msg.(*QmpInit) conn = init.conn handler = loop glog.Info("QMP initialzed, go into main QMP loop") //routine for get message go qmpReceiver(qc.qmp, qc.waitQmp, init.decoder) if len(buf) > 0 { go qmpCommander(qc.qmp, conn, buf[0], res) } case QMP_FINISH: finish := msg.(*QmpFinish) if !finish.success { timer.Stop() ctx.Hub <- &hypervisor.InitFailedEvent{ Reason: finish.reason["error"].(string), } handler = nil glog.Error("QMP initialize failed") } case QMP_TIMEOUT: ctx.Hub <- &hypervisor.InitFailedEvent{ Reason: "QMP Init timeout", } handler = nil glog.Error("QMP initialize timeout") case QMP_SESSION: glog.Info("got new session during initializing") buf = append(buf, msg.(*QmpSession)) } } handler = initializing for handler != nil { msg, ok := <-qc.qmp if !ok { glog.Info("QMP channel closed, Quit qmp handler") break } handler(msg) } }
func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemon, err error) { // set up the tmpDir to use a canonical path tmp, err := tempDir(config.Root) if err != nil { return nil, fmt.Errorf("Unable to get the TempDir under %s: %s", config.Root, err) } realTmp, err := fileutils.ReadSymlinkedDirectory(tmp) if err != nil { return nil, fmt.Errorf("Unable to get the full path to the TempDir (%s): %s", tmp, err) } os.Setenv("TMPDIR", realTmp) // get the canonical path to the Docker root directory var realRoot string if _, err := os.Stat(config.Root); err != nil && os.IsNotExist(err) { realRoot = config.Root } else { realRoot, err = fileutils.ReadSymlinkedDirectory(config.Root) if err != nil { return nil, fmt.Errorf("Unable to get the full path to root (%s): %s", config.Root, err) } } config.Root = realRoot // Create the root directory if it doesn't exists if err := os.MkdirAll(config.Root, 0700); err != nil && !os.IsExist(err) { return nil, err } // Set the default driver graphdriver.DefaultDriver = config.GraphDriver // Load storage driver driver, err := graphdriver.New(config.Root, config.GraphOptions) if err != nil { return nil, fmt.Errorf("error initializing graphdriver: %v", err) } glog.V(1).Infof("Using graph driver %s", driver) d := &Daemon{} d.driver = driver defer func() { if err != nil { if err := d.Shutdown(); err != nil { glog.Error(err) } } }() daemonRepo := path.Join(config.Root, "containers") if err := os.MkdirAll(daemonRepo, 0700); err != nil && !os.IsExist(err) { glog.Error(err.Error()) return nil, err } glog.Info("Creating images graph") g, err := graph.NewGraph(path.Join(config.Root, "graph"), d.driver) if err != nil { glog.Error(err.Error()) return nil, err } trustKey, err := api.LoadOrCreateTrustKey(config.TrustKeyPath) if err != nil { glog.Error(err.Error()) return nil, err } trustDir := path.Join(config.Root, "trust") if err := os.MkdirAll(trustDir, 0700); err != nil && !os.IsExist(err) { glog.Error(err.Error()) return nil, err } trustService, err := trust.NewTrustStore(trustDir) if err != nil { glog.Error(err.Error()) return nil, fmt.Errorf("could not create trust store: %s", err) } eventsService := events.New() glog.Info("Creating repository list") tagCfg := &graph.TagStoreConfig{ Graph: g, Key: trustKey, Registry: registryService, Events: eventsService, Trust: trustService, } repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+d.driver.String()), tagCfg) if err != nil { glog.Error(err.Error()) return nil, fmt.Errorf("Couldn't create Tag store: %s", err) } graphdbPath := path.Join(config.Root, "linkgraph.db") graph, err := graphdb.NewSqliteConn(graphdbPath) if err != nil { glog.Error(err.Error()) return nil, err } d.containerGraph = graph sysInfo := sysinfo.New(false) d.ID = trustKey.PublicKey().KeyID() d.repository = daemonRepo d.containers = &contStore{s: make(map[string]*Container)} d.graph = g d.repositories = repositories d.idIndex = truncindex.NewTruncIndex([]string{}) d.sysInfo = sysInfo d.config = config d.sysInitPath = "" d.defaultLogConfig = config.LogConfig d.RegistryService = registryService d.EventsService = eventsService d.root = config.Root if err := d.restore(); err != nil { return nil, err } return d, nil }
func (daemon *Daemon) restore() error { type cr struct { container *Container registered bool } var ( currentDriver = daemon.driver.String() containers = make(map[string]*cr) ) dir, err := ioutil.ReadDir(daemon.repository) if err != nil { return err } for _, v := range dir { id := v.Name() container, err := daemon.load(id) if err != nil { glog.Errorf("Failed to load container %v: %v", id, err) continue } // Ignore the container if it does not support the current driver being used by the graph if (container.Driver == "" && currentDriver == "aufs") || container.Driver == currentDriver { glog.V(1).Infof("Loaded container %v", container.ID) containers[container.ID] = &cr{container: container} } else { glog.V(1).Infof("Cannot load container %s because it was created with another graph driver.", container.ID) } } if entities := daemon.containerGraph.List("/", -1); entities != nil { for _, p := range entities.Paths() { e := entities[p] if c, ok := containers[e.ID()]; ok { c.registered = true } } } group := sync.WaitGroup{} for _, c := range containers { group.Add(1) go func(container *Container, registered bool) { defer group.Done() if !registered { // Try to set the default name for a container if it exists prior to links container.Name, err = daemon.generateNewName(container.ID) if err != nil { glog.V(1).Infof("Setting default id - %s", err) } } if err := daemon.register(container, false); err != nil { glog.V(1).Infof("Failed to register container %s: %s", container.ID, err) } // check the restart policy on the containers and restart any container with // the restart policy of "always" if daemon.config.AutoRestart && container.shouldRestart() { glog.V(1).Infof("Starting container %s", container.ID) if err := container.Start(); err != nil { glog.V(1).Infof("Failed to start container %s: %s", container.ID, err) } } }(c.container, c.registered) } group.Wait() glog.Info("Loading containers: done.") return nil }
func stateStarting(ctx *VmContext, ev VmEvent) { if processed := commonStateHandler(ctx, ev, true); processed { //processed by common } else if processed := deviceInitHandler(ctx, ev); processed { if ctx.deviceReady() { glog.V(1).Info("device ready, could run pod.") ctx.startPod() } } else if processed := initFailureHandler(ctx, ev); processed { ctx.shutdownVM(true, "Fail during init pod running environment") ctx.Become(stateTerminating, "TERMINATING") } else { switch ev.Event() { case EVENT_VM_START_FAILED: glog.Info("VM did not start up properly, go to cleaning up") if closed := ctx.onVmExit(true); !closed { ctx.Become(stateDestroying, "DESTROYING") } case EVENT_INIT_CONNECTED: glog.Info("begin to wait vm commands") ctx.reportVmRun() case COMMAND_RELEASE: glog.Info("pod starting, got release, please wait") ctx.reportBusy("") case COMMAND_ATTACH: ctx.attachCmd(ev.(*AttachCommand)) case COMMAND_WINDOWSIZE: cmd := ev.(*WindowSizeCommand) if ctx.userSpec.Tty { ctx.setWindowSize(cmd.ClientTag, cmd.Size) } case COMMAND_ACK: ack := ev.(*CommandAck) glog.V(1).Infof("[starting] got init ack to %d", ack.reply) if ack.reply == INIT_STARTPOD { ctx.unsetTimeout() var pinfo []byte = []byte{} persist, err := ctx.dump() if err == nil { buf, err := persist.serialize() if err == nil { pinfo = buf } } ctx.reportSuccess("Start POD success", pinfo) ctx.Become(stateRunning, "RUNNING") glog.Info("pod start success ", string(ack.msg)) } case ERROR_CMD_FAIL: ack := ev.(*CommandError) if ack.context.code == INIT_STARTPOD { reason := "Start POD failed" ctx.shutdownVM(true, reason) ctx.Become(stateTerminating, "TERMINATING") glog.Error(reason) } case EVENT_VM_TIMEOUT: reason := "Start POD timeout" ctx.shutdownVM(true, reason) ctx.Become(stateTerminating, "TERMINATING") glog.Error(reason) default: glog.Warning("got event during pod initiating") } } }
func waitCmdToInit(ctx *VmContext, init *net.UnixConn) { looping := true cmds := []*DecodedMessage{} var data []byte var timeout bool = false var index int = 0 var got int = 0 var pingTimer *time.Timer = nil var pongTimer *time.Timer = nil go waitInitAck(ctx, init) for looping { cmd, ok := <-ctx.vm if !ok { glog.Info("vm channel closed, quit") break } if cmd.code == INIT_ACK || cmd.code == INIT_ERROR { if len(cmds) > 0 { if cmds[0].code == INIT_DESTROYPOD { glog.Info("got response of shutdown command, last round of command to init") looping = false } if cmd.code == INIT_ACK { if cmds[0].code != INIT_PING { ctx.Hub <- &CommandAck{ reply: cmds[0].code, msg: cmd.message, } } } else { ctx.Hub <- &CommandError{ context: cmds[0], msg: cmd.message, } } cmds = cmds[1:] if pongTimer != nil { glog.V(1).Info("ack got, clear pong timer") pongTimer.Stop() pongTimer = nil } if pingTimer == nil { pingTimer = time.AfterFunc(30*time.Second, func() { defer func() { recover() }() glog.V(1).Info("Send ping message to init") ctx.vm <- &DecodedMessage{ code: INIT_PING, message: []byte{}, } pingTimer = nil }) } else { pingTimer.Reset(30 * time.Second) } } else { glog.Error("got ack but no command in queue") } } else if cmd.code == INIT_FINISHPOD { num := len(cmd.message) / 4 results := make([]uint32, num) for i := 0; i < num; i++ { results[i] = binary.BigEndian.Uint32(cmd.message[i*4 : i*4+4]) } for _, c := range cmds { if c.code == INIT_DESTROYPOD { glog.Info("got pod finish message after having send destroy message") looping = false ctx.Hub <- &CommandAck{ reply: c.code, } break } } glog.V(1).Infof("Pod finished, returned %d values", num) ctx.Hub <- &PodFinished{ result: results, } } else { if cmd.code == INIT_NEXT { glog.V(1).Infof("get command NEXT") got += int(binary.BigEndian.Uint32(cmd.message[0:4])) glog.V(1).Infof("send %d, receive %d", index, got) timeout = false if index == got { /* received the sent out message */ tmp := data[index:] data = tmp index = 0 got = 0 } } else { glog.V(1).Infof("send command %d to init, payload: '%s'.", cmd.code, string(cmd.message)) cmds = append(cmds, cmd) data = append(data, newVmMessage(cmd)...) timeout = true } if index == 0 && len(data) != 0 { var end int = len(data) if end > 512 { end = 512 } wrote, _ := init.Write(data[:end]) glog.V(1).Infof("write %d to init, payload: '%s'.", wrote, data[:end]) index += wrote } if timeout && pongTimer == nil { glog.V(1).Info("message sent, set pong timer") pongTimer = time.AfterFunc(30*time.Second, func() { ctx.Hub <- &Interrupted{Reason: "init not reply ping mesg"} }) } } } if pingTimer != nil { pingTimer.Stop() } if pongTimer != nil { pongTimer.Stop() } }