// Start builds the docker container from the tar file and launches it. func (dc *DockerContainer) Start() (err error) { s := path.Join(dc.Factory.SocketDir, randName()) dc.SocketPath = s + ".sock" dc.CidfilePath = s + ".cid" dc.RulesPath = dc.Factory.RulesPath dc.TaoChannel = util.NewUnixSingleReadWriteCloser(dc.SocketPath) defer func() { if err != nil { dc.TaoChannel.Close() dc.TaoChannel = nil } }() args := []string{"run", "--rm=true", "-v", dc.SocketPath + ":/tao"} args = append(args, "--cidfile", dc.CidfilePath) args = append(args, "--env", HostChannelTypeEnvVar+"="+"unix") args = append(args, "--env", HostSpecEnvVar+"="+"/tao") if dc.RulesPath != "" { args = append(args, "-v", dc.RulesPath+":/"+path.Base(dc.RulesPath)) } // ContainerArgs has a name plus args passed directly to docker, i.e. before // image name. Args are passed to the ENTRYPOINT within the Docker image, // i.e. after image name. // Note: Uid, Gid, Dir, and Env do not apply to docker hosted programs. if len(dc.spec.ContainerArgs) > 1 { args = append(args, dc.spec.ContainerArgs[1:]...) } args = append(args, dc.ImageName) args = append(args, dc.spec.Args...) dc.Cmd = exec.Command("docker", args...) dc.Cmd.Stdin = dc.spec.Stdin dc.Cmd.Stdout = dc.spec.Stdout dc.Cmd.Stderr = dc.spec.Stderr err = dc.Cmd.Start() if err != nil { return } // Reap the child when the process dies. go func() { sc := make(chan os.Signal, 1) signal.Notify(sc, syscall.SIGCHLD) <-sc dc.Cmd.Wait() signal.Stop(sc) time.Sleep(1 * time.Second) docker(nil, "rmi", dc.ImageName) dc.Done <- true os.Remove(dc.CidfilePath) close(dc.Done) // prevent any more blocking }() return }
// Start starts the the hosted process and returns a tao channel to it. func (p *HostedProcess) Start() (channel io.ReadWriteCloser, err error) { var extraFiles []*os.File var evar string switch p.Factory.channelType { case "pipe": // Get a pipe pair for communication with the child. var serverRead, clientRead, serverWrite, clientWrite *os.File serverRead, clientWrite, err = os.Pipe() if err != nil { return } defer clientWrite.Close() clientRead, serverWrite, err = os.Pipe() if err != nil { serverRead.Close() return } defer clientRead.Close() channel = util.NewPairReadWriteCloser(serverRead, serverWrite) extraFiles = []*os.File{clientRead, clientWrite} // fd 3, fd 4 // Note: ExtraFiles below ensures readfd=3, writefd=4 in child evar = HostSpecEnvVar + "=tao::RPC+tao::FDMessageChannel(3, 4)" case "unix": // Get a random name for the socket. nameBytes := make([]byte, sockNameLen) if _, err = rand.Read(nameBytes); err != nil { return } sockName := base64.URLEncoding.EncodeToString(nameBytes) sockPath := path.Join(p.Factory.socketPath, sockName) channel = util.NewUnixSingleReadWriteCloser(sockPath) if channel == nil { err = fmt.Errorf("Couldn't create a new Unix channel\n") return } evar = HostSpecEnvVar + "=" + sockPath default: err = fmt.Errorf("invalid channel type '%s'\n", p.Factory.channelType) return } defer func() { if err != nil { channel.Close() channel = nil } }() env := p.spec.Env if env == nil { env = os.Environ() } // Make sure that the child knows to use the right kind of channel. etvar := HostChannelTypeEnvVar + "=" + p.Factory.channelType replaced := false replacedType := false for i, pair := range env { if strings.HasPrefix(pair, HostSpecEnvVar+"=") { env[i] = evar replaced = true } if strings.HasPrefix(pair, HostChannelTypeEnvVar+"=") { env[i] = etvar replacedType = true } } if !replaced { env = append(env, evar) } if !replacedType { env = append(env, etvar) } if (p.spec.Uid == 0 || p.spec.Gid == 0) && !p.spec.Superuser { err = fmt.Errorf("Uid and Gid must be nonzero unless Superuser is set\n") return } wd := p.spec.Dir if wd == "" { wd = p.Tempdir } // Every hosted process is given its own process group (Setpgid=true). This // ensures that hosted processes will not be in orphaned process groups, // allowing them to receive job control signals (SIGTTIN, SIGTTOU, and // SIGTSTP). // // If this host is running in "daemon" mode, i.e. without a controlling tty // and in our own session and process group, then this host will be (a) the // parent of a process in the child's group, (b) in the same session, and // (c) not in the same group as the child, so it will serve as the anchor // that keeps the child process groups from being considered orphaned. // // If this host is running in "foreground" mode, i.e. with a controlling tty // and as part of our parent process's session but in our own process group, // then the same three conditions are satisified, so this host can still // serve as the anchor that keeps the child process groups from being // considered orphaned. (Note: We could also use Setpid=false in this case, // since the host would be part of the child process group and our parent // would then meet the requirements.) spa := &syscall.SysProcAttr{ Credential: &syscall.Credential{ Uid: uint32(p.spec.Uid), Gid: uint32(p.spec.Uid), }, // Setsid: true, // Create session. Setpgid: true, // Set process group ID to new pid (SYSV setpgrp) // Setctty: true, // Set controlling terminal to fd Ctty (only meaningful if Setsid is set) // Noctty: true, // Detach fd 0 from controlling terminal // Ctty: 0, // Controlling TTY fd (Linux only) } argv := []string{p.Argv0} argv = append(argv, p.spec.Args...) p.Cmd = exec.Cmd{ Path: p.Temppath, Dir: wd, Args: argv, Stdin: p.spec.Stdin, Stdout: p.spec.Stdout, Stderr: p.spec.Stderr, Env: env, ExtraFiles: extraFiles, SysProcAttr: spa, } if err = p.Cmd.Start(); err != nil { return } // Reap the child when the process dies. sc := make(chan os.Signal, 1) signal.Notify(sc, syscall.SIGCHLD) go func() { <-sc p.Cmd.Wait() signal.Stop(sc) os.RemoveAll(p.Tempdir) p.Done <- true close(p.Done) // prevent any more blocking }() // TODO(kwalsh) put channel into p, remove the struct in linux_host.go return }
// Start launches a QEMU/KVM CoreOS instance, connects to it with SSH to start // the LinuxHost on it, and returns the socket connection to that host. func (kcc *KvmCoreOSHostContainer) Start() (err error) { // Create the listening server before starting the connection. This lets // QEMU start right away. See the comments in Start, above, for why this // is. kcc.TaoChannel = util.NewUnixSingleReadWriteCloser(kcc.Cfg.SocketPath) defer func() { if err != nil { kcc.TaoChannel.Close() kcc.TaoChannel = nil } }() if err = kcc.startVM(); err != nil { return } // TODO(kwalsh) reap and clenaup when vm dies; see linux_process_factory.go // We need some way to wait for the socket to open before we can connect // to it and return the ReadWriteCloser for communication. Also we need // to connect by SSH to the instance once it comes up properly. For now, // we just wait for a timeout before trying to connect and listen. tc := time.After(10 * time.Second) // Set up an ssh client config to use to connect to CoreOS. conf := &ssh.ClientConfig{ // The CoreOS user for the SSH keys is currently always 'core' // on the virtual machine. User: "******", Auth: []ssh.AuthMethod{ssh.PublicKeys(kcc.Factory.PrivateKey)}, } glog.Info("Waiting about 10 seconds for qemu/coreos to start") <-tc hostPort := net.JoinHostPort("localhost", kcc.spec.Args[2]) glog.Info("Connecting to " + hostPort) client, err := ssh.Dial("tcp", hostPort, conf) if err != nil { err = fmt.Errorf("couldn't dial '%s': %s", hostPort, err) return } // We need to run a set of commands to set up the LinuxHost on the // remote system. // Mount the filesystem. glog.Info("Mounting /media/tao on the guest") mount, err := client.NewSession() //mount.Stdin = kcc.spec.Stdin //mount.Stdout = kcc.spec.Stdout //mount.Stderr = kcc.spec.Stderr if err != nil { err = fmt.Errorf("couldn't establish a mount session on SSH: %s", err) return } if err = mount.Run("sudo mkdir /media/tao && sudo mount -t 9p -o trans=virtio,version=9p2000.L tao /media/tao && sudo chmod -R 755 /media/tao"); err != nil { err = fmt.Errorf("couldn't mount the tao filesystem on the guest: %s", err) return } mount.Close() // Start the linux_host on the container. glog.Info("Starting linux_host on the guest") start, err := client.NewSession() start.Stdin = kcc.spec.Stdin start.Stdout = kcc.spec.Stdout start.Stderr = kcc.spec.Stderr if err != nil { err = fmt.Errorf("couldn't establish a start session on SSH: %s", err) return } if err = start.Start("sudo /media/tao/linux_host start -foreground -alsologtostderr -stacked -parent_type file -parent_spec 'tao::RPC+tao::FileMessageChannel(/dev/virtio-ports/tao)' -tao_domain /media/tao"); err != nil { err = fmt.Errorf("couldn't start linux_host on the guest: %s", err) return } start.Close() glog.Info("Hosted qemu/coreos/linux_host is ready") return }
// Start launches a QEMU/KVM CoreOS instance, connects to it with SSH to start // the LinuxHost on it, and returns the socket connection to that host. func (kcc *KvmCoreOSContainer) Start() (channel io.ReadWriteCloser, err error) { // The args must contain the directory to write the linux_host into, as // well as the port to use for SSH. if len(kcc.spec.Args) != 3 { glog.Errorf("Expected %d args, but got %d", 3, len(kcc.spec.Args)) for i, a := range kcc.spec.Args { glog.Errorf("Arg %d: %s", i, a) } err = errors.New("KVM/CoreOS guest Tao requires args: <linux_host image> <temp directory for linux_host> <SSH port>") return } // Build the new Config and start it. Make sure it has a random name so // it doesn't conflict with other virtual machines. sockName := getRandomFileName(nameLen) sockPath := path.Join(kcc.Factory.SocketPath, sockName) sshCfg := kcc.Factory.Cfg.SSHKeysCfg + "\n - " + string(kcc.Factory.PublicKey) // Create a new docker image from the filesystem tarball, and use it to // build a container and launch it. kcc.Cfg = &CoreOSConfig{ Name: getRandomFileName(nameLen), ImageFile: kcc.Factory.Cfg.ImageFile, // the VM image Memory: kcc.Factory.Cfg.Memory, RulesPath: kcc.Factory.Cfg.RulesPath, SSHKeysCfg: sshCfg, SocketPath: sockPath, } // Create the listening server before starting the connection. This lets // QEMU start right away. See the comments in Start, above, for why this // is. channel = util.NewUnixSingleReadWriteCloser(kcc.Cfg.SocketPath) defer func() { if err != nil { channel.Close() channel = nil } }() if err = kcc.startVM(); err != nil { return } // TODO(kwalsh) reap and clenaup when vm dies; see linux_process_factory.go // We need some way to wait for the socket to open before we can connect // to it and return the ReadWriteCloser for communication. Also we need // to connect by SSH to the instance once it comes up properly. For now, // we just wait for a timeout before trying to connect and listen. tc := time.After(10 * time.Second) // Set up an ssh client config to use to connect to CoreOS. conf := &ssh.ClientConfig{ // The CoreOS user for the SSH keys is currently always 'core' // on the virtual machine. User: "******", Auth: []ssh.AuthMethod{ssh.PublicKeys(kcc.Factory.PrivateKey)}, } glog.Info("Waiting for at most 10 seconds before trying to connect") <-tc hostPort := net.JoinHostPort("localhost", kcc.spec.Args[2]) client, err := ssh.Dial("tcp", hostPort, conf) if err != nil { err = fmt.Errorf("couldn't dial '%s': %s", hostPort, err) return } // We need to run a set of commands to set up the LinuxHost on the // remote system. // Mount the filesystem. mount, err := client.NewSession() mount.Stdin = kcc.spec.Stdin mount.Stdout = kcc.spec.Stdout mount.Stderr = kcc.spec.Stderr if err != nil { err = fmt.Errorf("couldn't establish a mount session on SSH: %s", err) return } if err = mount.Run("sudo mkdir /media/tao && sudo mount -t 9p -o trans=virtio,version=9p2000.L tao /media/tao && sudo chmod -R 755 /media/tao"); err != nil { err = fmt.Errorf("couldn't mount the tao filesystem on the guest: %s", err) return } mount.Close() // Start the linux_host on the container. start, err := client.NewSession() start.Stdin = kcc.spec.Stdin start.Stdout = kcc.spec.Stdout start.Stderr = kcc.spec.Stderr if err != nil { err = fmt.Errorf("couldn't establish a start session on SSH: %s", err) return } if err = start.Start("sudo /media/tao/linux_host --host_type stacked --host_spec 'tao::RPC+tao::FileMessageChannel(/dev/virtio-ports/tao)' --host_channel_type file --config_path /media/tao/tao.config"); err != nil { err = fmt.Errorf("couldn't start linux_host on the guest: %s", err) return } start.Close() return }
// Start launches a QEMU/KVM CoreOS instance, connects to it with SSH to start // the LinuxHost on it, and returns the socket connection to that host. func (kcc *KvmCoreOSContainer) Start() (err error) { // TODO(kwalsh) make ContainerArgs in HostedProgram spec be a map to // facilitate things like hostname, memory, ssh rules, etc. // Use random name to avoid conflicts with other virtual machines. kcc.VMName = randName() kcc.Memory = 1024 kcc.SSHAuthorizedKeys = []string{ "ssh-dss AAAAB3NzaC1kc3MAAACBAI3+32jaz6TR+CGCxsz/ggd6+W9zfCFmWHrWy7BfvtI8sW70752eJVlH806tkv8pvI8q+xbGjDpDkzYUBLS8ilfgPoGqzxg7DYB/1TQzUWPoncKqlUiX96xgvqwiL1MsT/rVn9MMRj7iixRrU+DJgOMw/RBoJ6xApNBybLy4SzQ9AAAAFQChgDI8sBoxAwCr4tBDGyrZSlO3VwAAAIEAjDo8AYlMoByOvjQzZZnElw7juxe14qT4eac++x+jEEnpjtefFRqZuN4FJq2oplcXpi1Z8D2CSsq0fFUGH9ej6Mv9gOE2uQUyg4DsfBvyOdNP8IKo0CuUDnRd4tWjGT8UvmDrAN+XcXnm/61+2gMkwmisyfYyGoSi85gXcU7BWGIAAACAP+wM/IpBfI8egYIXT596I1eJLLaLcWODYhym0GRoNnovLZUH6XS9dIU29RnD69NPjROkBOvux9X7PIxseFfA4TDKaKZbu6U3hdC0XJt2QK6ItnUdfN4l+PvHX/KcINTpA4wtsdPxOlDhpttErO+WnypkYM46Illml3DvOcXeQoI= kwalsh@oakham", "ssh-dss AAAAB3NzaC1kc3MAAACBAPlRunaTbzhvZ/zF/PFpdDSbSZgl/bJi35hSS8v/EVx/nXQJhfSEOMt0NDKajDdpryCywU7RnWiMGBaaGo36inIQPnxxrCLfO0fot9Cj9HMoSQTCvJ4P2GW7VR0VkoGNv2notOwJSabTOh7K8cUxxVZwd52bYeSH75y8w7/w6gcnAAAAFQDXJ4RQf9pC00ARgbE0y8L2p8ow9wAAAIBKXrzjr4QIeITPpZYu/x/3s4ecjQ02Wa6LoYOi+BMb/6HHJEGqyZ6pCP2UBXNPxZn+bf5WTUBiL+b5t22NwWoKFPVJE+glKhMjkfXqyYpnwugRJGnu1PEgjSZjL+6wRGBbBdkMWnrJfN11Zefiuzt+ebRSano6zQeqvzQo5ocBxAAAAIBiuEpiCp71X6uo78sTawOWyWJHHDBrBQN06cO7jDsVSYBgv5fxqNCAVctLmzYKye6Ptu5zgNiDUMt7ZjfVZaf8kFfWIxntVXZEaWibRFm1nOuSiSaobrMOd5usGRmmNNVMk6+CeBwRurob81xhpo6OwTa4kntEiyk7XdCqo75YPQ== family@oakham", } kcc.SSHForwardingPort = 2222 // TODO(kwalsh) remove hack kcc.SSHCommand = false kcc.MountFrom = nil kcc.MountTo = nil t := randName() kcc.SocketPath = path.Join(kcc.Factory.TempDir, t+".sock") kcc.LogPath = path.Join(kcc.Factory.TempDir, t+".log") kcc.CoreOSConfigPath = path.Join(kcc.Factory.TempDir, t+".config") kcc.PrivateImage = path.Join(kcc.Factory.TempDir, t+".img") // Copy the image cp := exec.Command("qemu-img", "create", "-f", "qcow2", "-o", "backing_file="+kcc.Factory.CoreOSImage, kcc.PrivateImage) if err := cp.Run(); err != nil { glog.Errorf("qemu-img error creating copy-on-write image: %s\n", err) return err } // Create the listening server before starting the connection. This lets // QEMU start right away. See the comments in Start, above, for why this // is. kcc.TaoChannel = util.NewUnixSingleReadWriteCloser(kcc.SocketPath) defer func() { if err != nil { kcc.TaoChannel.Close() kcc.TaoChannel = nil } }() kcc.StdioPath = path.Join(kcc.Factory.TempDir, t+".stdio") if !kcc.SSHCommand { kcc.StdioChannel = util.NewUnixSingleReadWriteCloser(kcc.StdioPath) defer func() { if err != nil { kcc.StdioChannel.Close() kcc.StdioChannel = nil } }() } if err = kcc.startVM(); err != nil { glog.Infof("Failed to start qemu: %s", err) return } // Reap the child when the process dies. sc := make(chan os.Signal, 1) signal.Notify(sc, syscall.SIGCHLD) go func() { <-sc kcc.QCmd.Wait() kcc.Cleanup() signal.Stop(sc) kcc.Done <- true close(kcc.Done) // prevent any more blocking }() if !kcc.SSHCommand { // Proxy stdio go io.Copy(kcc.spec.Stdout, kcc.StdioChannel) go io.Copy(kcc.StdioChannel, kcc.spec.Stdin) } else { // We need some way to wait for the socket to open before we can connect // to it and return the ReadWriteCloser for communication. Also we need // to connect by SSH to the instance once it comes up properly. For now, // we just wait for a timeout before trying to connect and listen. glog.Info("Waiting about 10 seconds for qemu/coreos to start") <-time.After(10 * time.Second) dest := net.JoinHostPort("localhost", strconv.Itoa(kcc.SSHForwardingPort)) glog.Info("Connecting to " + dest) conf := &ssh.ClientConfig{ User: "******", Auth: []ssh.AuthMethod{ssh.PublicKeys(kcc.Factory.PrivateKey)}, } var client *ssh.Client client, err = ssh.Dial("tcp", dest, conf) if err != nil { err = fmt.Errorf("couldn't dial '%s': %s", dest, err) return } glog.Info("Executing user command on the guest") kcc.SCmd, err = client.NewSession() if err != nil { err = fmt.Errorf("couldn't establish a start session on SSH: %s", err) return } kcc.SCmd.Stdin = kcc.spec.Stdin kcc.SCmd.Stdout = kcc.spec.Stdout kcc.SCmd.Stderr = kcc.spec.Stderr ctype := "file" cspec := "tao::RPC+tao::FileMessageChannel(/dev/virtio-ports/tao)" env := fmt.Sprintf("%s=%s %s=%s", HostChannelTypeEnvVar, ctype, HostSpecEnvVar, cspec) cmd := kcc.spec.Path args := "" for _, arg := range kcc.spec.Args { args += " " + sshQuote(arg) } shutdown := "/sbin/shutdown -h now" if err = kcc.SCmd.Start(env + " " + cmd + args + " ; " + shutdown); err != nil { err = fmt.Errorf("couldn't start user command on the guest: %s", err) return } // Reap the child when the ssh session dies. sc := make(chan os.Signal, 1) go func() { kcc.SCmd.Wait() // Give the guest a moment to shutdown cleanly. <-time.After(3 * time.Second) kcc.Cleanup() signal.Stop(sc) kcc.Done <- true close(kcc.Done) // prevent any more blocking }() } return }