// Start builds the docker container from the tar file and launches it.
func (dc *DockerContainer) Start() (err error) {
s := path.Join(dc.Factory.SocketDir, randName())
dc.SocketPath = s + ".sock"
dc.CidfilePath = s + ".cid"
dc.RulesPath = dc.Factory.RulesPath
dc.TaoChannel = util.NewUnixSingleReadWriteCloser(dc.SocketPath)
defer func() {
if err != nil {
dc.TaoChannel.Close()
dc.TaoChannel = nil
}
}()
args := []string{"run", "--rm=true", "-v", dc.SocketPath + ":/tao"}
args = append(args, "--cidfile", dc.CidfilePath)
args = append(args, "--env", HostChannelTypeEnvVar+"="+"unix")
args = append(args, "--env", HostSpecEnvVar+"="+"/tao")
if dc.RulesPath != "" {
args = append(args, "-v", dc.RulesPath+":/"+path.Base(dc.RulesPath))
}
// ContainerArgs has a name plus args passed directly to docker, i.e. before
// image name. Args are passed to the ENTRYPOINT within the Docker image,
// i.e. after image name.
// Note: Uid, Gid, Dir, and Env do not apply to docker hosted programs.
if len(dc.spec.ContainerArgs) > 1 {
args = append(args, dc.spec.ContainerArgs[1:]...)
}
args = append(args, dc.ImageName)
args = append(args, dc.spec.Args...)
dc.Cmd = exec.Command("docker", args...)
dc.Cmd.Stdin = dc.spec.Stdin
dc.Cmd.Stdout = dc.spec.Stdout
dc.Cmd.Stderr = dc.spec.Stderr
err = dc.Cmd.Start()
if err != nil {
return
}
// Reap the child when the process dies.
go func() {
sc := make(chan os.Signal, 1)
signal.Notify(sc, syscall.SIGCHLD)
<-sc
dc.Cmd.Wait()
signal.Stop(sc)
time.Sleep(1 * time.Second)
docker(nil, "rmi", dc.ImageName)
dc.Done <- true
os.Remove(dc.CidfilePath)
close(dc.Done) // prevent any more blocking
}()
return
}
// Start starts the the hosted process and returns a tao channel to it.
func (p *HostedProcess) Start() (channel io.ReadWriteCloser, err error) {
var extraFiles []*os.File
var evar string
switch p.Factory.channelType {
case "pipe":
// Get a pipe pair for communication with the child.
var serverRead, clientRead, serverWrite, clientWrite *os.File
serverRead, clientWrite, err = os.Pipe()
if err != nil {
return
}
defer clientWrite.Close()
clientRead, serverWrite, err = os.Pipe()
if err != nil {
serverRead.Close()
return
}
defer clientRead.Close()
channel = util.NewPairReadWriteCloser(serverRead, serverWrite)
extraFiles = []*os.File{clientRead, clientWrite} // fd 3, fd 4
// Note: ExtraFiles below ensures readfd=3, writefd=4 in child
evar = HostSpecEnvVar + "=tao::RPC+tao::FDMessageChannel(3, 4)"
case "unix":
// Get a random name for the socket.
nameBytes := make([]byte, sockNameLen)
if _, err = rand.Read(nameBytes); err != nil {
return
}
sockName := base64.URLEncoding.EncodeToString(nameBytes)
sockPath := path.Join(p.Factory.socketPath, sockName)
channel = util.NewUnixSingleReadWriteCloser(sockPath)
if channel == nil {
err = fmt.Errorf("Couldn't create a new Unix channel\n")
return
}
evar = HostSpecEnvVar + "=" + sockPath
default:
err = fmt.Errorf("invalid channel type '%s'\n", p.Factory.channelType)
return
}
defer func() {
if err != nil {
channel.Close()
channel = nil
}
}()
env := p.spec.Env
if env == nil {
env = os.Environ()
}
// Make sure that the child knows to use the right kind of channel.
etvar := HostChannelTypeEnvVar + "=" + p.Factory.channelType
replaced := false
replacedType := false
for i, pair := range env {
if strings.HasPrefix(pair, HostSpecEnvVar+"=") {
env[i] = evar
replaced = true
}
if strings.HasPrefix(pair, HostChannelTypeEnvVar+"=") {
env[i] = etvar
replacedType = true
}
}
if !replaced {
env = append(env, evar)
}
if !replacedType {
env = append(env, etvar)
}
if (p.spec.Uid == 0 || p.spec.Gid == 0) && !p.spec.Superuser {
err = fmt.Errorf("Uid and Gid must be nonzero unless Superuser is set\n")
return
}
wd := p.spec.Dir
if wd == "" {
wd = p.Tempdir
}
// Every hosted process is given its own process group (Setpgid=true). This
// ensures that hosted processes will not be in orphaned process groups,
// allowing them to receive job control signals (SIGTTIN, SIGTTOU, and
// SIGTSTP).
//
// If this host is running in "daemon" mode, i.e. without a controlling tty
// and in our own session and process group, then this host will be (a) the
// parent of a process in the child's group, (b) in the same session, and
// (c) not in the same group as the child, so it will serve as the anchor
// that keeps the child process groups from being considered orphaned.
//
// If this host is running in "foreground" mode, i.e. with a controlling tty
// and as part of our parent process's session but in our own process group,
// then the same three conditions are satisified, so this host can still
// serve as the anchor that keeps the child process groups from being
// considered orphaned. (Note: We could also use Setpid=false in this case,
// since the host would be part of the child process group and our parent
// would then meet the requirements.)
spa := &syscall.SysProcAttr{
Credential: &syscall.Credential{
Uid: uint32(p.spec.Uid),
Gid: uint32(p.spec.Uid),
},
// Setsid: true, // Create session.
Setpgid: true, // Set process group ID to new pid (SYSV setpgrp)
// Setctty: true, // Set controlling terminal to fd Ctty (only meaningful if Setsid is set)
// Noctty: true, // Detach fd 0 from controlling terminal
// Ctty: 0, // Controlling TTY fd (Linux only)
}
argv := []string{p.Argv0}
argv = append(argv, p.spec.Args...)
p.Cmd = exec.Cmd{
Path: p.Temppath,
Dir: wd,
Args: argv,
Stdin: p.spec.Stdin,
Stdout: p.spec.Stdout,
Stderr: p.spec.Stderr,
Env: env,
ExtraFiles: extraFiles,
SysProcAttr: spa,
}
if err = p.Cmd.Start(); err != nil {
return
}
// Reap the child when the process dies.
sc := make(chan os.Signal, 1)
signal.Notify(sc, syscall.SIGCHLD)
go func() {
<-sc
p.Cmd.Wait()
signal.Stop(sc)
os.RemoveAll(p.Tempdir)
p.Done <- true
close(p.Done) // prevent any more blocking
}()
// TODO(kwalsh) put channel into p, remove the struct in linux_host.go
return
}
// Start launches a QEMU/KVM CoreOS instance, connects to it with SSH to start
// the LinuxHost on it, and returns the socket connection to that host.
func (kcc *KvmCoreOSHostContainer) Start() (err error) {
// Create the listening server before starting the connection. This lets
// QEMU start right away. See the comments in Start, above, for why this
// is.
kcc.TaoChannel = util.NewUnixSingleReadWriteCloser(kcc.Cfg.SocketPath)
defer func() {
if err != nil {
kcc.TaoChannel.Close()
kcc.TaoChannel = nil
}
}()
if err = kcc.startVM(); err != nil {
return
}
// TODO(kwalsh) reap and clenaup when vm dies; see linux_process_factory.go
// We need some way to wait for the socket to open before we can connect
// to it and return the ReadWriteCloser for communication. Also we need
// to connect by SSH to the instance once it comes up properly. For now,
// we just wait for a timeout before trying to connect and listen.
tc := time.After(10 * time.Second)
// Set up an ssh client config to use to connect to CoreOS.
conf := &ssh.ClientConfig{
// The CoreOS user for the SSH keys is currently always 'core'
// on the virtual machine.
User: "******",
Auth: []ssh.AuthMethod{ssh.PublicKeys(kcc.Factory.PrivateKey)},
}
glog.Info("Waiting about 10 seconds for qemu/coreos to start")
<-tc
hostPort := net.JoinHostPort("localhost", kcc.spec.Args[2])
glog.Info("Connecting to " + hostPort)
client, err := ssh.Dial("tcp", hostPort, conf)
if err != nil {
err = fmt.Errorf("couldn't dial '%s': %s", hostPort, err)
return
}
// We need to run a set of commands to set up the LinuxHost on the
// remote system.
// Mount the filesystem.
glog.Info("Mounting /media/tao on the guest")
mount, err := client.NewSession()
//mount.Stdin = kcc.spec.Stdin
//mount.Stdout = kcc.spec.Stdout
//mount.Stderr = kcc.spec.Stderr
if err != nil {
err = fmt.Errorf("couldn't establish a mount session on SSH: %s", err)
return
}
if err = mount.Run("sudo mkdir /media/tao && sudo mount -t 9p -o trans=virtio,version=9p2000.L tao /media/tao && sudo chmod -R 755 /media/tao"); err != nil {
err = fmt.Errorf("couldn't mount the tao filesystem on the guest: %s", err)
return
}
mount.Close()
// Start the linux_host on the container.
glog.Info("Starting linux_host on the guest")
start, err := client.NewSession()
start.Stdin = kcc.spec.Stdin
start.Stdout = kcc.spec.Stdout
start.Stderr = kcc.spec.Stderr
if err != nil {
err = fmt.Errorf("couldn't establish a start session on SSH: %s", err)
return
}
if err = start.Start("sudo /media/tao/linux_host start -foreground -alsologtostderr -stacked -parent_type file -parent_spec 'tao::RPC+tao::FileMessageChannel(/dev/virtio-ports/tao)' -tao_domain /media/tao"); err != nil {
err = fmt.Errorf("couldn't start linux_host on the guest: %s", err)
return
}
start.Close()
glog.Info("Hosted qemu/coreos/linux_host is ready")
return
}
// Start launches a QEMU/KVM CoreOS instance, connects to it with SSH to start
// the LinuxHost on it, and returns the socket connection to that host.
func (kcc *KvmCoreOSContainer) Start() (channel io.ReadWriteCloser, err error) {
// The args must contain the directory to write the linux_host into, as
// well as the port to use for SSH.
if len(kcc.spec.Args) != 3 {
glog.Errorf("Expected %d args, but got %d", 3, len(kcc.spec.Args))
for i, a := range kcc.spec.Args {
glog.Errorf("Arg %d: %s", i, a)
}
err = errors.New("KVM/CoreOS guest Tao requires args: <linux_host image> <temp directory for linux_host> <SSH port>")
return
}
// Build the new Config and start it. Make sure it has a random name so
// it doesn't conflict with other virtual machines.
sockName := getRandomFileName(nameLen)
sockPath := path.Join(kcc.Factory.SocketPath, sockName)
sshCfg := kcc.Factory.Cfg.SSHKeysCfg + "\n - " + string(kcc.Factory.PublicKey)
// Create a new docker image from the filesystem tarball, and use it to
// build a container and launch it.
kcc.Cfg = &CoreOSConfig{
Name: getRandomFileName(nameLen),
ImageFile: kcc.Factory.Cfg.ImageFile, // the VM image
Memory: kcc.Factory.Cfg.Memory,
RulesPath: kcc.Factory.Cfg.RulesPath,
SSHKeysCfg: sshCfg,
SocketPath: sockPath,
}
// Create the listening server before starting the connection. This lets
// QEMU start right away. See the comments in Start, above, for why this
// is.
channel = util.NewUnixSingleReadWriteCloser(kcc.Cfg.SocketPath)
defer func() {
if err != nil {
channel.Close()
channel = nil
}
}()
if err = kcc.startVM(); err != nil {
return
}
// TODO(kwalsh) reap and clenaup when vm dies; see linux_process_factory.go
// We need some way to wait for the socket to open before we can connect
// to it and return the ReadWriteCloser for communication. Also we need
// to connect by SSH to the instance once it comes up properly. For now,
// we just wait for a timeout before trying to connect and listen.
tc := time.After(10 * time.Second)
// Set up an ssh client config to use to connect to CoreOS.
conf := &ssh.ClientConfig{
// The CoreOS user for the SSH keys is currently always 'core'
// on the virtual machine.
User: "******",
Auth: []ssh.AuthMethod{ssh.PublicKeys(kcc.Factory.PrivateKey)},
}
glog.Info("Waiting for at most 10 seconds before trying to connect")
<-tc
hostPort := net.JoinHostPort("localhost", kcc.spec.Args[2])
client, err := ssh.Dial("tcp", hostPort, conf)
if err != nil {
err = fmt.Errorf("couldn't dial '%s': %s", hostPort, err)
return
}
// We need to run a set of commands to set up the LinuxHost on the
// remote system.
// Mount the filesystem.
mount, err := client.NewSession()
mount.Stdin = kcc.spec.Stdin
mount.Stdout = kcc.spec.Stdout
mount.Stderr = kcc.spec.Stderr
if err != nil {
err = fmt.Errorf("couldn't establish a mount session on SSH: %s", err)
return
}
if err = mount.Run("sudo mkdir /media/tao && sudo mount -t 9p -o trans=virtio,version=9p2000.L tao /media/tao && sudo chmod -R 755 /media/tao"); err != nil {
err = fmt.Errorf("couldn't mount the tao filesystem on the guest: %s", err)
return
}
mount.Close()
// Start the linux_host on the container.
start, err := client.NewSession()
start.Stdin = kcc.spec.Stdin
start.Stdout = kcc.spec.Stdout
start.Stderr = kcc.spec.Stderr
if err != nil {
err = fmt.Errorf("couldn't establish a start session on SSH: %s", err)
return
}
if err = start.Start("sudo /media/tao/linux_host --host_type stacked --host_spec 'tao::RPC+tao::FileMessageChannel(/dev/virtio-ports/tao)' --host_channel_type file --config_path /media/tao/tao.config"); err != nil {
err = fmt.Errorf("couldn't start linux_host on the guest: %s", err)
return
}
start.Close()
return
}
// Start launches a QEMU/KVM CoreOS instance, connects to it with SSH to start
// the LinuxHost on it, and returns the socket connection to that host.
func (kcc *KvmCoreOSContainer) Start() (err error) {
// TODO(kwalsh) make ContainerArgs in HostedProgram spec be a map to
// facilitate things like hostname, memory, ssh rules, etc.
// Use random name to avoid conflicts with other virtual machines.
kcc.VMName = randName()
kcc.Memory = 1024
kcc.SSHAuthorizedKeys = []string{
"ssh-dss AAAAB3NzaC1kc3MAAACBAI3+32jaz6TR+CGCxsz/ggd6+W9zfCFmWHrWy7BfvtI8sW70752eJVlH806tkv8pvI8q+xbGjDpDkzYUBLS8ilfgPoGqzxg7DYB/1TQzUWPoncKqlUiX96xgvqwiL1MsT/rVn9MMRj7iixRrU+DJgOMw/RBoJ6xApNBybLy4SzQ9AAAAFQChgDI8sBoxAwCr4tBDGyrZSlO3VwAAAIEAjDo8AYlMoByOvjQzZZnElw7juxe14qT4eac++x+jEEnpjtefFRqZuN4FJq2oplcXpi1Z8D2CSsq0fFUGH9ej6Mv9gOE2uQUyg4DsfBvyOdNP8IKo0CuUDnRd4tWjGT8UvmDrAN+XcXnm/61+2gMkwmisyfYyGoSi85gXcU7BWGIAAACAP+wM/IpBfI8egYIXT596I1eJLLaLcWODYhym0GRoNnovLZUH6XS9dIU29RnD69NPjROkBOvux9X7PIxseFfA4TDKaKZbu6U3hdC0XJt2QK6ItnUdfN4l+PvHX/KcINTpA4wtsdPxOlDhpttErO+WnypkYM46Illml3DvOcXeQoI= kwalsh@oakham",
"ssh-dss AAAAB3NzaC1kc3MAAACBAPlRunaTbzhvZ/zF/PFpdDSbSZgl/bJi35hSS8v/EVx/nXQJhfSEOMt0NDKajDdpryCywU7RnWiMGBaaGo36inIQPnxxrCLfO0fot9Cj9HMoSQTCvJ4P2GW7VR0VkoGNv2notOwJSabTOh7K8cUxxVZwd52bYeSH75y8w7/w6gcnAAAAFQDXJ4RQf9pC00ARgbE0y8L2p8ow9wAAAIBKXrzjr4QIeITPpZYu/x/3s4ecjQ02Wa6LoYOi+BMb/6HHJEGqyZ6pCP2UBXNPxZn+bf5WTUBiL+b5t22NwWoKFPVJE+glKhMjkfXqyYpnwugRJGnu1PEgjSZjL+6wRGBbBdkMWnrJfN11Zefiuzt+ebRSano6zQeqvzQo5ocBxAAAAIBiuEpiCp71X6uo78sTawOWyWJHHDBrBQN06cO7jDsVSYBgv5fxqNCAVctLmzYKye6Ptu5zgNiDUMt7ZjfVZaf8kFfWIxntVXZEaWibRFm1nOuSiSaobrMOd5usGRmmNNVMk6+CeBwRurob81xhpo6OwTa4kntEiyk7XdCqo75YPQ== family@oakham",
}
kcc.SSHForwardingPort = 2222 // TODO(kwalsh) remove hack
kcc.SSHCommand = false
kcc.MountFrom = nil
kcc.MountTo = nil
t := randName()
kcc.SocketPath = path.Join(kcc.Factory.TempDir, t+".sock")
kcc.LogPath = path.Join(kcc.Factory.TempDir, t+".log")
kcc.CoreOSConfigPath = path.Join(kcc.Factory.TempDir, t+".config")
kcc.PrivateImage = path.Join(kcc.Factory.TempDir, t+".img")
// Copy the image
cp := exec.Command("qemu-img", "create", "-f", "qcow2", "-o", "backing_file="+kcc.Factory.CoreOSImage, kcc.PrivateImage)
if err := cp.Run(); err != nil {
glog.Errorf("qemu-img error creating copy-on-write image: %s\n", err)
return err
}
// Create the listening server before starting the connection. This lets
// QEMU start right away. See the comments in Start, above, for why this
// is.
kcc.TaoChannel = util.NewUnixSingleReadWriteCloser(kcc.SocketPath)
defer func() {
if err != nil {
kcc.TaoChannel.Close()
kcc.TaoChannel = nil
}
}()
kcc.StdioPath = path.Join(kcc.Factory.TempDir, t+".stdio")
if !kcc.SSHCommand {
kcc.StdioChannel = util.NewUnixSingleReadWriteCloser(kcc.StdioPath)
defer func() {
if err != nil {
kcc.StdioChannel.Close()
kcc.StdioChannel = nil
}
}()
}
if err = kcc.startVM(); err != nil {
glog.Infof("Failed to start qemu: %s", err)
return
}
// Reap the child when the process dies.
sc := make(chan os.Signal, 1)
signal.Notify(sc, syscall.SIGCHLD)
go func() {
<-sc
kcc.QCmd.Wait()
kcc.Cleanup()
signal.Stop(sc)
kcc.Done <- true
close(kcc.Done) // prevent any more blocking
}()
if !kcc.SSHCommand {
// Proxy stdio
go io.Copy(kcc.spec.Stdout, kcc.StdioChannel)
go io.Copy(kcc.StdioChannel, kcc.spec.Stdin)
} else {
// We need some way to wait for the socket to open before we can connect
// to it and return the ReadWriteCloser for communication. Also we need
// to connect by SSH to the instance once it comes up properly. For now,
// we just wait for a timeout before trying to connect and listen.
glog.Info("Waiting about 10 seconds for qemu/coreos to start")
<-time.After(10 * time.Second)
dest := net.JoinHostPort("localhost", strconv.Itoa(kcc.SSHForwardingPort))
glog.Info("Connecting to " + dest)
conf := &ssh.ClientConfig{
User: "******",
Auth: []ssh.AuthMethod{ssh.PublicKeys(kcc.Factory.PrivateKey)},
}
var client *ssh.Client
client, err = ssh.Dial("tcp", dest, conf)
if err != nil {
err = fmt.Errorf("couldn't dial '%s': %s", dest, err)
return
}
glog.Info("Executing user command on the guest")
kcc.SCmd, err = client.NewSession()
if err != nil {
err = fmt.Errorf("couldn't establish a start session on SSH: %s", err)
return
}
kcc.SCmd.Stdin = kcc.spec.Stdin
kcc.SCmd.Stdout = kcc.spec.Stdout
kcc.SCmd.Stderr = kcc.spec.Stderr
ctype := "file"
cspec := "tao::RPC+tao::FileMessageChannel(/dev/virtio-ports/tao)"
env := fmt.Sprintf("%s=%s %s=%s",
HostChannelTypeEnvVar, ctype,
HostSpecEnvVar, cspec)
cmd := kcc.spec.Path
args := ""
for _, arg := range kcc.spec.Args {
args += " " + sshQuote(arg)
}
shutdown := "/sbin/shutdown -h now"
if err = kcc.SCmd.Start(env + " " + cmd + args + " ; " + shutdown); err != nil {
err = fmt.Errorf("couldn't start user command on the guest: %s", err)
return
}
// Reap the child when the ssh session dies.
sc := make(chan os.Signal, 1)
go func() {
kcc.SCmd.Wait()
// Give the guest a moment to shutdown cleanly.
<-time.After(3 * time.Second)
kcc.Cleanup()
signal.Stop(sc)
kcc.Done <- true
close(kcc.Done) // prevent any more blocking
}()
}
return
}