// setup standard pipes so that the TTY of the calling runc process
// is not inherited by the container.
func createStdioPipes(p *libcontainer.Process, rootuid int) (*tty, error) {
var (
t = &tty{}
fds []int
)
r, w, err := os.Pipe()
if err != nil {
return nil, err
}
fds = append(fds, int(r.Fd()), int(w.Fd()))
go io.Copy(w, os.Stdin)
t.closers = append(t.closers, w)
p.Stdin = r
if r, w, err = os.Pipe(); err != nil {
return nil, err
}
fds = append(fds, int(r.Fd()), int(w.Fd()))
go io.Copy(os.Stdout, r)
p.Stdout = w
t.closers = append(t.closers, r)
if r, w, err = os.Pipe(); err != nil {
return nil, err
}
fds = append(fds, int(r.Fd()), int(w.Fd()))
go io.Copy(os.Stderr, r)
p.Stderr = w
t.closers = append(t.closers, r)
// change the ownership of the pipe fds incase we are in a user namespace.
for _, fd := range fds {
if err := syscall.Fchown(fd, rootuid, rootuid); err != nil {
return nil, err
}
}
return t, nil
}
// setup standard pipes so that the TTY of the calling runc process
// is not inherited by the container.
func createStdioPipes(p *libcontainer.Process, rootuid int) (*tty, error) {
i, err := p.InitializeIO(rootuid)
if err != nil {
return nil, err
}
t := &tty{
closers: []io.Closer{
i.Stdin,
i.Stdout,
i.Stderr,
},
}
// add the process's io to the post start closers if they support close
for _, cc := range []interface{}{
p.Stdin,
p.Stdout,
p.Stderr,
} {
if c, ok := cc.(io.Closer); ok {
t.postStart = append(t.postStart, c)
}
}
go func() {
io.Copy(i.Stdin, os.Stdin)
i.Stdin.Close()
}()
t.wg.Add(2)
go t.copyIO(os.Stdout, i.Stdout)
go t.copyIO(os.Stderr, i.Stderr)
return t, nil
}
func createTty(p *libcontainer.Process, rootuid int, consolePath string) (*tty, error) {
if consolePath != "" {
if err := p.ConsoleFromPath(consolePath); err != nil {
return nil, err
}
return &tty{}, nil
}
console, err := p.NewConsole(rootuid)
if err != nil {
return nil, err
}
go io.Copy(console, os.Stdin)
go io.Copy(os.Stdout, console)
state, err := term.SetRawTerminal(os.Stdin.Fd())
if err != nil {
return nil, fmt.Errorf("failed to set the terminal from the stdin: %v", err)
}
t := &tty{
console: console,
state: state,
closers: []io.Closer{
console,
},
}
return t, nil
}
func waitProcess(p *libcontainer.Process, t *testing.T) {
_, file, line, _ := runtime.Caller(1)
status, err := p.Wait()
if err != nil {
t.Fatalf("%s:%d: unexpected error: %s\n\n", filepath.Base(file), line, err.Error())
}
if !status.Success() {
t.Fatalf("%s:%d: unexpected status: %s\n\n", filepath.Base(file), line, status.String())
}
}
func createPidFile(path string, process *libcontainer.Process) error {
pid, err := process.Pid()
if err != nil {
return err
}
f, err := os.Create(path)
if err != nil {
return err
}
defer f.Close()
_, err = fmt.Fprintf(f, "%d", pid)
return err
}
// we have to use this type of signal handler because there is a memory leak if we
// wait and reap with SICHLD.
func handleSignals(process *libcontainer.Process, tty *tty) {
sigc := make(chan os.Signal, 10)
signal.Notify(sigc)
tty.resize()
for sig := range sigc {
switch sig {
case syscall.SIGWINCH:
tty.resize()
default:
process.Signal(sig)
}
}
}
func createPidFile(path string, process *libcontainer.Process) error {
pid, err := process.Pid()
if err != nil {
return err
}
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
if err != nil {
return err
}
defer f.Close()
_, err = fmt.Fprintf(f, "%d", pid)
return err
}
func dupStdio(process *libcontainer.Process, rootuid int) error {
process.Stdin = os.Stdin
process.Stdout = os.Stdout
process.Stderr = os.Stderr
for _, fd := range []uintptr{
os.Stdin.Fd(),
os.Stdout.Fd(),
os.Stderr.Fd(),
} {
if err := syscall.Fchown(int(fd), rootuid, rootuid); err != nil {
return err
}
}
return nil
}
// setup standard pipes so that the TTY of the calling runc process
// is not inherited by the container.
func createStdioPipes(p *libcontainer.Process, rootuid int) (*tty, error) {
i, err := p.InitializeIO(rootuid)
if err != nil {
return nil, err
}
t := &tty{
closers: []io.Closer{
i.Stdin,
i.Stdout,
i.Stderr,
},
}
go io.Copy(i.Stdin, os.Stdin)
go io.Copy(os.Stdout, i.Stdout)
go io.Copy(os.Stderr, i.Stderr)
return t, nil
}
// createPidFile creates a file with the processes pid inside it atomically
// it creates a temp file with the paths filename + '.' infront of it
// then renames the file
func createPidFile(path string, process *libcontainer.Process) error {
pid, err := process.Pid()
if err != nil {
return err
}
var (
tmpDir = filepath.Dir(path)
tmpName = filepath.Join(tmpDir, fmt.Sprintf(".%s", filepath.Base(path)))
)
f, err := os.OpenFile(tmpName, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, 0666)
if err != nil {
return err
}
_, err = fmt.Fprintf(f, "%d", pid)
f.Close()
if err != nil {
return err
}
return os.Rename(tmpName, path)
}
func waitInPIDHost(p *libcontainer.Process, c libcontainer.Container) func() (*os.ProcessState, error) {
return func() (*os.ProcessState, error) {
pid, err := p.Pid()
if err != nil {
return nil, err
}
process, err := os.FindProcess(pid)
s, err := process.Wait()
if err != nil {
execErr, ok := err.(*exec.ExitError)
if !ok {
return s, err
}
s = execErr.ProcessState
}
killCgroupProcs(c)
p.Wait()
return s, err
}
}
// forward handles the main signal event loop forwarding, resizing, or reaping depending
// on the signal received.
func (h *signalHandler) forward(process *libcontainer.Process) (int, error) {
// make sure we know the pid of our main process so that we can return
// after it dies.
pid1, err := process.Pid()
if err != nil {
return -1, err
}
// perform the initial tty resize.
h.tty.resize()
for s := range h.signals {
switch s {
case syscall.SIGWINCH:
h.tty.resize()
case syscall.SIGCHLD:
exits, err := h.reap()
if err != nil {
logrus.Error(err)
}
for _, e := range exits {
logrus.WithFields(logrus.Fields{
"pid": e.pid,
"status": e.status,
}).Debug("process exited")
if e.pid == pid1 {
// call Wait() on the process even though we already have the exit
// status because we must ensure that any of the go specific process
// fun such as flushing pipes are complete before we return.
process.Wait()
return e.status, nil
}
}
default:
logrus.Debugf("sending signal to process %s", s)
if err := syscall.Kill(pid1, s.(syscall.Signal)); err != nil {
logrus.Error(err)
}
}
}
return -1, nil
}
func (t *tty) recvtty(process *libcontainer.Process, detach bool) error {
console, err := process.GetConsole()
if err != nil {
return err
}
if !detach {
go io.Copy(console, os.Stdin)
t.wg.Add(1)
go t.copyIO(os.Stdout, console)
state, err := term.SetRawTerminal(os.Stdin.Fd())
if err != nil {
return fmt.Errorf("failed to set the terminal from the stdin: %v", err)
}
t.state = state
}
t.console = console
t.closers = []io.Closer{console}
return nil
}
// setupIO modifies the given process config according to the options.
func setupIO(process *libcontainer.Process, rootuid, rootgid int, createTTY, detach bool) (*tty, error) {
// This is entirely handled by recvtty.
if createTTY {
process.Stdin = nil
process.Stdout = nil
process.Stderr = nil
return &tty{}, nil
}
// When we detach, we just dup over stdio and call it a day. There's no
// requirement that we set up anything nice for our caller or the
// container.
if detach {
if err := dupStdio(process, rootuid, rootgid); err != nil {
return nil, err
}
return &tty{}, nil
}
// XXX: This doesn't sit right with me. It's ugly.
return createStdioPipes(process, rootuid, rootgid)
}
func createTty(p *libcontainer.Process, rootuid int) (*tty, error) {
console, err := p.NewConsole(rootuid)
if err != nil {
return nil, err
}
go io.Copy(console, os.Stdin)
go io.Copy(os.Stdout, console)
state, err := term.SetRawTerminal(os.Stdin.Fd())
if err != nil {
return nil, err
}
t := &tty{
console: console,
state: state,
closers: []io.Closer{
console,
},
}
p.Stderr = nil
p.Stdout = nil
p.Stdin = nil
return t, nil
}
// setup standard pipes so that the TTY of the calling runc process
// is not inherited by the container.
func createStdioPipes(p *libcontainer.Process) (*tty, error) {
t := &tty{}
r, w, err := os.Pipe()
if err != nil {
return nil, err
}
go io.Copy(w, os.Stdin)
t.closers = append(t.closers, w)
p.Stdin = r
if r, w, err = os.Pipe(); err != nil {
return nil, err
}
go io.Copy(os.Stdout, r)
p.Stdout = w
t.closers = append(t.closers, r)
if r, w, err = os.Pipe(); err != nil {
return nil, err
}
go io.Copy(os.Stderr, r)
p.Stderr = w
t.closers = append(t.closers, r)
return t, nil
}
func setupPipes(container *configs.Config, processConfig *execdriver.ProcessConfig, p *libcontainer.Process, pipes *execdriver.Pipes) error {
var term execdriver.Terminal
var err error
if processConfig.Tty {
rootuid, err := container.HostUID()
if err != nil {
return err
}
cons, err := p.NewConsole(rootuid)
if err != nil {
return err
}
term, err = NewTtyConsole(cons, pipes)
} else {
p.Stdout = pipes.Stdout
p.Stderr = pipes.Stderr
r, w, err := os.Pipe()
if err != nil {
return err
}
if pipes.Stdin != nil {
go func() {
io.Copy(w, pipes.Stdin)
w.Close()
}()
p.Stdin = r
}
term = &execdriver.StdConsole{}
}
if err != nil {
return err
}
processConfig.Terminal = term
return nil
}
func setupPipes(container *configs.Config, processConfig *execdriver.ProcessConfig, p *libcontainer.Process, pipes *execdriver.Pipes, wg *sync.WaitGroup) ([]io.WriteCloser, error) {
writers := []io.WriteCloser{}
rootuid, err := container.HostUID()
if err != nil {
return writers, err
}
if processConfig.Tty {
cons, err := p.NewConsole(rootuid)
if err != nil {
return writers, err
}
term, err := NewTtyConsole(cons, pipes)
if err != nil {
return writers, err
}
processConfig.Terminal = term
return writers, nil
}
// not a tty--set up stdio pipes
term := &execdriver.StdConsole{}
processConfig.Terminal = term
// if we are not in a user namespace, there is no reason to go through
// the hassle of setting up os-level pipes with proper (remapped) ownership
// so we will do the prior shortcut for non-userns containers
if rootuid == 0 {
p.Stdout = pipes.Stdout
p.Stderr = pipes.Stderr
r, w, err := os.Pipe()
if err != nil {
return writers, err
}
if pipes.Stdin != nil {
go func() {
io.Copy(w, pipes.Stdin)
w.Close()
}()
p.Stdin = r
}
return writers, nil
}
// if we have user namespaces enabled (rootuid != 0), we will set
// up os pipes for stderr, stdout, stdin so we can chown them to
// the proper ownership to allow for proper access to the underlying
// fds
var fds []uintptr
copyPipes := func(out io.Writer, in io.ReadCloser) {
defer wg.Done()
io.Copy(out, in)
in.Close()
}
//setup stdout
r, w, err := os.Pipe()
if err != nil {
w.Close()
return writers, err
}
writers = append(writers, w)
fds = append(fds, r.Fd(), w.Fd())
if pipes.Stdout != nil {
wg.Add(1)
go copyPipes(pipes.Stdout, r)
}
term.Closers = append(term.Closers, r)
p.Stdout = w
//setup stderr
r, w, err = os.Pipe()
if err != nil {
w.Close()
return writers, err
}
writers = append(writers, w)
fds = append(fds, r.Fd(), w.Fd())
if pipes.Stderr != nil {
wg.Add(1)
go copyPipes(pipes.Stderr, r)
}
term.Closers = append(term.Closers, r)
p.Stderr = w
//setup stdin
r, w, err = os.Pipe()
if err != nil {
r.Close()
return writers, err
}
fds = append(fds, r.Fd(), w.Fd())
if pipes.Stdin != nil {
go func() {
io.Copy(w, pipes.Stdin)
w.Close()
}()
p.Stdin = r
}
for _, fd := range fds {
if err := syscall.Fchown(int(fd), rootuid, rootuid); err != nil {
return writers, fmt.Errorf("Failed to chown pipes fd: %v", err)
}
}
return writers, nil
}
func (r *runner) terminate(p *libcontainer.Process) {
p.Signal(syscall.SIGKILL)
p.Wait()
}
func TestEnter(t *testing.T) {
if testing.Short() {
return
}
root, err := newTestRoot()
ok(t, err)
defer os.RemoveAll(root)
rootfs, err := newRootfs()
ok(t, err)
defer remove(rootfs)
config := newTemplateConfig(rootfs)
container, err := factory.Create("test", config)
ok(t, err)
defer container.Destroy()
// Execute a first process in the container
stdinR, stdinW, err := os.Pipe()
ok(t, err)
var stdout, stdout2 bytes.Buffer
pconfig := libcontainer.Process{
Args: []string{"sh", "-c", "cat && readlink /proc/self/ns/pid"},
Env: standardEnvironment,
Stdin: stdinR,
Stdout: &stdout,
}
err = container.Start(&pconfig)
stdinR.Close()
defer stdinW.Close()
ok(t, err)
pid, err := pconfig.Pid()
ok(t, err)
// Execute another process in the container
stdinR2, stdinW2, err := os.Pipe()
ok(t, err)
pconfig2 := libcontainer.Process{
Env: standardEnvironment,
}
pconfig2.Args = []string{"sh", "-c", "cat && readlink /proc/self/ns/pid"}
pconfig2.Stdin = stdinR2
pconfig2.Stdout = &stdout2
err = container.Start(&pconfig2)
stdinR2.Close()
defer stdinW2.Close()
ok(t, err)
pid2, err := pconfig2.Pid()
ok(t, err)
processes, err := container.Processes()
ok(t, err)
n := 0
for i := range processes {
if processes[i] == pid || processes[i] == pid2 {
n++
}
}
if n != 2 {
t.Fatal("unexpected number of processes", processes, pid, pid2)
}
// Wait processes
stdinW2.Close()
waitProcess(&pconfig2, t)
stdinW.Close()
waitProcess(&pconfig, t)
// Check that both processes live in the same pidns
pidns := string(stdout.Bytes())
ok(t, err)
pidns2 := string(stdout2.Bytes())
ok(t, err)
if pidns != pidns2 {
t.Fatal("The second process isn't in the required pid namespace", pidns, pidns2)
}
}
func TestCheckpoint(t *testing.T) {
if testing.Short() {
return
}
root, err := newTestRoot()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
rootfs, err := newRootfs()
if err != nil {
t.Fatal(err)
}
defer remove(rootfs)
config := newTemplateConfig(rootfs)
config.Mounts = append(config.Mounts, &configs.Mount{
Destination: "/sys/fs/cgroup",
Device: "cgroup",
Flags: defaultMountFlags | syscall.MS_RDONLY,
})
factory, err := libcontainer.New(root, libcontainer.Cgroupfs)
if err != nil {
t.Fatal(err)
}
container, err := factory.Create("test", config)
if err != nil {
t.Fatal(err)
}
defer container.Destroy()
stdinR, stdinW, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
var stdout bytes.Buffer
pconfig := libcontainer.Process{
Cwd: "/",
Args: []string{"cat"},
Env: standardEnvironment,
Stdin: stdinR,
Stdout: &stdout,
}
err = container.Start(&pconfig)
stdinR.Close()
defer stdinW.Close()
if err != nil {
t.Fatal(err)
}
pid, err := pconfig.Pid()
if err != nil {
t.Fatal(err)
}
process, err := os.FindProcess(pid)
if err != nil {
t.Fatal(err)
}
imagesDir, err := ioutil.TempDir("", "criu")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(imagesDir)
checkpointOpts := &libcontainer.CriuOpts{
ImagesDirectory: imagesDir,
WorkDirectory: imagesDir,
}
dumpLog := filepath.Join(checkpointOpts.WorkDirectory, "dump.log")
restoreLog := filepath.Join(checkpointOpts.WorkDirectory, "restore.log")
if err := container.Checkpoint(checkpointOpts); err != nil {
showFile(t, dumpLog)
t.Fatal(err)
}
state, err := container.Status()
if err != nil {
t.Fatal(err)
}
if state != libcontainer.Running {
t.Fatal("Unexpected state checkpoint: ", state)
}
stdinW.Close()
_, err = process.Wait()
if err != nil {
t.Fatal(err)
}
// reload the container
container, err = factory.Load("test")
if err != nil {
t.Fatal(err)
}
restoreStdinR, restoreStdinW, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
restoreProcessConfig := &libcontainer.Process{
Cwd: "/",
Stdin: restoreStdinR,
Stdout: &stdout,
}
err = container.Restore(restoreProcessConfig, checkpointOpts)
restoreStdinR.Close()
defer restoreStdinW.Close()
if err != nil {
showFile(t, restoreLog)
t.Fatal(err)
}
state, err = container.Status()
if err != nil {
t.Fatal(err)
}
if state != libcontainer.Running {
t.Fatal("Unexpected restore state: ", state)
}
pid, err = restoreProcessConfig.Pid()
if err != nil {
t.Fatal(err)
}
process, err = os.FindProcess(pid)
if err != nil {
t.Fatal(err)
}
_, err = restoreStdinW.WriteString("Hello!")
if err != nil {
t.Fatal(err)
}
restoreStdinW.Close()
s, err := process.Wait()
if err != nil {
t.Fatal(err)
}
if !s.Success() {
t.Fatal(s.String(), pid)
}
output := string(stdout.Bytes())
if !strings.Contains(output, "Hello!") {
t.Fatal("Did not restore the pipe correctly:", output)
}
}
func setupPipes(container *configs.Config, processConfig *execdriver.ProcessConfig, p *libcontainer.Process, pipes *execdriver.Pipes) error {
rootuid, err := container.HostUID()
if err != nil {
return err
}
if processConfig.Tty {
cons, err := p.NewConsole(rootuid)
if err != nil {
return err
}
term, err := NewTtyConsole(cons, pipes)
if err != nil {
return err
}
processConfig.Terminal = term
return nil
}
// not a tty--set up stdio pipes
term := &execdriver.StdConsole{}
processConfig.Terminal = term
// if we are not in a user namespace, there is no reason to go through
// the hassle of setting up os-level pipes with proper (remapped) ownership
// so we will do the prior shortcut for non-userns containers
if rootuid == 0 {
p.Stdout = pipes.Stdout
p.Stderr = pipes.Stderr
r, w, err := os.Pipe()
if err != nil {
return err
}
if pipes.Stdin != nil {
go func() {
io.Copy(w, pipes.Stdin)
w.Close()
}()
p.Stdin = r
}
return nil
}
// if we have user namespaces enabled (rootuid != 0), we will set
// up os pipes for stderr, stdout, stdin so we can chown them to
// the proper ownership to allow for proper access to the underlying
// fds
var fds []int
//setup stdout
r, w, err := os.Pipe()
if err != nil {
return err
}
fds = append(fds, int(r.Fd()), int(w.Fd()))
if pipes.Stdout != nil {
go io.Copy(pipes.Stdout, r)
}
term.Closers = append(term.Closers, r)
p.Stdout = w
//setup stderr
r, w, err = os.Pipe()
if err != nil {
return err
}
fds = append(fds, int(r.Fd()), int(w.Fd()))
if pipes.Stderr != nil {
go io.Copy(pipes.Stderr, r)
}
term.Closers = append(term.Closers, r)
p.Stderr = w
//setup stdin
r, w, err = os.Pipe()
if err != nil {
return err
}
fds = append(fds, int(r.Fd()), int(w.Fd()))
if pipes.Stdin != nil {
go func() {
io.Copy(w, pipes.Stdin)
w.Close()
}()
p.Stdin = r
}
for _, fd := range fds {
if err := syscall.Fchown(fd, rootuid, rootuid); err != nil {
return fmt.Errorf("Failed to chown pipes fd: %v", err)
}
}
return nil
}
func TestCheckpoint(t *testing.T) {
if testing.Short() {
return
}
root, err := newTestRoot()
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(root)
rootfs, err := newRootfs()
if err != nil {
t.Fatal(err)
}
defer remove(rootfs)
config := newTemplateConfig(rootfs)
factory, err := libcontainer.New(root, libcontainer.Cgroupfs)
if err != nil {
t.Fatal(err)
}
container, err := factory.Create("test", config)
if err != nil {
t.Fatal(err)
}
defer container.Destroy()
stdinR, stdinW, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
var stdout bytes.Buffer
pconfig := libcontainer.Process{
Args: []string{"cat"},
Env: standardEnvironment,
Stdin: stdinR,
Stdout: &stdout,
}
err = container.Start(&pconfig)
stdinR.Close()
defer stdinW.Close()
if err != nil {
t.Fatal(err)
}
pid, err := pconfig.Pid()
if err != nil {
t.Fatal(err)
}
process, err := os.FindProcess(pid)
if err != nil {
t.Fatal(err)
}
imagesDir, err := ioutil.TempDir("", "criu")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(imagesDir)
checkpointOpts := &libcontainer.CriuOpts{
ImagesDirectory: imagesDir,
WorkDirectory: imagesDir,
}
if err := container.Checkpoint(checkpointOpts); err != nil {
t.Fatal(err)
}
state, err := container.Status()
if err != nil {
t.Fatal(err)
}
if state != libcontainer.Checkpointed {
t.Fatal("Unexpected state: ", state)
}
stdinW.Close()
_, err = process.Wait()
if err != nil {
t.Fatal(err)
}
// reload the container
container, err = factory.Load("test")
if err != nil {
t.Fatal(err)
}
restoreStdinR, restoreStdinW, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
restoreProcessConfig := &libcontainer.Process{
Stdin: restoreStdinR,
Stdout: &stdout,
}
err = container.Restore(restoreProcessConfig, &libcontainer.CriuOpts{
ImagesDirectory: imagesDir,
})
restoreStdinR.Close()
defer restoreStdinW.Close()
state, err = container.Status()
if err != nil {
t.Fatal(err)
}
if state != libcontainer.Running {
t.Fatal("Unexpected state: ", state)
}
pid, err = restoreProcessConfig.Pid()
if err != nil {
t.Fatal(err)
}
process, err = os.FindProcess(pid)
if err != nil {
t.Fatal(err)
}
_, err = restoreStdinW.WriteString("Hello!")
if err != nil {
t.Fatal(err)
}
restoreStdinW.Close()
s, err := process.Wait()
if err != nil {
t.Fatal(err)
}
if !s.Success() {
t.Fatal(s.String(), pid)
}
output := string(stdout.Bytes())
if !strings.Contains(output, "Hello!") {
t.Fatal("Did not restore the pipe correctly:", output)
}
}