// formatQcow2 formats a partition with the default linux filesystem type.
func formatQcow2(dev string) error {
// make an ext4 filesystem
p := process("mkfs")
cmd := &exec.Cmd{
Path: p,
Args: []string{
p,
dev,
},
Env: nil,
Dir: "",
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return err
}
log.LogAll(stdout, log.INFO, "mkfs")
log.LogAll(stderr, log.INFO, "mkfs")
log.Debug("formatting with with cmd: %v", cmd)
return cmd.Run()
}
// DisconnectDevice disconnects a given NBD using qemu-nbd.
func DisconnectDevice(dev string) error {
// disconnect nbd
p := process("qemu-nbd")
cmd := &exec.Cmd{
Path: p,
Args: []string{
p,
"-d",
dev,
},
Env: nil,
Dir: "",
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return err
}
log.LogAll(stdout, log.INFO, "qemu-nbd")
log.LogAll(stderr, log.ERROR, "qemu-nbd")
log.Debug("disconnecting nbd with cmd: %v", cmd)
return cmd.Run()
}
// extlinuxMBR installs the specified master boot record in the partition table
// for the provided device.
func extlinuxMBR(dev, mbr string) error {
// dd the mbr image
p := process("dd")
cmd := &exec.Cmd{
Path: p,
Args: []string{
p,
fmt.Sprintf("if=%v", mbr),
"conv=notrunc",
"bs=440",
"count=1",
fmt.Sprintf("of=%v", dev),
},
Env: nil,
Dir: "",
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return err
}
log.LogAll(stdout, log.INFO, "dd")
log.LogAll(stderr, log.INFO, "dd")
log.Debug("installing mbr with cmd: %v", cmd)
return cmd.Run()
}
// umountQcow2 unmounts qcow2 image that was previously mounted with
// mountQcow2.
func umountQcow2(path string) error {
// unmount
p := process("umount")
cmd := &exec.Cmd{
Path: p,
Args: []string{
p,
path,
},
Env: nil,
Dir: "",
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return err
}
log.LogAll(stdout, log.INFO, "umount")
log.LogAll(stderr, log.ERROR, "umount")
log.Debug("unmounting with cmd: %v", cmd)
return cmd.Run()
}
// copyQcow2 recursively copies files from src to dst using cp.
func copyQcow2(src, dst string) error {
// copy everything over
p := process("cp")
cmd := &exec.Cmd{
Path: p,
Args: []string{
p,
"-a",
"-v",
src + "/.",
dst,
},
Env: nil,
Dir: "",
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return err
}
log.LogAll(stdout, log.INFO, "cp")
log.LogAll(stderr, log.ERROR, "cp")
log.Debug("copy with with cmd: %v", cmd)
return cmd.Run()
}
// Debootstrap will invoke the debootstrap tool with a target build directory
// in build_path, using configuration from c.
func Debootstrap(buildPath string, c vmconfig.Config) error {
p := process("debootstrap")
// build debootstrap parameters
var args []string
args = append(args, "--variant=minbase")
args = append(args, fmt.Sprintf("--include=%v", strings.Join(c.Packages, ",")))
args = append(args, *f_branch)
args = append(args, buildPath)
args = append(args, *f_debian_mirror)
log.Debugln("args:", args)
cmd := exec.Command(p, args...)
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return err
}
log.LogAll(stdout, log.INFO, "debootstrap")
log.LogAll(stderr, log.ERROR, "debootstrap")
err = cmd.Run()
if err != nil {
return err
}
return nil
}
// PostBuildCommands invokes any commands listed in the postbuild variable
// of a config file. It does so by copying the entire string of the postbuild
// variable into a bash script under /tmp of the build directory, and then
// executing it with bash inside of a chroot. Post build commands are executed
// in depth-first order.
func PostBuildCommands(buildPath string, c vmconfig.Config) error {
for _, pb := range c.Postbuilds {
log.Debugln("postbuild:", pb)
tmpfile := buildPath + "/tmp/postbuild.bash"
ioutil.WriteFile(tmpfile, []byte(pb), 0770)
p := process("chroot")
cmd := exec.Command(p, buildPath, "/bin/bash", "/tmp/postbuild.bash")
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return err
}
log.LogAll(stdout, log.INFO, "postbuild")
log.LogAll(stderr, log.ERROR, "postbuild")
err = cmd.Run()
if err != nil {
return err
}
os.Remove(tmpfile)
}
return nil
}
// createQcow2 creates a target qcow2 image using qemu-img. Size specifies the
// size of the image in bytes but optional suffixes such as "K" and "G" can be
// used. See qemu-img(8) for details.
func createQcow2(target, size string) error {
// create our qcow image
p := process("qemu-img")
cmd := &exec.Cmd{
Path: p,
Args: []string{
p,
"create",
"-f",
"qcow2",
target,
size,
},
Env: nil,
Dir: "",
}
log.Debug("creating disk image with cmd: %v", cmd)
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return err
}
log.LogAll(stdout, log.INFO, "qemu-img")
log.LogAll(stderr, log.ERROR, "qemu-img")
return cmd.Run()
}
// BuildRootFS generates simple rootfs a from the stage 1 directory.
func BuildRootFS(buildPath string, c vmconfig.Config) error {
targetName := strings.Split(filepath.Base(c.Path), ".")[0] + "_rootfs"
log.Debugln("using target name:", targetName)
err := os.Mkdir(targetName, 0666)
if err != nil {
return err
}
p := process("cp")
cmd := exec.Command(p, "-r", "-v", buildPath+"/.", targetName)
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return err
}
log.LogAll(stdout, log.INFO, "cp")
log.LogAll(stderr, log.ERROR, "cp")
err = cmd.Run()
if err != nil {
return err
}
return nil
}
func transcode(in, out string) error {
p := "ffmpeg"
var args []string
args = append(args, "-f")
args = append(args, "mjpeg")
args = append(args, "-r")
args = append(args, "10") // minimega uses 10 frames per second
args = append(args, "-i")
args = append(args, fmt.Sprintf("http://localhost:%v/%v", *f_port, in))
args = append(args, out)
log.Debugln("args:", args)
cmd := exec.Command(p, args...)
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return err
}
log.LogAll(stdout, log.INFO, "ffmpeg")
log.LogAll(stderr, log.INFO, "ffmpeg")
err = cmd.Run()
if err != nil {
return err
}
return nil
}
// ConnectImage exports a image using the NBD protocol using the qemu-nbd. If
// successful, returns the NBD device.
func ConnectImage(image string) (string, error) {
var nbdPath string
var err error
for i := 0; i < maxConnectRetries; i++ {
nbdPath, err = GetDevice()
if err != ErrNoDeviceAvailable {
break
}
log.Debug("all nbds in use, sleeping before retrying")
time.Sleep(time.Second * 10)
}
if err != nil {
return "", err
}
// connect it to qemu-nbd
p := process("qemu-nbd")
cmd := &exec.Cmd{
Path: p,
Args: []string{
p,
"-c",
nbdPath,
image,
},
Env: nil,
Dir: "",
}
log.Debug("connecting to nbd with cmd: %v", cmd)
stdout, err := cmd.StdoutPipe()
if err != nil {
return "", err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return "", err
}
log.LogAll(stdout, log.INFO, "qemu-nbd")
log.LogAll(stderr, log.ERROR, "qemu-nbd")
err = cmd.Run()
if err != nil {
return "", err
}
return nbdPath, nil
}
// Overlays copies any overlay directories indicated in c into the build
// directory build_path. Overlays are copied in depth-first order, so that
// the oldest parent overlay data is copied in first. This allows a child
// to overwrite any overlay data created by a parent.
func Overlays(buildPath string, c vmconfig.Config) error {
// copy the overlays in order
for i, o := range c.Overlays {
log.Infoln("copying overlay:", o)
var sourcePath string
// check if overlay exists as absolute path or relative to cwd
if _, err := os.Stat(o); os.IsNotExist(err) {
// it doesn't, so we'll check relative to config file
log.Debugln("overlay directory '%v' does not exist as an absolute path or relative to the current working directory.", o)
var path string
base := filepath.Base(o) // get base path of overlay directory
if i == len(c.Overlays)-1 { // if this is the last overlay, we'll check relative to c.Path
log.Debugln("non-parent overlay")
path = filepath.Join(filepath.Dir(c.Path), base)
} else { // if not, it's a parent overlay and we'll check relative to c.Parents[i]
log.Debugln("parent overlay")
path = filepath.Join(filepath.Dir(c.Parents[i]), base)
}
log.Debugln("checking path relative to config location: '%v'", path)
if _, err := os.Stat(path); os.IsNotExist(err) { // check if we can find overlay relative to config file
return err // nope
} else { // yep
sourcePath = path
}
} else {
sourcePath = o
}
p := process("cp")
cmd := exec.Command(p, "-r", "-v", sourcePath+"/.", buildPath)
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return err
}
log.LogAll(stdout, log.INFO, "cp")
log.LogAll(stderr, log.ERROR, "cp")
err = cmd.Run()
if err != nil {
return err
}
}
return nil
}
// extlinux installs the SYSLINUX bootloader using extlinux. Path should be the
// root directory for the filesystem. extlinux also writes out a
// minimega-specific configuration file for SYSLINUX.
func extlinux(path string) error {
// install extlinux
p := process("extlinux")
cmd := &exec.Cmd{
Path: p,
Args: []string{
p,
"--install",
filepath.Join(path, "boot"),
},
Env: nil,
Dir: "",
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return err
}
log.LogAll(stdout, log.INFO, "extlinux")
log.LogAll(stderr, log.INFO, "extlinux")
log.Debug("installing bootloader with cmd: %v", cmd)
err = cmd.Run()
if err != nil {
return err
}
// write out the bootloader config, but first figure out the kernel and
// initrd files in /boot
filepath.Walk(filepath.Join(path, "boot"), kernelWalker)
if kernelName == "" {
return fmt.Errorf("could not find kernel name")
}
if initrdName == "" {
return fmt.Errorf("could not find initrd name")
}
extlinuxConfig := fmt.Sprintf("DEFAULT minimegalinux\nLABEL minimegalinux\nSAY booting minimegalinux\nLINUX /boot/%v\nAPPEND root=/dev/sda1\nINITRD /boot/%v", kernelName, initrdName)
return ioutil.WriteFile(filepath.Join(path, "/boot/extlinux.conf"), []byte(extlinuxConfig), os.FileMode(0660))
}
// BuildTargets generates the initrd and kernel files as the last stage of the
// build process. It does so by writing a find/cpio/gzip command as a script
// to a temporary file and executing that in a bash shell. The output filenames
// are equal to the base name of the input config file. So a config called
// 'my_vm.conf' will generate 'my_vm.initrd' and 'my_vm.kernel'. The kernel
// image is the one found in /boot of the build directory.
func BuildTargets(buildPath string, c vmconfig.Config) error {
targetName := strings.Split(filepath.Base(c.Path), ".")[0]
log.Debugln("using target name:", targetName)
wd, err := os.Getwd()
if err != nil {
return err
}
targetInitrd := fmt.Sprintf("%v/%v.initrd", wd, targetName)
targetKernel := fmt.Sprintf("%v/%v.kernel", wd, targetName)
f, err := ioutil.TempFile("", "vmbetter_cpio")
if err != nil {
return err
}
eName := f.Name()
initrdCommand := fmt.Sprintf("cd %v && find . -print0 | cpio --quiet --null -ov --format=newc | gzip -9 > %v\ncp boot/vmlinu* %v", buildPath, targetInitrd, targetKernel)
f.WriteString(initrdCommand)
f.Close()
log.Debugln("initrd command:", initrdCommand)
p := process("bash")
cmd := exec.Command(p, eName)
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return err
}
log.LogAll(stdout, log.INFO, "cpio")
// the cpio command outputs regular stuff to stderr, so i have a hack to push all output to the INFO level, instead of INFO/ERROR
log.LogAll(stderr, log.INFO, "cpio")
err = cmd.Run()
if err != nil {
return err
}
os.Remove(eName)
return nil
}
// mountQcow2 mounts a partition to a temporary directory. If successful,
// returns the path to that temporary directory.
func mountQcow2(dev string) (string, error) {
// mount the filesystem
mountPath, err := ioutil.TempDir("", "vmbetter_mount_")
if err != nil {
log.Fatalln("cannot create temporary directory:", err)
}
log.Debugln("using mount path:", mountPath)
p := process("mount")
cmd := &exec.Cmd{
Path: p,
Args: []string{
p,
dev,
mountPath,
},
Env: nil,
Dir: "",
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return "", err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return "", err
}
log.LogAll(stdout, log.INFO, "mount")
log.LogAll(stderr, log.ERROR, "mount")
log.Debug("mounting with with cmd: %v", cmd)
err = cmd.Run()
if err != nil {
return "", err
}
return mountPath, nil
}
// partitionQcow2 partitions the provided device creating one primary partition
// that is the size of the whole device and bootable.
func partitionQcow2(dev string) error {
// partition with fdisk
p := process("fdisk")
cmd := &exec.Cmd{
Path: p,
Args: []string{
p,
dev,
},
Env: nil,
Dir: "",
}
sIn, err := cmd.StdinPipe()
if err != nil {
return err
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return err
}
log.LogAll(stdout, log.INFO, "fdisk")
log.LogAll(stderr, log.INFO, "fdisk")
log.Debug("partitioning with cmd: %v", cmd)
err = cmd.Start()
if err != nil {
return err
}
io.WriteString(sIn, "n\np\n1\n\n\na\n1\nw\n")
return cmd.Wait()
}
// BuildISO generates a bootable ISO from the stage 1 directory.
func BuildISO(buildPath string, c vmconfig.Config) error {
targetName := strings.Split(filepath.Base(c.Path), ".")[0]
log.Debugln("using target name:", targetName)
// Set up a temporary directory
tdir, err := ioutil.TempDir("", targetName)
if err != nil {
return err
}
liveDir := tdir + "/image/live/"
isolinuxDir := tdir + "/image/isolinux/"
err = os.MkdirAll(liveDir, os.ModeDir|0755)
if err != nil {
return err
}
err = os.MkdirAll(isolinuxDir, os.ModeDir|0755)
if err != nil {
return err
}
// Get the kernel path we'll be using
matches, err := filepath.Glob(buildPath + "/boot/vmlinu*")
if err != nil {
return err
}
if len(matches) == 0 {
return errors.New("couldn't find kernel")
}
kernel := matches[0]
// Get the initrd path
matches, err = filepath.Glob(buildPath + "/boot/initrd*")
if err != nil {
return err
}
if len(matches) == 0 {
return errors.New("couldn't find initrd")
}
initrd := matches[0]
log.Debugln("copy kernel")
// Copy the kernel and initrd into the appropriate places
p := process("cp")
cmd := exec.Command(p, kernel, liveDir+"vmlinuz")
err = cmd.Run()
if err != nil {
return err
}
cmd = exec.Command(p, initrd, liveDir+"initrd")
err = cmd.Run()
if err != nil {
return err
}
log.Debugln("copy isolinux")
// Copy over the ISOLINUX stuff
matches, err = filepath.Glob(*f_isolinux + "/*")
if err != nil {
return err
}
for _, m := range matches {
cmd = exec.Command(p, m, isolinuxDir)
err = cmd.Run()
if err != nil {
return err
}
}
log.Debugln("make squashfs")
// Now compress the chroot
p = process("mksquashfs")
cmd = exec.Command(p, buildPath, liveDir+"filesystem.squashfs", "-e", "boot")
err = cmd.Run()
if err != nil {
return err
}
log.Debugln("genisoimage")
// Finally, run genisoimage
//genisoimage -rational-rock -volid "Minimega" -cache-inodes -joliet -full-iso9660-filenames -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table -output ../minimega.iso .
p = process("genisoimage")
cmd = exec.Command(p, "-rational-rock", "-volid", "\"Minimega\"", "-cache-inodes", "-joliet", "-full-iso9660-filenames", "-b", "isolinux/isolinux.bin", "-c", "isolinux/boot.cat", "-no-emul-boot", "-boot-load-size", "4", "-boot-info-table", "-output", targetName+".iso", tdir+"/image")
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Fatalln(err)
}
stderr, err := cmd.StderrPipe()
if err != nil {
log.Fatalln(err)
}
log.LogAll(stdout, log.INFO, "genisoimage")
log.LogAll(stderr, log.ERROR, "genisoimage")
err = cmd.Run()
if err != nil {
return err
}
// clean up
err = os.RemoveAll(tdir)
if err != nil {
return err
}
return nil
}
// launch is the low-level launch function for Container VMs. The caller should
// hold the VM's lock.
func (vm *ContainerVM) launch() error {
log.Info("launching vm: %v", vm.ID)
err := containerInit()
if err != nil {
log.Errorln(err)
vm.setError(err)
return err
}
if !containerInitSuccess {
err = fmt.Errorf("cgroups are not initialized, cannot continue")
log.Errorln(err)
vm.setError(err)
return err
}
// If this is the first time launching the VM, do the final configuration
// check, create a directory for it, and setup the FS.
if vm.State == VM_BUILDING {
if err := os.MkdirAll(vm.instancePath, os.FileMode(0700)); err != nil {
teardownf("unable to create VM dir: %v", err)
}
if vm.Snapshot {
if err := vm.overlayMount(); err != nil {
log.Error("overlayMount: %v", err)
vm.setError(err)
return err
}
} else {
vm.effectivePath = vm.FSPath
}
}
// write the config for this vm
config := vm.BaseConfig.String() + vm.ContainerConfig.String()
writeOrDie(vm.path("config"), config)
writeOrDie(vm.path("name"), vm.Name)
// the child process will communicate with a fake console using pipes
// to mimic stdio, and a fourth pipe for logging before the child execs
// into the init program
// two additional pipes are needed to synchronize freezing the child
// before it enters the container
parentLog, childLog, err := os.Pipe()
if err != nil {
log.Error("pipe: %v", err)
vm.setError(err)
return err
}
parentSync1, childSync1, err := os.Pipe()
if err != nil {
log.Error("pipe: %v", err)
vm.setError(err)
return err
}
childSync2, parentSync2, err := os.Pipe()
if err != nil {
log.Error("pipe: %v", err)
vm.setError(err)
return err
}
// create the uuid path that will bind mount into sysfs in the
// container
uuidPath := vm.path("uuid")
ioutil.WriteFile(uuidPath, []byte(vm.UUID+"\n"), 0400)
// create fifos
for i := 0; i < vm.Fifos; i++ {
p := vm.path(fmt.Sprintf("fifo%v", i))
if err = syscall.Mkfifo(p, 0660); err != nil {
log.Error("fifo: %v", err)
vm.setError(err)
return err
}
}
// 0 : minimega binary
// 1 : CONTAINER
// 2 : instance path
// 3 : vm id
// 4 : hostname ("CONTAINER_NONE" if none)
// 5 : filesystem path
// 6 : memory in megabytes
// 7 : uuid
// 8 : number of fifos
// 9 : init program (relative to filesystem path)
// 10: init args
hn := vm.Hostname
if hn == "" {
hn = CONTAINER_NONE
}
preinit := vm.Preinit
if preinit == "" {
preinit = CONTAINER_NONE
}
args := []string{
os.Args[0],
"-base",
*f_base,
CONTAINER_MAGIC,
vm.instancePath,
fmt.Sprintf("%v", vm.ID),
hn,
vm.effectivePath,
vm.Memory,
uuidPath,
fmt.Sprintf("%v", vm.Fifos),
preinit,
}
args = append(args, vm.Init...)
// launch the container
cmd := &exec.Cmd{
Path: "/proc/self/exe",
Args: args,
ExtraFiles: []*os.File{
childLog,
childSync1,
childSync2,
},
SysProcAttr: &syscall.SysProcAttr{
Cloneflags: uintptr(CONTAINER_FLAGS),
},
}
// Start the child and give it a pty
pseudotty, err := pty.Start(cmd)
if err != nil {
vm.overlayUnmount()
log.Error("start container: %v", err)
vm.setError(err)
return err
}
vm.pid = cmd.Process.Pid
log.Debug("vm %v has pid %v", vm.ID, vm.pid)
// log the child
childLog.Close()
log.LogAll(parentLog, log.DEBUG, "containerShim")
go vm.console(pseudotty)
// TODO: add affinity funcs for containers
// vm.CheckAffinity()
// network creation for containers happens /after/ the container is
// started, as we need the PID in order to attach a veth to the container
// side of the network namespace. That means that unlike kvm vms, we MUST
// create/destroy taps on launch/kill boundaries (kvm destroys taps on
// flush).
if err = vm.launchNetwork(); err != nil {
log.Errorln(err)
}
childSync1.Close()
if err == nil {
// wait for the freezer notification
var buf = make([]byte, 1)
parentSync1.Read(buf)
err = vm.freeze()
parentSync2.Close()
} else {
parentSync1.Close()
parentSync2.Close()
}
ccPath := filepath.Join(vm.effectivePath, "cc")
if err == nil {
// connect cc. Note that we have a local err here because we don't want
// to prevent the VM from continuing to launch, even if we can't
// connect to cc.
if err := ccNode.ListenUnix(ccPath); err != nil {
log.Warn("unable to connect to cc for vm %v: %v", vm.ID, err)
}
}
if err != nil {
// Some error occurred.. clean up the process
cmd.Process.Kill()
vm.setError(err)
return err
}
// Channel to signal when the process has exited
errChan := make(chan error)
// Create goroutine to wait for process to exit
go func() {
defer close(errChan)
errChan <- cmd.Wait()
}()
go func() {
cgroupFreezerPath := filepath.Join(*f_cgroup, "freezer", "minimega", fmt.Sprintf("%v", vm.ID))
cgroupMemoryPath := filepath.Join(*f_cgroup, "memory", "minimega", fmt.Sprintf("%v", vm.ID))
cgroupDevicesPath := filepath.Join(*f_cgroup, "devices", "minimega", fmt.Sprintf("%v", vm.ID))
sendKillAck := false
select {
case err := <-errChan:
log.Info("VM %v exited", vm.ID)
vm.lock.Lock()
defer vm.lock.Unlock()
// we don't need to check the error for a clean kill,
// as there's no way to get here if we killed it.
if err != nil {
log.Error("kill container: %v", err)
vm.setError(err)
}
case <-vm.kill:
log.Info("Killing VM %v", vm.ID)
vm.lock.Lock()
defer vm.lock.Unlock()
cmd.Process.Kill()
// containers cannot exit unless thawed, so thaw it if necessary
if err := vm.thaw(); err != nil {
log.Errorln(err)
vm.setError(err)
}
// wait for the taskset to actually exit (from uninterruptible
// sleep state).
for {
t, err := ioutil.ReadFile(filepath.Join(cgroupFreezerPath, "tasks"))
if err != nil {
log.Errorln(err)
vm.setError(err)
break
}
if len(t) == 0 {
break
}
count := strings.Count(string(t), "\n")
log.Info("waiting on %d tasks for VM %v", count, vm.ID)
time.Sleep(100 * time.Millisecond)
}
// drain errChan
for err := range errChan {
log.Debug("kill container: %v", err)
}
sendKillAck = true // wait to ack until we've cleaned up
}
if vm.ptyUnixListener != nil {
vm.ptyUnixListener.Close()
}
if vm.ptyTCPListener != nil {
vm.ptyTCPListener.Close()
}
// cleanup cc domain socket
ccNode.CloseUnix(ccPath)
vm.unlinkNetns()
for _, net := range vm.Networks {
br, err := getBridge(net.Bridge)
if err != nil {
log.Error("get bridge: %v", err)
} else {
br.DestroyTap(net.Tap)
}
}
// clean up the cgroup directory
if err := os.Remove(cgroupFreezerPath); err != nil {
log.Errorln(err)
}
if err := os.Remove(cgroupMemoryPath); err != nil {
log.Errorln(err)
}
if err := os.Remove(cgroupDevicesPath); err != nil {
log.Errorln(err)
}
if vm.State != VM_ERROR {
// Set to QUIT unless we've already been put into the error state
vm.setState(VM_QUIT)
}
if sendKillAck {
killAck <- vm.ID
}
}()
return nil
}
func (vm *ContainerVM) launch(ack chan int) {
log.Info("launching vm: %v", vm.ID)
if !cgroupInitialized {
err := containerInit()
if err != nil {
log.Errorln(err)
vm.setState(VM_ERROR)
ack <- vm.ID
return
}
cgroupInitialized = true
}
s := vm.GetState()
// don't repeat the preamble if we're just in the quit state
if s != VM_QUIT && !vm.launchPreamble(ack) {
vm.setState(VM_ERROR)
ack <- vm.ID
return
}
vm.setState(VM_BUILDING)
// write the config for this vm
writeOrDie(filepath.Join(vm.instancePath, "config"), vm.Config().String())
writeOrDie(filepath.Join(vm.instancePath, "name"), vm.Name)
var waitChan = make(chan int)
// clear taps, we may have come from the quit state
for i := range vm.Networks {
vm.Networks[i].Tap = ""
}
if vm.Snapshot {
err := vm.overlayMount()
if err != nil {
log.Error("overlayMount: %v", err)
vm.setState(VM_ERROR)
ack <- vm.ID
return
}
} else {
vm.effectivePath = vm.FSPath
}
// the child process will communicate with a fake console using pipes
// to mimic stdio, and a fourth pipe for logging before the child execs
// into the init program
// two additional pipes are needed to synchronize freezing the child
// before it enters the container
parentLog, childLog, err := os.Pipe()
if err != nil {
log.Error("pipe: %v", err)
vm.setState(VM_ERROR)
ack <- vm.ID
return
}
childStdin, parentStdin, err := os.Pipe()
if err != nil {
log.Error("pipe: %v", err)
vm.setState(VM_ERROR)
ack <- vm.ID
return
}
parentStdout, childStdout, err := os.Pipe()
if err != nil {
log.Error("pipe: %v", err)
vm.setState(VM_ERROR)
ack <- vm.ID
return
}
parentStderr, childStderr, err := os.Pipe()
if err != nil {
log.Error("pipe: %v", err)
vm.setState(VM_ERROR)
ack <- vm.ID
return
}
parentSync1, childSync1, err := os.Pipe()
if err != nil {
log.Error("pipe: %v", err)
vm.setState(VM_ERROR)
ack <- vm.ID
return
}
childSync2, parentSync2, err := os.Pipe()
if err != nil {
log.Error("pipe: %v", err)
vm.setState(VM_ERROR)
ack <- vm.ID
return
}
// create the uuid path that will bind mount into sysfs in the
// container
uuidPath := filepath.Join(vm.instancePath, "uuid")
ioutil.WriteFile(uuidPath, []byte(vm.UUID+"\n"), 0400)
// create fifos
for i := 0; i < vm.Fifos; i++ {
p := filepath.Join(vm.instancePath, fmt.Sprintf("fifo%v", i))
err := syscall.Mkfifo(p, 0660)
if err != nil {
log.Error("fifo: %v", err)
vm.setState(VM_ERROR)
ack <- vm.ID
return
}
}
// 0 : minimega binary
// 1 : CONTAINER
// 2 : instance path
// 3 : vm id
// 4 : hostname ("CONTAINER_NONE" if none)
// 5 : filesystem path
// 6 : memory in megabytes
// 7 : uuid
// 8 : number of fifos
// 9 : init program (relative to filesystem path)
// 10: init args
hn := vm.Hostname
if hn == "" {
hn = CONTAINER_NONE
}
args := []string{
os.Args[0],
CONTAINER_MAGIC,
vm.instancePath,
fmt.Sprintf("%v", vm.ID),
hn,
vm.effectivePath,
vm.Memory,
uuidPath,
fmt.Sprintf("%v", vm.Fifos),
}
args = append(args, vm.Init...)
// launch the container
cmd := &exec.Cmd{
Path: "/proc/self/exe",
Args: args,
Env: nil,
Dir: "",
ExtraFiles: []*os.File{
childLog,
childSync1,
childSync2,
childStdin,
childStdout,
childStderr,
},
SysProcAttr: &syscall.SysProcAttr{
Cloneflags: uintptr(CONTAINER_FLAGS),
},
}
err = cmd.Start()
if err != nil {
vm.overlayUnmount()
log.Error("start container: %v", err)
vm.setState(VM_ERROR)
ack <- vm.ID
return
}
vm.pid = cmd.Process.Pid
log.Debug("vm %v has pid %v", vm.ID, vm.pid)
// log the child
childLog.Close()
log.LogAll(parentLog, log.DEBUG, "containerShim")
go vm.console(parentStdin, parentStdout, parentStderr)
go func() {
err := cmd.Wait()
vm.setState(VM_QUIT)
if err != nil {
if err.Error() != "signal: killed" { // because we killed it
log.Error("kill container: %v", err)
vm.setState(VM_ERROR)
}
}
waitChan <- vm.ID
}()
// we can't just return on error at this point because we'll
// leave dangling goroutines, we have to clean up on failure
success := true
sendKillAck := false
// TODO: add affinity funcs for containers
// vm.CheckAffinity()
// network creation for containers happens /after/ the container is
// started, as we need the PID in order to attach a veth to the
// container side of the network namespace.
// create and add taps if we are associated with any networks
// expose the network namespace to iptool
err = vm.symlinkNetns()
if err != nil {
log.Error("symlinkNetns: %v", err)
vm.setState(VM_ERROR)
cmd.Process.Kill()
<-waitChan
success = false
}
if success {
for i := range vm.Networks {
net := &vm.Networks[i]
b, err := getBridge(net.Bridge)
if err != nil {
log.Error("get bridge: %v", err)
vm.setState(VM_ERROR)
cmd.Process.Kill()
<-waitChan
success = false
break
}
net.Tap, err = b.ContainerTapCreate(net.VLAN, vm.netns, net.MAC, i)
if err != nil {
log.Error("create tap: %v", err)
vm.setState(VM_ERROR)
cmd.Process.Kill()
<-waitChan
success = false
break
}
updates := make(chan ipmac.IP)
go func(vm *ContainerVM, net *NetConfig) {
defer close(updates)
for {
// TODO: need to acquire VM lock?
select {
case update := <-updates:
if update.IP4 != "" {
net.IP4 = update.IP4
} else if net.IP6 != "" && strings.HasPrefix(update.IP6, "fe80") {
log.Debugln("ignoring link-local over existing IPv6 address")
} else if update.IP6 != "" {
net.IP6 = update.IP6
}
case <-vm.kill:
b.iml.DelMac(net.MAC)
return
}
}
}(vm, net)
b.iml.AddMac(net.MAC, updates)
}
}
if success {
if len(vm.Networks) > 0 {
taps := []string{}
for _, net := range vm.Networks {
taps = append(taps, net.Tap)
}
err := ioutil.WriteFile(filepath.Join(vm.instancePath, "taps"), []byte(strings.Join(taps, "\n")), 0666)
if err != nil {
log.Error("write instance taps file: %v", err)
vm.setState(VM_ERROR)
cmd.Process.Kill()
<-waitChan
success = false
}
}
}
childSync1.Close()
if success {
// wait for the freezer notification
var buf = make([]byte, 1)
parentSync1.Read(buf)
freezer := filepath.Join(CGROUP_PATH, fmt.Sprintf("%v", vm.ID), "freezer.state")
err = ioutil.WriteFile(freezer, []byte("FROZEN"), 0644)
if err != nil {
log.Error("freezer: %v", err)
vm.setState(VM_ERROR)
cmd.Process.Kill()
<-waitChan
success = false
}
parentSync2.Close()
} else {
parentSync1.Close()
parentSync2.Close()
}
// connect cc
ccPath := filepath.Join(vm.effectivePath, "cc")
err = ccNode.ListenUnix(ccPath)
if err != nil {
log.Errorln(err)
}
ack <- vm.ID
if success {
select {
case <-waitChan:
log.Info("VM %v exited", vm.ID)
case <-vm.kill:
log.Info("Killing VM %v", vm.ID)
cmd.Process.Kill()
// containers cannot return unless thawed, so thaw the
// process if necessary
freezer := filepath.Join(CGROUP_PATH, fmt.Sprintf("%v", vm.ID), "freezer.state")
err = ioutil.WriteFile(freezer, []byte("THAWED"), 0644)
if err != nil {
log.Error("freezer: %v", err)
vm.setState(VM_ERROR)
<-waitChan
}
<-waitChan
sendKillAck = true // wait to ack until we've cleaned up
}
}
err = ccNode.CloseUDS(ccPath)
if err != nil {
log.Errorln(err)
}
vm.listener.Close()
vm.unlinkNetns()
for _, net := range vm.Networks {
b, err := getBridge(net.Bridge)
if err != nil {
log.Error("get bridge: %v", err)
} else {
b.ContainerTapDestroy(net.VLAN, net.Tap)
}
}
// clean up the cgroup directory
cgroupPath := filepath.Join(CGROUP_PATH, fmt.Sprintf("%v", vm.ID))
err = os.Remove(cgroupPath)
if err != nil {
log.Errorln(err)
}
// umount the overlay, if any
if vm.Snapshot {
vm.overlayUnmount()
}
if sendKillAck {
killAck <- vm.ID
}
}
func main() {
flag.Usage = usage
flag.Parse()
logSetup()
if flag.NArg() != 1 {
usage()
os.Exit(1)
}
externalCheck()
// stage 1 and stage 2 flags are mutually exclusive
if *f_stage1 && *f_stage2 != "" {
log.Fatalln("-1 cannot be used with -2")
}
// find any other dependent configs and get an ordered list of those
configfile := flag.Arg(0)
log.Debugln("using config:", configfile)
config, err := vmconfig.ReadConfig(configfile)
if err != nil {
log.Fatalln(err)
} else {
log.Debugln("read config:", config)
}
// If we're doing a LiveCD, we need to add the live-boot package
if *f_iso {
config.Packages = append(config.Packages, "live-boot")
}
var buildPath string
// stage 1
if *f_stage2 == "" {
// create a build path
buildPath, err = ioutil.TempDir("", "vmbetter_build_")
if err != nil {
log.Fatalln("cannot create temporary directory:", err)
}
log.Debugln("using build path:", buildPath)
// invoke debootstrap
fmt.Println("invoking debootstrap (this may take a while)...")
err = Debootstrap(buildPath, config)
if err != nil {
log.Fatalln(err)
}
// copy any overlay into place in reverse order of opened dependencies
fmt.Println("copying overlays")
err = Overlays(buildPath, config)
if err != nil {
log.Fatalln(err)
}
//
// stage 1 complete
//
if *f_stage1 {
stage1Target := strings.Split(filepath.Base(config.Path), ".")[0] + "_stage1"
log.Infoln("writing stage 1 target", stage1Target)
err = os.Mkdir(stage1Target, 0666)
if err != nil {
log.Fatalln(err)
}
p := process("cp")
cmd := exec.Command(p, "-r", "-v", buildPath+"/.", stage1Target)
stdout, err := cmd.StdoutPipe()
if err != nil {
log.Fatalln(err)
}
stderr, err := cmd.StderrPipe()
if err != nil {
log.Fatalln(err)
}
log.LogAll(stdout, log.INFO, "cp")
log.LogAll(stderr, log.ERROR, "cp")
err = cmd.Run()
if err != nil {
log.Fatalln(err)
}
}
} else {
buildPath = *f_stage2
}
// stage 2
if *f_stage2 != "" || !*f_stage1 {
// call post build chroot commands in reverse order as well
fmt.Println("executing post-build commands")
err = PostBuildCommands(buildPath, config)
if err != nil {
log.Fatalln(err)
}
// build the image file
fmt.Println("building target files")
if *f_qcow {
err = Buildqcow2(buildPath, config)
} else if *f_iso {
err = BuildISO(buildPath, config)
} else {
err = BuildTargets(buildPath, config)
}
if err != nil {
log.Fatalln(err)
}
}
// cleanup?
if !*f_noclean && *f_stage2 == "" {
fmt.Println("cleaning up")
err = os.RemoveAll(buildPath)
if err != nil {
log.Errorln(err)
}
}
fmt.Println("done")
}