// Returns the duration of a given kbrecording file
func getDuration(filename string) time.Duration {
d := 0
f, _ := os.Open(filename)
scanner := bufio.NewScanner(f)
for scanner.Scan() {
s := strings.SplitN(scanner.Text(), ":", 2)
// Ignore comments in the vnc file
if s[0] == "#" {
continue
}
i, err := strconv.Atoi(s[0])
if err != nil {
log.Errorln(err)
return 0
}
d += i
}
duration, err := time.ParseDuration(strconv.Itoa(d) + "ns")
if err != nil {
log.Errorln(err)
return 0
}
return duration
}
// client connection handler and transport. Messages on chan out are sent to
// the ron server. Incoming messages are put on the message queue to be routed
// by the mux. The entry to handler() also creates the tunnel transport.
func (c *Client) handler() {
log.Debug("ron handler")
// create a tunnel
stop := make(chan bool)
defer func() { stop <- true }()
go c.handleTunnel(false, stop)
enc := gob.NewEncoder(c.conn)
dec := gob.NewDecoder(c.conn)
// handle client i/o
go func() {
for {
m := <-c.out
err := enc.Encode(m)
if err != nil {
log.Errorln(err)
return
}
}
}()
for {
var m Message
err := dec.Decode(&m)
if err != nil {
log.Errorln(err)
return
}
c.in <- &m
}
}
func vncWsHandler(ws *websocket.Conn) {
// URL should be of the form `/ws/<vm_name>`
path := strings.Trim(ws.Config().Location.Path, "/")
fields := strings.Split(path, "/")
if len(fields) != 2 {
return
}
vmName := fields[1]
vms := GlobalVMs()
vm, err := vms.findKvmVM(vmName)
if err != nil {
log.Errorln(err)
return
}
// Undocumented "feature" of websocket -- need to set to PayloadType in
// order for a direct io.Copy to work.
ws.PayloadType = websocket.BinaryFrame
// connect to the remote host
rhost := fmt.Sprintf("%v:%v", vm.Host, vm.VNCPort)
remote, err := net.Dial("tcp", rhost)
if err != nil {
log.Errorln(err)
return
}
defer remote.Close()
go io.Copy(ws, remote)
io.Copy(remote, ws)
log.Info("ws client disconnected from %v", rhost)
}
// return a file to a client requesting it via the clients GetFile() call
func (s *Server) sendFile(m *Message) {
log.Debug("ron sendFile: %v", m.Filename)
filename := filepath.Join(s.path, m.Filename)
info, err := os.Stat(filename)
if err != nil {
e := fmt.Errorf("file %v does not exist: %v", filename, err)
m.Error = e.Error()
log.Errorln(e)
} else if info.IsDir() {
e := fmt.Errorf("file %v is a directory", filename)
m.Error = e.Error()
log.Errorln(e)
} else {
// read the file
m.File, err = ioutil.ReadFile(filename)
if err != nil {
e := fmt.Errorf("file %v: %v", filename, err)
m.Error = e.Error()
log.Errorln(e)
}
}
// route this message ourselves instead of using the mux, because we
// want the type to still be FILE
s.clientLock.Lock()
defer s.clientLock.Unlock()
if c, ok := s.clients[m.UUID]; ok {
c.out <- m
} else {
log.Error("no such client %v", m.UUID)
}
}
func (t *Tunnel) handleRemote(m *tunnelMessage) {
host := m.Host
port := m.Port
TID := m.TID
// attempt to connect to the host/port
conn, err := net.Dial("tcp", fmt.Sprintf("%v:%v", host, port))
if err == nil {
in := t.chans.add(TID)
go t.transfer(in, conn, TID)
return
}
log.Errorln(err)
resp := &tunnelMessage{
Type: CLOSED,
TID: TID,
Error: err.Error(),
}
if err := t.sendMessage(resp); err != nil {
log.Errorln(err)
}
}
// dirHandler serves a directory listing for the requested path, rooted at basePath.
func dirHandler(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/favicon.ico" {
http.Error(w, "not found", 404)
return
}
const base = "."
name := filepath.Join(base, r.URL.Path)
if isDoc(name) {
err := renderDoc(w, name)
if err != nil {
log.Errorln(err)
http.Error(w, err.Error(), 500)
}
return
}
if isDir, err := dirList(w, name); err != nil {
log.Errorln(err)
http.Error(w, err.Error(), 500)
return
} else if isDir {
return
}
http.FileServer(http.Dir(*f_root)).ServeHTTP(w, r)
}
func kill(pid int) {
Client.Lock()
defer Client.Unlock()
if pid == -1 {
// Wildcard
log.Info("killing all processes")
for _, p := range Client.Processes {
if err := p.process.Kill(); err != nil {
log.Errorln(err)
}
}
return
}
log.Info("killing PID %v", pid)
if p, ok := Client.Processes[pid]; ok {
if err := p.process.Kill(); err != nil {
log.Errorln(err)
}
return
}
log.Error("no such process: %v", pid)
}
func containerFifos(vmFSPath string, vmInstancePath string, vmFifos int) error {
err := os.Mkdir(filepath.Join(vmFSPath, "/dev/fifos"), 0755)
if err != nil {
log.Errorln(err)
return nil
}
for i := 0; i < vmFifos; i++ {
src := filepath.Join(vmInstancePath, fmt.Sprintf("fifo%v", i))
_, err := os.Stat(src)
if err != nil {
log.Errorln(err)
return err
}
dst := filepath.Join(vmFSPath, fmt.Sprintf("/dev/fifos/fifo%v", i))
// dst must exist for bind mounting to work
f, err := os.Create(dst)
if err != nil {
log.Errorln(err)
return err
}
f.Close()
log.Debug("bind mounting: %v -> %v", src, dst)
err = syscall.Mount(src, dst, "", syscall.MS_BIND, "")
if err != nil {
log.Errorln(err)
return err
}
}
return nil
}
func (vm *KvmVM) connectVNC() error {
l, err := net.Listen("tcp", "")
if err != nil {
return err
}
// Keep track of shim so that we can close it later
vm.vncShim = l
vm.VNCPort = l.Addr().(*net.TCPAddr).Port
ns := fmt.Sprintf("%v:%v", vm.Namespace, vm.Name)
go func() {
defer l.Close()
for {
// Sit waiting for new connections
remote, err := l.Accept()
if err != nil && strings.Contains(err.Error(), "use of closed network connection") {
return
} else if err != nil {
log.Errorln(err)
return
}
go func() {
defer remote.Close()
// Dial domain socket
local, err := net.Dial("unix", vm.path("vnc"))
if err != nil {
log.Error("unable to dial vm vnc: %v", err)
return
}
defer local.Close()
// copy local -> remote
go io.Copy(remote, local)
// Reads will implicitly copy from remote -> local
tee := io.TeeReader(remote, local)
for {
// Read
msg, err := vnc.ReadClientMessage(tee)
if err != nil {
if err == io.EOF || strings.Contains(err.Error(), "closed network") {
break
}
log.Errorln(err)
}
if r, ok := vncKBRecording[ns]; ok {
r.RecordMessage(msg)
}
}
}()
}
}()
return nil
}
// recvFiles retrieves a list of files from the ron server by requesting each
// one individually.
func recvFiles(files []*ron.File) {
start := time.Now()
var size int64
for _, v := range files {
log.Info("requesting file %v", v)
dst := filepath.Join(*f_path, "files", v.Name)
if _, err := os.Stat(dst); err == nil {
// file exists (TODO: overwrite?)
log.Info("skipping %v -- already exists")
continue
}
m := &ron.Message{
Type: ron.MESSAGE_FILE,
UUID: Client.UUID,
Filename: v.Name,
}
if err := sendMessage(m); err != nil {
log.Error("send failed: %v", err)
return
}
resp := <-Client.fileChan
if resp.Filename != v.Name {
log.Error("filename mismatch: %v != %v", resp.Filename, v.Name)
continue
}
if resp.Error != "" {
log.Error("%v", resp.Error)
continue
}
dir := filepath.Dir(dst)
if err := os.MkdirAll(dir, os.FileMode(0770)); err != nil {
log.Errorln(err)
continue
}
if err := ioutil.WriteFile(dst, resp.File, v.Perm); err != nil {
log.Errorln(err)
continue
}
size += int64(len(resp.File))
}
d := time.Since(start)
rate := (float64(size) / 1024 / d.Seconds())
log.Debug("received %v bytes in %v (%v KBps)", size, d, rate)
return
}
func runCommand(command []string, background bool) (string, string) {
var stdout bytes.Buffer
var stderr bytes.Buffer
path, err := exec.LookPath(command[0])
if err != nil {
log.Errorln(err)
return "", err.Error()
}
cmd := &exec.Cmd{
Path: path,
Args: command,
Stdout: &stdout,
Stderr: &stderr,
}
log.Info("executing: %v", command)
if background {
log.Debug("starting in background")
if err := cmd.Start(); err != nil {
log.Errorln(err)
return "", stderr.String()
}
pid := cmd.Process.Pid
Client.Lock()
defer Client.Unlock()
Client.Processes[pid] = &Process{
PID: pid,
Command: command,
process: cmd.Process,
}
go func() {
cmd.Wait()
log.Info("command exited: %v", command)
if stdout.Len() > 0 {
log.Info(stdout.String())
}
if stderr.Len() > 0 {
log.Info(stderr.String())
}
Client.Lock()
defer Client.Unlock()
delete(Client.Processes, pid)
}()
return "", ""
}
if err := cmd.Run(); err != nil {
log.Errorln(err)
}
return stdout.String(), stderr.String()
}
func sshHandleChannel(conn net.Conn, newChannel ssh.NewChannel) {
// Channels have a type, depending on the application level protocol
// intended. In the case of a shell, the type is "session" and ServerShell
// may be used to present a simple terminal interface.
if newChannel.ChannelType() != "session" {
newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
return
}
channel, requests, err := newChannel.Accept()
if err != nil {
log.Errorln(err)
return
}
// Sessions have out-of-band requests such as "shell", "pty-req" and "env".
// Here we handle only the "shell" request.
go func(in <-chan *ssh.Request) {
for req := range in {
ok := false
switch req.Type {
case "shell":
ok = true
if len(req.Payload) > 0 {
// We don't accept any commands, only the default shell.
ok = false
}
case "pty-req":
ok = true
}
req.Reply(ok, nil)
}
}(requests)
term := terminal.NewTerminal(channel, "> ")
go func() {
defer channel.Close()
for {
line, err := term.ReadLine()
start := time.Now().UnixNano()
if err != nil {
if err != io.EOF {
log.Errorln(err)
}
return
}
sshReportChan <- uint64(len(line))
// just echo the message
log.Debugln("ssh received: ", line)
term.Write([]byte(line))
term.Write([]byte{'\r', '\n'})
stop := time.Now().UnixNano()
log.Info("ssh %v %vns", conn.RemoteAddr(), uint64(stop-start))
}
}()
}
func (t *Tunnel) handle(in chan *tunnelMessage, conn net.Conn, TID int32) {
// begin forwarding until an error occurs
go func() {
for {
select {
case <-t.quit:
conn.Close()
return
case m := <-in:
if m.Type == CLOSED {
if m.Error != "" {
log.Errorln(m.Error)
conn.Close()
break
}
}
_, err := conn.Write(m.Data)
if err != nil {
log.Errorln(err)
conn.Close()
t.out <- &tunnelMessage{
Type: CLOSED,
TID: TID,
Error: err.Error(),
}
break
}
}
}
}()
for {
var buf = make([]byte, BUFFER_SIZE)
n, err := conn.Read(buf)
if err != nil {
conn.Close()
closeMessage := &tunnelMessage{
Type: CLOSED,
TID: TID,
}
if err != io.EOF && !strings.Contains(err.Error(), errClosing) {
log.Errorln(err)
closeMessage.Error = err.Error()
}
t.out <- closeMessage
t.unregisterTID(TID)
break
}
m := &tunnelMessage{
Type: DATA,
TID: TID,
Data: buf[:n],
}
t.out <- m
}
}
// aggressively cleanup container cruff, called by the nuke api
func containerNuke() {
// walk minimega cgroups for tasks, killing each one
cgroupFreezer := filepath.Join(*f_cgroup, "freezer", "minimega")
cgroupMemory := filepath.Join(*f_cgroup, "memory", "minimega")
cgroupDevices := filepath.Join(*f_cgroup, "devices", "minimega")
cgroups := []string{cgroupFreezer, cgroupMemory, cgroupDevices}
for _, cgroup := range cgroups {
if _, err := os.Stat(cgroup); err == nil {
err := filepath.Walk(cgroup, containerNukeWalker)
if err != nil {
log.Errorln(err)
}
}
}
// Allow udev to sync
time.Sleep(time.Second * 1)
// umount megamount_*, this include overlayfs mounts
d, err := ioutil.ReadFile("/proc/mounts")
if err != nil {
log.Errorln(err)
} else {
mounts := strings.Split(string(d), "\n")
for _, m := range mounts {
if strings.Contains(m, "megamount") {
mount := strings.Split(m, " ")[1]
if err := syscall.Unmount(mount, 0); err != nil {
log.Error("overlay unmount %s: %v", m, err)
}
}
}
}
containerCleanCgroupDirs()
// remove meganet_* from /var/run/netns
if _, err := os.Stat("/var/run/netns"); err == nil {
netns, err := ioutil.ReadDir("/var/run/netns")
if err != nil {
log.Errorln(err)
} else {
for _, n := range netns {
if strings.Contains(n.Name(), "meganet") {
err := os.Remove(filepath.Join("/var/run/netns", n.Name()))
if err != nil {
log.Errorln(err)
}
}
}
}
}
}
func containerTeardown() {
if cgroupInitialized {
err := os.Remove(CGROUP_PATH)
if err != nil {
log.Errorln(err)
}
err = syscall.Unmount(CGROUP_ROOT, 0)
if err != nil {
log.Errorln(err)
}
}
}
func commandSocketHandle(c net.Conn) {
var err error
enc := json.NewEncoder(c)
dec := json.NewDecoder(c)
outer:
for err == nil {
var cmd *minicli.Command
cmd, err = readLocalCommand(dec)
if err != nil {
if err != io.EOF {
// Must be incompatible versions of minimega... F***
log.Errorln(err)
}
break
}
err = nil
var prevResp minicli.Responses
if cmd != nil {
// HAX: Don't record the read command
if hasCommand(cmd, "read") {
cmd.Record = false
}
// HAX: Work around so that we can add the more boolean
for resp := range runCommand(cmd) {
if prevResp != nil {
err = sendLocalResp(enc, prevResp, true)
if err != nil {
break outer
}
}
prevResp = resp
}
}
if err == nil {
err = sendLocalResp(enc, prevResp, false)
}
}
if err != nil {
if err == io.EOF {
log.Infoln("command client disconnected")
} else {
log.Errorln(err)
}
}
}
// runCommand runs a command through a JSON pipe.
func runCommand(cmd Command) chan *localResponse {
conn, err := net.Dial("unix", path.Join(*f_minimega, "minimega"))
if err != nil {
log.Errorln(err)
return nil
}
enc := json.NewEncoder(conn)
dec := json.NewDecoder(conn)
log.Debug("encoding command: %v", cmd)
err = enc.Encode(cmd)
if err != nil {
log.Errorln("local command json encode: %v", err)
return nil
}
log.Debugln("encoded command:", cmd)
respChan := make(chan *localResponse)
go func() {
defer close(respChan)
for {
var r localResponse
err = dec.Decode(&r)
if err != nil {
if err == io.EOF {
log.Infoln("server disconnected")
return
}
log.Errorln("local command json decode: %v", err)
return
}
respChan <- &r
if !r.More {
log.Debugln("got last message")
break
} else {
log.Debugln("expecting more data")
}
}
}()
return respChan
}
func (a SortEth) Less(i, j int) bool {
// sort on the integer part of an interface name, which we assume will
// be in the form of string+int
idxI, err := strconv.Atoi(strings.TrimLeft(a[i], trim))
if err != nil {
log.Errorln(err)
return false
}
idxJ, err := strconv.Atoi(strings.TrimLeft(a[j], trim))
if err != nil {
log.Errorln(err)
return false
}
return idxI < idxJ
}
// a filename completer for goreadline that searches for the file: prefix,
// attempts to find matching files, and returns an array of candidates.
func iomCompleter(line string) []string {
f := strings.Fields(line)
if len(f) == 0 {
return nil
}
last := f[len(f)-1]
if strings.HasPrefix(last, IOM_HELPER_MATCH) {
fileprefix := strings.TrimPrefix(last, IOM_HELPER_MATCH)
matches := iom.Info(fileprefix + "*")
log.Debug("got raw matches: %v", matches)
// we need to clean up matches to collapse directories, unless
// there is a directory common prefix, in which case we
// collapse offset by the number of common directories.
dlcp := lcp(matches)
didx := strings.LastIndex(dlcp, string(filepath.Separator))
drel := ""
if didx > 0 {
drel = dlcp[:didx]
}
log.Debug("dlcp: %v, drel: %v", dlcp, drel)
if len(fileprefix) < len(drel) {
r := IOM_HELPER_MATCH + drel + string(filepath.Separator)
return []string{r, r + "0"} // hack to prevent readline from fastforwarding beyond the directory name
}
var finalMatches []string
for _, v := range matches {
if strings.Contains(v, "*") {
continue
}
r, err := filepath.Rel(drel, v)
if err != nil {
log.Errorln(err)
return nil
}
dir := filepath.Dir(r)
if dir == "." {
finalMatches = append(finalMatches, IOM_HELPER_MATCH+v)
continue
}
paths := strings.Split(dir, string(filepath.Separator))
found := false
for _, d := range finalMatches {
if d == paths[0]+string(filepath.Separator) {
found = true
break
}
}
if !found {
finalMatches = append(finalMatches, IOM_HELPER_MATCH+filepath.Join(drel, paths[0])+string(filepath.Separator))
}
}
return finalMatches
}
return nil
}
func teardown() {
// Clear namespace so that we hit all the VMs
SetNamespace("")
vncClear()
clearAllCaptures()
vms.Kill(Wildcard)
dnsmasqKillAll()
ksmDisable()
vms.Flush()
vms.CleanDirs()
containerTeardown()
if err := bridgesDestroy(); err != nil {
log.Errorln(err)
}
commandSocketRemove()
goreadline.Rlcleanup()
if err := os.Remove(filepath.Join(*f_base, "minimega.pid")); err != nil {
log.Fatalln(err)
}
if cpuProfileOut != nil {
pprof.StopCPUProfile()
cpuProfileOut.Close()
}
os.Exit(0)
}
// readFile reads the file by name and returns a message that can be sent back
// to the client.
func (s *Server) readFile(f string) *Message {
log.Debug("readFile: %v", f)
filename := filepath.Join(s.path, f)
m := &Message{
Type: MESSAGE_FILE,
Filename: f,
}
info, err := os.Stat(filename)
if err != nil {
m.Error = fmt.Sprintf("file %v does not exist: %v", filename, err)
} else if info.IsDir() {
m.Error = fmt.Sprintf("file %v is a directory", filename)
} else {
// read the file
m.File, err = ioutil.ReadFile(filename)
if err != nil {
m.Error = fmt.Sprintf("file %v: %v", filename, err)
}
}
if m.Error != "" {
log.Errorln(m.Error)
}
return m
}
// Starts a Ron server on the specified port
func (s *Server) Listen(port int) error {
ln, err := net.Listen("tcp", fmt.Sprintf(":%v", port))
if err != nil {
return err
}
go func() {
for {
conn, err := ln.Accept()
if err != nil {
log.Errorln(err)
return
}
log.Debug("new connection from: %v", conn.RemoteAddr())
go func() {
addr := conn.RemoteAddr()
s.clientHandler(conn)
log.Debug("disconnected from: %v", addr)
}()
}
}()
return nil
}
// return names of bridges as shown in f_base/bridges. Optionally include
// bridges that existed before minimega was launched
func nukeBridgeNames(preExist bool) []string {
var ret []string
b, err := os.Open(filepath.Join(*f_base, "bridges"))
if os.IsNotExist(err) {
return nil
} else if err != nil {
log.Errorln(err)
return nil
}
scanner := bufio.NewScanner(b)
// skip the first line
scanner.Scan()
for scanner.Scan() {
f := strings.Fields(scanner.Text())
log.Debugln(f)
if len(f) <= 2 {
continue
}
if (f[1] == "true" && preExist) || f[1] == "false" {
ret = append(ret, f[0])
}
}
log.Debug("nukeBridgeNames got: %v", ret)
return ret
}
func commandSocketRemove() {
f := filepath.Join(*f_base, "minimega")
err := os.Remove(f)
if err != nil {
log.Errorln(err)
}
}
// Get the VM info from all hosts optionally applying column/row filters.
// Returns a map with keys for the hostnames and values as the tabular data
// from the host.
func globalVmInfo() map[string]VMs {
cmdStr := "vm info"
res := map[string]VMs{}
cmd := minicli.MustCompile(cmdStr)
cmd.Record = false
for resps := range runCommandGlobally(cmd) {
for _, resp := range resps {
if resp.Error != "" {
log.Errorln(resp.Error)
continue
}
switch data := resp.Data.(type) {
case VMs:
res[resp.Host] = data
default:
log.Error("unknown data field in vm info")
}
}
}
return res
}
func commandSocketRemove() {
f := *f_base + "minimega"
err := os.Remove(f)
if err != nil {
log.Errorln(err)
}
}
func sendCommand(s string) string {
if strings.TrimSpace(s) == "" {
return ""
}
log.Debug("sendCommand: %v", s)
mm, err := dial(*f_minimega)
if err != nil {
log.Errorln(err)
return err.Error()
}
defer mm.Close()
cmd := &minicli.Command{Original: s}
var responses string
for resp := range mm.Run(cmd) {
if r := resp.Rendered; r != "" {
responses += r + "\n"
}
if e := resp.Resp.Error(); e != "" {
responses += e + "\n"
}
}
return responses
}
func webHosts(w http.ResponseWriter, r *http.Request) {
table := htmlTable{
Header: []string{},
Tabular: [][]interface{}{},
ID: "example",
Class: "hover",
}
cmd := minicli.MustCompile("host")
cmd.Record = false
for resps := range runCommandGlobally(cmd) {
for _, resp := range resps {
if resp.Error != "" {
log.Errorln(resp.Error)
continue
}
if len(table.Header) == 0 && len(resp.Header) > 0 {
table.Header = append(table.Header, resp.Header...)
}
for _, row := range resp.Tabular {
res := []interface{}{}
for _, v := range row {
res = append(res, v)
}
table.Tabular = append(table.Tabular, res)
}
}
}
webRenderTemplate(w, "hosts.html", table)
}
func meshageHandler() {
for {
m := <-meshageCommandChan
go func() {
mCmd := m.Body.(meshageCommand)
cmd, err := minicli.Compile(mCmd.Original)
if err != nil {
log.Error("invalid command from mesh: `%s`", mCmd.Original)
return
}
resps := []minicli.Responses{}
for resp := range runCommand(cmd) {
resps = append(resps, resp)
}
if len(resps) > 1 || len(resps[0]) > 1 {
// This should never happen because the only commands that
// return multiple responses are `read` and `mesh send` which
// aren't supposed to be sent across meshage.
log.Error("unsure how to process multiple responses!!")
}
resp := meshageResponse{Response: *resps[0][0], TID: mCmd.TID}
recipient := []string{m.Source}
_, err = meshageNode.Set(recipient, resp)
if err != nil {
log.Errorln(err)
}
}()
}
}
// Handle incoming "get file info" messages by looking up if we have the file
// and responding with the number of parts or a NACK. Also process directories
// and globs, populating the Glob field of the IOMMessage if needed.
func (iom *IOMeshage) handleInfo(m *IOMMessage) {
// do we have this file, rooted at iom.base?
resp := IOMMessage{
From: iom.node.Name(),
Type: TYPE_RESPONSE,
Filename: m.Filename,
TID: m.TID,
}
glob, parts, err := iom.fileInfo(filepath.Join(iom.base, m.Filename))
if err != nil {
resp.ACK = false
} else if len(glob) == 1 && glob[0] == m.Filename {
resp.ACK = true
resp.Part = parts
fi, err := os.Stat(filepath.Join(iom.base, m.Filename))
if err != nil {
resp.ACK = false
} else {
resp.Perm = fi.Mode() & os.ModePerm
}
if log.WillLog(log.DEBUG) {
log.Debugln("handleInfo found file with parts: ", resp.Part)
}
} else {
// populate Glob
resp.ACK = true
resp.Glob = glob
}
_, err = iom.node.Set([]string{m.From}, resp)
if err != nil {
log.Errorln("handleInfo: sending message: ", err)
}
}