// Starts the connector listening on the specified source // TODO: should have mechanism for stopping this, and probably handing off the connections to another // routine to insert into the map func (c *Connector) serve() { defer c.wg.Done() for { if c.listener == nil { log.Debugf("attach connector: listener closed") break } conn, err := c.listener.Accept() select { case <-c.listenerQuit: log.Debugf("attach connector: serve exitting") return default: } if err != nil { log.Errorf("Error waiting for incoming connection: %s", errors.ErrorStack(err)) continue } log.Info("attach connector: Received incoming connection") go c.processIncoming(conn) } }
func (m *Manager) Attach(op trace.Operation, disk *types.VirtualDisk) error { deviceList := object.VirtualDeviceList{} deviceList = append(deviceList, disk) changeSpec, err := deviceList.ConfigSpec(types.VirtualDeviceConfigSpecOperationAdd) if err != nil { return err } machineSpec := types.VirtualMachineConfigSpec{} machineSpec.DeviceChange = append(machineSpec.DeviceChange, changeSpec...) m.reconfig.Lock() _, err = m.vm.WaitForResult(op, func(ctx context.Context) (tasks.Task, error) { t, er := m.vm.Reconfigure(ctx, machineSpec) op.Debugf("Attach reconfigure task=%s", t.Reference()) return t, er }) m.reconfig.Unlock() if err != nil { op.Errorf("vmdk storage driver failed to attach disk: %s", errors.ErrorStack(err)) return errors.Trace(err) } return nil }
// Wait wraps govmomi operations and wait the operation to complete // Sample usage: // info, err := Wait(ctx, func(ctx) (*TaskInfo, error) { // return vm.Reconfigure(ctx, config) // }) func Wait(ctx context.Context, f func(context.Context) (Waiter, error)) error { task, err := f(ctx) if err != nil { cerr := errors.Errorf("Failed to invoke operation: %s", errors.ErrorStack(err)) log.Errorf(cerr.Error()) return cerr } err = task.Wait(ctx) if err != nil { cerr := errors.Errorf("Operation failed: %s", errors.ErrorStack(err)) log.Errorf(cerr.Error()) return cerr } return nil }
func (m *Manager) devicePathByURI(ctx context.Context, datastoreURI string) (string, error) { disk, err := findDisk(ctx, m.vm, datastoreURI) if err != nil { log.Debugf("findDisk failed for %s with %s", datastoreURI, errors.ErrorStack(err)) return "", errors.Trace(err) } return fmt.Sprintf(m.byPathFormat, *disk.UnitNumber), nil }
func (m *Manager) Detach(op trace.Operation, d *VirtualDisk) error { defer trace.End(trace.Begin(d.DevicePath)) op.Infof("Detaching disk %s", d.DevicePath) d.lock() defer d.unlock() if !d.Attached() { op.Infof("Disk %s is already detached", d.DevicePath) return nil } if err := d.canBeDetached(); err != nil { return errors.Trace(err) } spec := types.VirtualMachineConfigSpec{} disk, err := findDisk(op, m.vm, d.DatastoreURI) if err != nil { return errors.Trace(err) } config := []types.BaseVirtualDeviceConfigSpec{ &types.VirtualDeviceConfigSpec{ Device: disk, Operation: types.VirtualDeviceConfigSpecOperationRemove, }, } spec.DeviceChange = config m.reconfig.Lock() _, err = m.vm.WaitForResult(op, func(ctx context.Context) (tasks.Task, error) { t, er := m.vm.Reconfigure(ctx, spec) op.Debugf("Detach reconfigure task=%s", t.Reference()) return t, er }) m.reconfig.Unlock() if err != nil { op.Errorf(err.Error()) log.Warnf("detach for %s failed with %s", d.DevicePath, errors.ErrorStack(err)) return errors.Trace(err) } func() { select { case <-m.maxAttached: default: } }() return d.setDetached() }
// CreateAndAttach creates a new vmdk child from parent of the given size. // Returns a VirtualDisk corresponding to the created and attached disk. The // newDiskURI and parentURI are both Datastore URI paths in the form of // [datastoreN] /path/to/disk.vmdk. func (m *Manager) CreateAndAttach(ctx context.Context, newDiskURI, parentURI string, capacity int64, flags int) (*VirtualDisk, error) { defer trace.End(trace.Begin(newDiskURI)) // ensure we abide by max attached disks limits m.maxAttached <- true d, err := NewVirtualDisk(newDiskURI) if err != nil { return nil, errors.Trace(err) } spec := m.createDiskSpec(newDiskURI, parentURI, capacity, flags) log.Infof("Create/attach vmdk %s from parent %s", newDiskURI, parentURI) if err := m.vm.AddDevice(ctx, spec); err != nil { log.Errorf("vmdk storage driver failed to attach disk: %s", errors.ErrorStack(err)) return nil, errors.Trace(err) } log.Debugf("Mapping vmdk to pci device %s", newDiskURI) devicePath, err := m.devicePathByURI(ctx, newDiskURI) if err != nil { return nil, errors.Trace(err) } d.setAttached(devicePath) if err := waitForPath(ctx, devicePath); err != nil { log.Infof("waitForPath failed for %s with %s", newDiskURI, errors.ErrorStack(err)) // ensure that the disk is detached if it's the publish that's failed if detachErr := m.Detach(ctx, d); detachErr != nil { log.Debugf("detach(%s) failed with %s", newDiskURI, errors.ErrorStack(detachErr)) } return nil, errors.Trace(err) } return d, nil }
// CreateAndAttach creates a new vmdk child from parent of the given size. // Returns a VirtualDisk corresponding to the created and attached disk. The // newDiskURI and parentURI are both Datastore URI paths in the form of // [datastoreN] /path/to/disk.vmdk. func (m *Manager) CreateAndAttach(op trace.Operation, newDiskURI, parentURI string, capacity int64, flags int) (*VirtualDisk, error) { defer trace.End(trace.Begin(newDiskURI)) // ensure we abide by max attached disks limits m.maxAttached <- true d, err := NewVirtualDisk(newDiskURI) if err != nil { return nil, errors.Trace(err) } spec := m.createDiskSpec(newDiskURI, parentURI, capacity, flags) op.Infof("Create/attach vmdk %s from parent %s", newDiskURI, parentURI) err = m.Attach(op, spec) if err != nil { return nil, errors.Trace(err) } op.Debugf("Mapping vmdk to pci device %s", newDiskURI) devicePath, err := m.devicePathByURI(op, newDiskURI) if err != nil { return nil, errors.Trace(err) } d.setAttached(devicePath) if err := waitForPath(op, devicePath); err != nil { op.Infof("waitForPath failed for %s with %s", newDiskURI, errors.ErrorStack(err)) // ensure that the disk is detached if it's the publish that's failed if detachErr := m.Detach(op, d); detachErr != nil { op.Debugf("detach(%s) failed with %s", newDiskURI, errors.ErrorStack(detachErr)) } return nil, errors.Trace(err) } return d, nil }
func main() { app := cli.NewApp() app.Name = filepath.Base(os.Args[0]) app.Usage = "Install/remove VIC UI plugin" app.EnableBashCompletion = true ui := ui.NewUI() app.Commands = []cli.Command{ { Name: "install", Usage: "Install UI plugin", Action: ui.Install, Flags: ui.Flags(), }, { Name: "remove", Usage: "Remove UI plugin", Action: ui.Remove, Flags: ui.Flags(), }, { Name: "version", Usage: "Show VIC version information", Action: showVersion, }, } if Version != "" { app.Version = fmt.Sprintf("%s-%s-%s", Version, BuildID, CommitID) } else { app.Version = fmt.Sprintf("%s-%s", BuildID, CommitID) } logs := []io.Writer{app.Writer} // Open log file f, err := os.OpenFile(LogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644) if err != nil { fmt.Fprintf(os.Stderr, "Error opening logfile %s: %v\n", LogFile, err) } else { defer f.Close() logs = append(logs, f) } // Initiliaze logger with default TextFormatter log.SetFormatter(&log.TextFormatter{ForceColors: true, FullTimestamp: true}) // SetOutput to io.MultiWriter so that we can log to stdout and a file log.SetOutput(io.MultiWriter(logs...)) if err := app.Run(os.Args); err != nil { log.Errorf("--------------------") log.Errorf("%s failed: %s\n", app.Name, errors.ErrorStack(err)) os.Exit(1) } }
// WaitForResult wraps govmomi operations and wait the operation to complete. // Return the operation result // Sample usage: // info, err := WaitForResult(ctx, func(ctx) (*TaskInfo, error) { // return vm.Reconfigure(ctx, config) // }) func WaitForResult(ctx context.Context, f func(context.Context) (ResultWaiter, error)) (*types.TaskInfo, error) { task, err := f(ctx) if err != nil { cerr := errors.Errorf("Failed to invoke operation: %s", errors.ErrorStack(err)) log.Errorf(cerr.Error()) return nil, cerr } info, err := task.WaitForResult(ctx, nil) if err != nil { cerr := errors.Errorf("Operation failed: %s", errors.ErrorStack(err)) if info != nil && info.Error != nil { cerr = errors.Errorf("%s - (%s)", cerr, info.Error) } log.Errorf(cerr.Error()) return nil, cerr } return info, nil }
func (m *Manager) Detach(ctx context.Context, d *VirtualDisk) error { defer trace.End(trace.Begin(d.DevicePath)) log.Infof("Detaching disk %s", d.DevicePath) d.lock() defer d.unlock() if !d.Attached() { log.Infof("Disk %s is already detached", d.DevicePath) return nil } if err := d.canBeDetached(); err != nil { return errors.Trace(err) } spec := types.VirtualMachineConfigSpec{} disk, err := findDisk(ctx, m.vm, d.DatastoreURI) if err != nil { return errors.Trace(err) } config := []types.BaseVirtualDeviceConfigSpec{ &types.VirtualDeviceConfigSpec{ Device: disk, Operation: types.VirtualDeviceConfigSpecOperationRemove, }, } spec.DeviceChange = config err = tasks.Wait(ctx, func(ctx context.Context) (tasks.Waiter, error) { return m.vm.Reconfigure(ctx, spec) }) if err != nil { log.Warnf("detach for %s failed with %s", d.DevicePath, errors.ErrorStack(err)) return errors.Trace(err) } func() { select { case <-m.maxAttached: default: } }() return d.setDetached() }
// Start starts the TCP listener. func (n *Server) Start() error { defer trace.End(trace.Begin("")) log.Infof("Attach server listening on %s:%d", n.ip, n.port) addr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%d", n.ip, n.port)) n.l, err = net.ListenTCP("tcp", addr) if err != nil { err = fmt.Errorf("Attach server error %s: %s", addr, errors.ErrorStack(err)) log.Errorf("%s", err) return err } // starts serving requests immediately n.connServer = NewConnector(n.l) return nil }
// Find the disk by name attached to the given vm. func findDisk(op trace.Operation, vm *vm.VirtualMachine, name string) (*types.VirtualDisk, error) { defer trace.End(trace.Begin(vm.String())) log.Debugf("Looking for attached disk matching filename %s", name) devices, err := vm.Device(op) if err != nil { return nil, fmt.Errorf("Failed to refresh devices for vm: %s", errors.ErrorStack(err)) } candidates := devices.Select(func(device types.BaseVirtualDevice) bool { db := device.GetVirtualDevice().Backing if db == nil { return false } backing, ok := device.GetVirtualDevice().Backing.(*types.VirtualDiskFlatVer2BackingInfo) if !ok { return false } log.Debugf("backing file name %s", backing.VirtualDeviceFileBackingInfo.FileName) match := strings.HasSuffix(backing.VirtualDeviceFileBackingInfo.FileName, name) if match { log.Debugf("Found candidate disk for %s at %s", name, backing.VirtualDeviceFileBackingInfo.FileName) } return match }) if len(candidates) == 0 { log.Warnf("No disks match name: %s", name) return nil, os.ErrNotExist } if len(candidates) > 1 { return nil, errors.Errorf("Too many disks match name: %s", name) } return candidates[0].(*types.VirtualDisk), nil }
// ensures that a paravirtual scsi controller is present and determines the // base path of disks attached to it returns a handle to the controller and a // format string, with a single decimal for the disk unit number which will // result in the /dev/disk/by-path path func verifyParavirtualScsiController(op trace.Operation, vm *vm.VirtualMachine) (*types.ParaVirtualSCSIController, string, error) { devices, err := vm.Device(op) if err != nil { log.Errorf("vmware driver failed to retrieve device list for VM %s: %s", vm, errors.ErrorStack(err)) return nil, "", errors.Trace(err) } controller, ok := devices.PickController((*types.ParaVirtualSCSIController)(nil)).(*types.ParaVirtualSCSIController) if controller == nil || !ok { err = errors.Errorf("vmware driver failed to find a paravirtual SCSI controller - ensure setup ran correctly") log.Error(err.Error()) return nil, "", errors.Trace(err) } // build the base path // first we determine which label we're looking for (requires VMW hardware version >=10) targetLabel := fmt.Sprintf("SCSI%d", controller.BusNumber) log.Debugf("Looking for scsi controller with label %s", targetLabel) pciBase := "/sys/bus/pci/devices" pciBus, err := os.Open(pciBase) if err != nil { log.Errorf("Failed to open %s for reading: %s", pciBase, errors.ErrorStack(err)) return controller, "", errors.Trace(err) } defer pciBus.Close() pciDevices, err := pciBus.Readdirnames(0) if err != nil { log.Errorf("Failed to read contents of %s: %s", pciBase, errors.ErrorStack(err)) return controller, "", errors.Trace(err) } var buf = make([]byte, len(targetLabel)) var controllerName string for _, n := range pciDevices { nlabel := fmt.Sprintf("%s/%s/label", pciBase, n) flabel, err := os.Open(nlabel) if err != nil { if !os.IsNotExist(err) { log.Errorf("Unable to read label from %s: %s", nlabel, errors.ErrorStack(err)) } continue } defer flabel.Close() _, err = flabel.Read(buf) if err != nil { log.Errorf("Unable to read label from %s: %s", nlabel, errors.ErrorStack(err)) continue } if targetLabel == string(buf) { // we've found our controller controllerName = n log.Debugf("Found pvscsi controller directory: %s", controllerName) break } } if controllerName == "" { err := errors.Errorf("Failed to locate pvscsi controller directory") log.Errorf(err.Error()) return controller, "", errors.Trace(err) } formatString := fmt.Sprintf("/dev/disk/by-path/pci-%s-scsi-0:0:%%d:0", controllerName) log.Debugf("Disk location format: %s", formatString) return controller, formatString, nil }
func main() { app := cli.NewApp() app.Name = filepath.Base(os.Args[0]) app.Usage = "Create and manage Virtual Container Hosts" app.EnableBashCompletion = true create := create.NewCreate() uninstall := uninstall.NewUninstall() inspect := inspect.NewInspect() list := list.NewList() upgrade := upgrade.NewUpgrade() debug := debug.NewDebug() app.Commands = []cli.Command{ { Name: "create", Usage: "Deploy VCH", Action: create.Run, Flags: create.Flags(), }, { Name: "delete", Usage: "Delete VCH and associated resources", Action: uninstall.Run, Flags: uninstall.Flags(), }, { Name: "ls", Usage: "List VCHs", Action: list.Run, Flags: list.Flags(), }, { Name: "inspect", Usage: "Inspect VCH", Action: inspect.Run, Flags: inspect.Flags(), }, { Name: "upgrade", Usage: "Upgrade VCH to latest version", Action: upgrade.Run, Flags: upgrade.Flags(), }, { Name: "version", Usage: "Show VIC version information", Action: showVersion, }, { Name: "debug", Usage: "Debug VCH", Action: debug.Run, Flags: debug.Flags(), }, } app.Version = version.GetBuild().ShortVersion() logs := []io.Writer{app.Writer} // Open log file f, err := os.OpenFile(LogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644) if err != nil { fmt.Fprintf(os.Stderr, "Error opening logfile %s: %v\n", LogFile, err) } else { defer f.Close() logs = append(logs, f) } // Initiliaze logger with default TextFormatter log.SetFormatter(&log.TextFormatter{ForceColors: true, FullTimestamp: true}) // SetOutput to io.MultiWriter so that we can log to stdout and a file log.SetOutput(io.MultiWriter(logs...)) if err := app.Run(os.Args); err != nil { log.Errorf("--------------------") log.Errorf("%s failed: %s\n", app.Name, errors.ErrorStack(err)) os.Exit(1) } }
// takes the base connection, determines the ID of the source and stashes it in the map func (c *Connector) processIncoming(conn net.Conn) { var err error defer func() { if err != nil && conn != nil { conn.Close() } }() for { if conn == nil { log.Infof("attach connector: connection closed") return } serial.PurgeIncoming(conn) // TODO needs timeout handling. This could take 30s. // Timeout for client handshake should be reasonably small. // Server will try to drain a buffer and if the buffer doesn't contain // 2 or more bytes it will just wait, so client should timeout. // However, if timeout is too short, client will flood server with Syn requests. ctx, cancel := context.WithTimeout(context.TODO(), time.Second) deadline, ok := ctx.Deadline() if ok { conn.SetReadDeadline(deadline) } if err = serial.HandshakeClient(conn, c.debug); err == nil { conn.SetReadDeadline(time.Time{}) log.Debugf("attach connector: New connection") cancel() break } else if err == io.EOF { log.Debugf("caught EOF") conn.Close() return } else if _, ok := err.(*serial.HandshakeError); ok { log.Debugf("HandshakeClient: %v", err) } else { log.Errorf("HandshakeClient: %v", err) } } callback := func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil } config := &ssh.ClientConfig{ User: "******", HostKeyCallback: callback, } log.Debugf("Initiating ssh handshake with new connection attempt") var ( ccon ssh.Conn newchan <-chan ssh.NewChannel request <-chan *ssh.Request ) ccon, newchan, request, err = ssh.NewClientConn(conn, "", config) if err != nil { log.Errorf("SSH connection could not be established: %s", errors.ErrorStack(err)) return } client := ssh.NewClient(ccon, newchan, request) var ids []string ids, err = SSHls(client) if err != nil { log.Errorf("SSH connection could not be established: %s", errors.ErrorStack(err)) return } var si SessionInteraction for _, id := range ids { si, err = SSHAttach(client, id) if err != nil { log.Errorf("SSH connection could not be established (id=%s): %s", id, errors.ErrorStack(err)) return } log.Infof("Established connection with container VM: %s", id) c.mutex.Lock() connection := &Connection{ spty: si, id: id, } c.connections[connection.id] = connection c.cond.Broadcast() c.mutex.Unlock() } return }
// takes the base connection, determines the ID of the source and stashes it in the map func (c *Connector) processIncoming(conn net.Conn) { var err error defer func() { if err != nil && conn != nil { conn.Close() } }() for { if conn == nil { log.Infof("attach connector: connection closed") return } serial.PurgeIncoming(conn) // TODO needs timeout handling. This could take 30s. // This needs to timeout with a *longer* wait than the ticker set on // the tether side (in tether_linux.go) or alignment may not happen. // The PL sends the first SYN in the handshake and if the tether is not // waiting, the handshake may never succeed. ctx, cancel := context.WithTimeout(context.TODO(), 50*time.Millisecond) if err = serial.HandshakeClient(ctx, conn); err == nil { log.Debugf("attach connector: New connection") cancel() break } else if err == io.EOF { log.Debugf("caught EOF") conn.Close() return } } callback := func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil } config := &ssh.ClientConfig{ User: "******", HostKeyCallback: callback, } log.Debugf("Initiating ssh handshake with new connection attempt") var ( ccon ssh.Conn newchan <-chan ssh.NewChannel request <-chan *ssh.Request ) ccon, newchan, request, err = ssh.NewClientConn(conn, "", config) if err != nil { log.Errorf("SSH connection could not be established: %s", errors.ErrorStack(err)) return } client := ssh.NewClient(ccon, newchan, request) var ids []string ids, err = SSHls(client) if err != nil { log.Errorf("SSH connection could not be established: %s", errors.ErrorStack(err)) return } var si SessionInteraction for _, id := range ids { si, err = SSHAttach(client, id) if err != nil { log.Errorf("SSH connection could not be established (id=%s): %s", id, errors.ErrorStack(err)) return } log.Infof("Established connection with container VM: %s", id) c.mutex.Lock() connection := &Connection{ spty: si, id: id, } c.connections[connection.id] = connection c.cond.Broadcast() c.mutex.Unlock() } return }