// Starts the connector listening on the specified source
// TODO: should have mechanism for stopping this, and probably handing off the connections to another
// routine to insert into the map
func (c *Connector) serve() {
defer c.wg.Done()
for {
if c.listener == nil {
log.Debugf("attach connector: listener closed")
break
}
conn, err := c.listener.Accept()
select {
case <-c.listenerQuit:
log.Debugf("attach connector: serve exitting")
return
default:
}
if err != nil {
log.Errorf("Error waiting for incoming connection: %s", errors.ErrorStack(err))
continue
}
log.Info("attach connector: Received incoming connection")
go c.processIncoming(conn)
}
}
func (m *Manager) Attach(op trace.Operation, disk *types.VirtualDisk) error {
deviceList := object.VirtualDeviceList{}
deviceList = append(deviceList, disk)
changeSpec, err := deviceList.ConfigSpec(types.VirtualDeviceConfigSpecOperationAdd)
if err != nil {
return err
}
machineSpec := types.VirtualMachineConfigSpec{}
machineSpec.DeviceChange = append(machineSpec.DeviceChange, changeSpec...)
m.reconfig.Lock()
_, err = m.vm.WaitForResult(op, func(ctx context.Context) (tasks.Task, error) {
t, er := m.vm.Reconfigure(ctx, machineSpec)
op.Debugf("Attach reconfigure task=%s", t.Reference())
return t, er
})
m.reconfig.Unlock()
if err != nil {
op.Errorf("vmdk storage driver failed to attach disk: %s", errors.ErrorStack(err))
return errors.Trace(err)
}
return nil
}
// Wait wraps govmomi operations and wait the operation to complete
// Sample usage:
// info, err := Wait(ctx, func(ctx) (*TaskInfo, error) {
// return vm.Reconfigure(ctx, config)
// })
func Wait(ctx context.Context, f func(context.Context) (Waiter, error)) error {
task, err := f(ctx)
if err != nil {
cerr := errors.Errorf("Failed to invoke operation: %s", errors.ErrorStack(err))
log.Errorf(cerr.Error())
return cerr
}
err = task.Wait(ctx)
if err != nil {
cerr := errors.Errorf("Operation failed: %s", errors.ErrorStack(err))
log.Errorf(cerr.Error())
return cerr
}
return nil
}
func (m *Manager) devicePathByURI(ctx context.Context, datastoreURI string) (string, error) {
disk, err := findDisk(ctx, m.vm, datastoreURI)
if err != nil {
log.Debugf("findDisk failed for %s with %s", datastoreURI, errors.ErrorStack(err))
return "", errors.Trace(err)
}
return fmt.Sprintf(m.byPathFormat, *disk.UnitNumber), nil
}
func (m *Manager) Detach(op trace.Operation, d *VirtualDisk) error {
defer trace.End(trace.Begin(d.DevicePath))
op.Infof("Detaching disk %s", d.DevicePath)
d.lock()
defer d.unlock()
if !d.Attached() {
op.Infof("Disk %s is already detached", d.DevicePath)
return nil
}
if err := d.canBeDetached(); err != nil {
return errors.Trace(err)
}
spec := types.VirtualMachineConfigSpec{}
disk, err := findDisk(op, m.vm, d.DatastoreURI)
if err != nil {
return errors.Trace(err)
}
config := []types.BaseVirtualDeviceConfigSpec{
&types.VirtualDeviceConfigSpec{
Device: disk,
Operation: types.VirtualDeviceConfigSpecOperationRemove,
},
}
spec.DeviceChange = config
m.reconfig.Lock()
_, err = m.vm.WaitForResult(op, func(ctx context.Context) (tasks.Task, error) {
t, er := m.vm.Reconfigure(ctx, spec)
op.Debugf("Detach reconfigure task=%s", t.Reference())
return t, er
})
m.reconfig.Unlock()
if err != nil {
op.Errorf(err.Error())
log.Warnf("detach for %s failed with %s", d.DevicePath, errors.ErrorStack(err))
return errors.Trace(err)
}
func() {
select {
case <-m.maxAttached:
default:
}
}()
return d.setDetached()
}
// CreateAndAttach creates a new vmdk child from parent of the given size.
// Returns a VirtualDisk corresponding to the created and attached disk. The
// newDiskURI and parentURI are both Datastore URI paths in the form of
// [datastoreN] /path/to/disk.vmdk.
func (m *Manager) CreateAndAttach(ctx context.Context, newDiskURI,
parentURI string,
capacity int64, flags int) (*VirtualDisk, error) {
defer trace.End(trace.Begin(newDiskURI))
// ensure we abide by max attached disks limits
m.maxAttached <- true
d, err := NewVirtualDisk(newDiskURI)
if err != nil {
return nil, errors.Trace(err)
}
spec := m.createDiskSpec(newDiskURI, parentURI, capacity, flags)
log.Infof("Create/attach vmdk %s from parent %s", newDiskURI, parentURI)
if err := m.vm.AddDevice(ctx, spec); err != nil {
log.Errorf("vmdk storage driver failed to attach disk: %s", errors.ErrorStack(err))
return nil, errors.Trace(err)
}
log.Debugf("Mapping vmdk to pci device %s", newDiskURI)
devicePath, err := m.devicePathByURI(ctx, newDiskURI)
if err != nil {
return nil, errors.Trace(err)
}
d.setAttached(devicePath)
if err := waitForPath(ctx, devicePath); err != nil {
log.Infof("waitForPath failed for %s with %s", newDiskURI, errors.ErrorStack(err))
// ensure that the disk is detached if it's the publish that's failed
if detachErr := m.Detach(ctx, d); detachErr != nil {
log.Debugf("detach(%s) failed with %s", newDiskURI, errors.ErrorStack(detachErr))
}
return nil, errors.Trace(err)
}
return d, nil
}
// CreateAndAttach creates a new vmdk child from parent of the given size.
// Returns a VirtualDisk corresponding to the created and attached disk. The
// newDiskURI and parentURI are both Datastore URI paths in the form of
// [datastoreN] /path/to/disk.vmdk.
func (m *Manager) CreateAndAttach(op trace.Operation, newDiskURI,
parentURI string,
capacity int64, flags int) (*VirtualDisk, error) {
defer trace.End(trace.Begin(newDiskURI))
// ensure we abide by max attached disks limits
m.maxAttached <- true
d, err := NewVirtualDisk(newDiskURI)
if err != nil {
return nil, errors.Trace(err)
}
spec := m.createDiskSpec(newDiskURI, parentURI, capacity, flags)
op.Infof("Create/attach vmdk %s from parent %s", newDiskURI, parentURI)
err = m.Attach(op, spec)
if err != nil {
return nil, errors.Trace(err)
}
op.Debugf("Mapping vmdk to pci device %s", newDiskURI)
devicePath, err := m.devicePathByURI(op, newDiskURI)
if err != nil {
return nil, errors.Trace(err)
}
d.setAttached(devicePath)
if err := waitForPath(op, devicePath); err != nil {
op.Infof("waitForPath failed for %s with %s", newDiskURI, errors.ErrorStack(err))
// ensure that the disk is detached if it's the publish that's failed
if detachErr := m.Detach(op, d); detachErr != nil {
op.Debugf("detach(%s) failed with %s", newDiskURI, errors.ErrorStack(detachErr))
}
return nil, errors.Trace(err)
}
return d, nil
}
func main() {
app := cli.NewApp()
app.Name = filepath.Base(os.Args[0])
app.Usage = "Install/remove VIC UI plugin"
app.EnableBashCompletion = true
ui := ui.NewUI()
app.Commands = []cli.Command{
{
Name: "install",
Usage: "Install UI plugin",
Action: ui.Install,
Flags: ui.Flags(),
},
{
Name: "remove",
Usage: "Remove UI plugin",
Action: ui.Remove,
Flags: ui.Flags(),
},
{
Name: "version",
Usage: "Show VIC version information",
Action: showVersion,
},
}
if Version != "" {
app.Version = fmt.Sprintf("%s-%s-%s", Version, BuildID, CommitID)
} else {
app.Version = fmt.Sprintf("%s-%s", BuildID, CommitID)
}
logs := []io.Writer{app.Writer}
// Open log file
f, err := os.OpenFile(LogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
fmt.Fprintf(os.Stderr, "Error opening logfile %s: %v\n", LogFile, err)
} else {
defer f.Close()
logs = append(logs, f)
}
// Initiliaze logger with default TextFormatter
log.SetFormatter(&log.TextFormatter{ForceColors: true, FullTimestamp: true})
// SetOutput to io.MultiWriter so that we can log to stdout and a file
log.SetOutput(io.MultiWriter(logs...))
if err := app.Run(os.Args); err != nil {
log.Errorf("--------------------")
log.Errorf("%s failed: %s\n", app.Name, errors.ErrorStack(err))
os.Exit(1)
}
}
// WaitForResult wraps govmomi operations and wait the operation to complete.
// Return the operation result
// Sample usage:
// info, err := WaitForResult(ctx, func(ctx) (*TaskInfo, error) {
// return vm.Reconfigure(ctx, config)
// })
func WaitForResult(ctx context.Context, f func(context.Context) (ResultWaiter, error)) (*types.TaskInfo, error) {
task, err := f(ctx)
if err != nil {
cerr := errors.Errorf("Failed to invoke operation: %s", errors.ErrorStack(err))
log.Errorf(cerr.Error())
return nil, cerr
}
info, err := task.WaitForResult(ctx, nil)
if err != nil {
cerr := errors.Errorf("Operation failed: %s", errors.ErrorStack(err))
if info != nil && info.Error != nil {
cerr = errors.Errorf("%s - (%s)", cerr, info.Error)
}
log.Errorf(cerr.Error())
return nil, cerr
}
return info, nil
}
func (m *Manager) Detach(ctx context.Context, d *VirtualDisk) error {
defer trace.End(trace.Begin(d.DevicePath))
log.Infof("Detaching disk %s", d.DevicePath)
d.lock()
defer d.unlock()
if !d.Attached() {
log.Infof("Disk %s is already detached", d.DevicePath)
return nil
}
if err := d.canBeDetached(); err != nil {
return errors.Trace(err)
}
spec := types.VirtualMachineConfigSpec{}
disk, err := findDisk(ctx, m.vm, d.DatastoreURI)
if err != nil {
return errors.Trace(err)
}
config := []types.BaseVirtualDeviceConfigSpec{
&types.VirtualDeviceConfigSpec{
Device: disk,
Operation: types.VirtualDeviceConfigSpecOperationRemove,
},
}
spec.DeviceChange = config
err = tasks.Wait(ctx, func(ctx context.Context) (tasks.Waiter, error) {
return m.vm.Reconfigure(ctx, spec)
})
if err != nil {
log.Warnf("detach for %s failed with %s", d.DevicePath, errors.ErrorStack(err))
return errors.Trace(err)
}
func() {
select {
case <-m.maxAttached:
default:
}
}()
return d.setDetached()
}
// Start starts the TCP listener.
func (n *Server) Start() error {
defer trace.End(trace.Begin(""))
log.Infof("Attach server listening on %s:%d", n.ip, n.port)
addr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%d", n.ip, n.port))
n.l, err = net.ListenTCP("tcp", addr)
if err != nil {
err = fmt.Errorf("Attach server error %s: %s", addr, errors.ErrorStack(err))
log.Errorf("%s", err)
return err
}
// starts serving requests immediately
n.connServer = NewConnector(n.l)
return nil
}
// Find the disk by name attached to the given vm.
func findDisk(op trace.Operation, vm *vm.VirtualMachine, name string) (*types.VirtualDisk, error) {
defer trace.End(trace.Begin(vm.String()))
log.Debugf("Looking for attached disk matching filename %s", name)
devices, err := vm.Device(op)
if err != nil {
return nil, fmt.Errorf("Failed to refresh devices for vm: %s", errors.ErrorStack(err))
}
candidates := devices.Select(func(device types.BaseVirtualDevice) bool {
db := device.GetVirtualDevice().Backing
if db == nil {
return false
}
backing, ok := device.GetVirtualDevice().Backing.(*types.VirtualDiskFlatVer2BackingInfo)
if !ok {
return false
}
log.Debugf("backing file name %s", backing.VirtualDeviceFileBackingInfo.FileName)
match := strings.HasSuffix(backing.VirtualDeviceFileBackingInfo.FileName, name)
if match {
log.Debugf("Found candidate disk for %s at %s", name, backing.VirtualDeviceFileBackingInfo.FileName)
}
return match
})
if len(candidates) == 0 {
log.Warnf("No disks match name: %s", name)
return nil, os.ErrNotExist
}
if len(candidates) > 1 {
return nil, errors.Errorf("Too many disks match name: %s", name)
}
return candidates[0].(*types.VirtualDisk), nil
}
// ensures that a paravirtual scsi controller is present and determines the
// base path of disks attached to it returns a handle to the controller and a
// format string, with a single decimal for the disk unit number which will
// result in the /dev/disk/by-path path
func verifyParavirtualScsiController(op trace.Operation, vm *vm.VirtualMachine) (*types.ParaVirtualSCSIController, string, error) {
devices, err := vm.Device(op)
if err != nil {
log.Errorf("vmware driver failed to retrieve device list for VM %s: %s", vm, errors.ErrorStack(err))
return nil, "", errors.Trace(err)
}
controller, ok := devices.PickController((*types.ParaVirtualSCSIController)(nil)).(*types.ParaVirtualSCSIController)
if controller == nil || !ok {
err = errors.Errorf("vmware driver failed to find a paravirtual SCSI controller - ensure setup ran correctly")
log.Error(err.Error())
return nil, "", errors.Trace(err)
}
// build the base path
// first we determine which label we're looking for (requires VMW hardware version >=10)
targetLabel := fmt.Sprintf("SCSI%d", controller.BusNumber)
log.Debugf("Looking for scsi controller with label %s", targetLabel)
pciBase := "/sys/bus/pci/devices"
pciBus, err := os.Open(pciBase)
if err != nil {
log.Errorf("Failed to open %s for reading: %s", pciBase, errors.ErrorStack(err))
return controller, "", errors.Trace(err)
}
defer pciBus.Close()
pciDevices, err := pciBus.Readdirnames(0)
if err != nil {
log.Errorf("Failed to read contents of %s: %s", pciBase, errors.ErrorStack(err))
return controller, "", errors.Trace(err)
}
var buf = make([]byte, len(targetLabel))
var controllerName string
for _, n := range pciDevices {
nlabel := fmt.Sprintf("%s/%s/label", pciBase, n)
flabel, err := os.Open(nlabel)
if err != nil {
if !os.IsNotExist(err) {
log.Errorf("Unable to read label from %s: %s", nlabel, errors.ErrorStack(err))
}
continue
}
defer flabel.Close()
_, err = flabel.Read(buf)
if err != nil {
log.Errorf("Unable to read label from %s: %s", nlabel, errors.ErrorStack(err))
continue
}
if targetLabel == string(buf) {
// we've found our controller
controllerName = n
log.Debugf("Found pvscsi controller directory: %s", controllerName)
break
}
}
if controllerName == "" {
err := errors.Errorf("Failed to locate pvscsi controller directory")
log.Errorf(err.Error())
return controller, "", errors.Trace(err)
}
formatString := fmt.Sprintf("/dev/disk/by-path/pci-%s-scsi-0:0:%%d:0", controllerName)
log.Debugf("Disk location format: %s", formatString)
return controller, formatString, nil
}
func main() {
app := cli.NewApp()
app.Name = filepath.Base(os.Args[0])
app.Usage = "Create and manage Virtual Container Hosts"
app.EnableBashCompletion = true
create := create.NewCreate()
uninstall := uninstall.NewUninstall()
inspect := inspect.NewInspect()
list := list.NewList()
upgrade := upgrade.NewUpgrade()
debug := debug.NewDebug()
app.Commands = []cli.Command{
{
Name: "create",
Usage: "Deploy VCH",
Action: create.Run,
Flags: create.Flags(),
},
{
Name: "delete",
Usage: "Delete VCH and associated resources",
Action: uninstall.Run,
Flags: uninstall.Flags(),
},
{
Name: "ls",
Usage: "List VCHs",
Action: list.Run,
Flags: list.Flags(),
},
{
Name: "inspect",
Usage: "Inspect VCH",
Action: inspect.Run,
Flags: inspect.Flags(),
},
{
Name: "upgrade",
Usage: "Upgrade VCH to latest version",
Action: upgrade.Run,
Flags: upgrade.Flags(),
},
{
Name: "version",
Usage: "Show VIC version information",
Action: showVersion,
},
{
Name: "debug",
Usage: "Debug VCH",
Action: debug.Run,
Flags: debug.Flags(),
},
}
app.Version = version.GetBuild().ShortVersion()
logs := []io.Writer{app.Writer}
// Open log file
f, err := os.OpenFile(LogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
fmt.Fprintf(os.Stderr, "Error opening logfile %s: %v\n", LogFile, err)
} else {
defer f.Close()
logs = append(logs, f)
}
// Initiliaze logger with default TextFormatter
log.SetFormatter(&log.TextFormatter{ForceColors: true, FullTimestamp: true})
// SetOutput to io.MultiWriter so that we can log to stdout and a file
log.SetOutput(io.MultiWriter(logs...))
if err := app.Run(os.Args); err != nil {
log.Errorf("--------------------")
log.Errorf("%s failed: %s\n", app.Name, errors.ErrorStack(err))
os.Exit(1)
}
}
// takes the base connection, determines the ID of the source and stashes it in the map
func (c *Connector) processIncoming(conn net.Conn) {
var err error
defer func() {
if err != nil && conn != nil {
conn.Close()
}
}()
for {
if conn == nil {
log.Infof("attach connector: connection closed")
return
}
serial.PurgeIncoming(conn)
// TODO needs timeout handling. This could take 30s.
// Timeout for client handshake should be reasonably small.
// Server will try to drain a buffer and if the buffer doesn't contain
// 2 or more bytes it will just wait, so client should timeout.
// However, if timeout is too short, client will flood server with Syn requests.
ctx, cancel := context.WithTimeout(context.TODO(), time.Second)
deadline, ok := ctx.Deadline()
if ok {
conn.SetReadDeadline(deadline)
}
if err = serial.HandshakeClient(conn, c.debug); err == nil {
conn.SetReadDeadline(time.Time{})
log.Debugf("attach connector: New connection")
cancel()
break
} else if err == io.EOF {
log.Debugf("caught EOF")
conn.Close()
return
} else if _, ok := err.(*serial.HandshakeError); ok {
log.Debugf("HandshakeClient: %v", err)
} else {
log.Errorf("HandshakeClient: %v", err)
}
}
callback := func(hostname string, remote net.Addr, key ssh.PublicKey) error {
return nil
}
config := &ssh.ClientConfig{
User: "******",
HostKeyCallback: callback,
}
log.Debugf("Initiating ssh handshake with new connection attempt")
var (
ccon ssh.Conn
newchan <-chan ssh.NewChannel
request <-chan *ssh.Request
)
ccon, newchan, request, err = ssh.NewClientConn(conn, "", config)
if err != nil {
log.Errorf("SSH connection could not be established: %s", errors.ErrorStack(err))
return
}
client := ssh.NewClient(ccon, newchan, request)
var ids []string
ids, err = SSHls(client)
if err != nil {
log.Errorf("SSH connection could not be established: %s", errors.ErrorStack(err))
return
}
var si SessionInteraction
for _, id := range ids {
si, err = SSHAttach(client, id)
if err != nil {
log.Errorf("SSH connection could not be established (id=%s): %s", id, errors.ErrorStack(err))
return
}
log.Infof("Established connection with container VM: %s", id)
c.mutex.Lock()
connection := &Connection{
spty: si,
id: id,
}
c.connections[connection.id] = connection
c.cond.Broadcast()
c.mutex.Unlock()
}
return
}
// takes the base connection, determines the ID of the source and stashes it in the map
func (c *Connector) processIncoming(conn net.Conn) {
var err error
defer func() {
if err != nil && conn != nil {
conn.Close()
}
}()
for {
if conn == nil {
log.Infof("attach connector: connection closed")
return
}
serial.PurgeIncoming(conn)
// TODO needs timeout handling. This could take 30s.
// This needs to timeout with a *longer* wait than the ticker set on
// the tether side (in tether_linux.go) or alignment may not happen.
// The PL sends the first SYN in the handshake and if the tether is not
// waiting, the handshake may never succeed.
ctx, cancel := context.WithTimeout(context.TODO(), 50*time.Millisecond)
if err = serial.HandshakeClient(ctx, conn); err == nil {
log.Debugf("attach connector: New connection")
cancel()
break
} else if err == io.EOF {
log.Debugf("caught EOF")
conn.Close()
return
}
}
callback := func(hostname string, remote net.Addr, key ssh.PublicKey) error {
return nil
}
config := &ssh.ClientConfig{
User: "******",
HostKeyCallback: callback,
}
log.Debugf("Initiating ssh handshake with new connection attempt")
var (
ccon ssh.Conn
newchan <-chan ssh.NewChannel
request <-chan *ssh.Request
)
ccon, newchan, request, err = ssh.NewClientConn(conn, "", config)
if err != nil {
log.Errorf("SSH connection could not be established: %s", errors.ErrorStack(err))
return
}
client := ssh.NewClient(ccon, newchan, request)
var ids []string
ids, err = SSHls(client)
if err != nil {
log.Errorf("SSH connection could not be established: %s", errors.ErrorStack(err))
return
}
var si SessionInteraction
for _, id := range ids {
si, err = SSHAttach(client, id)
if err != nil {
log.Errorf("SSH connection could not be established (id=%s): %s", id, errors.ErrorStack(err))
return
}
log.Infof("Established connection with container VM: %s", id)
c.mutex.Lock()
connection := &Connection{
spty: si,
id: id,
}
c.connections[connection.id] = connection
c.cond.Broadcast()
c.mutex.Unlock()
}
return
}