// Create creates a disk without a parent (and doesn't attach it).
func (m *Manager) Create(ctx context.Context, newDiskURI string,
capacityKB int64) (*VirtualDisk, error) {
defer trace.End(trace.Begin(newDiskURI))
vdm := object.NewVirtualDiskManager(m.vm.Client())
d, err := NewVirtualDisk(newDiskURI)
if err != nil {
return nil, errors.Trace(err)
}
spec := &types.FileBackedVirtualDiskSpec{
VirtualDiskSpec: types.VirtualDiskSpec{
DiskType: string(types.VirtualDiskTypeThin),
AdapterType: string(types.VirtualDiskAdapterTypeLsiLogic),
},
CapacityKb: capacityKB,
}
log.Infof("Creating vmdk for layer or volume %s", d.DatastoreURI)
err = tasks.Wait(ctx, func(ctx context.Context) (tasks.Waiter, error) {
return vdm.CreateVirtualDisk(ctx, d.DatastoreURI, nil, spec)
})
if err != nil {
return nil, errors.Trace(err)
}
return d, nil
}
func (m *Manager) Detach(op trace.Operation, d *VirtualDisk) error {
defer trace.End(trace.Begin(d.DevicePath))
op.Infof("Detaching disk %s", d.DevicePath)
d.lock()
defer d.unlock()
if !d.Attached() {
op.Infof("Disk %s is already detached", d.DevicePath)
return nil
}
if err := d.canBeDetached(); err != nil {
return errors.Trace(err)
}
spec := types.VirtualMachineConfigSpec{}
disk, err := findDisk(op, m.vm, d.DatastoreURI)
if err != nil {
return errors.Trace(err)
}
config := []types.BaseVirtualDeviceConfigSpec{
&types.VirtualDeviceConfigSpec{
Device: disk,
Operation: types.VirtualDeviceConfigSpecOperationRemove,
},
}
spec.DeviceChange = config
m.reconfig.Lock()
_, err = m.vm.WaitForResult(op, func(ctx context.Context) (tasks.Task, error) {
t, er := m.vm.Reconfigure(ctx, spec)
op.Debugf("Detach reconfigure task=%s", t.Reference())
return t, er
})
m.reconfig.Unlock()
if err != nil {
op.Errorf(err.Error())
log.Warnf("detach for %s failed with %s", d.DevicePath, errors.ErrorStack(err))
return errors.Trace(err)
}
func() {
select {
case <-m.maxAttached:
default:
}
}()
return d.setDetached()
}
func (m *Manager) Detach(ctx context.Context, d *VirtualDisk) error {
defer trace.End(trace.Begin(d.DevicePath))
log.Infof("Detaching disk %s", d.DevicePath)
d.lock()
defer d.unlock()
if !d.Attached() {
log.Infof("Disk %s is already detached", d.DevicePath)
return nil
}
if err := d.canBeDetached(); err != nil {
return errors.Trace(err)
}
spec := types.VirtualMachineConfigSpec{}
disk, err := findDisk(ctx, m.vm, d.DatastoreURI)
if err != nil {
return errors.Trace(err)
}
config := []types.BaseVirtualDeviceConfigSpec{
&types.VirtualDeviceConfigSpec{
Device: disk,
Operation: types.VirtualDeviceConfigSpecOperationRemove,
},
}
spec.DeviceChange = config
err = tasks.Wait(ctx, func(ctx context.Context) (tasks.Waiter, error) {
return m.vm.Reconfigure(ctx, spec)
})
if err != nil {
log.Warnf("detach for %s failed with %s", d.DevicePath, errors.ErrorStack(err))
return errors.Trace(err)
}
func() {
select {
case <-m.maxAttached:
default:
}
}()
return d.setDetached()
}
func (m *Manager) Attach(op trace.Operation, disk *types.VirtualDisk) error {
deviceList := object.VirtualDeviceList{}
deviceList = append(deviceList, disk)
changeSpec, err := deviceList.ConfigSpec(types.VirtualDeviceConfigSpecOperationAdd)
if err != nil {
return err
}
machineSpec := types.VirtualMachineConfigSpec{}
machineSpec.DeviceChange = append(machineSpec.DeviceChange, changeSpec...)
m.reconfig.Lock()
_, err = m.vm.WaitForResult(op, func(ctx context.Context) (tasks.Task, error) {
t, er := m.vm.Reconfigure(ctx, machineSpec)
op.Debugf("Attach reconfigure task=%s", t.Reference())
return t, er
})
m.reconfig.Unlock()
if err != nil {
op.Errorf("vmdk storage driver failed to attach disk: %s", errors.ErrorStack(err))
return errors.Trace(err)
}
return nil
}
func (m *Manager) devicePathByURI(ctx context.Context, datastoreURI string) (string, error) {
disk, err := findDisk(ctx, m.vm, datastoreURI)
if err != nil {
log.Debugf("findDisk failed for %s with %s", datastoreURI, errors.ErrorStack(err))
return "", errors.Trace(err)
}
return fmt.Sprintf(m.byPathFormat, *disk.UnitNumber), nil
}
// CreateAndAttach creates a new vmdk child from parent of the given size.
// Returns a VirtualDisk corresponding to the created and attached disk. The
// newDiskURI and parentURI are both Datastore URI paths in the form of
// [datastoreN] /path/to/disk.vmdk.
func (m *Manager) CreateAndAttach(ctx context.Context, newDiskURI,
parentURI string,
capacity int64, flags int) (*VirtualDisk, error) {
defer trace.End(trace.Begin(newDiskURI))
// ensure we abide by max attached disks limits
m.maxAttached <- true
d, err := NewVirtualDisk(newDiskURI)
if err != nil {
return nil, errors.Trace(err)
}
spec := m.createDiskSpec(newDiskURI, parentURI, capacity, flags)
log.Infof("Create/attach vmdk %s from parent %s", newDiskURI, parentURI)
if err := m.vm.AddDevice(ctx, spec); err != nil {
log.Errorf("vmdk storage driver failed to attach disk: %s", errors.ErrorStack(err))
return nil, errors.Trace(err)
}
log.Debugf("Mapping vmdk to pci device %s", newDiskURI)
devicePath, err := m.devicePathByURI(ctx, newDiskURI)
if err != nil {
return nil, errors.Trace(err)
}
d.setAttached(devicePath)
if err := waitForPath(ctx, devicePath); err != nil {
log.Infof("waitForPath failed for %s with %s", newDiskURI, errors.ErrorStack(err))
// ensure that the disk is detached if it's the publish that's failed
if detachErr := m.Detach(ctx, d); detachErr != nil {
log.Debugf("detach(%s) failed with %s", newDiskURI, errors.ErrorStack(detachErr))
}
return nil, errors.Trace(err)
}
return d, nil
}
// CreateAndAttach creates a new vmdk child from parent of the given size.
// Returns a VirtualDisk corresponding to the created and attached disk. The
// newDiskURI and parentURI are both Datastore URI paths in the form of
// [datastoreN] /path/to/disk.vmdk.
func (m *Manager) CreateAndAttach(op trace.Operation, newDiskURI,
parentURI string,
capacity int64, flags int) (*VirtualDisk, error) {
defer trace.End(trace.Begin(newDiskURI))
// ensure we abide by max attached disks limits
m.maxAttached <- true
d, err := NewVirtualDisk(newDiskURI)
if err != nil {
return nil, errors.Trace(err)
}
spec := m.createDiskSpec(newDiskURI, parentURI, capacity, flags)
op.Infof("Create/attach vmdk %s from parent %s", newDiskURI, parentURI)
err = m.Attach(op, spec)
if err != nil {
return nil, errors.Trace(err)
}
op.Debugf("Mapping vmdk to pci device %s", newDiskURI)
devicePath, err := m.devicePathByURI(op, newDiskURI)
if err != nil {
return nil, errors.Trace(err)
}
d.setAttached(devicePath)
if err := waitForPath(op, devicePath); err != nil {
op.Infof("waitForPath failed for %s with %s", newDiskURI, errors.ErrorStack(err))
// ensure that the disk is detached if it's the publish that's failed
if detachErr := m.Detach(op, d); detachErr != nil {
op.Debugf("detach(%s) failed with %s", newDiskURI, errors.ErrorStack(detachErr))
}
return nil, errors.Trace(err)
}
return d, nil
}
func NewDiskManager(ctx context.Context, session *session.Session) (*Manager, error) {
vm, err := guest.GetSelf(ctx, session)
if err != nil {
return nil, errors.Trace(err)
}
// create handle to the docker daemon VM as we need to mount disks on it
controller, byPathFormat, err := verifyParavirtualScsiController(ctx, vm)
if err != nil {
return nil, errors.Trace(err)
}
d := &Manager{
maxAttached: make(chan bool, MaxAttachedDisks),
vm: vm,
controller: controller,
byPathFormat: byPathFormat,
}
return d, nil
}
// ensures that a paravirtual scsi controller is present and determines the
// base path of disks attached to it returns a handle to the controller and a
// format string, with a single decimal for the disk unit number which will
// result in the /dev/disk/by-path path
func verifyParavirtualScsiController(op trace.Operation, vm *vm.VirtualMachine) (*types.ParaVirtualSCSIController, string, error) {
devices, err := vm.Device(op)
if err != nil {
log.Errorf("vmware driver failed to retrieve device list for VM %s: %s", vm, errors.ErrorStack(err))
return nil, "", errors.Trace(err)
}
controller, ok := devices.PickController((*types.ParaVirtualSCSIController)(nil)).(*types.ParaVirtualSCSIController)
if controller == nil || !ok {
err = errors.Errorf("vmware driver failed to find a paravirtual SCSI controller - ensure setup ran correctly")
log.Error(err.Error())
return nil, "", errors.Trace(err)
}
// build the base path
// first we determine which label we're looking for (requires VMW hardware version >=10)
targetLabel := fmt.Sprintf("SCSI%d", controller.BusNumber)
log.Debugf("Looking for scsi controller with label %s", targetLabel)
pciBase := "/sys/bus/pci/devices"
pciBus, err := os.Open(pciBase)
if err != nil {
log.Errorf("Failed to open %s for reading: %s", pciBase, errors.ErrorStack(err))
return controller, "", errors.Trace(err)
}
defer pciBus.Close()
pciDevices, err := pciBus.Readdirnames(0)
if err != nil {
log.Errorf("Failed to read contents of %s: %s", pciBase, errors.ErrorStack(err))
return controller, "", errors.Trace(err)
}
var buf = make([]byte, len(targetLabel))
var controllerName string
for _, n := range pciDevices {
nlabel := fmt.Sprintf("%s/%s/label", pciBase, n)
flabel, err := os.Open(nlabel)
if err != nil {
if !os.IsNotExist(err) {
log.Errorf("Unable to read label from %s: %s", nlabel, errors.ErrorStack(err))
}
continue
}
defer flabel.Close()
_, err = flabel.Read(buf)
if err != nil {
log.Errorf("Unable to read label from %s: %s", nlabel, errors.ErrorStack(err))
continue
}
if targetLabel == string(buf) {
// we've found our controller
controllerName = n
log.Debugf("Found pvscsi controller directory: %s", controllerName)
break
}
}
if controllerName == "" {
err := errors.Errorf("Failed to locate pvscsi controller directory")
log.Errorf(err.Error())
return controller, "", errors.Trace(err)
}
formatString := fmt.Sprintf("/dev/disk/by-path/pci-%s-scsi-0:0:%%d:0", controllerName)
log.Debugf("Disk location format: %s", formatString)
return controller, formatString, nil
}