// waitRemove blocks until either:
// a) the device registered at <device_set_prefix>-<hash> is removed,
// or b) the 10 second timeout expires.
func (devices *DeviceSet) waitRemove(devname string) error {
log.Debugf("[deviceset %s] waitRemove(%s)", devices.devicePrefix, devname)
defer log.Debugf("[deviceset %s] waitRemove(%s) END", devices.devicePrefix, devname)
i := 0
for ; i < 1000; i++ {
devinfo, err := devicemapper.GetInfo(devname)
if err != nil {
// If there is an error we assume the device doesn't exist.
// The error might actually be something else, but we can't differentiate.
return nil
}
if i%100 == 0 {
log.Debugf("Waiting for removal of %s: exists=%d", devname, devinfo.Exists)
}
if devinfo.Exists == 0 {
break
}
devices.Unlock()
time.Sleep(10 * time.Millisecond)
devices.Lock()
}
if i == 1000 {
return fmt.Errorf("Timeout while waiting for device %s to be removed", devname)
}
return nil
}
func (devices *DeviceSet) activateDeviceIfNeeded(info *DevInfo) error {
log.Debugf("activateDeviceIfNeeded(%v)", info.Hash)
if devinfo, _ := devicemapper.GetInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 {
return nil
}
return devicemapper.ActivateDevice(devices.getPoolDevName(), info.Name(), info.DeviceId, info.Size)
}
func (devices *DeviceSet) HasActivatedDevice(hash string) bool {
info, _ := devices.lookupDevice(hash)
if info == nil {
return false
}
info.lock.Lock()
defer info.lock.Unlock()
devices.Lock()
defer devices.Unlock()
devinfo, _ := devicemapper.GetInfo(info.Name())
return devinfo != nil && devinfo.Exists != 0
}
func (devices *DeviceSet) activateDeviceIfNeeded(info *DevInfo) error {
logrus.Debugf("activateDeviceIfNeeded(%v)", info.Hash)
// Make sure deferred removal on device is canceled, if one was
// scheduled.
if err := devices.cancelDeferredRemoval(info); err != nil {
return fmt.Errorf("Deivce Deferred Removal Cancellation Failed: %s", err)
}
if devinfo, _ := devicemapper.GetInfo(info.Name()); devinfo != nil && devinfo.Exists != 0 {
return nil
}
return devicemapper.ActivateDevice(devices.getPoolDevName(), info.Name(), info.DeviceId, info.Size)
}
func (devices *DeviceSet) deactivateDevice(info *DevInfo) error {
logrus.Debugf("[devmapper] deactivateDevice(%s)", info.Hash)
defer logrus.Debugf("[devmapper] deactivateDevice END(%s)", info.Hash)
devinfo, err := devicemapper.GetInfo(info.Name())
if err != nil {
return err
}
if devinfo.Exists != 0 {
if err := devices.removeDevice(info.Name()); err != nil {
return err
}
}
return nil
}
func (devices *DeviceSet) deactivatePool() error {
log.Debugf("[devmapper] deactivatePool()")
defer log.Debugf("[devmapper] deactivatePool END")
devname := devices.getPoolDevName()
devinfo, err := devicemapper.GetInfo(devname)
if err != nil {
return err
}
if d, err := devicemapper.GetDeps(devname); err == nil {
// Access to more Debug output
log.Debugf("[devmapper] devicemapper.GetDeps() %s: %#v", devname, d)
}
if devinfo.Exists != 0 {
return devicemapper.RemoveDevice(devname)
}
return nil
}
func (devices *DeviceSet) deleteDevice(info *DevInfo) error {
if devices.doBlkDiscard {
// This is a workaround for the kernel not discarding block so
// on the thin pool when we remove a thinp device, so we do it
// manually
if err := devices.activateDeviceIfNeeded(info); err == nil {
if err := devicemapper.BlockDeviceDiscard(info.DevName()); err != nil {
log.Debugf("Error discarding block on device: %s (ignoring)", err)
}
}
}
devinfo, _ := devicemapper.GetInfo(info.Name())
if devinfo != nil && devinfo.Exists != 0 {
if err := devices.removeDeviceAndWait(info.Name()); err != nil {
log.Debugf("Error removing device: %s", err)
return err
}
}
if err := devices.openTransaction(info.Hash, info.DeviceId); err != nil {
log.Debugf("Error opening transaction hash = %s deviceId = %d", "", info.DeviceId)
return err
}
if err := devicemapper.DeleteDevice(devices.getPoolDevName(), info.DeviceId); err != nil {
log.Debugf("Error deleting device: %s", err)
return err
}
if err := devices.unregisterDevice(info.DeviceId, info.Hash); err != nil {
return err
}
if err := devices.closeTransaction(); err != nil {
return err
}
devices.markDeviceIdFree(info.DeviceId)
return nil
}
func (devices *DeviceSet) deleteDevice(info *DevInfo) error {
if devices.doBlkDiscard {
// This is a workaround for the kernel not discarding block so
// on the thin pool when we remove a thinp device, so we do it
// manually
if err := devices.activateDeviceIfNeeded(info); err == nil {
if err := devicemapper.BlockDeviceDiscard(info.DevName()); err != nil {
log.Debugf("Error discarding block on device: %s (ignoring)", err)
}
}
}
devinfo, _ := devicemapper.GetInfo(info.Name())
if devinfo != nil && devinfo.Exists != 0 {
if err := devices.removeDeviceAndWait(info.Name()); err != nil {
log.Debugf("Error removing device: %s", err)
return err
}
}
if err := devicemapper.DeleteDevice(devices.getPoolDevName(), info.DeviceId); err != nil {
log.Debugf("Error deleting device: %s", err)
return err
}
devices.allocateTransactionId()
devices.devicesLock.Lock()
delete(devices.Devices, info.Hash)
devices.devicesLock.Unlock()
if err := devices.removeMetadata(info); err != nil {
devices.devicesLock.Lock()
devices.Devices[info.Hash] = info
devices.devicesLock.Unlock()
log.Debugf("Error removing meta data: %s", err)
return err
}
return nil
}
// waitClose blocks until either:
// a) the device registered at <device_set_prefix>-<hash> is closed,
// or b) the 10 second timeout expires.
func (devices *DeviceSet) waitClose(info *DevInfo) error {
i := 0
for ; i < 1000; i++ {
devinfo, err := devicemapper.GetInfo(info.Name())
if err != nil {
return err
}
if i%100 == 0 {
log.Debugf("Waiting for unmount of %s: opencount=%d", info.Hash, devinfo.OpenCount)
}
if devinfo.OpenCount == 0 {
break
}
devices.Unlock()
time.Sleep(10 * time.Millisecond)
devices.Lock()
}
if i == 1000 {
return fmt.Errorf("Timeout while waiting for device %s to close", info.Hash)
}
return nil
}
func (devices *DeviceSet) deactivateDevice(info *DevInfo) error {
log.Debugf("[devmapper] deactivateDevice(%s)", info.Hash)
defer log.Debugf("[devmapper] deactivateDevice END(%s)", info.Hash)
// Wait for the unmount to be effective,
// by watching the value of Info.OpenCount for the device
if err := devices.waitClose(info); err != nil {
log.Errorf("Warning: error waiting for device %s to close: %s", info.Hash, err)
}
devinfo, err := devicemapper.GetInfo(info.Name())
if err != nil {
return err
}
if devinfo.Exists != 0 {
if err := devices.removeDeviceAndWait(info.Name()); err != nil {
return err
}
}
return nil
}
func (devices *DeviceSet) initDevmapper(doInit bool) error {
if os.Getenv("DEBUG") != "" {
devicemapper.LogInitVerbose(devicemapper.LogLevelDebug)
} else {
devicemapper.LogInitVerbose(devicemapper.LogLevelWarn)
}
// give ourselves to libdm as a log handler
devicemapper.LogInit(devices)
_, err := devicemapper.GetDriverVersion()
if err != nil {
// Can't even get driver version, assume not supported
return graphdriver.ErrNotSupported
}
// https://github.com/docker/docker/issues/4036
if supported := devicemapper.UdevSetSyncSupport(true); !supported {
log.Warnf("WARNING: Udev sync is not supported. This will lead to unexpected behavior, data loss and errors")
}
log.Debugf("devicemapper: udev sync support: %v", devicemapper.UdevSyncSupported())
if err := os.MkdirAll(devices.metadataDir(), 0700); err != nil && !os.IsExist(err) {
return err
}
// Set the device prefix from the device id and inode of the docker root dir
st, err := os.Stat(devices.root)
if err != nil {
return fmt.Errorf("Error looking up dir %s: %s", devices.root, err)
}
sysSt := st.Sys().(*syscall.Stat_t)
// "reg-" stands for "regular file".
// In the future we might use "dev-" for "device file", etc.
// docker-maj,min[-inode] stands for:
// - Managed by docker
// - The target of this device is at major <maj> and minor <min>
// - If <inode> is defined, use that file inside the device as a loopback image. Otherwise use the device itself.
devices.devicePrefix = fmt.Sprintf("docker-%d:%d-%d", major(sysSt.Dev), minor(sysSt.Dev), sysSt.Ino)
log.Debugf("Generated prefix: %s", devices.devicePrefix)
// Check for the existence of the thin-pool device
log.Debugf("Checking for existence of the pool '%s'", devices.getPoolName())
info, err := devicemapper.GetInfo(devices.getPoolName())
if info == nil {
log.Debugf("Error device devicemapper.GetInfo: %s", err)
return err
}
// It seems libdevmapper opens this without O_CLOEXEC, and go exec will not close files
// that are not Close-on-exec, and lxc-start will die if it inherits any unexpected files,
// so we add this badhack to make sure it closes itself
setCloseOnExec("/dev/mapper/control")
// Make sure the sparse images exist in <root>/devicemapper/data and
// <root>/devicemapper/metadata
createdLoopback := false
// If the pool doesn't exist, create it
if info.Exists == 0 && devices.thinPoolDevice == "" {
log.Debugf("Pool doesn't exist. Creating it.")
var (
dataFile *os.File
metadataFile *os.File
)
if devices.dataDevice == "" {
// Make sure the sparse images exist in <root>/devicemapper/data
hasData := devices.hasImage("data")
if !doInit && !hasData {
return errors.New("Loopback data file not found")
}
if !hasData {
createdLoopback = true
}
data, err := devices.ensureImage("data", devices.dataLoopbackSize)
if err != nil {
log.Debugf("Error device ensureImage (data): %s", err)
return err
}
dataFile, err = devicemapper.AttachLoopDevice(data)
if err != nil {
return err
}
devices.dataLoopFile = data
devices.dataDevice = dataFile.Name()
} else {
dataFile, err = os.OpenFile(devices.dataDevice, os.O_RDWR, 0600)
if err != nil {
return err
}
}
defer dataFile.Close()
if devices.metadataDevice == "" {
// Make sure the sparse images exist in <root>/devicemapper/metadata
hasMetadata := devices.hasImage("metadata")
if !doInit && !hasMetadata {
return errors.New("Loopback metadata file not found")
}
if !hasMetadata {
createdLoopback = true
}
metadata, err := devices.ensureImage("metadata", devices.metaDataLoopbackSize)
if err != nil {
log.Debugf("Error device ensureImage (metadata): %s", err)
return err
}
metadataFile, err = devicemapper.AttachLoopDevice(metadata)
if err != nil {
return err
}
devices.metadataLoopFile = metadata
devices.metadataDevice = metadataFile.Name()
} else {
metadataFile, err = os.OpenFile(devices.metadataDevice, os.O_RDWR, 0600)
if err != nil {
return err
}
}
defer metadataFile.Close()
if err := devicemapper.CreatePool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil {
return err
}
}
// If we didn't just create the data or metadata image, we need to
// load the transaction id and migrate old metadata
if !createdLoopback {
if err = devices.initMetaData(); err != nil {
return err
}
}
// Right now this loads only NextDeviceId. If there is more metatadata
// down the line, we might have to move it earlier.
if err = devices.loadDeviceSetMetaData(); err != nil {
return err
}
// Setup the base image
if doInit {
if err := devices.setupBaseImage(); err != nil {
log.Debugf("Error device setupBaseImage: %s", err)
return err
}
}
return nil
}