// Detaches the specified persistent disk device from node, verifies that it is detached, and retries if it fails.
// This function is intended to be called asynchronously as a go routine.
func detachDiskAndVerify(c *awsElasticBlockStoreCleaner) {
glog.V(5).Infof("detachDiskAndVerify(...) for pd %q. Will block for pending operations", c.volumeID)
defer runtime.HandleCrash()
// Block execution until any pending attach/detach operations for this PD have completed
attachDetachMutex.LockKey(c.volumeID)
defer attachDetachMutex.UnlockKey(c.volumeID)
glog.V(5).Infof("detachDiskAndVerify(...) for pd %q. Awake and ready to execute.", c.volumeID)
var awsCloud *aws.AWSCloud
for numRetries := 0; numRetries < maxRetries; numRetries++ {
var err error
if awsCloud == nil {
awsCloud, err = getCloudProvider()
if err != nil || awsCloud == nil {
// Retry on error. See issue #11321
glog.Errorf("Error getting AWSCloudProvider while detaching PD %q: %v", c.volumeID, err)
time.Sleep(errorSleepDuration)
continue
}
}
if numRetries > 0 {
glog.Warningf("Retrying detach for EBS Disk %q (retry count=%v).", c.volumeID, numRetries)
}
devicePath, err := awsCloud.DetachDisk(c.volumeID, c.plugin.host.GetHostName())
if err != nil {
glog.Errorf("Error detaching PD %q: %v", c.volumeID, err)
time.Sleep(errorSleepDuration)
continue
}
devicePaths := getDiskByIdPaths(c.awsElasticBlockStore, devicePath)
for numChecks := 0; numChecks < maxChecks; numChecks++ {
allPathsRemoved, err := verifyAllPathsRemoved(devicePaths)
if err != nil {
// Log error, if any, and continue checking periodically.
glog.Errorf("Error verifying EBS Disk (%q) is detached: %v", c.volumeID, err)
} else if allPathsRemoved {
// All paths to the PD have been successfully removed
unmountPDAndRemoveGlobalPath(c)
glog.Infof("Successfully detached EBS Disk %q.", c.volumeID)
return
}
// Sleep then check again
glog.V(3).Infof("Waiting for EBS Disk %q to detach.", c.volumeID)
time.Sleep(checkSleepDuration)
}
}
glog.Errorf("Failed to detach EBS Disk %q. One or more mount paths was not removed.", c.volumeID)
}
