// Detaches the specified persistent disk device from node, verifies that it is detached, and retries if it fails. // This function is intended to be called asynchronously as a go routine. func detachDiskAndVerify(c *awsElasticBlockStoreCleaner) { glog.V(5).Infof("detachDiskAndVerify(...) for pd %q. Will block for pending operations", c.volumeID) defer runtime.HandleCrash() // Block execution until any pending attach/detach operations for this PD have completed attachDetachMutex.LockKey(c.volumeID) defer attachDetachMutex.UnlockKey(c.volumeID) glog.V(5).Infof("detachDiskAndVerify(...) for pd %q. Awake and ready to execute.", c.volumeID) var awsCloud *aws.AWSCloud for numRetries := 0; numRetries < maxRetries; numRetries++ { var err error if awsCloud == nil { awsCloud, err = getCloudProvider() if err != nil || awsCloud == nil { // Retry on error. See issue #11321 glog.Errorf("Error getting AWSCloudProvider while detaching PD %q: %v", c.volumeID, err) time.Sleep(errorSleepDuration) continue } } if numRetries > 0 { glog.Warningf("Retrying detach for EBS Disk %q (retry count=%v).", c.volumeID, numRetries) } devicePath, err := awsCloud.DetachDisk(c.volumeID, c.plugin.host.GetHostName()) if err != nil { glog.Errorf("Error detaching PD %q: %v", c.volumeID, err) time.Sleep(errorSleepDuration) continue } devicePaths := getDiskByIdPaths(c.awsElasticBlockStore, devicePath) for numChecks := 0; numChecks < maxChecks; numChecks++ { allPathsRemoved, err := verifyAllPathsRemoved(devicePaths) if err != nil { // Log error, if any, and continue checking periodically. glog.Errorf("Error verifying EBS Disk (%q) is detached: %v", c.volumeID, err) } else if allPathsRemoved { // All paths to the PD have been successfully removed unmountPDAndRemoveGlobalPath(c) glog.Infof("Successfully detached EBS Disk %q.", c.volumeID) return } // Sleep then check again glog.V(3).Infof("Waiting for EBS Disk %q to detach.", c.volumeID) time.Sleep(checkSleepDuration) } } glog.Errorf("Failed to detach EBS Disk %q. One or more mount paths was not removed.", c.volumeID) }
// Attaches the specified persistent disk device to node, verifies that it is attached, and retries if it fails. func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.String) (string, error) { var awsCloud *aws.AWSCloud var attachError error for numRetries := 0; numRetries < maxRetries; numRetries++ { var err error if awsCloud == nil { awsCloud, err = getCloudProvider(b.awsElasticBlockStore.plugin) if err != nil || awsCloud == nil { // Retry on error. See issue #11321 glog.Errorf("Error getting AWSCloudProvider while detaching PD %q: %v", b.volumeID, err) time.Sleep(errorSleepDuration) continue } } if numRetries > 0 { glog.Warningf("Retrying attach for EBS Disk %q (retry count=%v).", b.volumeID, numRetries) } var devicePath string devicePath, attachError = awsCloud.AttachDisk(b.volumeID, "", b.readOnly) if attachError != nil { glog.Errorf("Error attaching PD %q: %v", b.volumeID, attachError) time.Sleep(errorSleepDuration) continue } devicePaths := getDiskByIdPaths(b.awsElasticBlockStore, devicePath) for numChecks := 0; numChecks < maxChecks; numChecks++ { path, err := verifyDevicePath(devicePaths) if err != nil { // Log error, if any, and continue checking periodically. See issue #11321 glog.Errorf("Error verifying EBS Disk (%q) is attached: %v", b.volumeID, err) } else if path != "" { // A device path has successfully been created for the PD glog.Infof("Successfully attached EBS Disk %q.", b.volumeID) return path, nil } // Sleep then check again glog.V(3).Infof("Waiting for EBS Disk %q to attach.", b.volumeID) time.Sleep(checkSleepDuration) } } if attachError != nil { return "", fmt.Errorf("Could not attach EBS Disk %q: %v", b.volumeID, attachError) } return "", fmt.Errorf("Could not attach EBS Disk %q. Timeout waiting for mount paths to be created.", b.volumeID) }