Example #1
0
// Detaches the specified persistent disk device from node, verifies that it is detached, and retries if it fails.
// This function is intended to be called asynchronously as a go routine.
func detachDiskAndVerify(c *awsElasticBlockStoreCleaner) {
	glog.V(5).Infof("detachDiskAndVerify(...) for pd %q. Will block for pending operations", c.volumeID)
	defer runtime.HandleCrash()

	// Block execution until any pending attach/detach operations for this PD have completed
	attachDetachMutex.LockKey(c.volumeID)
	defer attachDetachMutex.UnlockKey(c.volumeID)

	glog.V(5).Infof("detachDiskAndVerify(...) for pd %q. Awake and ready to execute.", c.volumeID)

	var awsCloud *aws.AWSCloud
	for numRetries := 0; numRetries < maxRetries; numRetries++ {
		var err error
		if awsCloud == nil {
			awsCloud, err = getCloudProvider()
			if err != nil || awsCloud == nil {
				// Retry on error. See issue #11321
				glog.Errorf("Error getting AWSCloudProvider while detaching PD %q: %v", c.volumeID, err)
				time.Sleep(errorSleepDuration)
				continue
			}
		}

		if numRetries > 0 {
			glog.Warningf("Retrying detach for EBS Disk %q (retry count=%v).", c.volumeID, numRetries)
		}

		devicePath, err := awsCloud.DetachDisk(c.volumeID, c.plugin.host.GetHostName())
		if err != nil {
			glog.Errorf("Error detaching PD %q: %v", c.volumeID, err)
			time.Sleep(errorSleepDuration)
			continue
		}

		devicePaths := getDiskByIdPaths(c.awsElasticBlockStore, devicePath)

		for numChecks := 0; numChecks < maxChecks; numChecks++ {
			allPathsRemoved, err := verifyAllPathsRemoved(devicePaths)
			if err != nil {
				// Log error, if any, and continue checking periodically.
				glog.Errorf("Error verifying EBS Disk (%q) is detached: %v", c.volumeID, err)
			} else if allPathsRemoved {
				// All paths to the PD have been successfully removed
				unmountPDAndRemoveGlobalPath(c)
				glog.Infof("Successfully detached EBS Disk %q.", c.volumeID)
				return
			}

			// Sleep then check again
			glog.V(3).Infof("Waiting for EBS Disk %q to detach.", c.volumeID)
			time.Sleep(checkSleepDuration)
		}

	}

	glog.Errorf("Failed to detach EBS Disk %q. One or more mount paths was not removed.", c.volumeID)
}
Example #2
0
// Attaches the specified persistent disk device to node, verifies that it is attached, and retries if it fails.
func attachDiskAndVerify(b *awsElasticBlockStoreBuilder, xvdBeforeSet sets.String) (string, error) {
	var awsCloud *aws.AWSCloud
	var attachError error

	for numRetries := 0; numRetries < maxRetries; numRetries++ {
		var err error
		if awsCloud == nil {
			awsCloud, err = getCloudProvider(b.awsElasticBlockStore.plugin)
			if err != nil || awsCloud == nil {
				// Retry on error. See issue #11321
				glog.Errorf("Error getting AWSCloudProvider while detaching PD %q: %v", b.volumeID, err)
				time.Sleep(errorSleepDuration)
				continue
			}
		}

		if numRetries > 0 {
			glog.Warningf("Retrying attach for EBS Disk %q (retry count=%v).", b.volumeID, numRetries)
		}

		var devicePath string
		devicePath, attachError = awsCloud.AttachDisk(b.volumeID, "", b.readOnly)
		if attachError != nil {
			glog.Errorf("Error attaching PD %q: %v", b.volumeID, attachError)
			time.Sleep(errorSleepDuration)
			continue
		}

		devicePaths := getDiskByIdPaths(b.awsElasticBlockStore, devicePath)

		for numChecks := 0; numChecks < maxChecks; numChecks++ {
			path, err := verifyDevicePath(devicePaths)
			if err != nil {
				// Log error, if any, and continue checking periodically. See issue #11321
				glog.Errorf("Error verifying EBS Disk (%q) is attached: %v", b.volumeID, err)
			} else if path != "" {
				// A device path has successfully been created for the PD
				glog.Infof("Successfully attached EBS Disk %q.", b.volumeID)
				return path, nil
			}

			// Sleep then check again
			glog.V(3).Infof("Waiting for EBS Disk %q to attach.", b.volumeID)
			time.Sleep(checkSleepDuration)
		}
	}

	if attachError != nil {
		return "", fmt.Errorf("Could not attach EBS Disk %q: %v", b.volumeID, attachError)
	}
	return "", fmt.Errorf("Could not attach EBS Disk %q. Timeout waiting for mount paths to be created.", b.volumeID)
}