Exemple #1
0
func (rc *reconciler) reconciliationLoopFunc() func() {
	return func() {
		// Detaches are triggered before attaches so that volumes referenced by
		// pods that are rescheduled to a different node are detached first.

		// Ensure volumes that should be detached are detached.
		for _, attachedVolume := range rc.actualStateOfWorld.GetAttachedVolumes() {
			if !rc.desiredStateOfWorld.VolumeExists(
				attachedVolume.VolumeName, attachedVolume.NodeName) {
				// Volume exists in actual state of world but not desired

				// Mark desire to detach
				timeElapsed, err := rc.actualStateOfWorld.MarkDesireToDetach(attachedVolume.VolumeName, attachedVolume.NodeName)
				if err != nil {
					glog.Errorf("Unexpected error actualStateOfWorld.MarkDesireToDetach(): %v", err)
				}

				// Update Node Status to indicate volume is no longer safe to mount.
				err = rc.nodeStatusUpdater.UpdateNodeStatuses()
				if err != nil {
					// Skip detaching this volume if unable to update node status
					glog.Infof("UpdateNodeStatuses failed with: %v", err)
					continue
				}

				if !attachedVolume.MountedByNode {
					glog.V(5).Infof("Attempting to start DetachVolume for volume %q from node %q", attachedVolume.VolumeName, attachedVolume.NodeName)
					err := rc.attacherDetacher.DetachVolume(attachedVolume.AttachedVolume, true /* verifySafeToDetach */, rc.actualStateOfWorld)
					if err == nil {
						glog.Infof("Started DetachVolume for volume %q from node %q", attachedVolume.VolumeName, attachedVolume.NodeName)
					}
					if err != nil &&
						!goroutinemap.IsAlreadyExists(err) &&
						!goroutinemap.IsExponentialBackoff(err) {
						// Ignore goroutinemap.IsAlreadyExists && goroutinemap.IsExponentialBackoff errors, they are expected.
						// Log all other errors.
						glog.Errorf(
							"operationExecutor.DetachVolume failed to start for volume %q (spec.Name: %q) from node %q with err: %v",
							attachedVolume.VolumeName,
							attachedVolume.VolumeSpec.Name(),
							attachedVolume.NodeName,
							err)
					}
				} else {
					// If volume is not safe to detach (is mounted) wait a max amount of time before detaching any way.
					if timeElapsed > rc.maxWaitForUnmountDuration {
						glog.V(5).Infof("Attempting to start DetachVolume for volume %q from node %q. Volume is not safe to detach, but maxWaitForUnmountDuration expired.", attachedVolume.VolumeName, attachedVolume.NodeName)
						err := rc.attacherDetacher.DetachVolume(attachedVolume.AttachedVolume, false /* verifySafeToDetach */, rc.actualStateOfWorld)
						if err == nil {
							glog.Infof("Started DetachVolume for volume %q from node %q due to maxWaitForUnmountDuration expiry.", attachedVolume.VolumeName, attachedVolume.NodeName)
						}
						if err != nil &&
							!goroutinemap.IsAlreadyExists(err) &&
							!goroutinemap.IsExponentialBackoff(err) {
							// Ignore goroutinemap.IsAlreadyExists && goroutinemap.IsExponentialBackoff errors, they are expected.
							// Log all other errors.
							glog.Errorf(
								"operationExecutor.DetachVolume failed to start (maxWaitForUnmountDuration expiry) for volume %q (spec.Name: %q) from node %q with err: %v",
								attachedVolume.VolumeName,
								attachedVolume.VolumeSpec.Name(),
								attachedVolume.NodeName,
								err)
						}
					}
				}
			}
		}

		// Ensure volumes that should be attached are attached.
		for _, volumeToAttach := range rc.desiredStateOfWorld.GetVolumesToAttach() {
			if rc.actualStateOfWorld.VolumeNodeExists(
				volumeToAttach.VolumeName, volumeToAttach.NodeName) {
				// Volume/Node exists, touch it to reset detachRequestedTime
				glog.V(12).Infof("Volume %q/Node %q is attached--touching.", volumeToAttach.VolumeName, volumeToAttach.NodeName)
				_, err := rc.actualStateOfWorld.AddVolumeNode(
					volumeToAttach.VolumeSpec, volumeToAttach.NodeName, "" /* devicePath */)
				if err != nil {
					glog.Errorf("Unexpected error on actualStateOfWorld.AddVolumeNode(): %v", err)
				}
			} else {
				// Volume/Node doesn't exist, spawn a goroutine to attach it
				glog.V(5).Infof("Attempting to start AttachVolume for volume %q to node %q", volumeToAttach.VolumeName, volumeToAttach.NodeName)
				err := rc.attacherDetacher.AttachVolume(volumeToAttach.VolumeToAttach, rc.actualStateOfWorld)
				if err == nil {
					glog.Infof("Started AttachVolume for volume %q to node %q", volumeToAttach.VolumeName, volumeToAttach.NodeName)
				}
				if err != nil &&
					!goroutinemap.IsAlreadyExists(err) &&
					!goroutinemap.IsExponentialBackoff(err) {
					// Ignore goroutinemap.IsAlreadyExists && goroutinemap.IsExponentialBackoff errors, they are expected.
					// Log all other errors.
					glog.Errorf(
						"operationExecutor.AttachVolume failed to start for volume %q (spec.Name: %q) to node %q with err: %v",
						volumeToAttach.VolumeName,
						volumeToAttach.VolumeSpec.Name(),
						volumeToAttach.NodeName,
						err)
				}
			}
		}

		// Update Node Status
		err := rc.nodeStatusUpdater.UpdateNodeStatuses()
		if err != nil {
			glog.Infof("UpdateNodeStatuses failed with: %v", err)
		}
	}
}
Exemple #2
0
func (rc *reconciler) reconciliationLoopFunc() func() {
	return func() {
		// Unmounts are triggered before mounts so that a volume that was
		// referenced by a pod that was deleted and is now referenced by another
		// pod is unmounted from the first pod before being mounted to the new
		// pod.

		// Ensure volumes that should be unmounted are unmounted.
		for _, mountedVolume := range rc.actualStateOfWorld.GetMountedVolumes() {
			if !rc.desiredStateOfWorld.PodExistsInVolume(mountedVolume.PodName, mountedVolume.VolumeName) {
				// Volume is mounted, unmount it
				glog.V(12).Infof("Attempting to start UnmountVolume for volume %q (spec.Name: %q) from pod %q (UID: %q).",
					mountedVolume.VolumeName,
					mountedVolume.OuterVolumeSpecName,
					mountedVolume.PodName,
					mountedVolume.PodUID)
				err := rc.operationExecutor.UnmountVolume(
					mountedVolume.MountedVolume, rc.actualStateOfWorld)
				if err != nil &&
					!goroutinemap.IsAlreadyExists(err) &&
					!goroutinemap.IsExponentialBackoff(err) {
					// Ignore goroutinemap.IsAlreadyExists and goroutinemap.IsExponentialBackoff errors, they are expected.
					// Log all other errors.
					glog.Errorf(
						"operationExecutor.UnmountVolume failed for volume %q (spec.Name: %q) pod %q (UID: %q) controllerAttachDetachEnabled: %v with err: %v",
						mountedVolume.VolumeName,
						mountedVolume.OuterVolumeSpecName,
						mountedVolume.PodName,
						mountedVolume.PodUID,
						rc.controllerAttachDetachEnabled,
						err)
				}
				if err == nil {
					glog.Infof("UnmountVolume operation started for volume %q (spec.Name: %q) from pod %q (UID: %q).",
						mountedVolume.VolumeName,
						mountedVolume.OuterVolumeSpecName,
						mountedVolume.PodName,
						mountedVolume.PodUID)
				}
			}
		}

		// Ensure volumes that should be attached/mounted are attached/mounted.
		for _, volumeToMount := range rc.desiredStateOfWorld.GetVolumesToMount() {
			volMounted, devicePath, err := rc.actualStateOfWorld.PodExistsInVolume(volumeToMount.PodName, volumeToMount.VolumeName)
			volumeToMount.DevicePath = devicePath
			if cache.IsVolumeNotAttachedError(err) {
				if rc.controllerAttachDetachEnabled || !volumeToMount.PluginIsAttachable {
					// Volume is not attached (or doesn't implement attacher), kubelet attach is disabled, wait
					// for controller to finish attaching volume.
					glog.V(12).Infof("Attempting to start VerifyControllerAttachedVolume for volume %q (spec.Name: %q) pod %q (UID: %q)",
						volumeToMount.VolumeName,
						volumeToMount.VolumeSpec.Name(),
						volumeToMount.PodName,
						volumeToMount.Pod.UID)
					err := rc.operationExecutor.VerifyControllerAttachedVolume(
						volumeToMount.VolumeToMount,
						rc.hostName,
						rc.actualStateOfWorld)
					if err != nil &&
						!goroutinemap.IsAlreadyExists(err) &&
						!goroutinemap.IsExponentialBackoff(err) {
						// Ignore goroutinemap.IsAlreadyExists and goroutinemap.IsExponentialBackoff errors, they are expected.
						// Log all other errors.
						glog.Errorf(
							"operationExecutor.VerifyControllerAttachedVolume failed for volume %q (spec.Name: %q) pod %q (UID: %q) controllerAttachDetachEnabled: %v with err: %v",
							volumeToMount.VolumeName,
							volumeToMount.VolumeSpec.Name(),
							volumeToMount.PodName,
							volumeToMount.Pod.UID,
							rc.controllerAttachDetachEnabled,
							err)
					}
					if err == nil {
						glog.Infof("VerifyControllerAttachedVolume operation started for volume %q (spec.Name: %q) pod %q (UID: %q)",
							volumeToMount.VolumeName,
							volumeToMount.VolumeSpec.Name(),
							volumeToMount.PodName,
							volumeToMount.Pod.UID)
					}
				} else {
					// Volume is not attached to node, kubelet attach is enabled, volume implements an attacher,
					// so attach it
					volumeToAttach := operationexecutor.VolumeToAttach{
						VolumeName: volumeToMount.VolumeName,
						VolumeSpec: volumeToMount.VolumeSpec,
						NodeName:   rc.hostName,
					}
					glog.V(12).Infof("Attempting to start AttachVolume for volume %q (spec.Name: %q)  pod %q (UID: %q)",
						volumeToMount.VolumeName,
						volumeToMount.VolumeSpec.Name(),
						volumeToMount.PodName,
						volumeToMount.Pod.UID)
					err := rc.operationExecutor.AttachVolume(volumeToAttach, rc.actualStateOfWorld)
					if err != nil &&
						!goroutinemap.IsAlreadyExists(err) &&
						!goroutinemap.IsExponentialBackoff(err) {
						// Ignore goroutinemap.IsAlreadyExists and goroutinemap.IsExponentialBackoff errors, they are expected.
						// Log all other errors.
						glog.Errorf(
							"operationExecutor.AttachVolume failed for volume %q (spec.Name: %q) pod %q (UID: %q) controllerAttachDetachEnabled: %v with err: %v",
							volumeToMount.VolumeName,
							volumeToMount.VolumeSpec.Name(),
							volumeToMount.PodName,
							volumeToMount.Pod.UID,
							rc.controllerAttachDetachEnabled,
							err)
					}
					if err == nil {
						glog.Infof("AttachVolume operation started for volume %q (spec.Name: %q) pod %q (UID: %q)",
							volumeToMount.VolumeName,
							volumeToMount.VolumeSpec.Name(),
							volumeToMount.PodName,
							volumeToMount.Pod.UID)
					}
				}
			} else if !volMounted || cache.IsRemountRequiredError(err) {
				// Volume is not mounted, or is already mounted, but requires remounting
				remountingLogStr := ""
				if cache.IsRemountRequiredError(err) {
					remountingLogStr = "Volume is already mounted to pod, but remount was requested."
				}
				glog.V(12).Infof("Attempting to start MountVolume for volume %q (spec.Name: %q) to pod %q (UID: %q). %s",
					volumeToMount.VolumeName,
					volumeToMount.VolumeSpec.Name(),
					volumeToMount.PodName,
					volumeToMount.Pod.UID,
					remountingLogStr)
				err := rc.operationExecutor.MountVolume(
					rc.waitForAttachTimeout,
					volumeToMount.VolumeToMount,
					rc.actualStateOfWorld)
				if err != nil &&
					!goroutinemap.IsAlreadyExists(err) &&
					!goroutinemap.IsExponentialBackoff(err) {
					// Ignore goroutinemap.IsAlreadyExists and goroutinemap.IsExponentialBackoff errors, they are expected.
					// Log all other errors.
					glog.Errorf(
						"operationExecutor.MountVolume failed for volume %q (spec.Name: %q) pod %q (UID: %q) controllerAttachDetachEnabled: %v with err: %v",
						volumeToMount.VolumeName,
						volumeToMount.VolumeSpec.Name(),
						volumeToMount.PodName,
						volumeToMount.Pod.UID,
						rc.controllerAttachDetachEnabled,
						err)
				}
				if err == nil {
					glog.Infof("MountVolume operation started for volume %q (spec.Name: %q) to pod %q (UID: %q). %s",
						volumeToMount.VolumeName,
						volumeToMount.VolumeSpec.Name(),
						volumeToMount.PodName,
						volumeToMount.Pod.UID,
						remountingLogStr)
				}
			}
		}

		// Ensure devices that should be detached/unmounted are detached/unmounted.
		for _, attachedVolume := range rc.actualStateOfWorld.GetUnmountedVolumes() {
			if !rc.desiredStateOfWorld.VolumeExists(attachedVolume.VolumeName) {
				if attachedVolume.GloballyMounted {
					// Volume is globally mounted to device, unmount it
					glog.V(12).Infof("Attempting to start UnmountDevice for volume %q (spec.Name: %q)",
						attachedVolume.VolumeName,
						attachedVolume.VolumeSpec.Name())
					err := rc.operationExecutor.UnmountDevice(
						attachedVolume.AttachedVolume, rc.actualStateOfWorld, rc.mounter)
					if err != nil &&
						!goroutinemap.IsAlreadyExists(err) &&
						!goroutinemap.IsExponentialBackoff(err) {
						// Ignore goroutinemap.IsAlreadyExists and goroutinemap.IsExponentialBackoff errors, they are expected.
						// Log all other errors.
						glog.Errorf(
							"operationExecutor.UnmountDevice failed for volume %q (spec.Name: %q) controllerAttachDetachEnabled: %v with err: %v",
							attachedVolume.VolumeName,
							attachedVolume.VolumeSpec.Name(),
							rc.controllerAttachDetachEnabled,
							err)
					}
					if err == nil {
						glog.Infof("UnmountDevice operation started for volume %q (spec.Name: %q)",
							attachedVolume.VolumeName,
							attachedVolume.VolumeSpec.Name())
					}
				} else {
					// Volume is attached to node, detach it
					if rc.controllerAttachDetachEnabled || !attachedVolume.PluginIsAttachable {
						// Kubelet not responsible for detaching or this volume has a non-attachable volume plugin,
						// so just remove it to actualStateOfWorld without attach.
						rc.actualStateOfWorld.MarkVolumeAsDetached(
							attachedVolume.VolumeName, rc.hostName)
					} else {
						// Only detach if kubelet detach is enabled
						glog.V(12).Infof("Attempting to start DetachVolume for volume %q (spec.Name: %q)",
							attachedVolume.VolumeName,
							attachedVolume.VolumeSpec.Name())
						err := rc.operationExecutor.DetachVolume(
							attachedVolume.AttachedVolume, false /* verifySafeToDetach */, rc.actualStateOfWorld)
						if err != nil &&
							!goroutinemap.IsAlreadyExists(err) &&
							!goroutinemap.IsExponentialBackoff(err) {
							// Ignore goroutinemap.IsAlreadyExists && goroutinemap.IsExponentialBackoff errors, they are expected.
							// Log all other errors.
							glog.Errorf(
								"operationExecutor.DetachVolume failed for volume %q (spec.Name: %q) controllerAttachDetachEnabled: %v with err: %v",
								attachedVolume.VolumeName,
								attachedVolume.VolumeSpec.Name(),
								rc.controllerAttachDetachEnabled,
								err)
						}
						if err == nil {
							glog.Infof("DetachVolume operation started for volume %q (spec.Name: %q)",
								attachedVolume.VolumeName,
								attachedVolume.VolumeSpec.Name())
						}
					}
				}
			}
		}
	}
}