// executeExecNewPod executes a ExecNewPod hook by creating a new pod based on // the hook parameters and deployment. The pod is then synchronously watched // until the pod completes, and if the pod failed, an error is returned. // // The hook pod inherits the following from the container the hook refers to: // // * Environment (hook keys take precedence) // * Working directory // * Resources func (e *HookExecutor) executeExecNewPod(hook *deployapi.LifecycleHook, deployment *kapi.ReplicationController, label string) error { // Build a pod spec from the hook config and deployment podSpec, err := makeHookPod(hook, deployment, label) if err != nil { return err } // Try to create the pod. pod, err := e.PodClient.CreatePod(deployment.Namespace, podSpec) if err != nil { if !kerrors.IsAlreadyExists(err) { return fmt.Errorf("couldn't create lifecycle pod for %s: %v", deployutil.LabelForDeployment(deployment), err) } } else { glog.V(0).Infof("Created lifecycle pod %s for deployment %s", pod.Name, deployutil.LabelForDeployment(deployment)) } stopChannel := make(chan struct{}) defer close(stopChannel) nextPod := e.PodClient.PodWatch(pod.Namespace, pod.Name, pod.ResourceVersion, stopChannel) glog.V(0).Infof("Waiting for hook pod %s/%s to complete", pod.Namespace, pod.Name) for { pod := nextPod() switch pod.Status.Phase { case kapi.PodSucceeded: return nil case kapi.PodFailed: return fmt.Errorf(pod.Status.Message) } } }
func (c *DeploymentController) cancelDeployerPods(deployment *kapi.ReplicationController) error { deployerPods, err := c.podClient.getDeployerPodsFor(deployment.Namespace, deployment.Name) if err != nil { return fmt.Errorf("couldn't fetch deployer pods for %s while trying to cancel deployment: %v", deployutil.LabelForDeployment(deployment), err) } glog.V(4).Infof("Cancelling %d deployer pods for deployment %s", len(deployerPods), deployutil.LabelForDeployment(deployment)) zeroDelay := int64(1) anyCancelled := false for _, deployerPod := range deployerPods { // Set the ActiveDeadlineSeconds on the pod so it's terminated very soon. if deployerPod.Spec.ActiveDeadlineSeconds == nil || *deployerPod.Spec.ActiveDeadlineSeconds != zeroDelay { deployerPod.Spec.ActiveDeadlineSeconds = &zeroDelay if _, err := c.podClient.updatePod(deployerPod.Namespace, &deployerPod); err != nil { if config, decodeErr := c.decodeConfig(deployment); decodeErr == nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "FailedCancellation", "Error cancelling deployer pod %s for deployment %s: %v", deployerPod.Name, deployutil.LabelForDeployment(deployment), err) } else { c.recorder.Eventf(deployment, kapi.EventTypeWarning, "FailedCancellation", "Error cancelling deployer pod %s for deployment %s: %v", deployerPod.Name, deployutil.LabelForDeployment(deployment), err) } return fmt.Errorf("couldn't cancel deployer pod %s for deployment %s: %v", deployerPod.Name, deployutil.LabelForDeployment(deployment), err) } anyCancelled = true glog.V(4).Infof("Cancelled deployer pod %s for deployment %s", deployerPod.Name, deployutil.LabelForDeployment(deployment)) } } if anyCancelled { if config, decodeErr := c.decodeConfig(deployment); decodeErr == nil && len(deployerPods) > 0 { c.recorder.Eventf(config, kapi.EventTypeNormal, "Cancelled", "Cancelled deployer pods for deployment %s", deployutil.LabelForDeployment(deployment)) } else if len(deployerPods) > 0 { c.recorder.Eventf(deployment, kapi.EventTypeNormal, "Cancelled", "Cancelled deployer pods") } } return nil }
// Accept implements UpdateAcceptor. func (c *AcceptNewlyObservedReadyPods) Accept(deployment *kapi.ReplicationController) error { // Make a pod store to poll and ensure it gets cleaned up. podStore, stopStore := c.getDeploymentPodStore(deployment) defer close(stopStore) // Start checking for pod updates. glog.V(0).Infof("Waiting %.f seconds for pods owned by deployment %q to become ready (checking every %.f seconds; %d pods previously accepted)", c.timeout.Seconds(), deployutil.LabelForDeployment(deployment), c.interval.Seconds(), c.acceptedPods.Len()) err := wait.Poll(c.interval, c.timeout, func() (done bool, err error) { // Check for pod readiness. unready := kutil.NewStringSet() for _, obj := range podStore.List() { pod := obj.(*kapi.Pod) // Skip previously accepted pods; we only want to verify newly observed // and unaccepted pods. if c.acceptedPods.Has(pod.Name) { continue } if kapi.IsPodReady(pod) { // If the pod is ready, track it as accepted. c.acceptedPods.Insert(pod.Name) } else { // Otherwise, track it as unready. unready.Insert(pod.Name) } } // Check to see if we're done. if unready.Len() == 0 { glog.V(0).Infof("All pods ready for %s", deployutil.LabelForDeployment(deployment)) return true, nil } // Otherwise, try again later. glog.V(4).Infof("Still waiting for %d pods to become ready for deployment %s", unready.Len(), deployutil.LabelForDeployment(deployment)) return false, nil }) // Handle acceptance failure. if err != nil { if err == wait.ErrWaitTimeout { return fmt.Errorf("pods for deployment %q took longer than %.f seconds to become ready", deployutil.LabelForDeployment(deployment), c.timeout.Seconds()) } return fmt.Errorf("pod readiness check failed for deployment %q: %v", deployutil.LabelForDeployment(deployment), err) } return nil }
// executeExecNewPod executes a ExecNewPod hook by creating a new pod based on // the hook parameters and deployment. The pod is then synchronously watched // until the pod completes, and if the pod failed, an error is returned. // // The hook pod inherits the following from the container the hook refers to: // // * Environment (hook keys take precedence) // * Working directory // * Resources func (e *HookExecutor) executeExecNewPod(hook *deployapi.LifecycleHook, deployment *kapi.ReplicationController, label string) error { config, err := deployutil.DecodeDeploymentConfig(deployment, e.codec) if err != nil { return err } // Build a pod spec from the hook config and deployment podSpec, err := makeHookPod(hook, deployment, &config.Template.Strategy, label) if err != nil { return err } // Try to create the pod. pod, err := e.podClient.CreatePod(deployment.Namespace, podSpec) if err != nil { if !kerrors.IsAlreadyExists(err) { return fmt.Errorf("couldn't create lifecycle pod for %s: %v", deployutil.LabelForDeployment(deployment), err) } } else { glog.V(0).Infof("Created lifecycle pod %s/%s for deployment %s", pod.Namespace, pod.Name, deployutil.LabelForDeployment(deployment)) } stopChannel := make(chan struct{}) defer close(stopChannel) nextPod := e.podClient.PodWatch(pod.Namespace, pod.Name, pod.ResourceVersion, stopChannel) // Wait for the hook pod to reach a terminal phase. Start reading logs as // soon as the pod enters a usable phase. var updatedPod *kapi.Pod var once sync.Once wg := &sync.WaitGroup{} wg.Add(1) glog.V(0).Infof("Watching logs for hook pod %s/%s while awaiting completion", pod.Namespace, pod.Name) waitLoop: for { updatedPod = nextPod() switch updatedPod.Status.Phase { case kapi.PodRunning: go once.Do(func() { e.readPodLogs(pod, wg) }) case kapi.PodSucceeded, kapi.PodFailed: go once.Do(func() { e.readPodLogs(pod, wg) }) break waitLoop } } // The pod is finished, wait for all logs to be consumed before returning. wg.Wait() if updatedPod.Status.Phase == kapi.PodFailed { return fmt.Errorf(updatedPod.Status.Message) } return nil }
func (c *DeploymentController) cancelDeployerPods(deployment *kapi.ReplicationController) error { deployerPods, err := c.podClient.getDeployerPodsFor(deployment.Namespace, deployment.Name) if err != nil { return fmt.Errorf("couldn't fetch deployer pods for %s while trying to cancel deployment: %v", deployutil.LabelForDeployment(deployment), err) } glog.V(4).Infof("Cancelling %d deployer pods for deployment %s", len(deployerPods), deployutil.LabelForDeployment(deployment)) zeroDelay := int64(1) cleanedAll := len(deployerPods) > 0 for _, deployerPod := range deployerPods { // Set the ActiveDeadlineSeconds on the pod so it's terminated very soon. if deployerPod.Spec.ActiveDeadlineSeconds == nil || *deployerPod.Spec.ActiveDeadlineSeconds != zeroDelay { deployerPod.Spec.ActiveDeadlineSeconds = &zeroDelay if _, err := c.podClient.updatePod(deployerPod.Namespace, &deployerPod); err != nil { cleanedAll = false utilruntime.HandleError(fmt.Errorf("couldn't cancel deployer pod %s for deployment %s: %v", deployerPod.Name, deployutil.LabelForDeployment(deployment), err)) } glog.V(4).Infof("Cancelled deployer pod %s for deployment %s", deployerPod.Name, deployutil.LabelForDeployment(deployment)) } } if cleanedAll { c.emitDeploymentEvent(deployment, kapi.EventTypeNormal, "Cancelled", "Cancelled all deployer pods") } return nil }
func (c *DeploymentTriggerController) updateDeploymentConfig(old, cur interface{}) { newDc := cur.(*deployapi.DeploymentConfig) oldDc := old.(*deployapi.DeploymentConfig) // A periodic relist will send update events for all known deployment configs. if newDc.ResourceVersion == oldDc.ResourceVersion { return } // No need to enqueue deployment configs that have no triggers or are paused. if len(newDc.Spec.Triggers) == 0 || newDc.Spec.Paused { return } // We don't want to compete with the main deployment config controller. Let's process this // config once it's synced. Note that this does not eliminate conflicts between the two // controllers because the main controller is constantly updating deployment configs as // owning replication controllers and pods are updated. if !deployutil.HasSynced(newDc, newDc.Generation) { return } // Enqueue the deployment config if it hasn't been deployed yet. if newDc.Status.LatestVersion == 0 { c.enqueueDeploymentConfig(newDc) return } // Compare deployment config templates before enqueueing. This reduces the amount of times // we will try to instantiate a deployment config at the expense of duplicating some of the // work that the instantiate endpoint is already doing but I think this is fine. shouldInstantiate := true latestRc, err := c.rcLister.ReplicationControllers(newDc.Namespace).Get(deployutil.LatestDeploymentNameForConfig(newDc)) if err != nil { // If we get an error here it may be due to the rc cache lagging behind. In such a case // just defer to the api server (instantiate REST) where we will retry this. glog.V(2).Infof("Cannot get latest rc for dc %s:%d (%v) - will defer to instantiate", deployutil.LabelForDeploymentConfig(newDc), newDc.Status.LatestVersion, err) } else { initial, err := deployutil.DecodeDeploymentConfig(latestRc, c.codec) if err != nil { glog.V(2).Infof("Cannot decode dc from replication controller %s: %v", deployutil.LabelForDeployment(latestRc), err) return } shouldInstantiate = !reflect.DeepEqual(newDc.Spec.Template, initial.Spec.Template) } if !shouldInstantiate { return } c.enqueueDeploymentConfig(newDc) }
func (c *DeploymentController) cleanupDeployerPods(deployment *kapi.ReplicationController) error { deployerPods, err := c.podClient.getDeployerPodsFor(deployment.Namespace, deployment.Name) if err != nil { return fmt.Errorf("couldn't fetch deployer pods for %q: %v", deployutil.LabelForDeployment(deployment), err) } cleanedAll := true for _, deployerPod := range deployerPods { if err := c.podClient.deletePod(deployerPod.Namespace, deployerPod.Name); err != nil && !kerrors.IsNotFound(err) { // if the pod deletion failed, then log the error and continue // we will try to delete any remaining deployer pods and return an error later utilruntime.HandleError(fmt.Errorf("couldn't delete completed deployer pod %q for deployment %q: %v", deployerPod.Name, deployutil.LabelForDeployment(deployment), err)) cleanedAll = false } } if !cleanedAll { return actionableError(fmt.Sprintf("couldn't clean up all deployer pods for %s", deployment.Name)) } return nil }
func (c *DeploymentController) cleanupDeployerPods(deployment *kapi.ReplicationController) error { selector := deployutil.DeployerPodSelector(deployment.Name) deployerList, err := c.podStore.Pods(deployment.Namespace).List(selector) if err != nil { return fmt.Errorf("couldn't fetch deployer pods for %q: %v", deployutil.LabelForDeployment(deployment), err) } cleanedAll := true gracePeriod := int64(10) for _, deployerPod := range deployerList.Items { if err := c.pn.Pods(deployerPod.Namespace).Delete(deployerPod.Name, &kapi.DeleteOptions{GracePeriodSeconds: &gracePeriod}); err != nil && !kerrors.IsNotFound(err) { // if the pod deletion failed, then log the error and continue // we will try to delete any remaining deployer pods and return an error later utilruntime.HandleError(fmt.Errorf("couldn't delete completed deployer pod %q for deployment %q: %v", deployerPod.Name, deployutil.LabelForDeployment(deployment), err)) cleanedAll = false } } if !cleanedAll { return actionableError(fmt.Sprintf("couldn't clean up all deployer pods for %s", deployment.Name)) } return nil }
// Handle processes deployment and either creates a deployer pod or responds // to a terminal deployment status. Since this controller started using caches, // the provided rc MUST be deep-copied beforehand (see work() in factory.go). func (c *DeploymentController) Handle(deployment *kapi.ReplicationController) error { currentStatus := deployutil.DeploymentStatusFor(deployment) nextStatus := currentStatus deploymentScaled := false deployerPodName := deployutil.DeployerPodNameForDeployment(deployment.Name) deployer, deployerErr := c.getPod(deployment.Namespace, deployerPodName) switch currentStatus { case deployapi.DeploymentStatusNew: // If the deployment has been cancelled, don't create a deployer pod. // Instead try to delete any deployer pods found and transition the // deployment to Pending so that the deployment config controller // continues to see the deployment as in-flight. Eventually the deletion // of the deployer pod should cause a requeue of this deployment and // then it can be transitioned to Failed by this controller. if deployutil.IsDeploymentCancelled(deployment) { nextStatus = deployapi.DeploymentStatusPending if err := c.cleanupDeployerPods(deployment); err != nil { return err } break } // If the pod already exists, it's possible that a previous CreatePod // succeeded but the deployment state update failed and now we're re- // entering. Ensure that the pod is the one we created by verifying the // annotation on it, and throw a retryable error. if deployerErr != nil && !kerrors.IsNotFound(deployerErr) { return fmt.Errorf("couldn't fetch existing deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), deployerErr) } if deployerErr == nil && deployer != nil { // Do a stronger check to validate that the existing deployer pod is // actually for this deployment, and if not, fail this deployment. // // TODO: Investigate checking the container image of the running pod and // comparing with the intended deployer pod image. If we do so, we'll need // to ensure that changes to 'unrelated' pods don't result in updates to // the deployment. So, the image check will have to be done in other areas // of the code as well. if deployutil.DeploymentNameFor(deployer) != deployment.Name { nextStatus = deployapi.DeploymentStatusFailed deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedUnrelatedDeploymentExists c.emitDeploymentEvent(deployment, kapi.EventTypeWarning, "FailedCreate", fmt.Sprintf("Error creating deployer pod since another pod with the same name (%q) exists", deployer.Name)) glog.V(2).Infof("Couldn't create deployer pod for %s since an unrelated pod with the same name (%q) exists", deployutil.LabelForDeployment(deployment), deployer.Name) } else { // Update to pending relative to the existing validated deployer pod. deployment.Annotations[deployapi.DeploymentPodAnnotation] = deployer.Name nextStatus = deployapi.DeploymentStatusPending glog.V(4).Infof("Detected existing deployer pod %s for deployment %s", deployer.Name, deployutil.LabelForDeployment(deployment)) } // Don't try and re-create the deployer pod. break } if _, ok := deployment.Annotations[deployapi.DeploymentIgnorePodAnnotation]; ok { return nil } // Generate a deployer pod spec. deployerPod, err := c.makeDeployerPod(deployment) if err != nil { return fatalError(fmt.Sprintf("couldn't make deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err)) } // Create the deployer pod. deploymentPod, err := c.pn.Pods(deployment.Namespace).Create(deployerPod) // Retry on error. if err != nil { return actionableError(fmt.Sprintf("couldn't create deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err)) } deployment.Annotations[deployapi.DeploymentPodAnnotation] = deploymentPod.Name nextStatus = deployapi.DeploymentStatusPending glog.V(4).Infof("Created deployer pod %s for deployment %s", deploymentPod.Name, deployutil.LabelForDeployment(deployment)) case deployapi.DeploymentStatusPending, deployapi.DeploymentStatusRunning: switch { case kerrors.IsNotFound(deployerErr): nextStatus = deployapi.DeploymentStatusFailed // If the deployment is cancelled here then we deleted the deployer in a previous // resync of the deployment. if !deployutil.IsDeploymentCancelled(deployment) { deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus) deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedDeployerPodNoLongerExists c.emitDeploymentEvent(deployment, kapi.EventTypeWarning, "Failed", fmt.Sprintf("Deployer pod %q has gone missing", deployerPodName)) glog.V(4).Infof("Failing deployment %q because its deployer pod %q disappeared", deployutil.LabelForDeployment(deployment), deployerPodName) } case deployerErr != nil: // We'll try again later on resync. Continue to process cancellations. glog.V(4).Infof("Error getting deployer pod %s for deployment %s: %v", deployerPodName, deployutil.LabelForDeployment(deployment), deployerErr) default: /* err == nil */ // If the deployment has been cancelled, delete any deployer pods // found and transition the deployment to Pending so that the // deployment config controller continues to see the deployment // as in-flight. Eventually the deletion of the deployer pod should // cause a requeue of this deployment and then it can be transitioned // to Failed by this controller. if deployutil.IsDeploymentCancelled(deployment) { if err := c.cleanupDeployerPods(deployment); err != nil { return err } } } case deployapi.DeploymentStatusFailed: // Check for test deployment and ensure the deployment scale matches if config, err := deployutil.DecodeDeploymentConfig(deployment, c.codec); err == nil && config.Spec.Test { deploymentScaled = deployment.Spec.Replicas != 0 deployment.Spec.Replicas = 0 } // Try to cleanup once more a cancelled deployment in case hook pods // were created just after we issued the first cleanup request. if deployutil.IsDeploymentCancelled(deployment) { if err := c.cleanupDeployerPods(deployment); err != nil { return err } } case deployapi.DeploymentStatusComplete: // Check for test deployment and ensure the deployment scale matches if config, err := deployutil.DecodeDeploymentConfig(deployment, c.codec); err == nil && config.Spec.Test { deploymentScaled = deployment.Spec.Replicas != 0 deployment.Spec.Replicas = 0 } if err := c.cleanupDeployerPods(deployment); err != nil { return err } } if deployutil.CanTransitionPhase(currentStatus, nextStatus) || deploymentScaled { deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus) if _, err := c.rn.ReplicationControllers(deployment.Namespace).Update(deployment); err != nil { return fmt.Errorf("couldn't update deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err) } glog.V(4).Infof("Updated deployment %s status from %s to %s (scale: %d)", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus, deployment.Spec.Replicas) } return nil }
// Handle processes change triggers for config. func (c *DeploymentConfigChangeController) Handle(config *deployapi.DeploymentConfig) error { hasChangeTrigger := false for _, trigger := range config.Triggers { if trigger.Type == deployapi.DeploymentTriggerOnConfigChange { hasChangeTrigger = true break } } if !hasChangeTrigger { glog.V(4).Infof("Ignoring DeploymentConfig %s; no change triggers detected", deployutil.LabelForDeploymentConfig(config)) return nil } if config.LatestVersion == 0 { _, _, err := c.generateDeployment(config) if err != nil { if kerrors.IsConflict(err) { return fatalError(fmt.Sprintf("DeploymentConfig %s updated since retrieval; aborting trigger: %v", deployutil.LabelForDeploymentConfig(config), err)) } // TODO: This needs handled by setting some state within the API so // users know why the trigger isn't doing anything. // https://github.com/openshift/origin/issues/3526 return nil } glog.V(4).Infof("Created initial Deployment for DeploymentConfig %s", deployutil.LabelForDeploymentConfig(config)) return nil } latestDeploymentName := deployutil.LatestDeploymentNameForConfig(config) deployment, err := c.changeStrategy.getDeployment(config.Namespace, latestDeploymentName) if err != nil { // If there's no deployment for the latest config, we have no basis of // comparison. It's the responsibility of the deployment config controller // to make the deployment for the config, so return early. if kerrors.IsNotFound(err) { glog.V(2).Infof("Ignoring change for DeploymentConfig %s; no existing Deployment found", deployutil.LabelForDeploymentConfig(config)) return nil } return fmt.Errorf("couldn't retrieve Deployment for DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err) } deployedConfig, err := c.decodeConfig(deployment) if err != nil { return fatalError(fmt.Sprintf("error decoding DeploymentConfig from Deployment %s for DeploymentConfig %s: %v", deployutil.LabelForDeployment(deployment), deployutil.LabelForDeploymentConfig(config), err)) } newSpec, oldSpec := config.Template.ControllerTemplate.Template.Spec, deployedConfig.Template.ControllerTemplate.Template.Spec if kapi.Semantic.DeepEqual(oldSpec, newSpec) { glog.V(2).Infof("Ignoring DeploymentConfig change for %s (latestVersion=%d); same as Deployment %s", deployutil.LabelForDeploymentConfig(config), config.LatestVersion, deployutil.LabelForDeployment(deployment)) return nil } fromVersion, toVersion, err := c.generateDeployment(config) if err != nil { if kerrors.IsConflict(err) { return fatalError(fmt.Sprintf("DeploymentConfig %s updated since retrieval; aborting trigger: %v", deployutil.LabelForDeploymentConfig(config), err)) } return fmt.Errorf("couldn't generate deployment for DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err) } glog.V(4).Infof("Updated DeploymentConfig %s from version %d to %d for existing deployment %s", deployutil.LabelForDeploymentConfig(config), fromVersion, toVersion, deployutil.LabelForDeployment(deployment)) return nil }
// reconcileDeployments reconciles existing deployment replica counts which // could have diverged outside the deployment process (e.g. due to auto or // manual scaling, or partial deployments). The active deployment is the last // successful deployment, not necessarily the latest in terms of the config // version. The active deployment replica count should follow the config, and // all other deployments should be scaled to zero. // // Previously, scaling behavior was that the config replica count was used // only for initial deployments and the active deployment had to be scaled up // directly. To continue supporting that old behavior we must detect when the // deployment has been directly manipulated, and if so, preserve the directly // updated value and sync the config with the deployment. func (c *DeploymentConfigController) reconcileDeployments(existingDeployments []kapi.ReplicationController, config *deployapi.DeploymentConfig) error { latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments) if !latestIsDeployed { // We shouldn't be reconciling if the latest deployment hasn't been // created; this is enforced on the calling side, but double checking // can't hurt. return c.updateStatus(config, existingDeployments) } activeDeployment := deployutil.ActiveDeployment(existingDeployments) // Compute the replica count for the active deployment (even if the active // deployment doesn't exist). The active replica count is the value that // should be assigned to the config, to allow the replica propagation to // flow downward from the config. // // By default we'll assume the config replicas should be used to update the // active deployment except in special cases (like first sync or externally // updated deployments.) activeReplicas := config.Spec.Replicas source := "the deploymentConfig itself (no change)" activeDeploymentExists := activeDeployment != nil activeDeploymentIsLatest := activeDeploymentExists && activeDeployment.Name == latestDeployment.Name latestDesiredReplicas, latestHasDesiredReplicas := deployutil.DeploymentDesiredReplicas(latestDeployment) switch { case activeDeploymentExists && activeDeploymentIsLatest: // The active/latest deployment follows the config unless this is its first // sync or if an external change to the deployment replicas is detected. lastActiveReplicas, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment) if !hasLastActiveReplicas || lastActiveReplicas != activeDeployment.Spec.Replicas { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the latest/active deployment %q which was scaled directly or has not previously been synced", deployutil.LabelForDeployment(activeDeployment)) } case activeDeploymentExists && !activeDeploymentIsLatest: // The active/non-latest deployment follows the config if it was // previously synced; if this is the first sync, infer what the config // value should be based on either the latest desired or whatever the // deployment is currently scaled to. _, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment) if hasLastActiveReplicas { break } if latestHasDesiredReplicas { activeReplicas = latestDesiredReplicas source = fmt.Sprintf("the desired replicas of latest deployment %q which has not been previously synced", deployutil.LabelForDeployment(latestDeployment)) } else if activeDeployment.Spec.Replicas > 0 { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the active deployment %q which has not been previously synced", deployutil.LabelForDeployment(activeDeployment)) } case !activeDeploymentExists && latestHasDesiredReplicas: // If there's no active deployment, use the latest desired, if available. activeReplicas = latestDesiredReplicas source = fmt.Sprintf("the desired replicas of latest deployment %q with no active deployment", deployutil.LabelForDeployment(latestDeployment)) } // Bring the config in sync with the deployment. Once we know the config // accurately represents the desired replica count of the active deployment, // we can safely reconcile deployments. // // If the deployment config is test, never update the deployment config based // on deployments, since test behavior overrides user scaling. switch { case config.Spec.Replicas == activeReplicas: case config.Spec.Test: glog.V(4).Infof("Detected changed replicas for test deploymentConfig %q, ignoring that change", deployutil.LabelForDeploymentConfig(config)) default: copied, err := deployutil.DeploymentConfigDeepCopy(config) if err != nil { return err } oldReplicas := copied.Spec.Replicas copied.Spec.Replicas = activeReplicas config, err = c.dn.DeploymentConfigs(copied.Namespace).Update(copied) if err != nil { return err } glog.V(4).Infof("Synced deploymentConfig %q replicas from %d to %d based on %s", deployutil.LabelForDeploymentConfig(config), oldReplicas, activeReplicas, source) } // Reconcile deployments. The active deployment follows the config, and all // other deployments should be scaled to zero. var updatedDeployments []kapi.ReplicationController for i := range existingDeployments { deployment := existingDeployments[i] toAppend := deployment isActiveDeployment := activeDeployment != nil && deployment.Name == activeDeployment.Name oldReplicaCount := deployment.Spec.Replicas newReplicaCount := int32(0) if isActiveDeployment { newReplicaCount = activeReplicas } if config.Spec.Test { glog.V(4).Infof("Deployment config %q is test and deployment %q will be scaled down", deployutil.LabelForDeploymentConfig(config), deployutil.LabelForDeployment(&deployment)) newReplicaCount = 0 } lastReplicas, hasLastReplicas := deployutil.DeploymentReplicas(&deployment) // Only update if necessary. var copied *kapi.ReplicationController if !hasLastReplicas || newReplicaCount != oldReplicaCount || lastReplicas != newReplicaCount { if err := kclient.RetryOnConflict(kclient.DefaultBackoff, func() error { // refresh the replication controller version rc, err := c.rcStore.ReplicationControllers(deployment.Namespace).Get(deployment.Name) if err != nil { return err } copied, err = deployutil.DeploymentDeepCopy(rc) if err != nil { glog.V(2).Infof("Deep copy of deployment %q failed: %v", rc.Name, err) return err } copied.Spec.Replicas = newReplicaCount copied.Annotations[deployapi.DeploymentReplicasAnnotation] = strconv.Itoa(int(newReplicaCount)) _, err = c.rn.ReplicationControllers(copied.Namespace).Update(copied) return err }); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentScaleFailed", "Failed to scale deployment %q from %d to %d: %v", deployment.Name, oldReplicaCount, newReplicaCount, err) return err } // Only report scaling events if we changed the replica count. if oldReplicaCount != newReplicaCount { c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentScaled", "Scaled deployment %q from %d to %d", copied.Name, oldReplicaCount, newReplicaCount) } else { glog.V(4).Infof("Updated deployment %q replica annotation to match current replica count %d", deployutil.LabelForDeployment(copied), newReplicaCount) } toAppend = *copied } updatedDeployments = append(updatedDeployments, toAppend) } // As the deployment configuration has changed, we need to make sure to clean // up old deployments if we have now reached our deployment history quota if err := c.cleanupOldDeployments(existingDeployments, config); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCleanupFailed", "Couldn't clean up deployments: %v", err) } return c.updateStatus(config, updatedDeployments) }
// Get returns a streamer resource with the contents of the deployment log func (r *REST) Get(ctx kapi.Context, name string, opts runtime.Object) (runtime.Object, error) { // Ensure we have a namespace in the context namespace, ok := kapi.NamespaceFrom(ctx) if !ok { return nil, errors.NewBadRequest("namespace parameter required.") } // Validate DeploymentLogOptions deployLogOpts, ok := opts.(*deployapi.DeploymentLogOptions) if !ok { return nil, errors.NewBadRequest("did not get an expected options.") } if errs := validation.ValidateDeploymentLogOptions(deployLogOpts); len(errs) > 0 { return nil, errors.NewInvalid("deploymentLogOptions", "", errs) } // Fetch deploymentConfig and check latest version; if 0, there are no deployments // for this config config, err := r.ConfigGetter.DeploymentConfigs(namespace).Get(name) if err != nil { return nil, errors.NewNotFound("deploymentConfig", name) } desiredVersion := config.Status.LatestVersion if desiredVersion == 0 { return nil, errors.NewBadRequest(fmt.Sprintf("no deployment exists for deploymentConfig %q", config.Name)) } // Support retrieving logs for older deployments switch { case deployLogOpts.Version == nil: // Latest case *deployLogOpts.Version <= 0 || int(*deployLogOpts.Version) > config.Status.LatestVersion: // Invalid version return nil, errors.NewBadRequest(fmt.Sprintf("invalid version for deploymentConfig %q: %d", config.Name, *deployLogOpts.Version)) default: desiredVersion = int(*deployLogOpts.Version) } // Get desired deployment targetName := deployutil.DeploymentNameForConfigVersion(config.Name, desiredVersion) target, err := r.DeploymentGetter.ReplicationControllers(namespace).Get(targetName) if err != nil { return nil, err } // Check for deployment status; if it is new or pending, we will wait for it. If it is complete, // the deployment completed successfully and the deployer pod will be deleted so we will return a // success message. If it is running or failed, retrieve the log from the deployer pod. status := deployutil.DeploymentStatusFor(target) switch status { case deployapi.DeploymentStatusNew, deployapi.DeploymentStatusPending: if deployLogOpts.NoWait { glog.V(4).Infof("Deployment %s is in %s state. No logs to retrieve yet.", deployutil.LabelForDeployment(target), status) return &genericrest.LocationStreamer{}, nil } glog.V(4).Infof("Deployment %s is in %s state, waiting for it to start...", deployutil.LabelForDeployment(target), status) latest, ok, err := registry.WaitForRunningDeployment(r.DeploymentGetter, target, r.Timeout) if err != nil { return nil, errors.NewBadRequest(fmt.Sprintf("unable to wait for deployment %s to run: %v", deployutil.LabelForDeployment(target), err)) } if !ok { return nil, errors.NewTimeoutError(fmt.Sprintf("timed out waiting for deployment %s to start after %s", deployutil.LabelForDeployment(target), r.Timeout), 1) } if deployutil.DeploymentStatusFor(latest) == deployapi.DeploymentStatusComplete { // Deployer pod has been deleted, no logs to retrieve glog.V(4).Infof("Deployment %s was successful so the deployer pod is deleted. No logs to retrieve.", deployutil.LabelForDeployment(target)) return &genericrest.LocationStreamer{}, nil } case deployapi.DeploymentStatusComplete: // Deployer pod has been deleted, no logs to retrieve glog.V(4).Infof("Deployment %s was successful so the deployer pod is deleted. No logs to retrieve.", deployutil.LabelForDeployment(target)) return &genericrest.LocationStreamer{}, nil } // Setup url of the deployer pod deployPodName := deployutil.DeployerPodNameForDeployment(target.Name) logOpts := deployapi.DeploymentToPodLogOptions(deployLogOpts) location, transport, err := pod.LogLocation(r.PodGetter, r.ConnectionInfo, ctx, deployPodName, logOpts) if err != nil { return nil, errors.NewBadRequest(err.Error()) } return &genericrest.LocationStreamer{ Location: location, Transport: transport, ContentType: "text/plain", Flush: deployLogOpts.Follow, ResponseChecker: genericrest.NewGenericHttpResponseChecker("Pod", deployPodName), }, nil }
func (s *RollingDeploymentStrategy) Deploy(from *kapi.ReplicationController, to *kapi.ReplicationController, desiredReplicas int) error { config, err := deployutil.DecodeDeploymentConfig(to, s.codec) if err != nil { return fmt.Errorf("couldn't decode DeploymentConfig from deployment %s: %v", deployutil.LabelForDeployment(to), err) } params := config.Template.Strategy.RollingParams updateAcceptor := s.getUpdateAcceptor(time.Duration(*params.TimeoutSeconds) * time.Second) // If there's no prior deployment, delegate to another strategy since the // rolling updater only supports transitioning between two deployments. // // Hook support is duplicated here for now. When the rolling updater can // handle initial deployments, all of this code can go away. if from == nil { // Execute any pre-hook. if params.Pre != nil { err := s.hookExecutor.Execute(params.Pre, to, "prehook") if err != nil { return fmt.Errorf("Pre hook failed: %s", err) } glog.Infof("Pre hook finished") } // Execute the delegate strategy. err := s.initialStrategy.DeployWithAcceptor(from, to, desiredReplicas, updateAcceptor) if err != nil { return err } // Execute any post-hook. Errors are logged and ignored. if params.Post != nil { err := s.hookExecutor.Execute(params.Post, to, "posthook") if err != nil { util.HandleError(fmt.Errorf("post hook failed: %s", err)) } else { glog.Infof("Post hook finished") } } // All done. return nil } // Prepare for a rolling update. // Execute any pre-hook. if params.Pre != nil { err := s.hookExecutor.Execute(params.Pre, to, "prehook") if err != nil { return fmt.Errorf("pre hook failed: %s", err) } glog.Infof("Pre hook finished") } // HACK: Assign the source ID annotation that the rolling updater expects, // unless it already exists on the deployment. // // Related upstream issue: // https://github.com/kubernetes/kubernetes/pull/7183 err = wait.Poll(s.apiRetryPeriod, s.apiRetryTimeout, func() (done bool, err error) { existing, err := s.client.ReplicationControllers(to.Namespace).Get(to.Name) if err != nil { msg := fmt.Sprintf("couldn't look up deployment %s: %s", deployutil.LabelForDeployment(to), err) if kerrors.IsNotFound(err) { return false, fmt.Errorf("%s", msg) } // Try again. glog.Infof(msg) return false, nil } if _, hasSourceId := existing.Annotations[sourceIdAnnotation]; !hasSourceId { existing.Annotations[sourceIdAnnotation] = fmt.Sprintf("%s:%s", from.Name, from.ObjectMeta.UID) if _, err := s.client.ReplicationControllers(existing.Namespace).Update(existing); err != nil { msg := fmt.Sprintf("couldn't assign source annotation to deployment %s: %v", deployutil.LabelForDeployment(existing), err) if kerrors.IsNotFound(err) { return false, fmt.Errorf("%s", msg) } // Try again. glog.Infof(msg) return false, nil } } return true, nil }) if err != nil { return err } to, err = s.client.ReplicationControllers(to.Namespace).Get(to.Name) if err != nil { return err } // HACK: There's a validation in the rolling updater which assumes that when // an existing RC is supplied, it will have >0 replicas- a validation which // is then disregarded as the desired count is obtained from the annotation // on the RC. For now, fake it out by just setting replicas to 1. // // Related upstream issue: // https://github.com/kubernetes/kubernetes/pull/7183 to.Spec.Replicas = 1 // Perform a rolling update. rollingConfig := &kubectl.RollingUpdaterConfig{ Out: &rollingUpdaterWriter{}, OldRc: from, NewRc: to, UpdatePeriod: time.Duration(*params.UpdatePeriodSeconds) * time.Second, Interval: time.Duration(*params.IntervalSeconds) * time.Second, Timeout: time.Duration(*params.TimeoutSeconds) * time.Second, CleanupPolicy: kubectl.PreserveRollingUpdateCleanupPolicy, MaxSurge: params.MaxSurge, MaxUnavailable: params.MaxUnavailable, } err = s.rollingUpdate(rollingConfig) if err != nil { return err } // Execute any post-hook. Errors are logged and ignored. if params.Post != nil { err := s.hookExecutor.Execute(params.Post, to, "posthook") if err != nil { util.HandleError(fmt.Errorf("Post hook failed: %s", err)) } else { glog.Info("Post hook finished") } } return nil }
// Handle processes deployment and either creates a deployer pod or responds // to a terminal deployment status. func (c *DeploymentController) Handle(deployment *kapi.ReplicationController) error { currentStatus := deployutil.DeploymentStatusFor(deployment) nextStatus := currentStatus deploymentScaled := false switch currentStatus { case deployapi.DeploymentStatusNew: // If the deployment has been cancelled, don't create a deployer pod, and // transition to failed immediately. if deployutil.IsDeploymentCancelled(deployment) { nextStatus = deployapi.DeploymentStatusFailed break } // Generate a deployer pod spec. podTemplate, err := c.makeDeployerPod(deployment) if err != nil { return fatalError(fmt.Sprintf("couldn't make deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err)) } // Create the deployer pod. deploymentPod, err := c.podClient.createPod(deployment.Namespace, podTemplate) if err == nil { deployment.Annotations[deployapi.DeploymentPodAnnotation] = deploymentPod.Name nextStatus = deployapi.DeploymentStatusPending glog.V(4).Infof("Created pod %s for deployment %s", deploymentPod.Name, deployutil.LabelForDeployment(deployment)) break } // Retry on error. if !kerrors.IsAlreadyExists(err) { c.recorder.Eventf(deployment, kapi.EventTypeWarning, "FailedCreate", "Error creating deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err) return fmt.Errorf("couldn't create deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err) } // If the pod already exists, it's possible that a previous CreatePod // succeeded but the deployment state update failed and now we're re- // entering. Ensure that the pod is the one we created by verifying the // annotation on it, and throw a retryable error. existingPod, err := c.podClient.getPod(deployment.Namespace, deployutil.DeployerPodNameForDeployment(deployment.Name)) if err != nil { c.recorder.Eventf(deployment, kapi.EventTypeWarning, "FailedCreate", "Error getting existing deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err) return fmt.Errorf("couldn't fetch existing deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err) } // Do a stronger check to validate that the existing deployer pod is // actually for this deployment, and if not, fail this deployment. // // TODO: Investigate checking the container image of the running pod and // comparing with the intended deployer pod image. If we do so, we'll need // to ensure that changes to 'unrelated' pods don't result in updates to // the deployment. So, the image check will have to be done in other areas // of the code as well. if deployutil.DeploymentNameFor(existingPod) != deployment.Name { nextStatus = deployapi.DeploymentStatusFailed deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedUnrelatedDeploymentExists c.recorder.Eventf(deployment, kapi.EventTypeWarning, "FailedCreate", "Error creating deployer pod for %s since another pod with the same name (%q) exists", deployutil.LabelForDeployment(deployment), existingPod.Name) glog.V(2).Infof("Couldn't create deployer pod for %s since an unrelated pod with the same name (%q) exists", deployutil.LabelForDeployment(deployment), existingPod.Name) break } // Update to pending relative to the existing validated deployer pod. deployment.Annotations[deployapi.DeploymentPodAnnotation] = existingPod.Name nextStatus = deployapi.DeploymentStatusPending glog.V(4).Infof("Detected existing deployer pod %s for deployment %s", existingPod.Name, deployutil.LabelForDeployment(deployment)) case deployapi.DeploymentStatusPending, deployapi.DeploymentStatusRunning: // If the deployer pod has vanished, consider the deployment a failure. deployerPodName := deployutil.DeployerPodNameForDeployment(deployment.Name) if _, err := c.podClient.getPod(deployment.Namespace, deployerPodName); err != nil { if kerrors.IsNotFound(err) { nextStatus = deployapi.DeploymentStatusFailed deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus) deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedDeployerPodNoLongerExists c.recorder.Eventf(deployment, kapi.EventTypeWarning, "failed", "Deployer pod %q has gone missing", deployerPodName) glog.V(4).Infof("Failing deployment %q because its deployer pod %q disappeared", deployutil.LabelForDeployment(deployment), deployerPodName) break } else { // We'll try again later on resync. Continue to process cancellations. glog.V(2).Infof("Error getting deployer pod %s for deployment %s: %#v", deployerPodName, deployutil.LabelForDeployment(deployment), err) } } // If the deployment is cancelled, terminate any deployer/hook pods. // NOTE: Do not mark the deployment as Failed just yet. // The deployment will be marked as Failed by the deployer pod controller // when the deployer pod failure state is picked up // Also, it will scale down the failed deployment and scale back up // the last successful completed deployment if deployutil.IsDeploymentCancelled(deployment) { deployerPods, err := c.podClient.getDeployerPodsFor(deployment.Namespace, deployment.Name) if err != nil { return fmt.Errorf("couldn't fetch deployer pods for %s while trying to cancel deployment: %v", deployutil.LabelForDeployment(deployment), err) } glog.V(4).Infof("Cancelling %d deployer pods for deployment %s", len(deployerPods), deployutil.LabelForDeployment(deployment)) zeroDelay := int64(1) for _, deployerPod := range deployerPods { // Set the ActiveDeadlineSeconds on the pod so it's terminated very soon. if deployerPod.Spec.ActiveDeadlineSeconds == nil || *deployerPod.Spec.ActiveDeadlineSeconds != zeroDelay { deployerPod.Spec.ActiveDeadlineSeconds = &zeroDelay if _, err := c.podClient.updatePod(deployerPod.Namespace, &deployerPod); err != nil { c.recorder.Eventf(deployment, kapi.EventTypeWarning, "failedCancellation", "Error cancelling deployer pod %s for deployment %s: %v", deployerPod.Name, deployutil.LabelForDeployment(deployment), err) return fmt.Errorf("couldn't cancel deployer pod %s for deployment %s: %v", deployerPod.Name, deployutil.LabelForDeployment(deployment), err) } glog.V(4).Infof("Cancelled deployer pod %s for deployment %s", deployerPod.Name, deployutil.LabelForDeployment(deployment)) } } c.recorder.Eventf(deployment, kapi.EventTypeNormal, "Cancelled", "Cancelled deployment") } case deployapi.DeploymentStatusFailed: // Check for test deployment and ensure the deployment scale matches if config, err := c.decodeConfig(deployment); err == nil && config.Spec.Test { deploymentScaled = deployment.Spec.Replicas != 0 deployment.Spec.Replicas = 0 } case deployapi.DeploymentStatusComplete: // Check for test deployment and ensure the deployment scale matches if config, err := c.decodeConfig(deployment); err == nil && config.Spec.Test { deploymentScaled = deployment.Spec.Replicas != 0 deployment.Spec.Replicas = 0 } // now list any pods in the namespace that have the specified label deployerPods, err := c.podClient.getDeployerPodsFor(deployment.Namespace, deployment.Name) if err != nil { return fmt.Errorf("couldn't fetch deployer pods for %s after successful completion: %v", deployutil.LabelForDeployment(deployment), err) } if len(deployerPods) > 0 { glog.V(4).Infof("Deleting %d deployer pods for deployment %s", len(deployerPods), deployutil.LabelForDeployment(deployment)) } cleanedAll := true for _, deployerPod := range deployerPods { if err := c.podClient.deletePod(deployerPod.Namespace, deployerPod.Name); err != nil { if !kerrors.IsNotFound(err) { // if the pod deletion failed, then log the error and continue // we will try to delete any remaining deployer pods and return an error later utilruntime.HandleError(fmt.Errorf("couldn't delete completed deployer pod %s/%s for deployment %s: %v", deployment.Namespace, deployerPod.Name, deployutil.LabelForDeployment(deployment), err)) cleanedAll = false } // Already deleted } else { glog.V(4).Infof("Deleted completed deployer pod %s/%s for deployment %s", deployment.Namespace, deployerPod.Name, deployutil.LabelForDeployment(deployment)) } } if !cleanedAll { return fmt.Errorf("couldn't clean up all deployer pods for %s", deployutil.LabelForDeployment(deployment)) } } if currentStatus != nextStatus || deploymentScaled { deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus) if _, err := c.deploymentClient.updateDeployment(deployment.Namespace, deployment); err != nil { c.recorder.Eventf(deployment, kapi.EventTypeWarning, "FailedUpdate", "Error updating deployment %s status to %s", deployutil.LabelForDeployment(deployment), nextStatus) return fmt.Errorf("couldn't update deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err) } glog.V(4).Infof("Updated deployment %s status from %s to %s (scale: %d)", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus, deployment.Spec.Replicas) } return nil }
// DeployWithAcceptor scales down from and then scales up to. If // updateAcceptor is provided and the desired replica count is >1, the first // replica of to is rolled out and validated before performing the full scale // up. // // This is currently only used in conjunction with the rolling update strategy // for initial deployments. func (s *RecreateDeploymentStrategy) DeployWithAcceptor(from *kapi.ReplicationController, to *kapi.ReplicationController, desiredReplicas int, updateAcceptor strat.UpdateAcceptor) error { config, err := deployutil.DecodeDeploymentConfig(to, s.codec) if err != nil { return fmt.Errorf("couldn't decode config from deployment %s: %v", to.Name, err) } params := config.Spec.Strategy.RecreateParams retryParams := kubectl.NewRetryParams(s.retryPeriod, s.retryTimeout) waitParams := kubectl.NewRetryParams(s.retryPeriod, s.retryTimeout) // Execute any pre-hook. if params != nil && params.Pre != nil { if err := s.hookExecutor.Execute(params.Pre, to, "prehook"); err != nil { return fmt.Errorf("Pre hook failed: %s", err) } else { glog.Infof("Pre hook finished") } } // Scale down the from deployment. if from != nil { glog.Infof("Scaling %s down to zero", deployutil.LabelForDeployment(from)) _, err := s.scaleAndWait(from, 0, retryParams, waitParams) if err != nil { return fmt.Errorf("couldn't scale %s to 0: %v", deployutil.LabelForDeployment(from), err) } } // Scale up the to deployment. if desiredReplicas > 0 { // If an UpdateAcceptor is provided, scale up to 1 and validate the replica, // aborting if the replica isn't acceptable. if updateAcceptor != nil { glog.Infof("Scaling %s to 1 before performing acceptance check", deployutil.LabelForDeployment(to)) updatedTo, err := s.scaleAndWait(to, 1, retryParams, waitParams) if err != nil { return fmt.Errorf("couldn't scale %s to 1: %v", deployutil.LabelForDeployment(to), err) } glog.Infof("Performing acceptance check of %s", deployutil.LabelForDeployment(to)) if err := updateAcceptor.Accept(updatedTo); err != nil { return fmt.Errorf("update acceptor rejected %s: %v", deployutil.LabelForDeployment(to), err) } to = updatedTo } // Complete the scale up. if to.Spec.Replicas != desiredReplicas { glog.Infof("Scaling %s to %d", deployutil.LabelForDeployment(to), desiredReplicas) updatedTo, err := s.scaleAndWait(to, desiredReplicas, retryParams, waitParams) if err != nil { return fmt.Errorf("couldn't scale %s to %d: %v", deployutil.LabelForDeployment(to), desiredReplicas, err) } to = updatedTo } } // Execute any post-hook. Errors are logged and ignored. if params != nil && params.Post != nil { if err := s.hookExecutor.Execute(params.Post, to, "posthook"); err != nil { util.HandleError(fmt.Errorf("post hook failed: %s", err)) } else { glog.Infof("Post hook finished") } } glog.Infof("Deployment %s successfully made active", to.Name) return nil }
func (s *RollingDeploymentStrategy) Deploy(from *kapi.ReplicationController, to *kapi.ReplicationController, desiredReplicas int) error { config, err := deployutil.DecodeDeploymentConfig(to, s.codec) if err != nil { return fmt.Errorf("couldn't decode DeploymentConfig from deployment %s: %v", deployutil.LabelForDeployment(to), err) } params := config.Template.Strategy.RollingParams updateAcceptor := s.getUpdateAcceptor(time.Duration(*params.TimeoutSeconds) * time.Second) // If there's no prior deployment, delegate to another strategy since the // rolling updater only supports transitioning between two deployments. // // Hook support is duplicated here for now. When the rolling updater can // handle initial deployments, all of this code can go away. if from == nil { // Execute any pre-hook. if params.Pre != nil { err := s.hookExecutor.Execute(params.Pre, to, "prehook") if err != nil { return fmt.Errorf("Pre hook failed: %s", err) } glog.Infof("Pre hook finished") } // Execute the delegate strategy. err := s.initialStrategy.DeployWithAcceptor(from, to, desiredReplicas, updateAcceptor) if err != nil { return err } // Execute any post-hook. Errors are logged and ignored. if params.Post != nil { err := s.hookExecutor.Execute(params.Post, to, "posthook") if err != nil { util.HandleError(fmt.Errorf("post hook failed: %s", err)) } else { glog.Infof("Post hook finished") } } // All done. return nil } // Prepare for a rolling update. // Execute any pre-hook. if params.Pre != nil { err := s.hookExecutor.Execute(params.Pre, to, "prehook") if err != nil { return fmt.Errorf("pre hook failed: %s", err) } glog.Infof("Pre hook finished") } // HACK: Assign the source ID annotation that the rolling updater expects, // unless it already exists on the deployment. // // Related upstream issue: // https://github.com/GoogleCloudPlatform/kubernetes/pull/7183 to, err = s.client.GetReplicationController(to.Namespace, to.Name) if err != nil { return fmt.Errorf("couldn't look up deployment %s: %s", deployutil.LabelForDeployment(to), err) } if _, hasSourceId := to.Annotations[sourceIdAnnotation]; !hasSourceId { to.Annotations[sourceIdAnnotation] = fmt.Sprintf("%s:%s", from.Name, from.ObjectMeta.UID) if updated, err := s.client.UpdateReplicationController(to.Namespace, to); err != nil { return fmt.Errorf("couldn't assign source annotation to deployment %s: %v", deployutil.LabelForDeployment(to), err) } else { to = updated } } // HACK: There's a validation in the rolling updater which assumes that when // an existing RC is supplied, it will have >0 replicas- a validation which // is then disregarded as the desired count is obtained from the annotation // on the RC. For now, fake it out by just setting replicas to 1. // // Related upstream issue: // https://github.com/GoogleCloudPlatform/kubernetes/pull/7183 to.Spec.Replicas = 1 // Perform a rolling update. rollingConfig := &kubectl.RollingUpdaterConfig{ Out: &rollingUpdaterWriter{}, OldRc: from, NewRc: to, UpdatePeriod: time.Duration(*params.UpdatePeriodSeconds) * time.Second, Interval: time.Duration(*params.IntervalSeconds) * time.Second, Timeout: time.Duration(*params.TimeoutSeconds) * time.Second, UpdatePercent: params.UpdatePercent, CleanupPolicy: kubectl.PreserveRollingUpdateCleanupPolicy, UpdateAcceptor: updateAcceptor, } pct := "<nil>" if params.UpdatePercent != nil { pct = fmt.Sprintf("%d", *params.UpdatePercent) } glog.Infof("Starting rolling update from %s to %s (desired replicas: %d, updatePeriodSeconds=%ds, intervalSeconds=%ds, timeoutSeconds=%ds, updatePercent=%s%%)", deployutil.LabelForDeployment(from), deployutil.LabelForDeployment(to), desiredReplicas, *params.UpdatePeriodSeconds, *params.IntervalSeconds, *params.TimeoutSeconds, pct, ) if err := s.rollingUpdate(rollingConfig); err != nil { return err } // Execute any post-hook. Errors are logged and ignored. if params.Post != nil { err := s.hookExecutor.Execute(params.Post, to, "posthook") if err != nil { util.HandleError(fmt.Errorf("Post hook failed: %s", err)) } else { glog.Info("Post hook finished") } } return nil }
// Handle syncs pod's status with any associated deployment. func (c *DeployerPodController) Handle(pod *kapi.Pod) error { // Find the deployment associated with the deployer pod. deploymentName := deployutil.DeploymentNameFor(pod) if len(deploymentName) == 0 { return nil } // Reject updates to anything but the main deployer pod // TODO: Find a way to filter this on the watch side. if pod.Name != deployutil.DeployerPodNameForDeployment(deploymentName) { return nil } deployment, err := c.deploymentClient.getDeployment(pod.Namespace, deploymentName) // If the deployment for this pod has disappeared, we should clean up this // and any other deployer pods, then bail out. if err != nil { // Some retrieval error occured. Retry. if !kerrors.IsNotFound(err) { return fmt.Errorf("couldn't get deployment %s/%s which owns deployer pod %s/%s", pod.Namespace, deploymentName, pod.Name, pod.Namespace) } // Find all the deployer pods for the deployment (including this one). deployers, err := c.deployerPodsFor(pod.Namespace, deploymentName) if err != nil { // Retry. return fmt.Errorf("couldn't get deployer pods for %s: %v", deployutil.LabelForDeployment(deployment), err) } // Delete all deployers. for _, deployer := range deployers.Items { err := c.deletePod(deployer.Namespace, deployer.Name) if err != nil { if !kerrors.IsNotFound(err) { // TODO: Should this fire an event? glog.V(2).Infof("Couldn't delete orphaned deployer pod %s/%s: %v", deployer.Namespace, deployer.Name, err) } } else { // TODO: Should this fire an event? glog.V(2).Infof("Deleted orphaned deployer pod %s/%s", deployer.Namespace, deployer.Name) } } return nil } currentStatus := deployutil.DeploymentStatusFor(deployment) nextStatus := currentStatus switch pod.Status.Phase { case kapi.PodRunning: nextStatus = deployapi.DeploymentStatusRunning case kapi.PodSucceeded: // Detect failure based on the container state nextStatus = deployapi.DeploymentStatusComplete for _, info := range pod.Status.ContainerStatuses { if info.State.Termination != nil && info.State.Termination.ExitCode != 0 { nextStatus = deployapi.DeploymentStatusFailed } } case kapi.PodFailed: // if the deployment is already marked Failed, do not attempt clean up again if currentStatus != deployapi.DeploymentStatusFailed { // clean up will also update the deployment status to Failed // failure to clean up will result in retries and // the deployment will not be marked Failed // Note: this will prevent new deployments from being created for this config err := c.cleanupFailedDeployment(deployment) if err != nil { return transientError(fmt.Sprintf("couldn't clean up failed deployment: %v", err)) } } } if currentStatus != nextStatus { deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus) if _, err := c.deploymentClient.updateDeployment(deployment.Namespace, deployment); err != nil { if kerrors.IsNotFound(err) { return nil } return fmt.Errorf("couldn't update Deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err) } glog.V(4).Infof("Updated Deployment %s status from %s to %s", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus) } return nil }
func (c *DeployerPodController) cleanupFailedDeployment(deployment *kapi.ReplicationController) error { // Scale down the current failed deployment configName := deployutil.DeploymentConfigNameFor(deployment) existingDeployments, err := c.deploymentClient.listDeploymentsForConfig(deployment.Namespace, configName) if err != nil { return fmt.Errorf("couldn't list Deployments for DeploymentConfig %s: %v", configName, err) } desiredReplicas, ok := deployutil.DeploymentDesiredReplicas(deployment) if !ok { // if desired replicas could not be found, then log the error // and update the failed deployment // this cannot be treated as a transient error kutil.HandleError(fmt.Errorf("Could not determine desired replicas from %s to reset replicas for last completed deployment", deployutil.LabelForDeployment(deployment))) } if ok && len(existingDeployments.Items) > 0 { sort.Sort(deployutil.DeploymentsByLatestVersionDesc(existingDeployments.Items)) for index, existing := range existingDeployments.Items { // if a newer deployment exists: // - set the replicas for the current failed deployment to 0 // - there is no point in scaling up the last completed deployment // since that will be scaled down by the later deployment if index == 0 && existing.Name != deployment.Name { break } // the latest completed deployment is the one that needs to be scaled back up if deployutil.DeploymentStatusFor(&existing) == deployapi.DeploymentStatusComplete { if existing.Spec.Replicas == desiredReplicas { break } // scale back the completed deployment to the target of the failed deployment existing.Spec.Replicas = desiredReplicas if _, err := c.deploymentClient.updateDeployment(existing.Namespace, &existing); err != nil { if kerrors.IsNotFound(err) { return nil } return fmt.Errorf("couldn't update replicas to %d for deployment %s: %v", desiredReplicas, deployutil.LabelForDeployment(&existing), err) } glog.V(4).Infof("Updated replicas to %d for deployment %s", desiredReplicas, deployutil.LabelForDeployment(&existing)) break } } } // set the replicas for the failed deployment to 0 // and set the status to Failed deployment.Spec.Replicas = 0 deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusFailed) if _, err := c.deploymentClient.updateDeployment(deployment.Namespace, deployment); err != nil { if kerrors.IsNotFound(err) { return nil } return fmt.Errorf("couldn't scale down the deployment %s and mark it as failed: %v", deployutil.LabelForDeployment(deployment), err) } glog.V(4).Infof("Scaled down the deployment %s and marked it as failed", deployutil.LabelForDeployment(deployment)) return nil }
// reconcileDeployments reconciles existing deployment replica counts which // could have diverged outside the deployment process (e.g. due to auto or // manual scaling, or partial deployments). The active deployment is the last // successful deployment, not necessarily the latest in terms of the config // version. The active deployment replica count should follow the config, and // all other deployments should be scaled to zero. func (c *DeploymentConfigController) reconcileDeployments(existingDeployments []kapi.ReplicationController, config *deployapi.DeploymentConfig) error { activeDeployment := deployutil.ActiveDeployment(existingDeployments) // Reconcile deployments. The active deployment follows the config, and all // other deployments should be scaled to zero. var updatedDeployments []kapi.ReplicationController for i := range existingDeployments { deployment := existingDeployments[i] toAppend := deployment isActiveDeployment := activeDeployment != nil && deployment.Name == activeDeployment.Name oldReplicaCount := deployment.Spec.Replicas newReplicaCount := int32(0) if isActiveDeployment { newReplicaCount = config.Spec.Replicas } if config.Spec.Test { glog.V(4).Infof("Deployment config %q is test and deployment %q will be scaled down", deployutil.LabelForDeploymentConfig(config), deployutil.LabelForDeployment(&deployment)) newReplicaCount = 0 } // Only update if necessary. var copied *kapi.ReplicationController if newReplicaCount != oldReplicaCount { if err := kclient.RetryOnConflict(kclient.DefaultBackoff, func() error { // refresh the replication controller version rc, err := c.rcStore.ReplicationControllers(deployment.Namespace).Get(deployment.Name) if err != nil { return err } copied, err = deployutil.DeploymentDeepCopy(rc) if err != nil { glog.V(2).Infof("Deep copy of deployment %q failed: %v", rc.Name, err) return err } copied.Spec.Replicas = newReplicaCount copied, err = c.rn.ReplicationControllers(copied.Namespace).Update(copied) return err }); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "ReplicationControllerScaleFailed", "Failed to scale replication controler %q from %d to %d: %v", deployment.Name, oldReplicaCount, newReplicaCount, err) return err } c.recorder.Eventf(config, kapi.EventTypeNormal, "ReplicationControllerScaled", "Scaled replication controller %q from %d to %d", copied.Name, oldReplicaCount, newReplicaCount) toAppend = *copied } updatedDeployments = append(updatedDeployments, toAppend) } // As the deployment configuration has changed, we need to make sure to clean // up old deployments if we have now reached our deployment history quota if err := c.cleanupOldDeployments(updatedDeployments, config); err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "ReplicationControllerCleanupFailed", "Couldn't clean up replication controllers: %v", err) } return c.updateStatus(config, updatedDeployments) }
// Deploy starts the deployment process for rcName. func (d *Deployer) Deploy(namespace, rcName string) error { // Look up the new deployment. to, err := d.getDeployment(namespace, rcName) if err != nil { return fmt.Errorf("couldn't get deployment %s: %v", rcName, err) } // Decode the config from the deployment. config, err := deployutil.DecodeDeploymentConfig(to, kapi.Codecs.UniversalDecoder()) if err != nil { return fmt.Errorf("couldn't decode deployment config from deployment %s: %v", to.Name, err) } // Get a strategy for the deployment. s, err := d.strategyFor(config) if err != nil { return err } // New deployments must have a desired replica count. desiredReplicas, hasDesired := deployutil.DeploymentDesiredReplicas(to) if !hasDesired { return fmt.Errorf("deployment %s has already run to completion", to.Name) } // Find all deployments for the config. unsortedDeployments, err := d.getDeployments(namespace, config.Name) if err != nil { return fmt.Errorf("couldn't get controllers in namespace %s: %v", namespace, err) } deployments := unsortedDeployments.Items // Sort all the deployments by version. sort.Sort(deployutil.ByLatestVersionDesc(deployments)) // Find any last completed deployment. var from *kapi.ReplicationController for _, candidate := range deployments { if candidate.Name == to.Name { continue } if deployutil.IsCompleteDeployment(&candidate) { from = &candidate break } } if deployutil.DeploymentVersionFor(to) < deployutil.DeploymentVersionFor(from) { return fmt.Errorf("deployment %s is older than %s", to.Name, from.Name) } // Scale down any deployments which aren't the new or last deployment. for _, candidate := range deployments { // Skip the from/to deployments. if candidate.Name == to.Name { continue } if from != nil && candidate.Name == from.Name { continue } // Skip the deployment if it's already scaled down. if candidate.Spec.Replicas == 0 { continue } // Scale the deployment down to zero. retryWaitParams := kubectl.NewRetryParams(1*time.Second, 120*time.Second) if err := d.scaler.Scale(candidate.Namespace, candidate.Name, uint(0), &kubectl.ScalePrecondition{Size: -1, ResourceVersion: ""}, retryWaitParams, retryWaitParams); err != nil { fmt.Fprintf(d.errOut, "error: Couldn't scale down prior deployment %s: %v\n", deployutil.LabelForDeployment(&candidate), err) } else { fmt.Fprintf(d.out, "--> Scaled older deployment %s down\n", candidate.Name) } } if d.until == "start" { return strategy.NewConditionReachedErr("Ready to start deployment") } // Perform the deployment. if err := s.Deploy(from, to, int(desiredReplicas)); err != nil { return err } fmt.Fprintf(d.out, "--> Success\n") return nil }
// Handle syncs pod's status with any associated deployment. func (c *DeployerPodController) Handle(pod *kapi.Pod) error { // Find the deployment associated with the deployer pod. deploymentName := deployutil.DeploymentNameFor(pod) if len(deploymentName) == 0 { return nil } // Reject updates to anything but the main deployer pod // TODO: Find a way to filter this on the watch side. if pod.Name != deployutil.DeployerPodNameForDeployment(deploymentName) { return nil } deployment, err := c.deploymentClient.getDeployment(pod.Namespace, deploymentName) // If the deployment for this pod has disappeared, we should clean up this // and any other deployer pods, then bail out. if err != nil { // Some retrieval error occurred. Retry. if !kerrors.IsNotFound(err) { return fmt.Errorf("couldn't get deployment %s/%s which owns deployer pod %s/%s", pod.Namespace, deploymentName, pod.Name, pod.Namespace) } // Find all the deployer pods for the deployment (including this one). deployers, err := c.deployerPodsFor(pod.Namespace, deploymentName) if err != nil { // Retry. return fmt.Errorf("couldn't get deployer pods for %s: %v", deployutil.LabelForDeployment(deployment), err) } // Delete all deployers. for _, deployer := range deployers.Items { err := c.deletePod(deployer.Namespace, deployer.Name) if err != nil { if !kerrors.IsNotFound(err) { // TODO: Should this fire an event? glog.V(2).Infof("Couldn't delete orphaned deployer pod %s/%s: %v", deployer.Namespace, deployer.Name, err) } } else { // TODO: Should this fire an event? glog.V(2).Infof("Deleted orphaned deployer pod %s/%s", deployer.Namespace, deployer.Name) } } return nil } currentStatus := deployutil.DeploymentStatusFor(deployment) nextStatus := currentStatus switch pod.Status.Phase { case kapi.PodRunning: if !deployutil.IsTerminatedDeployment(deployment) { nextStatus = deployapi.DeploymentStatusRunning } case kapi.PodSucceeded: // Detect failure based on the container state nextStatus = deployapi.DeploymentStatusComplete for _, info := range pod.Status.ContainerStatuses { if info.State.Terminated != nil && info.State.Terminated.ExitCode != 0 { nextStatus = deployapi.DeploymentStatusFailed break } } // Sync the internal replica annotation with the target so that we can // distinguish deployer updates from other scaling events. deployment.Annotations[deployapi.DeploymentReplicasAnnotation] = deployment.Annotations[deployapi.DesiredReplicasAnnotation] if nextStatus == deployapi.DeploymentStatusComplete { delete(deployment.Annotations, deployapi.DesiredReplicasAnnotation) } case kapi.PodFailed: nextStatus = deployapi.DeploymentStatusFailed } if currentStatus != nextStatus { deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus) if _, err := c.deploymentClient.updateDeployment(deployment.Namespace, deployment); err != nil { if kerrors.IsNotFound(err) { return nil } return fmt.Errorf("couldn't update Deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err) } glog.V(4).Infof("Updated Deployment %s status from %s to %s", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus) } return nil }
// Handle syncs pod's status with any associated deployment. func (c *DeployerPodController) Handle(pod *kapi.Pod) error { // Find the deployment associated with the deployer pod. deploymentName := deployutil.DeploymentNameFor(pod) if len(deploymentName) == 0 { return nil } // Reject updates to anything but the main deployer pod // TODO: Find a way to filter this on the watch side. if pod.Name != deployutil.DeployerPodNameForDeployment(deploymentName) { return nil } deployment := &kapi.ReplicationController{ObjectMeta: kapi.ObjectMeta{Namespace: pod.Namespace, Name: deploymentName}} cached, exists, err := c.store.Get(deployment) if err == nil && exists { // Try to use the cache first. Trust hits and return them. deployment = cached.(*kapi.ReplicationController) } else { // Double-check with the master for cache misses/errors, since those // are rare and API calls are expensive but more reliable. deployment, err = c.kClient.ReplicationControllers(pod.Namespace).Get(deploymentName) } // If the deployment for this pod has disappeared, we should clean up this // and any other deployer pods, then bail out. if err != nil { // Some retrieval error occurred. Retry. if !kerrors.IsNotFound(err) { return fmt.Errorf("couldn't get deployment %s/%s which owns deployer pod %s/%s", pod.Namespace, deploymentName, pod.Name, pod.Namespace) } // Find all the deployer pods for the deployment (including this one). opts := kapi.ListOptions{LabelSelector: deployutil.DeployerPodSelector(deploymentName)} deployers, err := c.kClient.Pods(pod.Namespace).List(opts) if err != nil { // Retry. return fmt.Errorf("couldn't get deployer pods for %s: %v", deployutil.LabelForDeployment(deployment), err) } // Delete all deployers. for _, deployer := range deployers.Items { err := c.kClient.Pods(deployer.Namespace).Delete(deployer.Name, kapi.NewDeleteOptions(0)) if err != nil { if !kerrors.IsNotFound(err) { // TODO: Should this fire an event? glog.V(2).Infof("Couldn't delete orphaned deployer pod %s/%s: %v", deployer.Namespace, deployer.Name, err) } } else { // TODO: Should this fire an event? glog.V(2).Infof("Deleted orphaned deployer pod %s/%s", deployer.Namespace, deployer.Name) } } return nil } currentStatus := deployutil.DeploymentStatusFor(deployment) nextStatus := currentStatus switch pod.Status.Phase { case kapi.PodRunning: nextStatus = deployapi.DeploymentStatusRunning case kapi.PodSucceeded: nextStatus = deployapi.DeploymentStatusComplete config, decodeErr := c.decodeConfig(deployment) // If the deployment was cancelled just prior to the deployer pod succeeding // then we need to remove the cancel annotations from the complete deployment // and emit an event letting users know their cancellation failed. if deployutil.IsDeploymentCancelled(deployment) { delete(deployment.Annotations, deployapi.DeploymentCancelledAnnotation) delete(deployment.Annotations, deployapi.DeploymentStatusReasonAnnotation) if decodeErr == nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "FailedCancellation", "Deployment %q succeeded before cancel recorded", deployutil.LabelForDeployment(deployment)) } else { c.recorder.Event(deployment, kapi.EventTypeWarning, "FailedCancellation", "Succeeded before cancel recorded") } } // Sync the internal replica annotation with the target so that we can // distinguish deployer updates from other scaling events. deployment.Annotations[deployapi.DeploymentReplicasAnnotation] = deployment.Annotations[deployapi.DesiredReplicasAnnotation] if nextStatus == deployapi.DeploymentStatusComplete { delete(deployment.Annotations, deployapi.DesiredReplicasAnnotation) } // reset the size of any test container, since we are the ones updating the RC if decodeErr == nil && config.Spec.Test { deployment.Spec.Replicas = 0 } case kapi.PodFailed: nextStatus = deployapi.DeploymentStatusFailed // reset the size of any test container, since we are the ones updating the RC if config, err := c.decodeConfig(deployment); err == nil && config.Spec.Test { deployment.Spec.Replicas = 0 } } if deployutil.CanTransitionPhase(currentStatus, nextStatus) { deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus) if _, err := c.kClient.ReplicationControllers(deployment.Namespace).Update(deployment); err != nil { if kerrors.IsNotFound(err) { return nil } return fmt.Errorf("couldn't update Deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err) } glog.V(4).Infof("Updated deployment %s status from %s to %s (scale: %d)", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus, deployment.Spec.Replicas) // If the deployment was canceled, trigger a reconcilation of its deployment config // so that the latest complete deployment can immediately rollback in place of the // canceled deployment. if nextStatus == deployapi.DeploymentStatusFailed && deployutil.IsDeploymentCancelled(deployment) { // If we are unable to get the deployment config, then the deploymentconfig controller will // perform its duties once the resync interval forces the deploymentconfig to be reconciled. name := deployutil.DeploymentConfigNameFor(deployment) kclient.RetryOnConflict(kclient.DefaultRetry, func() error { config, err := c.client.DeploymentConfigs(deployment.Namespace).Get(name) if err != nil { return err } if config.Annotations == nil { config.Annotations = make(map[string]string) } config.Annotations[deployapi.DeploymentCancelledAnnotation] = strconv.Itoa(config.Status.LatestVersion) _, err = c.client.DeploymentConfigs(config.Namespace).Update(config) return err }) } } return nil }
// Get returns a streamer resource with the contents of the deployment log func (r *REST) Get(ctx kapi.Context, name string, opts runtime.Object) (runtime.Object, error) { // Ensure we have a namespace in the context namespace, ok := kapi.NamespaceFrom(ctx) if !ok { return nil, errors.NewBadRequest("namespace parameter required.") } // Validate DeploymentLogOptions deployLogOpts, ok := opts.(*deployapi.DeploymentLogOptions) if !ok { return nil, errors.NewBadRequest("did not get an expected options.") } if errs := validation.ValidateDeploymentLogOptions(deployLogOpts); len(errs) > 0 { return nil, errors.NewInvalid(deployapi.Kind("DeploymentLogOptions"), "", errs) } // Fetch deploymentConfig and check latest version; if 0, there are no deployments // for this config config, err := r.dn.DeploymentConfigs(namespace).Get(name) if err != nil { return nil, errors.NewNotFound(deployapi.Resource("deploymentconfig"), name) } desiredVersion := config.Status.LatestVersion if desiredVersion == 0 { return nil, errors.NewBadRequest(fmt.Sprintf("no deployment exists for deploymentConfig %q", config.Name)) } // Support retrieving logs for older deployments switch { case deployLogOpts.Version == nil: // Latest or previous if deployLogOpts.Previous { desiredVersion-- if desiredVersion < 1 { return nil, errors.NewBadRequest(fmt.Sprintf("no previous deployment exists for deploymentConfig %q", config.Name)) } } case *deployLogOpts.Version <= 0 || *deployLogOpts.Version > config.Status.LatestVersion: // Invalid version return nil, errors.NewBadRequest(fmt.Sprintf("invalid version for deploymentConfig %q: %d", config.Name, *deployLogOpts.Version)) default: desiredVersion = *deployLogOpts.Version } // Get desired deployment targetName := deployutil.DeploymentNameForConfigVersion(config.Name, desiredVersion) target, err := r.waitForExistingDeployment(namespace, targetName) if err != nil { return nil, err } podName := deployutil.DeployerPodNameForDeployment(target.Name) // Check for deployment status; if it is new or pending, we will wait for it. If it is complete, // the deployment completed successfully and the deployer pod will be deleted so we will return a // success message. If it is running or failed, retrieve the log from the deployer pod. status := deployutil.DeploymentStatusFor(target) switch status { case deployapi.DeploymentStatusNew, deployapi.DeploymentStatusPending: if deployLogOpts.NoWait { glog.V(4).Infof("Deployment %s is in %s state. No logs to retrieve yet.", deployutil.LabelForDeployment(target), status) return &genericrest.LocationStreamer{}, nil } glog.V(4).Infof("Deployment %s is in %s state, waiting for it to start...", deployutil.LabelForDeployment(target), status) if err := deployutil.WaitForRunningDeployerPod(r.pn, target, r.timeout); err != nil { return nil, errors.NewBadRequest(fmt.Sprintf("failed to run deployer pod %s: %v", podName, err)) } latest, ok, err := registry.WaitForRunningDeployment(r.rn, target, r.timeout) if err != nil { return nil, errors.NewBadRequest(fmt.Sprintf("unable to wait for deployment %s to run: %v", deployutil.LabelForDeployment(target), err)) } if !ok { return nil, errors.NewServerTimeout(kapi.Resource("ReplicationController"), "get", 2) } if deployutil.DeploymentStatusFor(latest) == deployapi.DeploymentStatusComplete { podName, err = r.returnApplicationPodName(target) if err != nil { return nil, err } } case deployapi.DeploymentStatusComplete: podName, err = r.returnApplicationPodName(target) if err != nil { return nil, err } } logOpts := deployapi.DeploymentToPodLogOptions(deployLogOpts) location, transport, err := pod.LogLocation(&podGetter{r.pn}, r.connInfo, ctx, podName, logOpts) if err != nil { return nil, errors.NewBadRequest(err.Error()) } return &genericrest.LocationStreamer{ Location: location, Transport: transport, ContentType: "text/plain", Flush: deployLogOpts.Follow, ResponseChecker: genericrest.NewGenericHttpResponseChecker(kapi.Resource("pod"), podName), }, nil }
// Deploy starts the deployment process for deploymentName. func (d *Deployer) Deploy(namespace, deploymentName string) error { // Look up the new deployment. to, err := d.getDeployment(namespace, deploymentName) if err != nil { return fmt.Errorf("couldn't get deployment %s/%s: %v", namespace, deploymentName, err) } // Decode the config from the deployment. config, err := deployutil.DecodeDeploymentConfig(to, latest.Codec) if err != nil { return fmt.Errorf("couldn't decode deployment config from deployment %s/%s: %v", to.Namespace, to.Name, err) } // Get a strategy for the deployment. strategy, err := d.strategyFor(config) if err != nil { return err } // New deployments must have a desired replica count. desiredReplicas, hasDesired := deployutil.DeploymentDesiredReplicas(to) if !hasDesired { return fmt.Errorf("deployment %s has no desired replica count", deployutil.LabelForDeployment(to)) } // Find all deployments for the config. unsortedDeployments, err := d.getDeployments(namespace, config.Name) if err != nil { return fmt.Errorf("couldn't get controllers in namespace %s: %v", namespace, err) } deployments := unsortedDeployments.Items // Sort all the deployments by version. sort.Sort(deployutil.ByLatestVersionDesc(deployments)) // Find any last completed deployment. var from *kapi.ReplicationController for _, candidate := range deployments { if candidate.Name == to.Name { continue } if deployutil.DeploymentStatusFor(&candidate) == deployapi.DeploymentStatusComplete { from = &candidate break } } // Scale down any deployments which aren't the new or last deployment. for _, candidate := range deployments { // Skip the from/to deployments. if candidate.Name == to.Name { continue } if from != nil && candidate.Name == from.Name { continue } // Skip the deployment if it's already scaled down. if candidate.Spec.Replicas == 0 { continue } // Scale the deployment down to zero. retryWaitParams := kubectl.NewRetryParams(1*time.Second, 120*time.Second) if err := d.scaler.Scale(candidate.Namespace, candidate.Name, uint(0), &kubectl.ScalePrecondition{Size: -1, ResourceVersion: ""}, retryWaitParams, retryWaitParams); err != nil { glog.Errorf("Couldn't scale down prior deployment %s: %v", deployutil.LabelForDeployment(&candidate), err) } else { glog.Infof("Scaled down prior deployment %s", deployutil.LabelForDeployment(&candidate)) } } // Perform the deployment. if from == nil { glog.Infof("Deploying %s for the first time (replicas: %d)", deployutil.LabelForDeployment(to), desiredReplicas) } else { glog.Infof("Deploying from %s to %s (replicas: %d)", deployutil.LabelForDeployment(from), deployutil.LabelForDeployment(to), desiredReplicas) } return strategy.Deploy(from, to, desiredReplicas) }
func (s *RollingDeploymentStrategy) Deploy(from *kapi.ReplicationController, to *kapi.ReplicationController, desiredReplicas int) error { config, err := deployutil.DecodeDeploymentConfig(to, s.decoder) if err != nil { return fmt.Errorf("couldn't decode DeploymentConfig from deployment %s: %v", deployutil.LabelForDeployment(to), err) } params := config.Spec.Strategy.RollingParams updateAcceptor := s.getUpdateAcceptor(time.Duration(*params.TimeoutSeconds)*time.Second, config.Spec.MinReadySeconds) // If there's no prior deployment, delegate to another strategy since the // rolling updater only supports transitioning between two deployments. // // Hook support is duplicated here for now. When the rolling updater can // handle initial deployments, all of this code can go away. if from == nil { // Execute any pre-hook. if params.Pre != nil { if err := s.hookExecutor.Execute(params.Pre, to, deployapi.PreHookPodSuffix, "pre"); err != nil { return fmt.Errorf("Pre hook failed: %s", err) } } // Execute the delegate strategy. err := s.initialStrategy.DeployWithAcceptor(from, to, desiredReplicas, updateAcceptor) if err != nil { return err } // Execute any post-hook. Errors are logged and ignored. if params.Post != nil { if err := s.hookExecutor.Execute(params.Post, to, deployapi.PostHookPodSuffix, "post"); err != nil { return fmt.Errorf("post hook failed: %s", err) } } // All done. return nil } // Record all warnings defer stratutil.RecordConfigWarnings(s.eventClient, from, s.decoder, s.out) defer stratutil.RecordConfigWarnings(s.eventClient, to, s.decoder, s.out) // Prepare for a rolling update. // Execute any pre-hook. if params.Pre != nil { if err := s.hookExecutor.Execute(params.Pre, to, deployapi.PreHookPodSuffix, "pre"); err != nil { return fmt.Errorf("pre hook failed: %s", err) } } if s.until == "pre" { return strat.NewConditionReachedErr("pre hook succeeded") } if s.until == "0%" { return strat.NewConditionReachedErr("Reached 0% (before rollout)") } // HACK: Assign the source ID annotation that the rolling updater expects, // unless it already exists on the deployment. // // Related upstream issue: // https://github.com/kubernetes/kubernetes/pull/7183 err = wait.Poll(s.apiRetryPeriod, s.apiRetryTimeout, func() (done bool, err error) { existing, err := s.rcClient.ReplicationControllers(to.Namespace).Get(to.Name) if err != nil { msg := fmt.Sprintf("couldn't look up deployment %s: %s", to.Name, err) if kerrors.IsNotFound(err) { return false, fmt.Errorf("%s", msg) } // Try again. fmt.Fprintln(s.errOut, "error:", msg) return false, nil } if _, hasSourceId := existing.Annotations[sourceIdAnnotation]; !hasSourceId { existing.Annotations[sourceIdAnnotation] = fmt.Sprintf("%s:%s", from.Name, from.ObjectMeta.UID) if _, err := s.rcClient.ReplicationControllers(existing.Namespace).Update(existing); err != nil { msg := fmt.Sprintf("couldn't assign source annotation to deployment %s: %v", existing.Name, err) if kerrors.IsNotFound(err) { return false, fmt.Errorf("%s", msg) } // Try again. fmt.Fprintln(s.errOut, "error:", msg) return false, nil } } return true, nil }) if err != nil { return err } to, err = s.rcClient.ReplicationControllers(to.Namespace).Get(to.Name) if err != nil { return err } // HACK: There's a validation in the rolling updater which assumes that when // an existing RC is supplied, it will have >0 replicas- a validation which // is then disregarded as the desired count is obtained from the annotation // on the RC. For now, fake it out by just setting replicas to 1. // // Related upstream issue: // https://github.com/kubernetes/kubernetes/pull/7183 to.Spec.Replicas = 1 // Perform a rolling update. rollingConfig := &kubectl.RollingUpdaterConfig{ Out: &rollingUpdaterWriter{w: s.out}, OldRc: from, NewRc: to, UpdatePeriod: time.Duration(*params.UpdatePeriodSeconds) * time.Second, Interval: time.Duration(*params.IntervalSeconds) * time.Second, Timeout: time.Duration(*params.TimeoutSeconds) * time.Second, MinReadySeconds: config.Spec.MinReadySeconds, CleanupPolicy: kubectl.PreserveRollingUpdateCleanupPolicy, MaxSurge: params.MaxSurge, MaxUnavailable: params.MaxUnavailable, OnProgress: func(oldRc, newRc *kapi.ReplicationController, percentage int) error { if expect, ok := strat.Percentage(s.until); ok && percentage >= expect { return strat.NewConditionReachedErr(fmt.Sprintf("Reached %s (currently %d%%)", s.until, percentage)) } return nil }, } if err := s.rollingUpdate(rollingConfig); err != nil { return err } // Execute any post-hook. if params.Post != nil { if err := s.hookExecutor.Execute(params.Post, to, deployapi.PostHookPodSuffix, "post"); err != nil { return fmt.Errorf("post hook failed: %s", err) } } return nil }
// reconcileDeployments reconciles existing deployment replica counts which // could have diverged outside the deployment process (e.g. due to auto or // manual scaling, or partial deployments). The active deployment is the last // successful deployment, not necessarily the latest in terms of the config // version. The active deployment replica count should follow the config, and // all other deployments should be scaled to zero. // // Previously, scaling behavior was that the config replica count was used // only for initial deployments and the active deployment had to be scaled up // directly. To continue supporting that old behavior we must detect when the // deployment has been directly manipulated, and if so, preserve the directly // updated value and sync the config with the deployment. func (c *DeploymentConfigController) reconcileDeployments(existingDeployments *kapi.ReplicationControllerList, config *deployapi.DeploymentConfig) error { latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments) if !latestIsDeployed { // We shouldn't be reconciling if the latest deployment hasn't been // created; this is enforced on the calling side, but double checking // can't hurt. return nil } activeDeployment := deployutil.ActiveDeployment(config, existingDeployments) // Compute the replica count for the active deployment (even if the active // deployment doesn't exist). The active replica count is the value that // should be assigned to the config, to allow the replica propagation to // flow downward from the config. // // By default we'll assume the config replicas should be used to update the // active deployment except in special cases (like first sync or externally // updated deployments.) activeReplicas := config.Spec.Replicas source := "the deploymentConfig itself (no change)" activeDeploymentExists := activeDeployment != nil activeDeploymentIsLatest := activeDeploymentExists && activeDeployment.Name == latestDeployment.Name latestDesiredReplicas, latestHasDesiredReplicas := deployutil.DeploymentDesiredReplicas(latestDeployment) switch { case activeDeploymentExists && activeDeploymentIsLatest: // The active/latest deployment follows the config unless this is its first // sync or if an external change to the deployment replicas is detected. lastActiveReplicas, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment) if !hasLastActiveReplicas || lastActiveReplicas != activeDeployment.Spec.Replicas { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the latest/active deployment %q which was scaled directly or has not previously been synced", deployutil.LabelForDeployment(activeDeployment)) } case activeDeploymentExists && !activeDeploymentIsLatest: // The active/non-latest deployment follows the config if it was // previously synced; if this is the first sync, infer what the config // value should be based on either the latest desired or whatever the // deployment is currently scaled to. _, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment) if hasLastActiveReplicas { break } if latestHasDesiredReplicas { activeReplicas = latestDesiredReplicas source = fmt.Sprintf("the desired replicas of latest deployment %q which has not been previously synced", deployutil.LabelForDeployment(latestDeployment)) } else if activeDeployment.Spec.Replicas > 0 { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the active deployment %q which has not been previously synced", deployutil.LabelForDeployment(activeDeployment)) } case !activeDeploymentExists && latestHasDesiredReplicas: // If there's no active deployment, use the latest desired, if available. activeReplicas = latestDesiredReplicas source = fmt.Sprintf("the desired replicas of latest deployment %q with no active deployment", deployutil.LabelForDeployment(latestDeployment)) } // Bring the config in sync with the deployment. Once we know the config // accurately represents the desired replica count of the active deployment, // we can safely reconcile deployments. // // If the deployment config is test, never update the deployment config based // on deployments, since test behavior overrides user scaling. switch { case config.Spec.Replicas == activeReplicas: case config.Spec.Test: glog.V(4).Infof("Detected changed replicas for test deploymentConfig %q, ignoring that change", deployutil.LabelForDeploymentConfig(config)) default: oldReplicas := config.Spec.Replicas config.Spec.Replicas = activeReplicas var err error config, err = c.osClient.DeploymentConfigs(config.Namespace).Update(config) if err != nil { return err } glog.V(4).Infof("Synced deploymentConfig %q replicas from %d to %d based on %s", deployutil.LabelForDeploymentConfig(config), oldReplicas, activeReplicas, source) } // Reconcile deployments. The active deployment follows the config, and all // other deployments should be scaled to zero. for _, deployment := range existingDeployments.Items { isActiveDeployment := activeDeployment != nil && deployment.Name == activeDeployment.Name oldReplicaCount := deployment.Spec.Replicas newReplicaCount := 0 if isActiveDeployment { newReplicaCount = activeReplicas } if config.Spec.Test { glog.V(4).Infof("Deployment config %q is test and deployment %q will be scaled down", deployutil.LabelForDeploymentConfig(config), deployutil.LabelForDeployment(&deployment)) newReplicaCount = 0 } lastReplicas, hasLastReplicas := deployutil.DeploymentReplicas(&deployment) // Only update if necessary. if !hasLastReplicas || newReplicaCount != oldReplicaCount || lastReplicas != newReplicaCount { deployment.Spec.Replicas = newReplicaCount deployment.Annotations[deployapi.DeploymentReplicasAnnotation] = strconv.Itoa(newReplicaCount) _, err := c.kubeClient.ReplicationControllers(deployment.Namespace).Update(&deployment) if err != nil { c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentScaleFailed", "Failed to scale deployment %q from %d to %d: %s", deployment.Name, oldReplicaCount, newReplicaCount, err) return err } // Only report scaling events if we changed the replica count. if oldReplicaCount != newReplicaCount { c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentScaled", "Scaled deployment %q from %d to %d", deployment.Name, oldReplicaCount, newReplicaCount) } else { glog.V(4).Infof("Updated deployment %q replica annotation to match current replica count %d", deployutil.LabelForDeployment(&deployment), newReplicaCount) } } } return c.updateStatus(config) }
// Handle processes config and creates a new deployment if necessary. func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error { // Inspect a deployment configuration every time the controller reconciles it details, existingDeployments, latestDeploymentExists, err := c.findDetails(config) if err != nil { return err } config, err = c.updateDetails(config, details) if err != nil { return transientError(err.Error()) } // Only deploy when the version has advanced past 0. if config.LatestVersion == 0 { glog.V(5).Infof("Waiting for first version of %s", deployutil.LabelForDeploymentConfig(config)) return nil } var inflightDeployment *kapi.ReplicationController for _, deployment := range existingDeployments.Items { deploymentStatus := deployutil.DeploymentStatusFor(&deployment) switch deploymentStatus { case deployapi.DeploymentStatusFailed, deployapi.DeploymentStatusComplete: // Previous deployment in terminal state - can ignore // Ignoring specific deployment states so that any newly introduced // deployment state will not be ignored default: if inflightDeployment == nil { inflightDeployment = &deployment continue } var deploymentForCancellation *kapi.ReplicationController if deployutil.DeploymentVersionFor(inflightDeployment) < deployutil.DeploymentVersionFor(&deployment) { deploymentForCancellation, inflightDeployment = inflightDeployment, &deployment } else { deploymentForCancellation = &deployment } deploymentForCancellation.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue deploymentForCancellation.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists if _, err := c.deploymentClient.updateDeployment(deploymentForCancellation.Namespace, deploymentForCancellation); err != nil { util.HandleError(fmt.Errorf("couldn't cancel deployment %s: %v", deployutil.LabelForDeployment(deploymentForCancellation), err)) } glog.V(4).Infof("Cancelled deployment %s for deployment config %s", deployutil.LabelForDeployment(deploymentForCancellation), deployutil.LabelForDeploymentConfig(config)) } } // if the latest deployment exists then nothing else needs to be done if latestDeploymentExists { return nil } // check to see if there are inflight deployments if inflightDeployment != nil { // raise a transientError so that the deployment config can be re-queued glog.V(4).Infof("Found previous inflight deployment for %s - will requeue", deployutil.LabelForDeploymentConfig(config)) return transientError(fmt.Sprintf("found previous inflight deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config))) } // Try and build a deployment for the config. deployment, err := c.makeDeployment(config) if err != nil { return fatalError(fmt.Sprintf("couldn't make deployment from (potentially invalid) deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err)) } // Compute the desired replicas for the deployment. Use the last completed // deployment's current replica count, or the config template if there is no // prior completed deployment available. desiredReplicas := config.Template.ControllerTemplate.Replicas if len(existingDeployments.Items) > 0 { sort.Sort(deployutil.ByLatestVersionDesc(existingDeployments.Items)) for _, existing := range existingDeployments.Items { if deployutil.DeploymentStatusFor(&existing) == deployapi.DeploymentStatusComplete { desiredReplicas = existing.Spec.Replicas glog.V(4).Infof("Desired replicas for %s set to %d based on prior completed deployment %s", deployutil.LabelForDeploymentConfig(config), desiredReplicas, existing.Name) break } } } deployment.Annotations[deployapi.DesiredReplicasAnnotation] = strconv.Itoa(desiredReplicas) // Create the deployment. if _, err := c.deploymentClient.createDeployment(config.Namespace, deployment); err == nil { glog.V(4).Infof("Created deployment for deployment config %s", deployutil.LabelForDeploymentConfig(config)) return nil } else { // If the deployment was already created, just move on. The cache could be stale, or another // process could have already handled this update. if errors.IsAlreadyExists(err) { glog.V(4).Infof("Deployment already exists for deployment config %s", deployutil.LabelForDeploymentConfig(config)) return nil } glog.Warningf("Cannot create latest deployment for deployment config %q: %v", deployutil.LabelForDeploymentConfig(config), err) return fmt.Errorf("couldn't create deployment for deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err) } }
// Handle processes change triggers for config. func (c *DeploymentConfigChangeController) Handle(config *deployapi.DeploymentConfig) error { if !deployutil.HasChangeTrigger(config) { glog.V(5).Infof("Ignoring DeploymentConfig %s; no change triggers detected", deployutil.LabelForDeploymentConfig(config)) return nil } if config.LatestVersion == 0 { _, _, err := c.generateDeployment(config) if err != nil { if kerrors.IsConflict(err) { return fatalError(fmt.Sprintf("DeploymentConfig %s updated since retrieval; aborting trigger: %v", deployutil.LabelForDeploymentConfig(config), err)) } glog.V(4).Infof("Couldn't create initial deployment for deploymentConfig %q: %v", deployutil.LabelForDeploymentConfig(config), err) return nil } glog.V(4).Infof("Created initial deployment for deploymentConfig %q", deployutil.LabelForDeploymentConfig(config)) return nil } latestDeploymentName := deployutil.LatestDeploymentNameForConfig(config) deployment, err := c.changeStrategy.getDeployment(config.Namespace, latestDeploymentName) if err != nil { // If there's no deployment for the latest config, we have no basis of // comparison. It's the responsibility of the deployment config controller // to make the deployment for the config, so return early. if kerrors.IsNotFound(err) { glog.V(5).Infof("Ignoring change for DeploymentConfig %s; no existing Deployment found", deployutil.LabelForDeploymentConfig(config)) return nil } return fmt.Errorf("couldn't retrieve Deployment for DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err) } deployedConfig, err := c.decodeConfig(deployment) if err != nil { return fatalError(fmt.Sprintf("error decoding DeploymentConfig from Deployment %s for DeploymentConfig %s: %v", deployutil.LabelForDeployment(deployment), deployutil.LabelForDeploymentConfig(config), err)) } // Detect template diffs, and return early if there aren't any changes. if kapi.Semantic.DeepEqual(config.Template.ControllerTemplate.Template, deployedConfig.Template.ControllerTemplate.Template) { glog.V(5).Infof("Ignoring DeploymentConfig change for %s (latestVersion=%d); same as Deployment %s", deployutil.LabelForDeploymentConfig(config), config.LatestVersion, deployutil.LabelForDeployment(deployment)) return nil } // There was a template diff, so generate a new config version. fromVersion, toVersion, err := c.generateDeployment(config) if err != nil { if kerrors.IsConflict(err) { return fatalError(fmt.Sprintf("DeploymentConfig %s updated since retrieval; aborting trigger: %v", deployutil.LabelForDeploymentConfig(config), err)) } return fmt.Errorf("couldn't generate deployment for DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err) } glog.V(4).Infof("Updated DeploymentConfig %s from version %d to %d for existing deployment %s", deployutil.LabelForDeploymentConfig(config), fromVersion, toVersion, deployutil.LabelForDeployment(deployment)) return nil }
// reconcileDeployments reconciles existing deployment replica counts which // could have diverged outside the deployment process (e.g. due to auto or // manual scaling, or partial deployments). The active deployment is the last // successful deployment, not necessarily the latest in terms of the config // version. The active deployment replica count should follow the config, and // all other deployments should be scaled to zero. // // Previously, scaling behavior was that the config replica count was used // only for initial deployments and the active deployment had to be scaled up // directly. To continue supporting that old behavior we must detect when the // deployment has been directly manipulated, and if so, preserve the directly // updated value and sync the config with the deployment. func (c *DeploymentConfigController) reconcileDeployments(existingDeployments *kapi.ReplicationControllerList, config *deployapi.DeploymentConfig) error { latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments) if !latestIsDeployed { // We shouldn't be reconciling if the latest deployment hasn't been // created; this is enforced on the calling side, but double checking // can't hurt. return nil } activeDeployment := deployutil.ActiveDeployment(config, existingDeployments) // Compute the replica count for the active deployment (even if the active // deployment doesn't exist). The active replica count is the value that // should be assigned to the config, to allow the replica propagation to // flow downward from the config. // // This takes into account resources predating the propagation behavior // change, as well as external modifications to the deployments (e.g. // scalers). activeReplicas := config.Template.ControllerTemplate.Replicas source := "the deploymentConfig itself (no change)" if activeDeployment != nil { activeDeploymentIsLatest := activeDeployment.Name == latestDeployment.Name lastActiveReplicas, hasLastActiveReplicas := deployutil.DeploymentReplicas(activeDeployment) if activeDeploymentIsLatest { if !hasLastActiveReplicas || lastActiveReplicas != activeDeployment.Spec.Replicas { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the latest/active deployment %q which was scaled directly or has not previously been synced", deployutil.LabelForDeployment(activeDeployment)) } } else { if hasLastActiveReplicas { if lastActiveReplicas != activeDeployment.Spec.Replicas && activeDeployment.Spec.Replicas > 0 { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the active deployment %q which was scaled directly", deployutil.LabelForDeployment(activeDeployment)) } } else { if activeDeployment.Spec.Replicas > 0 { activeReplicas = activeDeployment.Spec.Replicas source = fmt.Sprintf("the active deployment %q which has not been previously synced", deployutil.LabelForDeployment(activeDeployment)) } else { latestDesiredReplicas, latestHasDesiredReplicas := deployutil.DeploymentDesiredReplicas(latestDeployment) if latestHasDesiredReplicas { activeReplicas = latestDesiredReplicas source = fmt.Sprintf("the desired replicas of latest deployment %q which has not been previously synced", deployutil.LabelForDeployment(latestDeployment)) } } } } } else { latestDesiredReplicas, latestHasDesiredReplicas := deployutil.DeploymentDesiredReplicas(latestDeployment) if latestHasDesiredReplicas { activeReplicas = latestDesiredReplicas source = fmt.Sprintf("the desired replicas of latest deployment %q with no active deployment", deployutil.LabelForDeployment(latestDeployment)) } } // Bring the config in sync with the deployment. Once we know the config // accurately represents the desired replica count of the active deployment, // we can safely reconcile deployments. if config.Template.ControllerTemplate.Replicas != activeReplicas { oldReplicas := config.Template.ControllerTemplate.Replicas config.Template.ControllerTemplate.Replicas = activeReplicas _, err := c.osClient.DeploymentConfigs(config.Namespace).Update(config) if err != nil { return err } glog.V(4).Infof("Synced deploymentConfig %q replicas from %d to %d based on %s", deployutil.LabelForDeploymentConfig(config), oldReplicas, activeReplicas, source) } // Reconcile deployments. The active deployment follows the config, and all // other deployments should be scaled to zero. for _, deployment := range existingDeployments.Items { isActiveDeployment := activeDeployment != nil && deployment.Name == activeDeployment.Name oldReplicaCount := deployment.Spec.Replicas newReplicaCount := 0 if isActiveDeployment { newReplicaCount = activeReplicas } lastReplicas, hasLastReplicas := deployutil.DeploymentReplicas(&deployment) // Only update if necessary. if !hasLastReplicas || newReplicaCount != oldReplicaCount || lastReplicas != newReplicaCount { deployment.Spec.Replicas = newReplicaCount deployment.Annotations[deployapi.DeploymentReplicasAnnotation] = strconv.Itoa(newReplicaCount) _, err := c.kubeClient.ReplicationControllers(deployment.Namespace).Update(&deployment) if err != nil { c.recorder.Eventf(config, "DeploymentScaleFailed", "Failed to scale deployment %q from %d to %d: %s", deployment.Name, oldReplicaCount, newReplicaCount, err) return err } // Only report scaling events if we changed the replica count. if oldReplicaCount != newReplicaCount { c.recorder.Eventf(config, "DeploymentScaled", "Scaled deployment %q from %d to %d", deployment.Name, oldReplicaCount, newReplicaCount) } else { glog.V(4).Infof("Updated deployment %q replica annotation to match current replica count %d", deployutil.LabelForDeployment(&deployment), newReplicaCount) } } } return nil }