func (c *DeployerPodController) cleanupFailedDeployment(deployment *kapi.ReplicationController) error { // Scale down the current failed deployment configName := deployutil.DeploymentConfigNameFor(deployment) existingDeployments, err := c.deploymentClient.listDeploymentsForConfig(deployment.Namespace, configName) if err != nil { return fmt.Errorf("couldn't list Deployments for DeploymentConfig %s: %v", configName, err) } desiredReplicas, ok := deployutil.DeploymentDesiredReplicas(deployment) if !ok { // if desired replicas could not be found, then log the error // and update the failed deployment // this cannot be treated as a transient error kutil.HandleError(fmt.Errorf("Could not determine desired replicas from %s to reset replicas for last completed deployment", deployutil.LabelForDeployment(deployment))) } if ok && len(existingDeployments.Items) > 0 { sort.Sort(deployutil.DeploymentsByLatestVersionDesc(existingDeployments.Items)) for index, existing := range existingDeployments.Items { // if a newer deployment exists: // - set the replicas for the current failed deployment to 0 // - there is no point in scaling up the last completed deployment // since that will be scaled down by the later deployment if index == 0 && existing.Name != deployment.Name { break } // the latest completed deployment is the one that needs to be scaled back up if deployutil.DeploymentStatusFor(&existing) == deployapi.DeploymentStatusComplete { if existing.Spec.Replicas == desiredReplicas { break } // scale back the completed deployment to the target of the failed deployment existing.Spec.Replicas = desiredReplicas if _, err := c.deploymentClient.updateDeployment(existing.Namespace, &existing); err != nil { if kerrors.IsNotFound(err) { return nil } return fmt.Errorf("couldn't update replicas to %d for deployment %s: %v", desiredReplicas, deployutil.LabelForDeployment(&existing), err) } glog.V(4).Infof("Updated replicas to %d for deployment %s", desiredReplicas, deployutil.LabelForDeployment(&existing)) break } } } // set the replicas for the failed deployment to 0 // and set the status to Failed deployment.Spec.Replicas = 0 deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusFailed) if _, err := c.deploymentClient.updateDeployment(deployment.Namespace, deployment); err != nil { if kerrors.IsNotFound(err) { return nil } return fmt.Errorf("couldn't scale down the deployment %s and mark it as failed: %v", deployutil.LabelForDeployment(deployment), err) } glog.V(4).Infof("Scaled down the deployment %s and marked it as failed", deployutil.LabelForDeployment(deployment)) return nil }
// Deploy starts the deployment process for deploymentName. func (d *Deployer) Deploy(namespace, deploymentName string) error { // Look up the new deployment. to, err := d.getDeployment(namespace, deploymentName) if err != nil { return fmt.Errorf("couldn't get deployment %s/%s: %v", namespace, deploymentName, err) } // Decode the config from the deployment. config, err := deployutil.DecodeDeploymentConfig(to, latest.Codec) if err != nil { return fmt.Errorf("couldn't decode DeploymentConfig from deployment %s/%s: %v", to.Namespace, to.Name, err) } // Get a strategy for the deployment. strategy, err := d.strategyFor(config) if err != nil { return err } // New deployments must have a desired replica count. desiredReplicas, hasDesired := deployutil.DeploymentDesiredReplicas(to) if !hasDesired { return fmt.Errorf("deployment %s has no desired replica count", deployutil.LabelForDeployment(to)) } // Find all deployments for the config. unsortedDeployments, err := d.getDeployments(namespace, config.Name) if err != nil { return fmt.Errorf("couldn't get controllers in namespace %s: %v", namespace, err) } deployments := unsortedDeployments.Items // Sort all the deployments by version. sort.Sort(deployutil.DeploymentsByLatestVersionDesc(deployments)) // Find any last completed deployment. var from *kapi.ReplicationController for _, candidate := range deployments { if candidate.Name == to.Name { continue } if deployutil.DeploymentStatusFor(&candidate) == deployapi.DeploymentStatusComplete { from = &candidate break } } // Scale down any deployments which aren't the new or last deployment. for _, candidate := range deployments { // Skip the from/to deployments. if candidate.Name == to.Name { continue } if from != nil && candidate.Name == from.Name { continue } // Skip the deployment if it's already scaled down. if candidate.Spec.Replicas == 0 { continue } // Scale the deployment down to zero. retryWaitParams := kubectl.NewRetryParams(1*time.Second, 120*time.Second) if err := d.scaler.Scale(candidate.Namespace, candidate.Name, uint(0), &kubectl.ScalePrecondition{-1, ""}, retryWaitParams, retryWaitParams); err != nil { glog.Errorf("Couldn't scale down prior deployment %s: %v", deployutil.LabelForDeployment(&candidate), err) } else { glog.Infof("Scaled down prior deployment %s", deployutil.LabelForDeployment(&candidate)) } } // Perform the deployment. if from == nil { glog.Infof("Deploying %s for the first time (replicas: %d)", deployutil.LabelForDeployment(to), desiredReplicas) } else { glog.Infof("Deploying from %s to %s (replicas: %d)", deployutil.LabelForDeployment(from), deployutil.LabelForDeployment(to), desiredReplicas) } return strategy.Deploy(from, to, desiredReplicas) }
// Handle processes config and creates a new deployment if necessary. func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error { // Only deploy when the version has advanced past 0. if config.LatestVersion == 0 { glog.V(5).Infof("Waiting for first version of %s", deployutil.LabelForDeploymentConfig(config)) return nil } // Check if any existing inflight deployments (any non-terminal state). existingDeployments, err := c.deploymentClient.listDeploymentsForConfig(config.Namespace, config.Name) if err != nil { return fmt.Errorf("couldn't list Deployments for DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err) } var inflightDeployment *kapi.ReplicationController latestDeploymentExists := false for _, deployment := range existingDeployments.Items { // check if this is the latest deployment // we'll return after we've dealt with the multiple-active-deployments case if deployutil.DeploymentVersionFor(&deployment) == config.LatestVersion { latestDeploymentExists = true } deploymentStatus := deployutil.DeploymentStatusFor(&deployment) switch deploymentStatus { case deployapi.DeploymentStatusFailed, deployapi.DeploymentStatusComplete: // Previous deployment in terminal state - can ignore // Ignoring specific deployment states so that any newly introduced // deployment state will not be ignored default: if inflightDeployment == nil { inflightDeployment = &deployment continue } var deploymentForCancellation *kapi.ReplicationController if deployutil.DeploymentVersionFor(inflightDeployment) < deployutil.DeploymentVersionFor(&deployment) { deploymentForCancellation, inflightDeployment = inflightDeployment, &deployment } else { deploymentForCancellation = &deployment } deploymentForCancellation.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue deploymentForCancellation.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists if _, err := c.deploymentClient.updateDeployment(deploymentForCancellation.Namespace, deploymentForCancellation); err != nil { util.HandleError(fmt.Errorf("couldn't cancel Deployment %s: %v", deployutil.LabelForDeployment(deploymentForCancellation), err)) } glog.V(4).Infof("Cancelled Deployment %s for DeploymentConfig %s", deployutil.LabelForDeployment(deploymentForCancellation), deployutil.LabelForDeploymentConfig(config)) } } // if the latest deployment exists then nothing else needs to be done if latestDeploymentExists { return nil } // check to see if there are inflight deployments if inflightDeployment != nil { // raise a transientError so that the deployment config can be re-queued glog.V(4).Infof("Found previous inflight Deployment for %s - will requeue", deployutil.LabelForDeploymentConfig(config)) return transientError(fmt.Sprintf("found previous inflight Deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config))) } // Try and build a deployment for the config. deployment, err := c.makeDeployment(config) if err != nil { return fatalError(fmt.Sprintf("couldn't make Deployment from (potentially invalid) DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err)) } // Compute the desired replicas for the deployment. Use the last completed // deployment's current replica count, or the config template if there is no // prior completed deployment available. desiredReplicas := config.Template.ControllerTemplate.Replicas if len(existingDeployments.Items) > 0 { sort.Sort(deployutil.DeploymentsByLatestVersionDesc(existingDeployments.Items)) for _, existing := range existingDeployments.Items { if deployutil.DeploymentStatusFor(&existing) == deployapi.DeploymentStatusComplete { desiredReplicas = existing.Spec.Replicas glog.V(4).Infof("Desired replicas for %s set to %d based on prior completed deployment %s", deployutil.LabelForDeploymentConfig(config), desiredReplicas, existing.Name) break } } } deployment.Annotations[deployapi.DesiredReplicasAnnotation] = strconv.Itoa(desiredReplicas) // Create the deployment. if _, err := c.deploymentClient.createDeployment(config.Namespace, deployment); err == nil { glog.V(4).Infof("Created Deployment for DeploymentConfig %s", deployutil.LabelForDeploymentConfig(config)) return nil } else { // If the deployment was already created, just move on. The cache could be stale, or another // process could have already handled this update. if errors.IsAlreadyExists(err) { glog.V(4).Infof("Deployment already exists for DeploymentConfig %s", deployutil.LabelForDeploymentConfig(config)) return nil } // log an event if the deployment could not be created that the user can discover c.recorder.Eventf(config, "failedCreate", "Error creating: %v", err) return fmt.Errorf("couldn't create Deployment for DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err) } }