// findTargetDeployment finds the deployment which is the rollback target by
// searching for deployments associated with config. If desiredVersion is >0,
// the deployment matching desiredVersion will be returned. If desiredVersion
// is <=0, the last completed deployment which is older than the config's
// version will be returned.
func (o *RollbackOptions) findTargetDeployment(config *deployapi.DeploymentConfig, desiredVersion int) (*kapi.ReplicationController, error) {
// Find deployments for the config sorted by version descending.
deployments, err := o.kc.ReplicationControllers(config.Namespace).List(deployutil.ConfigSelector(config.Name))
if err != nil {
return nil, err
}
sort.Sort(deployutil.DeploymentsByLatestVersionDesc(deployments.Items))
// Find the target deployment for rollback. If a version was specified,
// use the version for a search. Otherwise, use the last completed
// deployment.
var target *kapi.ReplicationController
for _, deployment := range deployments.Items {
version := deployutil.DeploymentVersionFor(&deployment)
if desiredVersion > 0 {
if version == desiredVersion {
target = &deployment
break
}
} else {
if version < config.LatestVersion && deployutil.DeploymentStatusFor(&deployment) == deployapi.DeploymentStatusComplete {
target = &deployment
break
}
}
}
if target == nil {
return nil, fmt.Errorf("couldn't find deployment for rollback")
}
return target, nil
}
// cancel cancels any deployment process in progress for config.
func (o *DeployOptions) cancel(config *deployapi.DeploymentConfig, out io.Writer) error {
deployments, err := o.kubeClient.ReplicationControllers(config.Namespace).List(deployutil.ConfigSelector(config.Name))
if err != nil {
return err
}
if len(deployments.Items) == 0 {
fmt.Fprintf(out, "There have been no deployments for %s/%s\n", config.Namespace, config.Name)
return nil
}
sort.Sort(deployutil.DeploymentsByLatestVersionDesc(deployments.Items))
failedCancellations := []string{}
anyCancelled := false
for _, deployment := range deployments.Items {
status := deployutil.DeploymentStatusFor(&deployment)
switch status {
case deployapi.DeploymentStatusNew,
deployapi.DeploymentStatusPending,
deployapi.DeploymentStatusRunning:
if deployutil.IsDeploymentCancelled(&deployment) {
continue
}
deployment.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue
deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledByUser
_, err := o.kubeClient.ReplicationControllers(deployment.Namespace).Update(&deployment)
if err == nil {
fmt.Fprintf(out, "Cancelled deployment #%d\n", config.LatestVersion)
anyCancelled = true
} else {
fmt.Fprintf(out, "Couldn't cancel deployment #%d (status: %s): %v\n", deployutil.DeploymentVersionFor(&deployment), status, err)
failedCancellations = append(failedCancellations, strconv.Itoa(deployutil.DeploymentVersionFor(&deployment)))
}
}
}
if len(failedCancellations) > 0 {
return fmt.Errorf("couldn't cancel deployment %s", strings.Join(failedCancellations, ", "))
}
if !anyCancelled {
latest := &deployments.Items[0]
timeAt := strings.ToLower(units.HumanDuration(time.Now().Sub(latest.CreationTimestamp.Time)))
fmt.Fprintf(out, "No deployments are in progress (latest deployment #%d %s %s ago)\n",
deployutil.DeploymentVersionFor(latest),
strings.ToLower(string(deployutil.DeploymentStatusFor(latest))),
timeAt)
}
return nil
}
// Deploy starts the deployment process for deploymentName.
func (d *Deployer) Deploy(namespace, deploymentName string) error {
// Look up the new deployment.
to, err := d.getDeployment(namespace, deploymentName)
if err != nil {
return fmt.Errorf("couldn't get deployment %s/%s: %v", namespace, deploymentName, err)
}
// Decode the config from the deployment.
config, err := deployutil.DecodeDeploymentConfig(to, latest.Codec)
if err != nil {
return fmt.Errorf("couldn't decode deployment config from deployment %s/%s: %v", to.Namespace, to.Name, err)
}
// Get a strategy for the deployment.
strategy, err := d.strategyFor(config)
if err != nil {
return err
}
// New deployments must have a desired replica count.
desiredReplicas, hasDesired := deployutil.DeploymentDesiredReplicas(to)
if !hasDesired {
return fmt.Errorf("deployment %s has no desired replica count", deployutil.LabelForDeployment(to))
}
// Find all deployments for the config.
unsortedDeployments, err := d.getDeployments(namespace, config.Name)
if err != nil {
return fmt.Errorf("couldn't get controllers in namespace %s: %v", namespace, err)
}
deployments := unsortedDeployments.Items
// Sort all the deployments by version.
sort.Sort(deployutil.DeploymentsByLatestVersionDesc(deployments))
// Find any last completed deployment.
var from *kapi.ReplicationController
for _, candidate := range deployments {
if candidate.Name == to.Name {
continue
}
if deployutil.DeploymentStatusFor(&candidate) == deployapi.DeploymentStatusComplete {
from = &candidate
break
}
}
// Scale down any deployments which aren't the new or last deployment.
for _, candidate := range deployments {
// Skip the from/to deployments.
if candidate.Name == to.Name {
continue
}
if from != nil && candidate.Name == from.Name {
continue
}
// Skip the deployment if it's already scaled down.
if candidate.Spec.Replicas == 0 {
continue
}
// Scale the deployment down to zero.
retryWaitParams := kubectl.NewRetryParams(1*time.Second, 120*time.Second)
if err := d.scaler.Scale(candidate.Namespace, candidate.Name, uint(0), &kubectl.ScalePrecondition{Size: -1, ResourceVersion: ""}, retryWaitParams, retryWaitParams); err != nil {
glog.Errorf("Couldn't scale down prior deployment %s: %v", deployutil.LabelForDeployment(&candidate), err)
} else {
glog.Infof("Scaled down prior deployment %s", deployutil.LabelForDeployment(&candidate))
}
}
// Perform the deployment.
if from == nil {
glog.Infof("Deploying %s for the first time (replicas: %d)", deployutil.LabelForDeployment(to), desiredReplicas)
} else {
glog.Infof("Deploying from %s to %s (replicas: %d)", deployutil.LabelForDeployment(from), deployutil.LabelForDeployment(to), desiredReplicas)
}
return strategy.Deploy(from, to, desiredReplicas)
}
func (c *DeployerPodController) cleanupFailedDeployment(deployment *kapi.ReplicationController) error {
// Scale down the current failed deployment
configName := deployutil.DeploymentConfigNameFor(deployment)
existingDeployments, err := c.deploymentClient.listDeploymentsForConfig(deployment.Namespace, configName)
if err != nil {
return fmt.Errorf("couldn't list Deployments for DeploymentConfig %s: %v", configName, err)
}
desiredReplicas, ok := deployutil.DeploymentDesiredReplicas(deployment)
if !ok {
// if desired replicas could not be found, then log the error
// and update the failed deployment
// this cannot be treated as a transient error
kutil.HandleError(fmt.Errorf("Could not determine desired replicas from %s to reset replicas for last completed deployment", deployutil.LabelForDeployment(deployment)))
}
if ok && len(existingDeployments.Items) > 0 {
sort.Sort(deployutil.DeploymentsByLatestVersionDesc(existingDeployments.Items))
for index, existing := range existingDeployments.Items {
// if a newer deployment exists:
// - set the replicas for the current failed deployment to 0
// - there is no point in scaling up the last completed deployment
// since that will be scaled down by the later deployment
if index == 0 && existing.Name != deployment.Name {
break
}
// the latest completed deployment is the one that needs to be scaled back up
if deployutil.DeploymentStatusFor(&existing) == deployapi.DeploymentStatusComplete {
if existing.Spec.Replicas == desiredReplicas {
break
}
// scale back the completed deployment to the target of the failed deployment
existing.Spec.Replicas = desiredReplicas
if _, err := c.deploymentClient.updateDeployment(existing.Namespace, &existing); err != nil {
if kerrors.IsNotFound(err) {
return nil
}
return fmt.Errorf("couldn't update replicas to %d for deployment %s: %v", desiredReplicas, deployutil.LabelForDeployment(&existing), err)
}
glog.V(4).Infof("Updated replicas to %d for deployment %s", desiredReplicas, deployutil.LabelForDeployment(&existing))
break
}
}
}
// set the replicas for the failed deployment to 0
// and set the status to Failed
deployment.Spec.Replicas = 0
deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusFailed)
if _, err := c.deploymentClient.updateDeployment(deployment.Namespace, deployment); err != nil {
if kerrors.IsNotFound(err) {
return nil
}
return fmt.Errorf("couldn't scale down the deployment %s and mark it as failed: %v", deployutil.LabelForDeployment(deployment), err)
}
glog.V(4).Infof("Scaled down the deployment %s and marked it as failed", deployutil.LabelForDeployment(deployment))
return nil
}
// Handle processes config and creates a new deployment if necessary.
func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error {
// Only deploy when the version has advanced past 0.
if config.LatestVersion == 0 {
glog.V(5).Infof("Waiting for first version of %s", deployutil.LabelForDeploymentConfig(config))
return nil
}
// Check if any existing inflight deployments (any non-terminal state).
existingDeployments, err := c.deploymentClient.listDeploymentsForConfig(config.Namespace, config.Name)
if err != nil {
return fmt.Errorf("couldn't list Deployments for DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err)
}
var inflightDeployment *kapi.ReplicationController
latestDeploymentExists := false
for _, deployment := range existingDeployments.Items {
// check if this is the latest deployment
// we'll return after we've dealt with the multiple-active-deployments case
if deployutil.DeploymentVersionFor(&deployment) == config.LatestVersion {
latestDeploymentExists = true
}
deploymentStatus := deployutil.DeploymentStatusFor(&deployment)
switch deploymentStatus {
case deployapi.DeploymentStatusFailed,
deployapi.DeploymentStatusComplete:
// Previous deployment in terminal state - can ignore
// Ignoring specific deployment states so that any newly introduced
// deployment state will not be ignored
default:
if inflightDeployment == nil {
inflightDeployment = &deployment
continue
}
var deploymentForCancellation *kapi.ReplicationController
if deployutil.DeploymentVersionFor(inflightDeployment) < deployutil.DeploymentVersionFor(&deployment) {
deploymentForCancellation, inflightDeployment = inflightDeployment, &deployment
} else {
deploymentForCancellation = &deployment
}
deploymentForCancellation.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue
deploymentForCancellation.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists
if _, err := c.deploymentClient.updateDeployment(deploymentForCancellation.Namespace, deploymentForCancellation); err != nil {
util.HandleError(fmt.Errorf("couldn't cancel Deployment %s: %v", deployutil.LabelForDeployment(deploymentForCancellation), err))
}
glog.V(4).Infof("Cancelled Deployment %s for DeploymentConfig %s", deployutil.LabelForDeployment(deploymentForCancellation), deployutil.LabelForDeploymentConfig(config))
}
}
// if the latest deployment exists then nothing else needs to be done
if latestDeploymentExists {
return nil
}
// check to see if there are inflight deployments
if inflightDeployment != nil {
// raise a transientError so that the deployment config can be re-queued
glog.V(4).Infof("Found previous inflight Deployment for %s - will requeue", deployutil.LabelForDeploymentConfig(config))
return transientError(fmt.Sprintf("found previous inflight Deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config)))
}
// Try and build a deployment for the config.
deployment, err := c.makeDeployment(config)
if err != nil {
return fatalError(fmt.Sprintf("couldn't make Deployment from (potentially invalid) DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err))
}
// Compute the desired replicas for the deployment. Use the last completed
// deployment's current replica count, or the config template if there is no
// prior completed deployment available.
desiredReplicas := config.Template.ControllerTemplate.Replicas
if len(existingDeployments.Items) > 0 {
sort.Sort(deployutil.DeploymentsByLatestVersionDesc(existingDeployments.Items))
for _, existing := range existingDeployments.Items {
if deployutil.DeploymentStatusFor(&existing) == deployapi.DeploymentStatusComplete {
desiredReplicas = existing.Spec.Replicas
glog.V(4).Infof("Desired replicas for %s set to %d based on prior completed deployment %s", deployutil.LabelForDeploymentConfig(config), desiredReplicas, existing.Name)
break
}
}
}
deployment.Annotations[deployapi.DesiredReplicasAnnotation] = strconv.Itoa(desiredReplicas)
// Create the deployment.
if _, err := c.deploymentClient.createDeployment(config.Namespace, deployment); err == nil {
glog.V(4).Infof("Created Deployment for DeploymentConfig %s", deployutil.LabelForDeploymentConfig(config))
return nil
} else {
// If the deployment was already created, just move on. The cache could be stale, or another
// process could have already handled this update.
if errors.IsAlreadyExists(err) {
glog.V(4).Infof("Deployment already exists for DeploymentConfig %s", deployutil.LabelForDeploymentConfig(config))
return nil
}
// log an event if the deployment could not be created that the user can discover
c.recorder.Eventf(config, "failedCreate", "Error creating: %v", err)
return fmt.Errorf("couldn't create Deployment for DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err)
}
}