// updateReplicaCount attempts to update the Status.Replicas of the given controller, with a single GET/PUT retry. func updateReplicaCount(rcClient unversionedcore.ReplicationControllerInterface, controller api.ReplicationController, numReplicas, numFullyLabeledReplicas int) (updateErr error) { // This is the steady state. It happens when the rc doesn't have any expectations, since // we do a periodic relist every 30s. If the generations differ but the replicas are // the same, a caller might've resized to the same replica count. if int(controller.Status.Replicas) == numReplicas && int(controller.Status.FullyLabeledReplicas) == numFullyLabeledReplicas && controller.Generation == controller.Status.ObservedGeneration { return nil } // Save the generation number we acted on, otherwise we might wrongfully indicate // that we've seen a spec update when we retry. // TODO: This can clobber an update if we allow multiple agents to write to the // same status. generation := controller.Generation var getErr error for i, rc := 0, &controller; ; i++ { glog.V(4).Infof(fmt.Sprintf("Updating replica count for rc: %s/%s, ", controller.Namespace, controller.Name) + fmt.Sprintf("replicas %d->%d (need %d), ", controller.Status.Replicas, numReplicas, controller.Spec.Replicas) + fmt.Sprintf("fullyLabeledReplicas %d->%d, ", controller.Status.FullyLabeledReplicas, numFullyLabeledReplicas) + fmt.Sprintf("sequence No: %v->%v", controller.Status.ObservedGeneration, generation)) rc.Status = api.ReplicationControllerStatus{Replicas: int32(numReplicas), FullyLabeledReplicas: int32(numFullyLabeledReplicas), ObservedGeneration: generation} _, updateErr = rcClient.UpdateStatus(rc) if updateErr == nil || i >= statusUpdateRetries { return updateErr } // Update the controller with the latest resource version for the next poll if rc, getErr = rcClient.Get(controller.Name); getErr != nil { // If the GET fails we can't trust status.Replicas anymore. This error // is bound to be more interesting than the update failure. return getErr } } }
// getOverlappingControllers finds rcs that this controller overlaps, as well as rcs overlapping this controller. func getOverlappingControllers(rcClient coreclient.ReplicationControllerInterface, rc *api.ReplicationController) ([]api.ReplicationController, error) { rcs, err := rcClient.List(api.ListOptions{}) if err != nil { return nil, fmt.Errorf("error getting replication controllers: %v", err) } var matchingRCs []api.ReplicationController rcLabels := labels.Set(rc.Spec.Selector) for _, controller := range rcs.Items { newRCLabels := labels.Set(controller.Spec.Selector) if labels.SelectorFromSet(newRCLabels).Matches(rcLabels) || labels.SelectorFromSet(rcLabels).Matches(newRCLabels) { matchingRCs = append(matchingRCs, controller) } } return matchingRCs, nil }
// WaitForADeployment waits for a deployment to fulfill either isOK or isFailed. // When isOK returns true, WaitForADeployment returns nil, when isFailed returns // true, WaitForADeployment returns an error including the deployment status. // WaitForADeployment waits for at most a certain timeout (non-configurable). func WaitForADeployment(client kcoreclient.ReplicationControllerInterface, name string, isOK, isFailed func(*kapi.ReplicationController) bool, oc *CLI) error { timeout := 15 * time.Minute // closing done signals that any pending operation should be aborted. done := make(chan struct{}) defer close(done) // okOrFailed returns whether a replication controller matches either of // the predicates isOK or isFailed, and the associated error in case of // failure. okOrFailed := func(rc *kapi.ReplicationController) (err error, matched bool) { if isOK(rc) { return nil, true } if isFailed(rc) { return fmt.Errorf("The deployment %q status is %q", name, rc.Annotations[deployapi.DeploymentStatusAnnotation]), true } return nil, false } // waitForDeployment waits until okOrFailed returns true or the done // channel is closed. waitForDeployment := func() (err error, retry bool) { requirement, err := labels.NewRequirement(deployapi.DeploymentConfigAnnotation, selection.Equals, sets.NewString(name)) if err != nil { return fmt.Errorf("unexpected error generating label selector: %v", err), false } list, err := client.List(kapi.ListOptions{LabelSelector: labels.NewSelector().Add(*requirement)}) if err != nil { return err, false } // multiple deployments are conceivable; so we look to see how the latest depoy does var lastRC *kapi.ReplicationController for _, rc := range list.Items { if lastRC == nil { lastRC = &rc continue } if compareResourceControllerNames(lastRC.GetName(), rc.GetName()) <= 0 { lastRC = &rc } } if lastRC != nil { err, matched := okOrFailed(lastRC) if matched { return err, false } } w, err := client.Watch(kapi.ListOptions{LabelSelector: labels.NewSelector().Add(*requirement), ResourceVersion: list.ResourceVersion}) if err != nil { return err, false } defer w.Stop() for { select { case val, ok := <-w.ResultChan(): if !ok { // watcher error, re-get and re-watch return nil, true } rc, ok := val.Object.(*kapi.ReplicationController) if !ok { continue } if lastRC == nil { lastRC = rc } // multiple deployments are conceivable; so we look to see how the latest deployment does if compareResourceControllerNames(lastRC.GetName(), rc.GetName()) <= 0 { lastRC = rc err, matched := okOrFailed(rc) if matched { return err, false } } case <-done: // no more time left, stop what we were doing, // do no retry. return nil, false } } } // errCh is buffered so the goroutine below never blocks on sending, // preventing a goroutine leak if we reach the timeout. errCh := make(chan error, 1) go func() { defer close(errCh) err, retry := waitForDeployment() for retry { err, retry = waitForDeployment() } errCh <- err }() select { case err := <-errCh: if err != nil { DumpDeploymentLogs(name, oc) } return err case <-time.After(timeout): DumpDeploymentLogs(name, oc) // end for timing issues where we miss watch updates return fmt.Errorf("timed out waiting for deployment %q after %v", name, timeout) } }