// updateReplicaSetStatus attempts to update the Status.Replicas of the given ReplicaSet, with a single GET/PUT retry. func updateReplicaSetStatus(c unversionedextensions.ReplicaSetInterface, rs extensions.ReplicaSet, newStatus extensions.ReplicaSetStatus) (updateErr error) { // This is the steady state. It happens when the ReplicaSet doesn't have any expectations, since // we do a periodic relist every 30s. If the generations differ but the replicas are // the same, a caller might've resized to the same replica count. if rs.Status.Replicas == newStatus.Replicas && rs.Status.FullyLabeledReplicas == newStatus.FullyLabeledReplicas && rs.Status.ReadyReplicas == newStatus.ReadyReplicas && rs.Status.AvailableReplicas == newStatus.AvailableReplicas && rs.Generation == rs.Status.ObservedGeneration && reflect.DeepEqual(rs.Status.Conditions, newStatus.Conditions) { return nil } // deep copy to avoid mutation now. // TODO this method need some work. Retry on conflict probably, though I suspect this is stomping status to something it probably shouldn't copyObj, err := api.Scheme.DeepCopy(rs) if err != nil { return err } rs = copyObj.(extensions.ReplicaSet) // Save the generation number we acted on, otherwise we might wrongfully indicate // that we've seen a spec update when we retry. // TODO: This can clobber an update if we allow multiple agents to write to the // same status. newStatus.ObservedGeneration = rs.Generation var getErr error for i, rs := 0, &rs; ; i++ { glog.V(4).Infof(fmt.Sprintf("Updating replica count for ReplicaSet: %s/%s, ", rs.Namespace, rs.Name) + fmt.Sprintf("replicas %d->%d (need %d), ", rs.Status.Replicas, newStatus.Replicas, rs.Spec.Replicas) + fmt.Sprintf("fullyLabeledReplicas %d->%d, ", rs.Status.FullyLabeledReplicas, newStatus.FullyLabeledReplicas) + fmt.Sprintf("readyReplicas %d->%d, ", rs.Status.ReadyReplicas, newStatus.ReadyReplicas) + fmt.Sprintf("availableReplicas %d->%d, ", rs.Status.AvailableReplicas, newStatus.AvailableReplicas) + fmt.Sprintf("sequence No: %v->%v", rs.Status.ObservedGeneration, newStatus.ObservedGeneration)) rs.Status = newStatus _, updateErr = c.UpdateStatus(rs) if updateErr == nil || i >= statusUpdateRetries { return updateErr } // Update the ReplicaSet with the latest resource version for the next poll if rs, getErr = c.Get(rs.Name); getErr != nil { // If the GET fails we can't trust status.Replicas anymore. This error // is bound to be more interesting than the update failure. return getErr } } }
// UpdateRSWithRetries updates a RS with given applyUpdate function. Note that RS not found error is ignored. // The returned bool value can be used to tell if the RS is actually updated. func UpdateRSWithRetries(rsClient unversionedextensions.ReplicaSetInterface, rs *extensions.ReplicaSet, applyUpdate updateRSFunc) (*extensions.ReplicaSet, bool, error) { var err error var rsUpdated bool oldRs := rs if err = wait.Poll(10*time.Millisecond, 1*time.Minute, func() (bool, error) { rs, err = rsClient.Get(oldRs.Name) if err != nil { return false, err } // Apply the update, then attempt to push it to the apiserver. if err = applyUpdate(rs); err != nil { return false, err } if rs, err = rsClient.Update(rs); err == nil { // Update successful. return true, nil } // TODO: don't retry on perm-failed errors and handle them gracefully // Update could have failed due to conflict error. Try again. return false, nil }); err == nil { // When there's no error, we've updated this RS. rsUpdated = true } // Handle returned error from wait poll if err == wait.ErrWaitTimeout { err = fmt.Errorf("timed out trying to update RS: %#v", oldRs) } // Ignore the RS not found error, but the RS isn't updated. if errors.IsNotFound(err) { glog.V(4).Infof("%s %s/%s is not found, skip updating it.", oldRs.Kind, oldRs.Namespace, oldRs.Name) err = nil } // Ignore the precondition violated error, but the RS isn't updated. if err == errorsutil.ErrPreconditionViolated { glog.V(4).Infof("%s %s/%s precondition doesn't hold, skip updating it.", oldRs.Kind, oldRs.Namespace, oldRs.Name) err = nil } // If the error is non-nil the returned RS cannot be trusted; if rsUpdated is false, the contoller isn't updated; // if the error is nil and rsUpdated is true, the returned RS contains the applied update. return rs, rsUpdated, err }