// updateReplicaCount attempts to update the Status.Replicas of the given ReplicaSet, with a single GET/PUT retry.
func updateReplicaCount(rsClient unversionedextensions.ReplicaSetInterface, rs extensions.ReplicaSet, numReplicas, numFullyLabeledReplicas, numReadyReplicas, numAvailableReplicas int) (updateErr error) {
// This is the steady state. It happens when the ReplicaSet doesn't have any expectations, since
// we do a periodic relist every 30s. If the generations differ but the replicas are
// the same, a caller might've resized to the same replica count.
if int(rs.Status.Replicas) == numReplicas &&
int(rs.Status.FullyLabeledReplicas) == numFullyLabeledReplicas &&
int(rs.Status.ReadyReplicas) == numReadyReplicas &&
int(rs.Status.AvailableReplicas) == numAvailableReplicas &&
rs.Generation == rs.Status.ObservedGeneration {
return nil
}
// deep copy to avoid mutation now.
// TODO this method need some work. Retry on conflict probably, though I suspect this is stomping status to something it probably shouldn't
copyObj, err := api.Scheme.DeepCopy(rs)
if err != nil {
return err
}
rs = copyObj.(extensions.ReplicaSet)
// Save the generation number we acted on, otherwise we might wrongfully indicate
// that we've seen a spec update when we retry.
// TODO: This can clobber an update if we allow multiple agents to write to the
// same status.
generation := rs.Generation
var getErr error
for i, rs := 0, &rs; ; i++ {
glog.V(4).Infof(fmt.Sprintf("Updating replica count for ReplicaSet: %s/%s, ", rs.Namespace, rs.Name) +
fmt.Sprintf("replicas %d->%d (need %d), ", rs.Status.Replicas, numReplicas, rs.Spec.Replicas) +
fmt.Sprintf("fullyLabeledReplicas %d->%d, ", rs.Status.FullyLabeledReplicas, numFullyLabeledReplicas) +
fmt.Sprintf("readyReplicas %d->%d, ", rs.Status.ReadyReplicas, numReadyReplicas) +
fmt.Sprintf("availableReplicas %d->%d, ", rs.Status.AvailableReplicas, numAvailableReplicas) +
fmt.Sprintf("sequence No: %v->%v", rs.Status.ObservedGeneration, generation))
rs.Status = extensions.ReplicaSetStatus{
Replicas: int32(numReplicas),
FullyLabeledReplicas: int32(numFullyLabeledReplicas),
ReadyReplicas: int32(numReadyReplicas),
AvailableReplicas: int32(numAvailableReplicas),
ObservedGeneration: generation,
}
_, updateErr = rsClient.UpdateStatus(rs)
if updateErr == nil || i >= statusUpdateRetries {
return updateErr
}
// Update the ReplicaSet with the latest resource version for the next poll
if rs, getErr = rsClient.Get(rs.Name); getErr != nil {
// If the GET fails we can't trust status.Replicas anymore. This error
// is bound to be more interesting than the update failure.
return getErr
}
}
}
// UpdateRSWithRetries updates a RS with given applyUpdate function. Note that RS not found error is ignored.
// The returned bool value can be used to tell if the RS is actually updated.
func UpdateRSWithRetries(rsClient unversionedextensions.ReplicaSetInterface, rs *extensions.ReplicaSet, applyUpdate updateRSFunc) (*extensions.ReplicaSet, bool, error) {
var err error
var rsUpdated bool
oldRs := rs
if err = wait.Poll(10*time.Millisecond, 1*time.Minute, func() (bool, error) {
rs, err = rsClient.Get(oldRs.Name)
if err != nil {
return false, err
}
// Apply the update, then attempt to push it to the apiserver.
if err = applyUpdate(rs); err != nil {
return false, err
}
if rs, err = rsClient.Update(rs); err == nil {
// Update successful.
return true, nil
}
// TODO: don't retry on perm-failed errors and handle them gracefully
// Update could have failed due to conflict error. Try again.
return false, nil
}); err == nil {
// When there's no error, we've updated this RS.
rsUpdated = true
}
// Handle returned error from wait poll
if err == wait.ErrWaitTimeout {
err = fmt.Errorf("timed out trying to update RS: %+v", oldRs)
}
// Ignore the RS not found error, but the RS isn't updated.
if errors.IsNotFound(err) {
glog.V(4).Infof("%s %s/%s is not found, skip updating it.", oldRs.Kind, oldRs.Namespace, oldRs.Name)
err = nil
}
// Ignore the precondition violated error, but the RS isn't updated.
if err == errorsutil.ErrPreconditionViolated {
glog.V(4).Infof("%s %s/%s precondition doesn't hold, skip updating it.", oldRs.Kind, oldRs.Namespace, oldRs.Name)
err = nil
}
// If the error is non-nil the returned RS cannot be trusted; if rsUpdated is false, the contoller isn't updated;
// if the error is nil and rsUpdated is true, the returned RS contains the applied update.
return rs, rsUpdated, err
}
