// RunIdle runs the idling command logic, taking a list of resources or services in a file, scaling the associated
// scalable resources to zero, and annotating the associated endpoints objects with the scalable resources to unidle
// when they receive traffic.
func (o *IdleOptions) RunIdle(f *clientcmd.Factory) error {
hadError := false
nowTime := time.Now().UTC()
// figure out which endpoints and resources we need to idle
byService, byScalable, err := o.calculateIdlableAnnotationsByService(f)
if err != nil {
if len(byService) == 0 || len(byScalable) == 0 {
return fmt.Errorf("no valid scalable resources found to idle: %v", err)
}
fmt.Fprintf(o.errOut, "warning: continuing on for valid scalable resources, but an error occurred while finding scalable resources to idle: %v", err)
}
oclient, _, kclient, err := f.Clients()
if err != nil {
return err
}
delegScaleGetter := osclient.NewDelegatingScaleNamespacer(oclient, kclient.Extensions())
dcGetter := deployclient.New(oclient.RESTClient)
scaleAnnotater := utilunidling.NewScaleAnnotater(delegScaleGetter, dcGetter, kclient.Core(), func(currentReplicas int32, annotations map[string]string) {
annotations[unidlingapi.IdledAtAnnotation] = nowTime.UTC().Format(time.RFC3339)
annotations[unidlingapi.PreviousScaleAnnotation] = fmt.Sprintf("%v", currentReplicas)
})
replicas := make(map[unidlingapi.CrossGroupObjectReference]int32, len(byScalable))
toScale := make(map[unidlingapi.CrossGroupObjectReference]scaleInfo)
mapper, typer := f.Object(false)
// first, collect the scale info
for scaleRef, svcName := range byScalable {
obj, scale, err := scaleAnnotater.GetObjectWithScale(svcName.Namespace, scaleRef)
if err != nil {
fmt.Fprintf(o.errOut, "error: unable to get scale for %s %s/%s, not marking that scalable as idled: %v\n", scaleRef.Kind, svcName.Namespace, scaleRef.Name, err)
svcInfo := byService[svcName]
delete(svcInfo.scaleRefs, scaleRef)
hadError = true
continue
}
replicas[scaleRef] = scale.Spec.Replicas
toScale[scaleRef] = scaleInfo{scale: scale, obj: obj, namespace: svcName.Namespace}
}
// annotate the endpoints objects to indicate which scalable resources need to be unidled on traffic
for serviceName, info := range byService {
if info.obj.Annotations == nil {
info.obj.Annotations = make(map[string]string)
}
refsWithScale, err := pairScalesWithScaleRefs(serviceName, info.obj.Annotations, info.scaleRefs, replicas)
if err != nil {
fmt.Fprintf(o.errOut, "error: unable to mark service %s as idled: %v", serviceName.String(), err)
continue
}
if !o.dryRun {
if len(info.scaleRefs) == 0 {
fmt.Fprintf(o.errOut, "error: no scalable resources marked as idled for service %s, not marking as idled\n", serviceName.String())
hadError = true
continue
}
metadata, err := meta.Accessor(info.obj)
if err != nil {
fmt.Fprintf(o.errOut, "error: unable to mark service %s as idled: %v", serviceName.String(), err)
hadError = true
continue
}
gvks, _, err := typer.ObjectKinds(info.obj)
if err != nil {
fmt.Fprintf(o.errOut, "error: unable to mark service %s as idled: %v", serviceName.String(), err)
hadError = true
continue
}
oldData, err := json.Marshal(info.obj)
if err != nil {
fmt.Fprintf(o.errOut, "error: unable to mark service %s as idled: %v", serviceName.String(), err)
hadError = true
continue
}
mapping, err := mapper.RESTMapping(gvks[0].GroupKind(), gvks[0].Version)
if err != nil {
fmt.Fprintf(o.errOut, "error: unable to mark service %s as idled: %v", serviceName.String(), err)
hadError = true
continue
}
if err = setIdleAnnotations(serviceName, info.obj.Annotations, refsWithScale, nowTime); err != nil {
fmt.Fprintf(o.errOut, "error: unable to mark service %s as idled: %v", serviceName.String(), err)
hadError = true
continue
}
if _, err := patchObj(info.obj, metadata, oldData, mapping, f); err != nil {
fmt.Fprintf(o.errOut, "error: unable to mark service %s as idled: %v", serviceName.String(), err)
hadError = true
continue
}
}
for _, scaleRef := range refsWithScale {
fmt.Fprintf(o.out, "Marked service %s to unidle resource %s %s/%s (unidle to %v replicas)\n", serviceName.String(), scaleRef.Kind, serviceName.Namespace, scaleRef.Name, scaleRef.Replicas)
}
}
// actually "idle" the scalable resources by scaling them down to zero
// (scale down to zero *after* we've applied the annotation so that we don't miss any traffic)
for scaleRef, info := range toScale {
idled := ""
if !o.dryRun {
info.scale.Spec.Replicas = 0
scaleUpdater := utilunidling.NewScaleUpdater(f.JSONEncoder(), info.namespace, dcGetter, kclient.Core())
if err := scaleAnnotater.UpdateObjectScale(scaleUpdater, info.namespace, scaleRef, info.obj, info.scale); err != nil {
fmt.Fprintf(o.errOut, "error: unable to scale %s %s/%s to 0, but still listed as target for unidling: %v\n", scaleRef.Kind, info.namespace, scaleRef.Name, err)
hadError = true
continue
}
} else {
idled = "(dry run)"
}
fmt.Fprintf(o.out, "Idled %s %s/%s %s\n", scaleRef.Kind, info.namespace, scaleRef.Name, idled)
}
if hadError {
return cmdutil.ErrExit
}
return nil
}
// handleRequest handles a single request to unidle. After checking the validity of the request,
// it will examine the endpoints in question to determine which scalables to scale, and will scale
// them and remove them from the endpoints' list of idled scalables. If it is unable to properly
// process the request, it will return a boolean indicating whether or not we should retry later,
// as well as an error (e.g. if we're unable to parse an annotation, retrying later won't help,
// so it will return false).
func (c *UnidlingController) handleRequest(info types.NamespacedName, lastFired time.Time) (bool, error) {
// fetch the endpoints associated with the service in question
targetEndpoints, err := c.endpointsNamespacer.Endpoints(info.Namespace).Get(info.Name)
if err != nil {
return true, fmt.Errorf("unable to retrieve endpoints: %v", err)
}
// make sure we actually were idled...
idledTimeRaw, wasIdled := targetEndpoints.Annotations[unidlingapi.IdledAtAnnotation]
if !wasIdled {
glog.V(5).Infof("UnidlingController received a NeedPods event for a service that was not idled, ignoring")
return false, nil
}
// ...and make sure this request was to wake up from the most recent idling, and not a previous one
idledTime, err := time.Parse(time.RFC3339, idledTimeRaw)
if err != nil {
// retrying here won't help, we're just stuck as idle since we can't get parse the idled time
return false, fmt.Errorf("unable to check idled-at time: %v", err)
}
if lastFired.Before(idledTime) {
glog.V(5).Infof("UnidlingController received an out-of-date NeedPods event, ignoring")
return false, nil
}
// TODO: ew, this is unversioned. Such is life when working with annotations.
var targetScalables []unidlingapi.RecordedScaleReference
if targetScalablesStr, hasTargetScalables := targetEndpoints.Annotations[unidlingapi.UnidleTargetAnnotation]; hasTargetScalables {
if err = json.Unmarshal([]byte(targetScalablesStr), &targetScalables); err != nil {
// retrying here won't help, we're just stuck as idled since we can't parse the idled scalables list
return false, fmt.Errorf("unable to unmarshal target scalable references: %v", err)
}
} else {
glog.V(4).Infof("Service %s/%s had no scalables to unidle", info.Namespace, info.Name)
targetScalables = []unidlingapi.RecordedScaleReference{}
}
targetScalablesSet := make(map[unidlingapi.RecordedScaleReference]struct{}, len(targetScalables))
for _, v := range targetScalables {
targetScalablesSet[v] = struct{}{}
}
deleteIdlingAnnotations := func(_ int32, annotations map[string]string) {
delete(annotations, unidlingapi.IdledAtAnnotation)
delete(annotations, unidlingapi.PreviousScaleAnnotation)
}
scaleAnnotater := unidlingutil.NewScaleAnnotater(c.scaleNamespacer, c.dcNamespacer, c.rcNamespacer, deleteIdlingAnnotations)
for _, scalableRef := range targetScalables {
var scale *kextapi.Scale
var obj runtime.Object
obj, scale, err = scaleAnnotater.GetObjectWithScale(info.Namespace, scalableRef.CrossGroupObjectReference)
if err != nil {
if errors.IsNotFound(err) {
utilruntime.HandleError(fmt.Errorf("%s %q does not exist, removing from list of scalables while unidling service %s/%s: %v", scalableRef.Kind, scalableRef.Name, info.Namespace, info.Name, err))
delete(targetScalablesSet, scalableRef)
} else {
utilruntime.HandleError(fmt.Errorf("Unable to get scale for %s %q while unidling service %s/%s, will try again later: %v", scalableRef.Kind, scalableRef.Name, info.Namespace, info.Name, err))
}
continue
}
if scale.Spec.Replicas > 0 {
glog.V(4).Infof("%s %q is not idle, skipping while unidling service %s/%s", scalableRef.Kind, scalableRef.Name, info.Namespace, info.Name)
continue
}
scale.Spec.Replicas = scalableRef.Replicas
if err = scaleAnnotater.UpdateObjectScale(info.Namespace, scalableRef.CrossGroupObjectReference, obj, scale); err != nil {
if errors.IsNotFound(err) {
utilruntime.HandleError(fmt.Errorf("%s %q does not exist, removing from list of scalables while unidling service %s/%s: %v", scalableRef.Kind, scalableRef.Name, info.Namespace, info.Name, err))
delete(targetScalablesSet, scalableRef)
} else {
utilruntime.HandleError(fmt.Errorf("Unable to scale up %s %q while unidling service %s/%s: %v", scalableRef.Kind, scalableRef.Name, info.Namespace, info.Name, err))
}
continue
} else {
glog.V(4).Infof("Scaled up %s %q while unidling service %s/%s", scalableRef.Kind, scalableRef.Name, info.Namespace, info.Name)
}
delete(targetScalablesSet, scalableRef)
}
newAnnotationList := make([]unidlingapi.RecordedScaleReference, 0, len(targetScalablesSet))
for k := range targetScalablesSet {
newAnnotationList = append(newAnnotationList, k)
}
if len(newAnnotationList) == 0 {
delete(targetEndpoints.Annotations, unidlingapi.UnidleTargetAnnotation)
delete(targetEndpoints.Annotations, unidlingapi.IdledAtAnnotation)
} else {
var newAnnotationBytes []byte
newAnnotationBytes, err = json.Marshal(newAnnotationList)
if err != nil {
utilruntime.HandleError(fmt.Errorf("unable to update/remove idle annotations from %s/%s: unable to marshal list of remaining scalables, removing list entirely: %v", info.Namespace, info.Name, err))
delete(targetEndpoints.Annotations, unidlingapi.UnidleTargetAnnotation)
delete(targetEndpoints.Annotations, unidlingapi.IdledAtAnnotation)
} else {
targetEndpoints.Annotations[unidlingapi.UnidleTargetAnnotation] = string(newAnnotationBytes)
}
}
if _, err = c.endpointsNamespacer.Endpoints(info.Namespace).Update(targetEndpoints); err != nil {
return true, fmt.Errorf("unable to update/remove idle annotations from %s/%s: %v", info.Namespace, info.Name, err)
}
return false, nil
}