// syncResourceQuota runs a complete sync of resource quota status across all known kinds
func (rq *ResourceQuotaController) syncResourceQuota(v1ResourceQuota v1.ResourceQuota) (err error) {
// quota is dirty if any part of spec hard limits differs from the status hard limits
dirty := !api.Semantic.DeepEqual(v1ResourceQuota.Spec.Hard, v1ResourceQuota.Status.Hard)
resourceQuota := api.ResourceQuota{}
if err := v1.Convert_v1_ResourceQuota_To_api_ResourceQuota(&v1ResourceQuota, &resourceQuota, nil); err != nil {
return err
}
// dirty tracks if the usage status differs from the previous sync,
// if so, we send a new usage with latest status
// if this is our first sync, it will be dirty by default, since we need track usage
dirty = dirty || (resourceQuota.Status.Hard == nil || resourceQuota.Status.Used == nil)
used := api.ResourceList{}
if resourceQuota.Status.Used != nil {
used = quota.Add(api.ResourceList{}, resourceQuota.Status.Used)
}
hardLimits := quota.Add(api.ResourceList{}, resourceQuota.Spec.Hard)
newUsage, err := quota.CalculateUsage(resourceQuota.Namespace, resourceQuota.Spec.Scopes, hardLimits, rq.registry)
if err != nil {
return err
}
for key, value := range newUsage {
used[key] = value
}
// ensure set of used values match those that have hard constraints
hardResources := quota.ResourceNames(hardLimits)
used = quota.Mask(used, hardResources)
// Create a usage object that is based on the quota resource version that will handle updates
// by default, we preserve the past usage observation, and set hard to the current spec
usage := api.ResourceQuota{
ObjectMeta: metav1.ObjectMeta{
Name: resourceQuota.Name,
Namespace: resourceQuota.Namespace,
ResourceVersion: resourceQuota.ResourceVersion,
Labels: resourceQuota.Labels,
Annotations: resourceQuota.Annotations},
Status: api.ResourceQuotaStatus{
Hard: hardLimits,
Used: used,
},
}
dirty = dirty || !quota.Equals(usage.Status.Used, resourceQuota.Status.Used)
// there was a change observed by this controller that requires we update quota
if dirty {
v1Usage := &v1.ResourceQuota{}
if err := v1.Convert_api_ResourceQuota_To_v1_ResourceQuota(&usage, v1Usage, nil); err != nil {
return err
}
_, err = rq.kubeClient.Core().ResourceQuotas(usage.Namespace).UpdateStatus(v1Usage)
return err
}
return nil
}
func TestServiceEvaluatorUsage(t *testing.T) {
kubeClient := fake.NewSimpleClientset()
evaluator := NewServiceEvaluator(kubeClient)
testCases := map[string]struct {
service *api.Service
usage api.ResourceList
}{
"loadbalancer": {
service: &api.Service{
Spec: api.ServiceSpec{
Type: api.ServiceTypeLoadBalancer,
},
},
usage: api.ResourceList{
api.ResourceServicesLoadBalancers: resource.MustParse("1"),
api.ResourceServices: resource.MustParse("1"),
},
},
"clusterip": {
service: &api.Service{
Spec: api.ServiceSpec{
Type: api.ServiceTypeClusterIP,
},
},
usage: api.ResourceList{
api.ResourceServices: resource.MustParse("1"),
},
},
"nodeports": {
service: &api.Service{
Spec: api.ServiceSpec{
Type: api.ServiceTypeNodePort,
},
},
usage: api.ResourceList{
api.ResourceServices: resource.MustParse("1"),
api.ResourceServicesNodePorts: resource.MustParse("1"),
},
},
}
for testName, testCase := range testCases {
actual := evaluator.Usage(testCase.service)
if !quota.Equals(testCase.usage, actual) {
t.Errorf("%s expected: %v, actual: %v", testName, testCase.usage, actual)
}
}
}
func TestPersistentVolumeClaimEvaluatorUsage(t *testing.T) {
validClaim := testVolumeClaim("foo", "ns", api.PersistentVolumeClaimSpec{
Selector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "key2",
Operator: "Exists",
},
},
},
AccessModes: []api.PersistentVolumeAccessMode{
api.ReadWriteOnce,
api.ReadOnlyMany,
},
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("10Gi"),
},
},
})
kubeClient := fake.NewSimpleClientset()
evaluator := NewPersistentVolumeClaimEvaluator(kubeClient, nil)
testCases := map[string]struct {
pvc *api.PersistentVolumeClaim
usage api.ResourceList
}{
"pvc-usage": {
pvc: validClaim,
usage: api.ResourceList{
api.ResourceRequestsStorage: resource.MustParse("10Gi"),
api.ResourcePersistentVolumeClaims: resource.MustParse("1"),
},
},
}
for testName, testCase := range testCases {
actual := evaluator.Usage(testCase.pvc)
if !quota.Equals(testCase.usage, actual) {
t.Errorf("%s expected: %v, actual: %v", testName, testCase.usage, actual)
}
}
}
// checkQuotas checks the admission atttributes against the passed quotas. If a quota applies, it will attempt to update it
// AFTER it has checked all the admissionAttributes. The method breaks down into phase like this:
// 0. make a copy of the quotas to act as a "running" quota so we know what we need to update and can still compare against the
// originals
// 1. check each admission attribute to see if it fits within *all* the quotas. If it doesn't fit, mark the waiter as failed
// and the running quota don't change. If it did fit, check to see if any quota was changed. It there was no quota change
// mark the waiter as succeeded. If some quota did change, update the running quotas
// 2. If no running quota was changed, return now since no updates are needed.
// 3. for each quota that has changed, attempt an update. If all updates succeeded, update all unset waiters to success status and return. If the some
// updates failed on conflict errors and we have retries left, re-get the failed quota from our cache for the latest version
// and recurse into this method with the subset. It's safe for us to evaluate ONLY the subset, because the other quota
// documents for these waiters have already been evaluated. Step 1, will mark all the ones that should already have succeeded.
func (e *quotaEvaluator) checkQuotas(quotas []api.ResourceQuota, admissionAttributes []*admissionWaiter, remainingRetries int) {
// yet another copy to compare against originals to see if we actually have deltas
originalQuotas := make([]api.ResourceQuota, len(quotas), len(quotas))
copy(originalQuotas, quotas)
atLeastOneChanged := false
for i := range admissionAttributes {
admissionAttribute := admissionAttributes[i]
newQuotas, err := e.checkRequest(quotas, admissionAttribute.attributes)
if err != nil {
admissionAttribute.result = err
continue
}
// if the new quotas are the same as the old quotas, then this particular one doesn't issue any updates
// that means that no quota docs applied, so it can get a pass
atLeastOneChangeForThisWaiter := false
for j := range newQuotas {
if !quota.Equals(originalQuotas[j].Status.Used, newQuotas[j].Status.Used) {
atLeastOneChanged = true
atLeastOneChangeForThisWaiter = true
break
}
}
if !atLeastOneChangeForThisWaiter {
admissionAttribute.result = nil
}
quotas = newQuotas
}
// if none of the requests changed anything, there's no reason to issue an update, just fail them all now
if !atLeastOneChanged {
return
}
// now go through and try to issue updates. Things get a little weird here:
// 1. check to see if the quota changed. If not, skip.
// 2. if the quota changed and the update passes, be happy
// 3. if the quota changed and the update fails, add the original to a retry list
var updatedFailedQuotas []api.ResourceQuota
var lastErr error
for i := range quotas {
newQuota := quotas[i]
// if this quota didn't have its status changed, skip it
if quota.Equals(originalQuotas[i].Status.Used, newQuota.Status.Used) {
continue
}
if err := e.quotaAccessor.UpdateQuotaStatus(&newQuota); err != nil {
updatedFailedQuotas = append(updatedFailedQuotas, newQuota)
lastErr = err
}
}
if len(updatedFailedQuotas) == 0 {
// all the updates succeeded. At this point, anything with the default deny error was just waiting to
// get a successful update, so we can mark and notify
for _, admissionAttribute := range admissionAttributes {
if IsDefaultDeny(admissionAttribute.result) {
admissionAttribute.result = nil
}
}
return
}
// at this point, errors are fatal. Update all waiters without status to failed and return
if remainingRetries <= 0 {
for _, admissionAttribute := range admissionAttributes {
if IsDefaultDeny(admissionAttribute.result) {
admissionAttribute.result = lastErr
}
}
return
}
// this retry logic has the same bug that its possible to be checking against quota in a state that never actually exists where
// you've added a new documented, then updated an old one, your resource matches both and you're only checking one
// updates for these quota names failed. Get the current quotas in the namespace, compare by name, check to see if the
// resource versions have changed. If not, we're going to fall through an fail everything. If they all have, then we can try again
newQuotas, err := e.quotaAccessor.GetQuotas(quotas[0].Namespace)
if err != nil {
// this means that updates failed. Anything with a default deny error has failed and we need to let them know
for _, admissionAttribute := range admissionAttributes {
if IsDefaultDeny(admissionAttribute.result) {
admissionAttribute.result = lastErr
}
}
return
}
// this logic goes through our cache to find the new version of all quotas that failed update. If something has been removed
// it is skipped on this retry. After all, you removed it.
quotasToCheck := []api.ResourceQuota{}
for _, newQuota := range newQuotas {
for _, oldQuota := range updatedFailedQuotas {
if newQuota.Name == oldQuota.Name {
quotasToCheck = append(quotasToCheck, newQuota)
break
}
}
}
e.checkQuotas(quotasToCheck, admissionAttributes, remainingRetries-1)
}
func NewResourceQuotaController(options *ResourceQuotaControllerOptions) *ResourceQuotaController {
// build the resource quota controller
rq := &ResourceQuotaController{
kubeClient: options.KubeClient,
queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
resyncPeriod: options.ResyncPeriod,
registry: options.Registry,
replenishmentControllers: []framework.ControllerInterface{},
}
if options.KubeClient != nil && options.KubeClient.Core().GetRESTClient().GetRateLimiter() != nil {
metrics.RegisterMetricAndTrackRateLimiterUsage("resource_quota_controller", options.KubeClient.Core().GetRESTClient().GetRateLimiter())
}
// set the synchronization handler
rq.syncHandler = rq.syncResourceQuotaFromKey
// build the controller that observes quota
rq.rqIndexer, rq.rqController = framework.NewIndexerInformer(
&cache.ListWatch{
ListFunc: func(options api.ListOptions) (runtime.Object, error) {
return rq.kubeClient.Core().ResourceQuotas(api.NamespaceAll).List(options)
},
WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
return rq.kubeClient.Core().ResourceQuotas(api.NamespaceAll).Watch(options)
},
},
&api.ResourceQuota{},
rq.resyncPeriod(),
framework.ResourceEventHandlerFuncs{
AddFunc: rq.enqueueResourceQuota,
UpdateFunc: func(old, cur interface{}) {
// We are only interested in observing updates to quota.spec to drive updates to quota.status.
// We ignore all updates to quota.Status because they are all driven by this controller.
// IMPORTANT:
// We do not use this function to queue up a full quota recalculation. To do so, would require
// us to enqueue all quota.Status updates, and since quota.Status updates involve additional queries
// that cannot be backed by a cache and result in a full query of a namespace's content, we do not
// want to pay the price on spurious status updates. As a result, we have a separate routine that is
// responsible for enqueue of all resource quotas when doing a full resync (enqueueAll)
oldResourceQuota := old.(*api.ResourceQuota)
curResourceQuota := cur.(*api.ResourceQuota)
if quota.Equals(curResourceQuota.Spec.Hard, oldResourceQuota.Spec.Hard) {
return
}
rq.enqueueResourceQuota(curResourceQuota)
},
// This will enter the sync loop and no-op, because the controller has been deleted from the store.
// Note that deleting a controller immediately after scaling it to 0 will not work. The recommended
// way of achieving this is by performing a `stop` operation on the controller.
DeleteFunc: rq.enqueueResourceQuota,
},
cache.Indexers{"namespace": cache.MetaNamespaceIndexFunc},
)
for _, groupKindToReplenish := range options.GroupKindsToReplenish {
controllerOptions := &ReplenishmentControllerOptions{
GroupKind: groupKindToReplenish,
ResyncPeriod: options.ReplenishmentResyncPeriod,
ReplenishmentFunc: rq.replenishQuota,
}
replenishmentController, err := options.ControllerFactory.NewController(controllerOptions)
if err != nil {
glog.Warningf("quota controller unable to replenish %s due to %v, changes only accounted during full resync", groupKindToReplenish, err)
} else {
rq.replenishmentControllers = append(rq.replenishmentControllers, replenishmentController)
}
}
return rq
}
// syncResourceQuota runs a complete sync of resource quota status across all known kinds
func (rq *ResourceQuotaController) syncResourceQuota(resourceQuota api.ResourceQuota) (err error) {
// quota is dirty if any part of spec hard limits differs from the status hard limits
dirty := !api.Semantic.DeepEqual(resourceQuota.Spec.Hard, resourceQuota.Status.Hard)
// dirty tracks if the usage status differs from the previous sync,
// if so, we send a new usage with latest status
// if this is our first sync, it will be dirty by default, since we need track usage
dirty = dirty || (resourceQuota.Status.Hard == nil || resourceQuota.Status.Used == nil)
// Create a usage object that is based on the quota resource version that will handle updates
// by default, we preserve the past usage observation, and set hard to the current spec
previousUsed := api.ResourceList{}
if resourceQuota.Status.Used != nil {
previousUsed = quota.Add(api.ResourceList{}, resourceQuota.Status.Used)
}
usage := api.ResourceQuota{
ObjectMeta: api.ObjectMeta{
Name: resourceQuota.Name,
Namespace: resourceQuota.Namespace,
ResourceVersion: resourceQuota.ResourceVersion,
Labels: resourceQuota.Labels,
Annotations: resourceQuota.Annotations},
Status: api.ResourceQuotaStatus{
Hard: quota.Add(api.ResourceList{}, resourceQuota.Spec.Hard),
Used: previousUsed,
},
}
// find the intersection between the hard resources on the quota
// and the resources this controller can track to know what we can
// look to measure updated usage stats for
hardResources := quota.ResourceNames(usage.Status.Hard)
potentialResources := []api.ResourceName{}
evaluators := rq.registry.Evaluators()
for _, evaluator := range evaluators {
potentialResources = append(potentialResources, evaluator.MatchesResources()...)
}
matchedResources := quota.Intersection(hardResources, potentialResources)
// sum the observed usage from each evaluator
newUsage := api.ResourceList{}
usageStatsOptions := quota.UsageStatsOptions{Namespace: resourceQuota.Namespace, Scopes: resourceQuota.Spec.Scopes}
for _, evaluator := range evaluators {
stats, err := evaluator.UsageStats(usageStatsOptions)
if err != nil {
return err
}
newUsage = quota.Add(newUsage, stats.Used)
}
// mask the observed usage to only the set of resources tracked by this quota
// merge our observed usage with the quota usage status
// if the new usage is different than the last usage, we will need to do an update
newUsage = quota.Mask(newUsage, matchedResources)
for key, value := range newUsage {
usage.Status.Used[key] = value
}
dirty = dirty || !quota.Equals(usage.Status.Used, resourceQuota.Status.Used)
// there was a change observed by this controller that requires we update quota
if dirty {
_, err = rq.kubeClient.Core().ResourceQuotas(usage.Namespace).UpdateStatus(&usage)
return err
}
return nil
}
func TestPodEvaluatorUsage(t *testing.T) {
kubeClient := fake.NewSimpleClientset()
evaluator := NewPodEvaluator(kubeClient, nil)
testCases := map[string]struct {
pod *api.Pod
usage api.ResourceList
}{
"init container CPU": {
pod: &api.Pod{
Spec: api.PodSpec{
InitContainers: []api.Container{{
Resources: api.ResourceRequirements{
Requests: api.ResourceList{api.ResourceCPU: resource.MustParse("1m")},
Limits: api.ResourceList{api.ResourceCPU: resource.MustParse("2m")},
},
}},
},
},
usage: api.ResourceList{
api.ResourceRequestsCPU: resource.MustParse("1m"),
api.ResourceLimitsCPU: resource.MustParse("2m"),
api.ResourcePods: resource.MustParse("1"),
api.ResourceCPU: resource.MustParse("1m"),
},
},
"init container MEM": {
pod: &api.Pod{
Spec: api.PodSpec{
InitContainers: []api.Container{{
Resources: api.ResourceRequirements{
Requests: api.ResourceList{api.ResourceMemory: resource.MustParse("1m")},
Limits: api.ResourceList{api.ResourceMemory: resource.MustParse("2m")},
},
}},
},
},
usage: api.ResourceList{
api.ResourceRequestsMemory: resource.MustParse("1m"),
api.ResourceLimitsMemory: resource.MustParse("2m"),
api.ResourcePods: resource.MustParse("1"),
api.ResourceMemory: resource.MustParse("1m"),
},
},
"container CPU": {
pod: &api.Pod{
Spec: api.PodSpec{
Containers: []api.Container{{
Resources: api.ResourceRequirements{
Requests: api.ResourceList{api.ResourceCPU: resource.MustParse("1m")},
Limits: api.ResourceList{api.ResourceCPU: resource.MustParse("2m")},
},
}},
},
},
usage: api.ResourceList{
api.ResourceRequestsCPU: resource.MustParse("1m"),
api.ResourceLimitsCPU: resource.MustParse("2m"),
api.ResourcePods: resource.MustParse("1"),
api.ResourceCPU: resource.MustParse("1m"),
},
},
"container MEM": {
pod: &api.Pod{
Spec: api.PodSpec{
Containers: []api.Container{{
Resources: api.ResourceRequirements{
Requests: api.ResourceList{api.ResourceMemory: resource.MustParse("1m")},
Limits: api.ResourceList{api.ResourceMemory: resource.MustParse("2m")},
},
}},
},
},
usage: api.ResourceList{
api.ResourceRequestsMemory: resource.MustParse("1m"),
api.ResourceLimitsMemory: resource.MustParse("2m"),
api.ResourcePods: resource.MustParse("1"),
api.ResourceMemory: resource.MustParse("1m"),
},
},
"init container maximums override sum of containers": {
pod: &api.Pod{
Spec: api.PodSpec{
InitContainers: []api.Container{
{
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceCPU: resource.MustParse("4"),
api.ResourceMemory: resource.MustParse("100M"),
},
Limits: api.ResourceList{
api.ResourceCPU: resource.MustParse("8"),
api.ResourceMemory: resource.MustParse("200M"),
},
},
},
{
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceCPU: resource.MustParse("1"),
api.ResourceMemory: resource.MustParse("50M"),
},
Limits: api.ResourceList{
api.ResourceCPU: resource.MustParse("2"),
api.ResourceMemory: resource.MustParse("100M"),
},
},
},
},
Containers: []api.Container{
{
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceCPU: resource.MustParse("1"),
api.ResourceMemory: resource.MustParse("50M"),
},
Limits: api.ResourceList{
api.ResourceCPU: resource.MustParse("2"),
api.ResourceMemory: resource.MustParse("100M"),
},
},
},
{
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceCPU: resource.MustParse("2"),
api.ResourceMemory: resource.MustParse("25M"),
},
Limits: api.ResourceList{
api.ResourceCPU: resource.MustParse("5"),
api.ResourceMemory: resource.MustParse("50M"),
},
},
},
},
},
},
usage: api.ResourceList{
api.ResourceRequestsCPU: resource.MustParse("4"),
api.ResourceRequestsMemory: resource.MustParse("100M"),
api.ResourceLimitsCPU: resource.MustParse("8"),
api.ResourceLimitsMemory: resource.MustParse("200M"),
api.ResourcePods: resource.MustParse("1"),
api.ResourceCPU: resource.MustParse("4"),
api.ResourceMemory: resource.MustParse("100M"),
},
},
}
for testName, testCase := range testCases {
actual := evaluator.Usage(testCase.pod)
if !quota.Equals(testCase.usage, actual) {
t.Errorf("%s expected: %v, actual: %v", testName, testCase.usage, actual)
}
}
}