// UpdateQuotaStatus the newQuota coming in will be incremented from the original. The difference between the original
// and the new is the amount to add to the namespace total, but the total status is the used value itself
func (e *clusterQuotaAccessor) UpdateQuotaStatus(newQuota *kapi.ResourceQuota) error {
clusterQuota, err := e.clusterQuotaLister.Get(newQuota.Name)
if err != nil {
return err
}
clusterQuota = e.checkCache(clusterQuota)
// make a copy
obj, err := kapi.Scheme.Copy(clusterQuota)
if err != nil {
return err
}
// re-assign objectmeta
clusterQuota = obj.(*quotaapi.ClusterResourceQuota)
clusterQuota.ObjectMeta = newQuota.ObjectMeta
clusterQuota.Namespace = ""
// determine change in usage
usageDiff := utilquota.Subtract(newQuota.Status.Used, clusterQuota.Status.Total.Used)
// update aggregate usage
clusterQuota.Status.Total.Used = newQuota.Status.Used
// update per namespace totals
oldNamespaceTotals, _ := clusterQuota.Status.Namespaces.Get(newQuota.Namespace)
namespaceTotalCopy, err := kapi.Scheme.DeepCopy(oldNamespaceTotals)
if err != nil {
return err
}
newNamespaceTotals := namespaceTotalCopy.(kapi.ResourceQuotaStatus)
newNamespaceTotals.Used = utilquota.Add(oldNamespaceTotals.Used, usageDiff)
clusterQuota.Status.Namespaces.Insert(newQuota.Namespace, newNamespaceTotals)
updatedQuota, err := e.clusterQuotaClient.ClusterResourceQuotas().UpdateStatus(clusterQuota)
if err != nil {
return err
}
e.updatedClusterQuotas.Add(clusterQuota.Name, updatedQuota)
return nil
}
// admitBlobWrite checks whether the blob does not exceed image quota, if set. Returns
// ErrAccessDenied error if the quota is exceeded.
func admitBlobWrite(ctx context.Context, repo *repository) error {
rqs, err := repo.quotaClient.ResourceQuotas(repo.namespace).List(kapi.ListOptions{})
if err != nil {
if kerrors.IsForbidden(err) {
context.GetLogger(ctx).Warnf("Cannot list resourcequotas because of outdated cluster roles: %v", err)
return nil
}
context.GetLogger(ctx).Errorf("Failed to list resourcequotas: %v", err)
return err
}
usage := kapi.ResourceList{
// we are about to tag a single image to an image stream
imageapi.ResourceImages: *resource.NewQuantity(1, resource.DecimalSI),
}
resources := quota.ResourceNames(usage)
for _, rq := range rqs.Items {
newUsage := quota.Add(usage, rq.Status.Used)
newUsage = quota.Mask(newUsage, resources)
requested := quota.Mask(rq.Spec.Hard, resources)
allowed, exceeded := quota.LessThanOrEqual(newUsage, requested)
if !allowed {
details := make([]string, len(exceeded))
by := quota.Subtract(newUsage, requested)
for i, r := range exceeded {
details[i] = fmt.Sprintf("%s limited to %s by %s", r, requested[r], by[r])
}
context.GetLogger(ctx).Error("Refusing to write blob exceeding quota: " + strings.Join(details, ", "))
return distribution.ErrAccessDenied
}
}
return nil
}
// checkRequest verifies that the request does not exceed any quota constraint. it returns back a copy of quotas not yet persisted
// that capture what the usage would be if the request succeeded. It return an error if the is insufficient quota to satisfy the request
func (e *quotaEvaluator) checkRequest(quotas []api.ResourceQuota, a admission.Attributes) ([]api.ResourceQuota, error) {
namespace := a.GetNamespace()
evaluators := e.registry.Evaluators()
evaluator, found := evaluators[a.GetKind().GroupKind()]
if !found {
return quotas, nil
}
op := a.GetOperation()
operationResources := evaluator.OperationResources(op)
if len(operationResources) == 0 {
return quotas, nil
}
// find the set of quotas that are pertinent to this request
// reject if we match the quota, but usage is not calculated yet
// reject if the input object does not satisfy quota constraints
// if there are no pertinent quotas, we can just return
inputObject := a.GetObject()
interestingQuotaIndexes := []int{}
for i := range quotas {
resourceQuota := quotas[i]
match := evaluator.Matches(&resourceQuota, inputObject)
if !match {
continue
}
hardResources := quota.ResourceNames(resourceQuota.Status.Hard)
evaluatorResources := evaluator.MatchesResources()
requiredResources := quota.Intersection(hardResources, evaluatorResources)
err := evaluator.Constraints(requiredResources, inputObject)
if err != nil {
return nil, admission.NewForbidden(a, fmt.Errorf("Failed quota: %s: %v", resourceQuota.Name, err))
}
if !hasUsageStats(&resourceQuota) {
return nil, admission.NewForbidden(a, fmt.Errorf("Status unknown for quota: %s", resourceQuota.Name))
}
interestingQuotaIndexes = append(interestingQuotaIndexes, i)
}
if len(interestingQuotaIndexes) == 0 {
return quotas, nil
}
// Usage of some resources cannot be counted in isolation. For example when
// the resource represents a number of unique references to external
// resource. In such a case an evaluator needs to process other objects in
// the same namespace which needs to be known.
if accessor, err := meta.Accessor(inputObject); namespace != "" && err == nil {
if accessor.GetNamespace() == "" {
accessor.SetNamespace(namespace)
}
}
// there is at least one quota that definitely matches our object
// as a result, we need to measure the usage of this object for quota
// on updates, we need to subtract the previous measured usage
// if usage shows no change, just return since it has no impact on quota
deltaUsage := evaluator.Usage(inputObject)
if admission.Update == op {
prevItem := a.GetOldObject()
if prevItem == nil {
return nil, admission.NewForbidden(a, fmt.Errorf("Unable to get previous usage since prior version of object was not found"))
}
// if we can definitively determine that this is not a case of "create on update",
// then charge based on the delta. Otherwise, bill the maximum
metadata, err := meta.Accessor(prevItem)
if err == nil && len(metadata.GetResourceVersion()) > 0 {
prevUsage := evaluator.Usage(prevItem)
deltaUsage = quota.Subtract(deltaUsage, prevUsage)
}
}
if quota.IsZero(deltaUsage) {
return quotas, nil
}
for _, index := range interestingQuotaIndexes {
resourceQuota := quotas[index]
hardResources := quota.ResourceNames(resourceQuota.Status.Hard)
requestedUsage := quota.Mask(deltaUsage, hardResources)
newUsage := quota.Add(resourceQuota.Status.Used, requestedUsage)
maskedNewUsage := quota.Mask(newUsage, quota.ResourceNames(requestedUsage))
if allowed, exceeded := quota.LessThanOrEqual(maskedNewUsage, resourceQuota.Status.Hard); !allowed {
failedRequestedUsage := quota.Mask(requestedUsage, exceeded)
failedUsed := quota.Mask(resourceQuota.Status.Used, exceeded)
failedHard := quota.Mask(resourceQuota.Status.Hard, exceeded)
return nil, admission.NewForbidden(a,
fmt.Errorf("Exceeded quota: %s, requested: %s, used: %s, limited: %s",
resourceQuota.Name,
prettyPrint(failedRequestedUsage),
prettyPrint(failedUsed),
prettyPrint(failedHard)))
}
// update to the new usage number
quotas[index].Status.Used = newUsage
}
return quotas, nil
}
// syncResourceQuotaFromKey syncs a quota key
func (c *ClusterQuotaReconcilationController) syncQuotaForNamespaces(originalQuota *quotaapi.ClusterResourceQuota, workItems []workItem) (error, []workItem /* to retry */) {
obj, err := kapi.Scheme.Copy(originalQuota)
if err != nil {
return err, workItems
}
quota := obj.(*quotaapi.ClusterResourceQuota)
// get the list of namespaces that match this cluster quota
matchingNamespaceNamesList, quotaSelector := c.clusterQuotaMapper.GetNamespacesFor(quota.Name)
if !kapi.Semantic.DeepEqual(quotaSelector, quota.Spec.Selector) {
return fmt.Errorf("mapping not up to date, have=%v need=%v", quotaSelector, quota.Spec.Selector), workItems
}
matchingNamespaceNames := sets.NewString(matchingNamespaceNamesList...)
reconcilationErrors := []error{}
retryItems := []workItem{}
for _, item := range workItems {
namespaceName := item.namespaceName
namespaceTotals, namespaceLoaded := quota.Status.Namespaces.Get(namespaceName)
if !matchingNamespaceNames.Has(namespaceName) {
if namespaceLoaded {
// remove this item from all totals
quota.Status.Total.Used = utilquota.Subtract(quota.Status.Total.Used, namespaceTotals.Used)
quota.Status.Namespaces.Remove(namespaceName)
}
continue
}
// if there's no work for us to do, do nothing
if !item.forceRecalculation && namespaceLoaded && kapi.Semantic.DeepEqual(namespaceTotals.Hard, quota.Spec.Quota.Hard) {
continue
}
actualUsage, err := quotaUsageCalculationFunc(namespaceName, quota.Spec.Quota.Scopes, quota.Spec.Quota.Hard, c.registry)
if err != nil {
// tally up errors, but calculate everything you can
reconcilationErrors = append(reconcilationErrors, err)
retryItems = append(retryItems, item)
continue
}
recalculatedStatus := kapi.ResourceQuotaStatus{
Used: actualUsage,
Hard: quota.Spec.Quota.Hard,
}
// subtract old usage, add new usage
quota.Status.Total.Used = utilquota.Subtract(quota.Status.Total.Used, namespaceTotals.Used)
quota.Status.Total.Used = utilquota.Add(quota.Status.Total.Used, recalculatedStatus.Used)
quota.Status.Namespaces.Insert(namespaceName, recalculatedStatus)
}
quota.Status.Total.Hard = quota.Spec.Quota.Hard
// if there's no change, no update, return early. NewAggregate returns nil on empty input
if kapi.Semantic.DeepEqual(quota, originalQuota) {
return kutilerrors.NewAggregate(reconcilationErrors), retryItems
}
if _, err := c.clusterQuotaClient.ClusterResourceQuotas().UpdateStatus(quota); err != nil {
return kutilerrors.NewAggregate(append(reconcilationErrors, err)), workItems
}
return kutilerrors.NewAggregate(reconcilationErrors), retryItems
}
// Admit makes admission decisions while enforcing quota
func (q *quotaAdmission) Admit(a admission.Attributes) (err error) {
// ignore all operations that correspond to sub-resource actions
if a.GetSubresource() != "" {
return nil
}
// if we do not know how to evaluate use for this kind, just ignore
evaluators := q.registry.Evaluators()
evaluator, found := evaluators[a.GetKind()]
if !found {
return nil
}
// for this kind, check if the operation could mutate any quota resources
// if no resources tracked by quota are impacted, then just return
op := a.GetOperation()
operationResources := evaluator.OperationResources(op)
if len(operationResources) == 0 {
return nil
}
// determine if there are any quotas in this namespace
// if there are no quotas, we don't need to do anything
namespace, name := a.GetNamespace(), a.GetName()
items, err := q.indexer.Index("namespace", &api.ResourceQuota{ObjectMeta: api.ObjectMeta{Namespace: namespace, Name: ""}})
if err != nil {
return admission.NewForbidden(a, fmt.Errorf("Error resolving quota."))
}
// if there are no items held in our indexer, check our live-lookup LRU, if that misses, do the live lookup to prime it.
if len(items) == 0 {
lruItemObj, ok := q.liveLookupCache.Get(a.GetNamespace())
if !ok || lruItemObj.(liveLookupEntry).expiry.Before(time.Now()) {
// TODO: If there are multiple operations at the same time and cache has just expired,
// this may cause multiple List operations being issued at the same time.
// If there is already in-flight List() for a given namespace, we should wait until
// it is finished and cache is updated instead of doing the same, also to avoid
// throttling - see #22422 for details.
liveList, err := q.client.Core().ResourceQuotas(namespace).List(api.ListOptions{})
if err != nil {
return admission.NewForbidden(a, err)
}
newEntry := liveLookupEntry{expiry: time.Now().Add(q.liveTTL)}
for i := range liveList.Items {
newEntry.items = append(newEntry.items, &liveList.Items[i])
}
q.liveLookupCache.Add(a.GetNamespace(), newEntry)
lruItemObj = newEntry
}
lruEntry := lruItemObj.(liveLookupEntry)
for i := range lruEntry.items {
items = append(items, lruEntry.items[i])
}
}
// if there are still no items, we can return
if len(items) == 0 {
return nil
}
// find the set of quotas that are pertinent to this request
// reject if we match the quota, but usage is not calculated yet
// reject if the input object does not satisfy quota constraints
// if there are no pertinent quotas, we can just return
inputObject := a.GetObject()
resourceQuotas := []*api.ResourceQuota{}
for i := range items {
resourceQuota := items[i].(*api.ResourceQuota)
match := evaluator.Matches(resourceQuota, inputObject)
if !match {
continue
}
hardResources := quota.ResourceNames(resourceQuota.Status.Hard)
evaluatorResources := evaluator.MatchesResources()
requiredResources := quota.Intersection(hardResources, evaluatorResources)
err := evaluator.Constraints(requiredResources, inputObject)
if err != nil {
return admission.NewForbidden(a, fmt.Errorf("Failed quota: %s: %v", resourceQuota.Name, err))
}
if !hasUsageStats(resourceQuota) {
return admission.NewForbidden(a, fmt.Errorf("Status unknown for quota: %s", resourceQuota.Name))
}
resourceQuotas = append(resourceQuotas, resourceQuota)
}
if len(resourceQuotas) == 0 {
return nil
}
// there is at least one quota that definitely matches our object
// as a result, we need to measure the usage of this object for quota
// on updates, we need to subtract the previous measured usage
// if usage shows no change, just return since it has no impact on quota
deltaUsage := evaluator.Usage(inputObject)
if admission.Update == op {
prevItem, err := evaluator.Get(namespace, name)
if err != nil {
return admission.NewForbidden(a, fmt.Errorf("Unable to get previous: %v", err))
}
prevUsage := evaluator.Usage(prevItem)
deltaUsage = quota.Subtract(deltaUsage, prevUsage)
}
if quota.IsZero(deltaUsage) {
return nil
}
// TODO: Move to a bucketing work queue
// If we guaranteed that we processed the request in order it was received to server, we would reduce quota conflicts.
// Until we have the bucketing work queue, we jitter requests and retry on conflict.
numRetries := 10
interval := time.Duration(rand.Int63n(90)+int64(10)) * time.Millisecond
// seed the retry loop with the initial set of quotas to process (should reduce each iteration)
resourceQuotasToProcess := resourceQuotas
for retry := 1; retry <= numRetries; retry++ {
// the list of quotas we will try again if there is a version conflict
tryAgain := []*api.ResourceQuota{}
// check that we pass all remaining quotas so we do not prematurely charge
// for each quota, mask the usage to the set of resources tracked by the quota
// if request + used > hard, return an error describing the failure
updatedUsage := map[string]api.ResourceList{}
for _, resourceQuota := range resourceQuotasToProcess {
hardResources := quota.ResourceNames(resourceQuota.Status.Hard)
requestedUsage := quota.Mask(deltaUsage, hardResources)
newUsage := quota.Add(resourceQuota.Status.Used, requestedUsage)
if allowed, exceeded := quota.LessThanOrEqual(newUsage, resourceQuota.Status.Hard); !allowed {
failedRequestedUsage := quota.Mask(requestedUsage, exceeded)
failedUsed := quota.Mask(resourceQuota.Status.Used, exceeded)
failedHard := quota.Mask(resourceQuota.Status.Hard, exceeded)
return admission.NewForbidden(a,
fmt.Errorf("Exceeded quota: %s, requested: %s, used: %s, limited: %s",
resourceQuota.Name,
prettyPrint(failedRequestedUsage),
prettyPrint(failedUsed),
prettyPrint(failedHard)))
}
updatedUsage[resourceQuota.Name] = newUsage
}
// update the status for each quota with its new usage
// if we get a conflict, get updated quota, and enqueue
for i, resourceQuota := range resourceQuotasToProcess {
newUsage := updatedUsage[resourceQuota.Name]
quotaToUpdate := &api.ResourceQuota{
ObjectMeta: api.ObjectMeta{
Name: resourceQuota.Name,
Namespace: resourceQuota.Namespace,
ResourceVersion: resourceQuota.ResourceVersion,
},
Status: api.ResourceQuotaStatus{
Hard: quota.Add(api.ResourceList{}, resourceQuota.Status.Hard),
Used: newUsage,
},
}
_, err = q.client.Core().ResourceQuotas(quotaToUpdate.Namespace).UpdateStatus(quotaToUpdate)
if err != nil {
if !errors.IsConflict(err) {
return admission.NewForbidden(a, fmt.Errorf("Unable to update quota status: %s %v", resourceQuota.Name, err))
}
// if we get a conflict, we get the latest copy of the quota documents that were not yet modified so we retry all with latest state.
for fetchIndex := i; fetchIndex < len(resourceQuotasToProcess); fetchIndex++ {
latestQuota, err := q.client.Core().ResourceQuotas(namespace).Get(resourceQuotasToProcess[fetchIndex].Name)
if err != nil {
return admission.NewForbidden(a, fmt.Errorf("Unable to get quota: %s %v", resourceQuotasToProcess[fetchIndex].Name, err))
}
tryAgain = append(tryAgain, latestQuota)
}
break
}
}
// all quotas were updated, so we can return
if len(tryAgain) == 0 {
return nil
}
// we have concurrent requests to update quota, so look to retry if needed
// next iteration, we need to process the items that have to try again
// pause the specified interval to encourage jitter
if retry == numRetries {
names := []string{}
for _, quota := range tryAgain {
names = append(names, quota.Name)
}
return admission.NewForbidden(a, fmt.Errorf("Unable to update status for quota: %s, ", strings.Join(names, ",")))
}
resourceQuotasToProcess = tryAgain
time.Sleep(interval)
}
return nil
}
