// PodConstraintsFunc verifies that all required resources are present on the pod func PodConstraintsFunc(required []api.ResourceName, object runtime.Object) error { pod, ok := object.(*api.Pod) if !ok { return fmt.Errorf("Unexpected input object %v", object) } // TODO: fix this when we have pod level cgroups // since we do not yet pod level requests/limits, we need to ensure each // container makes an explict request or limit for a quota tracked resource requiredSet := quota.ToSet(required) missingSet := sets.NewString() for i := range pod.Spec.Containers { requests := pod.Spec.Containers[i].Resources.Requests limits := pod.Spec.Containers[i].Resources.Limits containerUsage := podUsageHelper(requests, limits) containerSet := quota.ToSet(quota.ResourceNames(containerUsage)) if !containerSet.Equal(requiredSet) { difference := requiredSet.Difference(containerSet) missingSet.Insert(difference.List()...) } } if len(missingSet) == 0 { return nil } return fmt.Errorf("must specify %s", strings.Join(missingSet.List(), ",")) }
// replenishQuota is a replenishment function invoked by a controller to notify that a quota should be recalculated func (c *ClusterQuotaReconcilationController) replenishQuota(groupKind unversioned.GroupKind, namespace string, object runtime.Object) { // check if the quota controller can evaluate this kind, if not, ignore it altogether... evaluators := c.registry.Evaluators() evaluator, found := evaluators[groupKind] if !found { return } quotaNames, _ := c.clusterQuotaMapper.GetClusterQuotasFor(namespace) // only queue those quotas that are tracking a resource associated with this kind. matchedResources := evaluator.MatchesResources() for _, quotaName := range quotaNames { quota, err := c.clusterQuotaLister.Get(quotaName) if err != nil { // replenishment will be delayed, but we'll get back around to it later if it matters continue } resourceQuotaResources := utilquota.ResourceNames(quota.Status.Total.Hard) if len(utilquota.Intersection(matchedResources, resourceQuotaResources)) > 0 { // TODO: make this support targeted replenishment to a specific kind, right now it does a full recalc on that quota. c.forceCalculation(quotaName, namespace) } } }
// replenishQuota is a replenishment function invoked by a controller to notify that a quota should be recalculated func (rq *ResourceQuotaController) replenishQuota(groupKind unversioned.GroupKind, namespace string, object runtime.Object) { // check if the quota controller can evaluate this kind, if not, ignore it altogether... evaluators := rq.registry.Evaluators() evaluator, found := evaluators[groupKind] if !found { return } // check if this namespace even has a quota... indexKey := &api.ResourceQuota{} indexKey.Namespace = namespace resourceQuotas, err := rq.rqIndexer.Index("namespace", indexKey) if err != nil { glog.Errorf("quota controller could not find ResourceQuota associated with namespace: %s, could take up to %v before a quota replenishes", namespace, rq.resyncPeriod()) } if len(resourceQuotas) == 0 { return } // only queue those quotas that are tracking a resource associated with this kind. matchedResources := evaluator.MatchesResources() for i := range resourceQuotas { resourceQuota := resourceQuotas[i].(*api.ResourceQuota) resourceQuotaResources := quota.ResourceNames(resourceQuota.Status.Hard) if len(quota.Intersection(matchedResources, resourceQuotaResources)) > 0 { // TODO: make this support targeted replenishment to a specific kind, right now it does a full recalc on that quota. rq.enqueueResourceQuota(resourceQuota) } } }
// syncResourceQuota runs a complete sync of resource quota status across all known kinds func (rq *ResourceQuotaController) syncResourceQuota(v1ResourceQuota v1.ResourceQuota) (err error) { // quota is dirty if any part of spec hard limits differs from the status hard limits dirty := !api.Semantic.DeepEqual(v1ResourceQuota.Spec.Hard, v1ResourceQuota.Status.Hard) resourceQuota := api.ResourceQuota{} if err := v1.Convert_v1_ResourceQuota_To_api_ResourceQuota(&v1ResourceQuota, &resourceQuota, nil); err != nil { return err } // dirty tracks if the usage status differs from the previous sync, // if so, we send a new usage with latest status // if this is our first sync, it will be dirty by default, since we need track usage dirty = dirty || (resourceQuota.Status.Hard == nil || resourceQuota.Status.Used == nil) used := api.ResourceList{} if resourceQuota.Status.Used != nil { used = quota.Add(api.ResourceList{}, resourceQuota.Status.Used) } hardLimits := quota.Add(api.ResourceList{}, resourceQuota.Spec.Hard) newUsage, err := quota.CalculateUsage(resourceQuota.Namespace, resourceQuota.Spec.Scopes, hardLimits, rq.registry) if err != nil { return err } for key, value := range newUsage { used[key] = value } // ensure set of used values match those that have hard constraints hardResources := quota.ResourceNames(hardLimits) used = quota.Mask(used, hardResources) // Create a usage object that is based on the quota resource version that will handle updates // by default, we preserve the past usage observation, and set hard to the current spec usage := api.ResourceQuota{ ObjectMeta: metav1.ObjectMeta{ Name: resourceQuota.Name, Namespace: resourceQuota.Namespace, ResourceVersion: resourceQuota.ResourceVersion, Labels: resourceQuota.Labels, Annotations: resourceQuota.Annotations}, Status: api.ResourceQuotaStatus{ Hard: hardLimits, Used: used, }, } dirty = dirty || !quota.Equals(usage.Status.Used, resourceQuota.Status.Used) // there was a change observed by this controller that requires we update quota if dirty { v1Usage := &v1.ResourceQuota{} if err := v1.Convert_api_ResourceQuota_To_v1_ResourceQuota(&usage, v1Usage, nil); err != nil { return err } _, err = rq.kubeClient.Core().ResourceQuotas(usage.Namespace).UpdateStatus(v1Usage) return err } return nil }
// enforcePodContainerConstraints checks for required resources that are not set on this container and // adds them to missingSet. func enforcePodContainerConstraints(container *api.Container, requiredSet, missingSet sets.String) { requests := container.Resources.Requests limits := container.Resources.Limits containerUsage := podUsageHelper(requests, limits) containerSet := quota.ToSet(quota.ResourceNames(containerUsage)) if !containerSet.Equal(requiredSet) { difference := requiredSet.Difference(containerSet) missingSet.Insert(difference.List()...) } }
// WaitForResourceQuotaSync watches given resource quota until its usage is updated to desired level or a // timeout occurs. If successful, used quota values will be returned for expected resources. Otherwise an // ErrWaitTimeout will be returned. If expectedIsUpperLimit is true, given expected usage must compare greater // or equal to quota's usage, which is useful for expected usage increment. Otherwise expected usage must // compare lower or equal to quota's usage, which is useful for expected usage decrement. func WaitForResourceQuotaSync( client kclient.ResourceQuotaInterface, name string, expectedUsage kapi.ResourceList, expectedIsUpperLimit bool, timeout time.Duration, ) (kapi.ResourceList, error) { startTime := time.Now() endTime := startTime.Add(timeout) expectedResourceNames := quota.ResourceNames(expectedUsage) list, err := client.List(kapi.ListOptions{FieldSelector: fields.Set{"metadata.name": name}.AsSelector()}) if err != nil { return nil, err } for i := range list.Items { used := quota.Mask(list.Items[i].Status.Used, expectedResourceNames) if isUsageSynced(used, expectedUsage, expectedIsUpperLimit) { return used, nil } } rv := list.ResourceVersion w, err := client.Watch(kapi.ListOptions{FieldSelector: fields.Set{"metadata.name": name}.AsSelector(), ResourceVersion: rv}) if err != nil { return nil, err } defer w.Stop() for time.Now().Before(endTime) { select { case val, ok := <-w.ResultChan(): if !ok { // reget and re-watch continue } if rq, ok := val.Object.(*kapi.ResourceQuota); ok { used := quota.Mask(rq.Status.Used, expectedResourceNames) if isUsageSynced(used, expectedUsage, expectedIsUpperLimit) { return used, nil } } case <-time.After(endTime.Sub(time.Now())): return nil, wait.ErrWaitTimeout } } return nil, wait.ErrWaitTimeout }
func isLimitSynced(received, expected kapi.ResourceList) bool { resourceNames := quota.ResourceNames(expected) masked := quota.Mask(received, resourceNames) if len(masked) != len(expected) { return false } if le, _ := quota.LessThanOrEqual(masked, expected); !le { return false } if le, _ := quota.LessThanOrEqual(expected, masked); !le { return false } return true }
func isUsageSynced(received, expected kapi.ResourceList, expectedIsUpperLimit bool) bool { resourceNames := quota.ResourceNames(expected) masked := quota.Mask(received, resourceNames) if len(masked) != len(expected) { return false } if expectedIsUpperLimit { if le, _ := quota.LessThanOrEqual(masked, expected); !le { return false } } else { if le, _ := quota.LessThanOrEqual(expected, masked); !le { return false } } return true }
// Matches returns true if the quota matches the specified item. func Matches(resourceQuota *api.ResourceQuota, item runtime.Object, matchFunc MatchingResourceNamesFunc, scopeFunc MatchesScopeFunc) (bool, error) { if resourceQuota == nil { return false, fmt.Errorf("expected non-nil quota") } // verify the quota matches on at least one resource matchResource := len(matchFunc(quota.ResourceNames(resourceQuota.Status.Hard))) > 0 // by default, no scopes matches all matchScope := true for _, scope := range resourceQuota.Spec.Scopes { innerMatch, err := scopeFunc(scope, item) if err != nil { return false, err } matchScope = matchScope && innerMatch } return matchResource && matchScope, nil }
// PersistentVolumeClaimConstraintsFunc verifies that all required resources are present on the claim // In addition, it validates that the resources are valid (i.e. requests < limits) func PersistentVolumeClaimConstraintsFunc(required []api.ResourceName, object runtime.Object) error { pvc, ok := object.(*api.PersistentVolumeClaim) if !ok { return fmt.Errorf("unexpected input object %v", object) } requiredSet := quota.ToSet(required) missingSet := sets.NewString() pvcUsage := PersistentVolumeClaimUsageFunc(pvc) pvcSet := quota.ToSet(quota.ResourceNames(pvcUsage)) if diff := requiredSet.Difference(pvcSet); len(diff) > 0 { missingSet.Insert(diff.List()...) } if len(missingSet) == 0 { return nil } return fmt.Errorf("must specify %s", strings.Join(missingSet.List(), ",")) }
// ServiceConstraintsFunc verifies that all required resources are captured in service usage. func ServiceConstraintsFunc(required []api.ResourceName, object runtime.Object) error { service, ok := object.(*api.Service) if !ok { return fmt.Errorf("unexpected input object %v", object) } requiredSet := quota.ToSet(required) missingSet := sets.NewString() serviceUsage := ServiceUsageFunc(service) serviceSet := quota.ToSet(quota.ResourceNames(serviceUsage)) if diff := requiredSet.Difference(serviceSet); len(diff) > 0 { missingSet.Insert(diff.List()...) } if len(missingSet) == 0 { return nil } return fmt.Errorf("must specify %s", strings.Join(missingSet.List(), ",")) }
// Constraints verifies that all required resources are present on the item func (p *serviceEvaluator) Constraints(required []api.ResourceName, item runtime.Object) error { service, ok := item.(*api.Service) if !ok { return fmt.Errorf("unexpected input object %v", item) } requiredSet := quota.ToSet(required) missingSet := sets.NewString() serviceUsage, err := p.Usage(service) if err != nil { return err } serviceSet := quota.ToSet(quota.ResourceNames(serviceUsage)) if diff := requiredSet.Difference(serviceSet); len(diff) > 0 { missingSet.Insert(diff.List()...) } if len(missingSet) == 0 { return nil } return fmt.Errorf("must specify %s", strings.Join(missingSet.List(), ",")) }
// PodConstraintsFunc verifies that all required resources are present on the pod // In addition, it validates that the resources are valid (i.e. requests < limits) func PodConstraintsFunc(required []api.ResourceName, object runtime.Object) error { pod, ok := object.(*api.Pod) if !ok { return fmt.Errorf("Unexpected input object %v", object) } // Pod level resources are often set during admission control // As a consequence, we want to verify that resources are valid prior // to ever charging quota prematurely in case they are not. allErrs := field.ErrorList{} fldPath := field.NewPath("spec").Child("containers") for i, ctr := range pod.Spec.Containers { idxPath := fldPath.Index(i) allErrs = append(allErrs, validation.ValidateResourceRequirements(&ctr.Resources, idxPath.Child("resources"))...) } if len(allErrs) > 0 { return allErrs.ToAggregate() } // TODO: fix this when we have pod level cgroups // since we do not yet pod level requests/limits, we need to ensure each // container makes an explict request or limit for a quota tracked resource requiredSet := quota.ToSet(required) missingSet := sets.NewString() for i := range pod.Spec.Containers { requests := pod.Spec.Containers[i].Resources.Requests limits := pod.Spec.Containers[i].Resources.Limits containerUsage := podUsageHelper(requests, limits) containerSet := quota.ToSet(quota.ResourceNames(containerUsage)) if !containerSet.Equal(requiredSet) { difference := requiredSet.Difference(containerSet) missingSet.Insert(difference.List()...) } } if len(missingSet) == 0 { return nil } return fmt.Errorf("must specify %s", strings.Join(missingSet.List(), ",")) }
// Constraints verifies that all required resources are present on the item. func (p *pvcEvaluator) Constraints(required []api.ResourceName, item runtime.Object) error { pvc, ok := item.(*api.PersistentVolumeClaim) if !ok { return fmt.Errorf("unexpected input object %v", item) } // these are the items that we will be handling based on the objects actual storage-class pvcRequiredSet := append([]api.ResourceName{}, pvcResources...) if storageClassRef := util.GetClaimStorageClass(pvc); len(storageClassRef) > 0 { pvcRequiredSet = append(pvcRequiredSet, ResourceByStorageClass(storageClassRef, api.ResourcePersistentVolumeClaims)) pvcRequiredSet = append(pvcRequiredSet, ResourceByStorageClass(storageClassRef, api.ResourceRequestsStorage)) } // in effect, this will remove things from the required set that are not tied to this pvcs storage class // for example, if a quota has bronze and gold storage class items defined, we should not error a bronze pvc for not being gold. // but we should error a bronze pvc if it doesn't make a storage request size... requiredResources := quota.Intersection(required, pvcRequiredSet) requiredSet := quota.ToSet(requiredResources) // usage for this pvc will only include global pvc items + this storage class specific items pvcUsage, err := p.Usage(item) if err != nil { return err } // determine what required resources were not tracked by usage. missingSet := sets.NewString() pvcSet := quota.ToSet(quota.ResourceNames(pvcUsage)) if diff := requiredSet.Difference(pvcSet); len(diff) > 0 { missingSet.Insert(diff.List()...) } if len(missingSet) == 0 { return nil } return fmt.Errorf("must specify %s", strings.Join(missingSet.List(), ",")) }
// admitBlobWrite checks whether the blob does not exceed image quota, if set. Returns // ErrAccessDenied error if the quota is exceeded. func admitBlobWrite(ctx context.Context, repo *repository) error { rqs, err := repo.quotaClient.ResourceQuotas(repo.namespace).List(kapi.ListOptions{}) if err != nil { if kerrors.IsForbidden(err) { context.GetLogger(ctx).Warnf("Cannot list resourcequotas because of outdated cluster roles: %v", err) return nil } context.GetLogger(ctx).Errorf("Failed to list resourcequotas: %v", err) return err } usage := kapi.ResourceList{ // we are about to tag a single image to an image stream imageapi.ResourceImages: *resource.NewQuantity(1, resource.DecimalSI), } resources := quota.ResourceNames(usage) for _, rq := range rqs.Items { newUsage := quota.Add(usage, rq.Status.Used) newUsage = quota.Mask(newUsage, resources) requested := quota.Mask(rq.Spec.Hard, resources) allowed, exceeded := quota.LessThanOrEqual(newUsage, requested) if !allowed { details := make([]string, len(exceeded)) by := quota.Subtract(newUsage, requested) for i, r := range exceeded { details[i] = fmt.Sprintf("%s limited to %s by %s", r, requested[r], by[r]) } context.GetLogger(ctx).Error("Refusing to write blob exceeding quota: " + strings.Join(details, ", ")) return distribution.ErrAccessDenied } } return nil }
// syncResourceQuota runs a complete sync of resource quota status across all known kinds func (rq *ResourceQuotaController) syncResourceQuota(resourceQuota api.ResourceQuota) (err error) { // quota is dirty if any part of spec hard limits differs from the status hard limits dirty := !api.Semantic.DeepEqual(resourceQuota.Spec.Hard, resourceQuota.Status.Hard) // dirty tracks if the usage status differs from the previous sync, // if so, we send a new usage with latest status // if this is our first sync, it will be dirty by default, since we need track usage dirty = dirty || (resourceQuota.Status.Hard == nil || resourceQuota.Status.Used == nil) // Create a usage object that is based on the quota resource version that will handle updates // by default, we preserve the past usage observation, and set hard to the current spec previousUsed := api.ResourceList{} if resourceQuota.Status.Used != nil { previousUsed = quota.Add(api.ResourceList{}, resourceQuota.Status.Used) } usage := api.ResourceQuota{ ObjectMeta: api.ObjectMeta{ Name: resourceQuota.Name, Namespace: resourceQuota.Namespace, ResourceVersion: resourceQuota.ResourceVersion, Labels: resourceQuota.Labels, Annotations: resourceQuota.Annotations}, Status: api.ResourceQuotaStatus{ Hard: quota.Add(api.ResourceList{}, resourceQuota.Spec.Hard), Used: previousUsed, }, } // find the intersection between the hard resources on the quota // and the resources this controller can track to know what we can // look to measure updated usage stats for hardResources := quota.ResourceNames(usage.Status.Hard) potentialResources := []api.ResourceName{} evaluators := rq.registry.Evaluators() for _, evaluator := range evaluators { potentialResources = append(potentialResources, evaluator.MatchesResources()...) } matchedResources := quota.Intersection(hardResources, potentialResources) // sum the observed usage from each evaluator newUsage := api.ResourceList{} usageStatsOptions := quota.UsageStatsOptions{Namespace: resourceQuota.Namespace, Scopes: resourceQuota.Spec.Scopes} for _, evaluator := range evaluators { stats, err := evaluator.UsageStats(usageStatsOptions) if err != nil { return err } newUsage = quota.Add(newUsage, stats.Used) } // mask the observed usage to only the set of resources tracked by this quota // merge our observed usage with the quota usage status // if the new usage is different than the last usage, we will need to do an update newUsage = quota.Mask(newUsage, matchedResources) for key, value := range newUsage { usage.Status.Used[key] = value } dirty = dirty || !quota.Equals(usage.Status.Used, resourceQuota.Status.Used) // there was a change observed by this controller that requires we update quota if dirty { _, err = rq.kubeClient.Core().ResourceQuotas(usage.Namespace).UpdateStatus(&usage) return err } return nil }
func TestImageStreamEvaluatorUsageStats(t *testing.T) { for _, tc := range []struct { name string iss []imageapi.ImageStream namespace string expectedISCount int64 }{ { name: "no image stream", iss: []imageapi.ImageStream{}, namespace: "test", expectedISCount: 0, }, { name: "one image stream", iss: []imageapi.ImageStream{ { ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "onetag", }, }, }, namespace: "test", expectedISCount: 1, }, { name: "two image streams", iss: []imageapi.ImageStream{ { ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "is1", }, }, { ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "is2", }, }, }, namespace: "test", expectedISCount: 2, }, { name: "two image streams in different namespaces", iss: []imageapi.ImageStream{ { ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "is1", }, }, { ObjectMeta: kapi.ObjectMeta{ Namespace: "other", Name: "is2", }, }, }, namespace: "test", expectedISCount: 1, }, } { fakeClient := &testclient.Fake{} fakeClient.AddReactor("list", "imagestreams", imagetest.GetFakeImageStreamListHandler(t, tc.iss...)) evaluator := NewImageStreamEvaluator(fakeClient) stats, err := evaluator.UsageStats(kquota.UsageStatsOptions{Namespace: tc.namespace}) if err != nil { t.Errorf("[%s]: could not get usage stats for namespace %q: %v", tc.name, tc.namespace, err) continue } expectedUsage := imagetest.ExpectedResourceListFor(tc.expectedISCount) expectedResources := kquota.ResourceNames(expectedUsage) if len(stats.Used) != len(expectedResources) { t.Errorf("[%s]: got unexpected number of computed resources: %d != %d", tc.name, len(stats.Used), len(expectedResources)) } masked := kquota.Mask(stats.Used, expectedResources) if len(masked) != len(expectedResources) { for k := range stats.Used { if _, exists := masked[k]; !exists { t.Errorf("[%s]: got unexpected resource %q from Usage() method", tc.name, k) } } for _, k := range expectedResources { if _, exists := masked[k]; !exists { t.Errorf("[%s]: expected resource %q not computed", tc.name, k) } } } for rname, expectedValue := range expectedUsage { if v, exists := masked[rname]; exists { if v.Cmp(expectedValue) != 0 { t.Errorf("[%s]: got unexpected usage for %q: %s != %s", tc.name, rname, v.String(), expectedValue.String()) } } } } }
func TestImageStreamTagEvaluatorUsage(t *testing.T) { for _, tc := range []struct { name string iss []imageapi.ImageStream ist imageapi.ImageStreamTag expectedISCount int64 }{ { name: "empty image stream", iss: []imageapi.ImageStream{ { ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "is", }, Status: imageapi.ImageStreamStatus{}, }, }, ist: imageapi.ImageStreamTag{ ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "is:dest", }, Tag: &imageapi.TagReference{ Name: "dest", From: &kapi.ObjectReference{ Kind: "ImageStreamImage", Namespace: "shared", Name: "is@" + imagetest.MiscImageDigest, }, }, }, expectedISCount: 0, }, { name: "no image stream", ist: imageapi.ImageStreamTag{ ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "is:dest", }, Tag: &imageapi.TagReference{ Name: "dest", From: &kapi.ObjectReference{ Kind: "ImageStreamImage", Namespace: "shared", Name: "is@" + imagetest.MiscImageDigest, }, }, }, expectedISCount: 1, }, { name: "no image stream using image stream tag", ist: imageapi.ImageStreamTag{ ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "is:dest", }, Tag: &imageapi.TagReference{ Name: "dest", From: &kapi.ObjectReference{ Kind: "ImageStreamTag", Namespace: "shared", Name: "is:latest", }, }, }, expectedISCount: 1, }, { name: "no tag given", ist: imageapi.ImageStreamTag{ ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "is:dest", }, Image: imageapi.Image{ ObjectMeta: kapi.ObjectMeta{ Name: imagetest.MiscImageDigest, Annotations: map[string]string{imageapi.ManagedByOpenShiftAnnotation: "true"}, }, DockerImageReference: imagetest.MakeDockerImageReference("shared", "is", imagetest.MiscImageDigest), DockerImageManifest: imagetest.MiscImageDigest, }, }, expectedISCount: 1, }, { name: "missing from", ist: imageapi.ImageStreamTag{ ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "is:dest", }, Tag: &imageapi.TagReference{ Name: "dest", }, Image: imageapi.Image{ ObjectMeta: kapi.ObjectMeta{ Name: imagetest.MiscImageDigest, Annotations: map[string]string{imageapi.ManagedByOpenShiftAnnotation: "true"}, }, DockerImageReference: imagetest.MakeDockerImageReference("test", "dest", imagetest.MiscImageDigest), DockerImageManifest: imagetest.MiscImage, }, }, expectedISCount: 1, }, { name: "update existing tag", iss: []imageapi.ImageStream{ { ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "havingtag", }, Status: imageapi.ImageStreamStatus{ Tags: map[string]imageapi.TagEventList{ "latest": { Items: []imageapi.TagEvent{ { DockerImageReference: imagetest.MakeDockerImageReference("test", "havingtag", imagetest.BaseImageWith1LayerDigest), Image: imagetest.BaseImageWith1LayerDigest, }, }, }, }, }, }, }, ist: imageapi.ImageStreamTag{ ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "havingtag:latest", }, Tag: &imageapi.TagReference{ Name: "latest", From: &kapi.ObjectReference{ Kind: "ImageStreamImage", Namespace: "shared", Name: "is@" + imagetest.ChildImageWith2LayersDigest, }, }, }, expectedISCount: 0, }, { name: "add a new tag with 2 image streams", iss: []imageapi.ImageStream{ { ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "is", }, }, { ObjectMeta: kapi.ObjectMeta{ Namespace: "other", Name: "is2", }, }, }, ist: imageapi.ImageStreamTag{ ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "destis:latest", }, Tag: &imageapi.TagReference{ Name: "latest", From: &kapi.ObjectReference{ Kind: "ImageStreamTag", Namespace: "other", Name: "is2:latest", }, }, }, expectedISCount: 1, }, } { fakeClient := &testclient.Fake{} fakeClient.AddReactor("get", "imagestreams", imagetest.GetFakeImageStreamGetHandler(t, tc.iss...)) evaluator := NewImageStreamTagEvaluator(fakeClient, fakeClient) usage := evaluator.Usage(&tc.ist) expectedUsage := imagetest.ExpectedResourceListFor(tc.expectedISCount) expectedResources := kquota.ResourceNames(expectedUsage) if len(usage) != len(expectedUsage) { t.Errorf("[%s]: got unexpected number of computed resources: %d != %d", tc.name, len(usage), len(expectedResources)) } masked := kquota.Mask(usage, expectedResources) if len(masked) != len(expectedUsage) { for k := range usage { if _, exists := masked[k]; !exists { t.Errorf("[%s]: got unexpected resource %q from Usage() method", tc.name, k) } } for k := range expectedUsage { if _, exists := masked[k]; !exists { t.Errorf("[%s]: expected resource %q not computed", tc.name, k) } } } for rname, expectedValue := range expectedUsage { if v, exists := masked[rname]; exists { if v.Cmp(expectedValue) != 0 { t.Errorf("[%s]: got unexpected usage for %q: %s != %s", tc.name, rname, v.String(), expectedValue.String()) } } } } }
// Admit makes admission decisions while enforcing quota func (q *quotaAdmission) Admit(a admission.Attributes) (err error) { // ignore all operations that correspond to sub-resource actions if a.GetSubresource() != "" { return nil } // if we do not know how to evaluate use for this kind, just ignore evaluators := q.registry.Evaluators() evaluator, found := evaluators[a.GetKind()] if !found { return nil } // for this kind, check if the operation could mutate any quota resources // if no resources tracked by quota are impacted, then just return op := a.GetOperation() operationResources := evaluator.OperationResources(op) if len(operationResources) == 0 { return nil } // determine if there are any quotas in this namespace // if there are no quotas, we don't need to do anything namespace, name := a.GetNamespace(), a.GetName() items, err := q.indexer.Index("namespace", &api.ResourceQuota{ObjectMeta: api.ObjectMeta{Namespace: namespace, Name: ""}}) if err != nil { return admission.NewForbidden(a, fmt.Errorf("Error resolving quota.")) } // if there are no items held in our indexer, check our live-lookup LRU, if that misses, do the live lookup to prime it. if len(items) == 0 { lruItemObj, ok := q.liveLookupCache.Get(a.GetNamespace()) if !ok || lruItemObj.(liveLookupEntry).expiry.Before(time.Now()) { // TODO: If there are multiple operations at the same time and cache has just expired, // this may cause multiple List operations being issued at the same time. // If there is already in-flight List() for a given namespace, we should wait until // it is finished and cache is updated instead of doing the same, also to avoid // throttling - see #22422 for details. liveList, err := q.client.Core().ResourceQuotas(namespace).List(api.ListOptions{}) if err != nil { return admission.NewForbidden(a, err) } newEntry := liveLookupEntry{expiry: time.Now().Add(q.liveTTL)} for i := range liveList.Items { newEntry.items = append(newEntry.items, &liveList.Items[i]) } q.liveLookupCache.Add(a.GetNamespace(), newEntry) lruItemObj = newEntry } lruEntry := lruItemObj.(liveLookupEntry) for i := range lruEntry.items { items = append(items, lruEntry.items[i]) } } // if there are still no items, we can return if len(items) == 0 { return nil } // find the set of quotas that are pertinent to this request // reject if we match the quota, but usage is not calculated yet // reject if the input object does not satisfy quota constraints // if there are no pertinent quotas, we can just return inputObject := a.GetObject() resourceQuotas := []*api.ResourceQuota{} for i := range items { resourceQuota := items[i].(*api.ResourceQuota) match := evaluator.Matches(resourceQuota, inputObject) if !match { continue } hardResources := quota.ResourceNames(resourceQuota.Status.Hard) evaluatorResources := evaluator.MatchesResources() requiredResources := quota.Intersection(hardResources, evaluatorResources) err := evaluator.Constraints(requiredResources, inputObject) if err != nil { return admission.NewForbidden(a, fmt.Errorf("Failed quota: %s: %v", resourceQuota.Name, err)) } if !hasUsageStats(resourceQuota) { return admission.NewForbidden(a, fmt.Errorf("Status unknown for quota: %s", resourceQuota.Name)) } resourceQuotas = append(resourceQuotas, resourceQuota) } if len(resourceQuotas) == 0 { return nil } // there is at least one quota that definitely matches our object // as a result, we need to measure the usage of this object for quota // on updates, we need to subtract the previous measured usage // if usage shows no change, just return since it has no impact on quota deltaUsage := evaluator.Usage(inputObject) if admission.Update == op { prevItem, err := evaluator.Get(namespace, name) if err != nil { return admission.NewForbidden(a, fmt.Errorf("Unable to get previous: %v", err)) } prevUsage := evaluator.Usage(prevItem) deltaUsage = quota.Subtract(deltaUsage, prevUsage) } if quota.IsZero(deltaUsage) { return nil } // TODO: Move to a bucketing work queue // If we guaranteed that we processed the request in order it was received to server, we would reduce quota conflicts. // Until we have the bucketing work queue, we jitter requests and retry on conflict. numRetries := 10 interval := time.Duration(rand.Int63n(90)+int64(10)) * time.Millisecond // seed the retry loop with the initial set of quotas to process (should reduce each iteration) resourceQuotasToProcess := resourceQuotas for retry := 1; retry <= numRetries; retry++ { // the list of quotas we will try again if there is a version conflict tryAgain := []*api.ResourceQuota{} // check that we pass all remaining quotas so we do not prematurely charge // for each quota, mask the usage to the set of resources tracked by the quota // if request + used > hard, return an error describing the failure updatedUsage := map[string]api.ResourceList{} for _, resourceQuota := range resourceQuotasToProcess { hardResources := quota.ResourceNames(resourceQuota.Status.Hard) requestedUsage := quota.Mask(deltaUsage, hardResources) newUsage := quota.Add(resourceQuota.Status.Used, requestedUsage) if allowed, exceeded := quota.LessThanOrEqual(newUsage, resourceQuota.Status.Hard); !allowed { failedRequestedUsage := quota.Mask(requestedUsage, exceeded) failedUsed := quota.Mask(resourceQuota.Status.Used, exceeded) failedHard := quota.Mask(resourceQuota.Status.Hard, exceeded) return admission.NewForbidden(a, fmt.Errorf("Exceeded quota: %s, requested: %s, used: %s, limited: %s", resourceQuota.Name, prettyPrint(failedRequestedUsage), prettyPrint(failedUsed), prettyPrint(failedHard))) } updatedUsage[resourceQuota.Name] = newUsage } // update the status for each quota with its new usage // if we get a conflict, get updated quota, and enqueue for i, resourceQuota := range resourceQuotasToProcess { newUsage := updatedUsage[resourceQuota.Name] quotaToUpdate := &api.ResourceQuota{ ObjectMeta: api.ObjectMeta{ Name: resourceQuota.Name, Namespace: resourceQuota.Namespace, ResourceVersion: resourceQuota.ResourceVersion, }, Status: api.ResourceQuotaStatus{ Hard: quota.Add(api.ResourceList{}, resourceQuota.Status.Hard), Used: newUsage, }, } _, err = q.client.Core().ResourceQuotas(quotaToUpdate.Namespace).UpdateStatus(quotaToUpdate) if err != nil { if !errors.IsConflict(err) { return admission.NewForbidden(a, fmt.Errorf("Unable to update quota status: %s %v", resourceQuota.Name, err)) } // if we get a conflict, we get the latest copy of the quota documents that were not yet modified so we retry all with latest state. for fetchIndex := i; fetchIndex < len(resourceQuotasToProcess); fetchIndex++ { latestQuota, err := q.client.Core().ResourceQuotas(namespace).Get(resourceQuotasToProcess[fetchIndex].Name) if err != nil { return admission.NewForbidden(a, fmt.Errorf("Unable to get quota: %s %v", resourceQuotasToProcess[fetchIndex].Name, err)) } tryAgain = append(tryAgain, latestQuota) } break } } // all quotas were updated, so we can return if len(tryAgain) == 0 { return nil } // we have concurrent requests to update quota, so look to retry if needed // next iteration, we need to process the items that have to try again // pause the specified interval to encourage jitter if retry == numRetries { names := []string{} for _, quota := range tryAgain { names = append(names, quota.Name) } return admission.NewForbidden(a, fmt.Errorf("Unable to update status for quota: %s, ", strings.Join(names, ","))) } resourceQuotasToProcess = tryAgain time.Sleep(interval) } return nil }
// checkRequest verifies that the request does not exceed any quota constraint. it returns back a copy of quotas not yet persisted // that capture what the usage would be if the request succeeded. It return an error if the is insufficient quota to satisfy the request func (e *quotaEvaluator) checkRequest(quotas []api.ResourceQuota, a admission.Attributes) ([]api.ResourceQuota, error) { namespace := a.GetNamespace() evaluators := e.registry.Evaluators() evaluator, found := evaluators[a.GetKind().GroupKind()] if !found { return quotas, nil } op := a.GetOperation() operationResources := evaluator.OperationResources(op) if len(operationResources) == 0 { return quotas, nil } // find the set of quotas that are pertinent to this request // reject if we match the quota, but usage is not calculated yet // reject if the input object does not satisfy quota constraints // if there are no pertinent quotas, we can just return inputObject := a.GetObject() interestingQuotaIndexes := []int{} for i := range quotas { resourceQuota := quotas[i] match := evaluator.Matches(&resourceQuota, inputObject) if !match { continue } hardResources := quota.ResourceNames(resourceQuota.Status.Hard) evaluatorResources := evaluator.MatchesResources() requiredResources := quota.Intersection(hardResources, evaluatorResources) err := evaluator.Constraints(requiredResources, inputObject) if err != nil { return nil, admission.NewForbidden(a, fmt.Errorf("Failed quota: %s: %v", resourceQuota.Name, err)) } if !hasUsageStats(&resourceQuota) { return nil, admission.NewForbidden(a, fmt.Errorf("Status unknown for quota: %s", resourceQuota.Name)) } interestingQuotaIndexes = append(interestingQuotaIndexes, i) } if len(interestingQuotaIndexes) == 0 { return quotas, nil } // Usage of some resources cannot be counted in isolation. For example when // the resource represents a number of unique references to external // resource. In such a case an evaluator needs to process other objects in // the same namespace which needs to be known. if accessor, err := meta.Accessor(inputObject); namespace != "" && err == nil { if accessor.GetNamespace() == "" { accessor.SetNamespace(namespace) } } // there is at least one quota that definitely matches our object // as a result, we need to measure the usage of this object for quota // on updates, we need to subtract the previous measured usage // if usage shows no change, just return since it has no impact on quota deltaUsage := evaluator.Usage(inputObject) if admission.Update == op { prevItem := a.GetOldObject() if prevItem == nil { return nil, admission.NewForbidden(a, fmt.Errorf("Unable to get previous usage since prior version of object was not found")) } // if we can definitively determine that this is not a case of "create on update", // then charge based on the delta. Otherwise, bill the maximum metadata, err := meta.Accessor(prevItem) if err == nil && len(metadata.GetResourceVersion()) > 0 { prevUsage := evaluator.Usage(prevItem) deltaUsage = quota.Subtract(deltaUsage, prevUsage) } } if quota.IsZero(deltaUsage) { return quotas, nil } for _, index := range interestingQuotaIndexes { resourceQuota := quotas[index] hardResources := quota.ResourceNames(resourceQuota.Status.Hard) requestedUsage := quota.Mask(deltaUsage, hardResources) newUsage := quota.Add(resourceQuota.Status.Used, requestedUsage) maskedNewUsage := quota.Mask(newUsage, quota.ResourceNames(requestedUsage)) if allowed, exceeded := quota.LessThanOrEqual(maskedNewUsage, resourceQuota.Status.Hard); !allowed { failedRequestedUsage := quota.Mask(requestedUsage, exceeded) failedUsed := quota.Mask(resourceQuota.Status.Used, exceeded) failedHard := quota.Mask(resourceQuota.Status.Hard, exceeded) return nil, admission.NewForbidden(a, fmt.Errorf("Exceeded quota: %s, requested: %s, used: %s, limited: %s", resourceQuota.Name, prettyPrint(failedRequestedUsage), prettyPrint(failedUsed), prettyPrint(failedHard))) } // update to the new usage number quotas[index].Status.Used = newUsage } return quotas, nil }
func TestImageStreamImportEvaluatorUsage(t *testing.T) { for _, tc := range []struct { name string iss []imageapi.ImageStream isiSpec imageapi.ImageStreamImportSpec expectedISCount int64 }{ { name: "nothing to import", isiSpec: imageapi.ImageStreamImportSpec{ Import: true, }, }, { name: "dry run", isiSpec: imageapi.ImageStreamImportSpec{ Import: false, Repository: &imageapi.RepositoryImportSpec{ From: kapi.ObjectReference{ Kind: "DockerImage", Name: "docker.io/library/fedora", }, }, }, }, { name: "wrong from kind", isiSpec: imageapi.ImageStreamImportSpec{ Import: true, Repository: &imageapi.RepositoryImportSpec{ From: kapi.ObjectReference{ Kind: "ImageStreamImage", Namespace: "test", Name: imageapi.MakeImageStreamImageName("someis", imagetest.BaseImageWith1LayerDigest), }, }, }, expectedISCount: 1, }, { name: "import from repository to empty project", isiSpec: imageapi.ImageStreamImportSpec{ Import: true, Repository: &imageapi.RepositoryImportSpec{ From: kapi.ObjectReference{ Kind: "DockerImage", Name: "docker.io/fedora", }, }, }, expectedISCount: 1, }, { name: "import from repository to existing image stream", iss: []imageapi.ImageStream{ { ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "havingtag", }, }, { ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "is", }, }, }, isiSpec: imageapi.ImageStreamImportSpec{ Import: true, Repository: &imageapi.RepositoryImportSpec{ From: kapi.ObjectReference{ Kind: "DockerImage", Name: "docker.io/fedora", }, }, }, // target image stream already exists expectedISCount: 0, }, { name: "import from repository to non-empty project", iss: []imageapi.ImageStream{ { ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "spec", }, }, }, isiSpec: imageapi.ImageStreamImportSpec{ Import: true, Repository: &imageapi.RepositoryImportSpec{ From: kapi.ObjectReference{ Kind: "DockerImage", Name: "docker.io/library/fedora", }, }, }, expectedISCount: 1, }, { name: "import images", isiSpec: imageapi.ImageStreamImportSpec{ Import: true, Images: []imageapi.ImageImportSpec{ { From: kapi.ObjectReference{ Kind: "DockerImage", Name: "docker.io/library/fedora:f23", }, }, }, }, expectedISCount: 1, }, { name: "import image and repository", isiSpec: imageapi.ImageStreamImportSpec{ Import: true, Images: []imageapi.ImageImportSpec{ { From: kapi.ObjectReference{ Kind: "DockerImage", Name: "docker.io/centos:latest", }, }, }, Repository: &imageapi.RepositoryImportSpec{ From: kapi.ObjectReference{ Kind: "DockerImage", Name: "docker.io/library/fedora", }, }, }, expectedISCount: 1, }, } { fakeClient := &testclient.Fake{} fakeClient.AddReactor("get", "imagestreams", imagetest.GetFakeImageStreamGetHandler(t, tc.iss...)) evaluator := NewImageStreamImportEvaluator(fakeClient) isi := &imageapi.ImageStreamImport{ ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "is", }, Spec: tc.isiSpec, } usage := evaluator.Usage(isi) expectedUsage := imagetest.ExpectedResourceListFor(tc.expectedISCount) expectedResources := kquota.ResourceNames(expectedUsage) if len(usage) != len(expectedResources) { t.Errorf("[%s]: got unexpected number of computed resources: %d != %d", tc.name, len(usage), len(expectedResources)) } masked := kquota.Mask(usage, expectedResources) if len(masked) != len(expectedUsage) { for k := range usage { if _, exists := masked[k]; !exists { t.Errorf("[%s]: got unexpected resource %q from Usage() method", tc.name, k) } } for k := range expectedUsage { if _, exists := masked[k]; !exists { t.Errorf("[%s]: expected resource %q not computed", tc.name, k) } } } for rname, expectedValue := range expectedUsage { if v, exists := masked[rname]; exists { if v.Cmp(expectedValue) != 0 { t.Errorf("[%s]: got unexpected usage for %q: %s != %s", tc.name, rname, v.String(), expectedValue.String()) } } } } }
func TestImageStreamEvaluatorUsage(t *testing.T) { for _, tc := range []struct { name string iss []imageapi.ImageStream expectedISCount int64 }{ { name: "new image stream", expectedISCount: 1, }, { name: "image stream already exists", iss: []imageapi.ImageStream{ { ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "is", }, }, }, expectedISCount: 1, }, { name: "new image stream in non-empty project", iss: []imageapi.ImageStream{ { ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "existing", }, }, }, expectedISCount: 1, }, } { newIS := &imageapi.ImageStream{ ObjectMeta: kapi.ObjectMeta{ Namespace: "test", Name: "is", }, } fakeClient := &testclient.Fake{} fakeClient.AddReactor("get", "imagestreams", imagetest.GetFakeImageStreamGetHandler(t, tc.iss...)) evaluator := NewImageStreamEvaluator(fakeClient) usage := evaluator.Usage(newIS) expectedUsage := imagetest.ExpectedResourceListFor(tc.expectedISCount) expectedResources := kquota.ResourceNames(expectedUsage) if len(usage) != len(expectedResources) { t.Errorf("[%s]: got unexpected number of computed resources: %d != %d", tc.name, len(usage), len(expectedResources)) } masked := kquota.Mask(usage, expectedResources) if len(masked) != len(expectedUsage) { for k := range usage { if _, exists := masked[k]; !exists { t.Errorf("[%s]: got unexpected resource %q from Usage() method", tc.name, k) } } for k := range expectedUsage { if _, exists := masked[k]; !exists { t.Errorf("[%s]: expected resource %q not computed", tc.name, k) } } } for rname, expectedValue := range expectedUsage { if v, exists := masked[rname]; exists { if v.Cmp(expectedValue) != 0 { t.Errorf("[%s]: got unexpected usage for %q: %s != %s", tc.name, rname, v.String(), expectedValue.String()) } } } } }