// WaitForResourceQuotaSync watches given resource quota until its usage is updated to desired level or a
// timeout occurs. If successful, used quota values will be returned for expected resources. Otherwise an
// ErrWaitTimeout will be returned. If expectedIsUpperLimit is true, given expected usage must compare greater
// or equal to quota's usage, which is useful for expected usage increment. Otherwise expected usage must
// compare lower or equal to quota's usage, which is useful for expected usage decrement.
func WaitForResourceQuotaSync(
client kclient.ResourceQuotaInterface,
name string,
expectedUsage kapi.ResourceList,
expectedIsUpperLimit bool,
timeout time.Duration,
) (kapi.ResourceList, error) {
startTime := time.Now()
endTime := startTime.Add(timeout)
expectedResourceNames := quota.ResourceNames(expectedUsage)
list, err := client.List(kapi.ListOptions{FieldSelector: fields.Set{"metadata.name": name}.AsSelector()})
if err != nil {
return nil, err
}
for i := range list.Items {
used := quota.Mask(list.Items[i].Status.Used, expectedResourceNames)
if isUsageSynced(used, expectedUsage, expectedIsUpperLimit) {
return used, nil
}
}
rv := list.ResourceVersion
w, err := client.Watch(kapi.ListOptions{FieldSelector: fields.Set{"metadata.name": name}.AsSelector(), ResourceVersion: rv})
if err != nil {
return nil, err
}
defer w.Stop()
for time.Now().Before(endTime) {
select {
case val, ok := <-w.ResultChan():
if !ok {
// reget and re-watch
continue
}
if rq, ok := val.Object.(*kapi.ResourceQuota); ok {
used := quota.Mask(rq.Status.Used, expectedResourceNames)
if isUsageSynced(used, expectedUsage, expectedIsUpperLimit) {
return used, nil
}
}
case <-time.After(endTime.Sub(time.Now())):
return nil, wait.ErrWaitTimeout
}
}
return nil, wait.ErrWaitTimeout
}
// syncResourceQuota runs a complete sync of resource quota status across all known kinds
func (rq *ResourceQuotaController) syncResourceQuota(v1ResourceQuota v1.ResourceQuota) (err error) {
// quota is dirty if any part of spec hard limits differs from the status hard limits
dirty := !api.Semantic.DeepEqual(v1ResourceQuota.Spec.Hard, v1ResourceQuota.Status.Hard)
resourceQuota := api.ResourceQuota{}
if err := v1.Convert_v1_ResourceQuota_To_api_ResourceQuota(&v1ResourceQuota, &resourceQuota, nil); err != nil {
return err
}
// dirty tracks if the usage status differs from the previous sync,
// if so, we send a new usage with latest status
// if this is our first sync, it will be dirty by default, since we need track usage
dirty = dirty || (resourceQuota.Status.Hard == nil || resourceQuota.Status.Used == nil)
used := api.ResourceList{}
if resourceQuota.Status.Used != nil {
used = quota.Add(api.ResourceList{}, resourceQuota.Status.Used)
}
hardLimits := quota.Add(api.ResourceList{}, resourceQuota.Spec.Hard)
newUsage, err := quota.CalculateUsage(resourceQuota.Namespace, resourceQuota.Spec.Scopes, hardLimits, rq.registry)
if err != nil {
return err
}
for key, value := range newUsage {
used[key] = value
}
// ensure set of used values match those that have hard constraints
hardResources := quota.ResourceNames(hardLimits)
used = quota.Mask(used, hardResources)
// Create a usage object that is based on the quota resource version that will handle updates
// by default, we preserve the past usage observation, and set hard to the current spec
usage := api.ResourceQuota{
ObjectMeta: metav1.ObjectMeta{
Name: resourceQuota.Name,
Namespace: resourceQuota.Namespace,
ResourceVersion: resourceQuota.ResourceVersion,
Labels: resourceQuota.Labels,
Annotations: resourceQuota.Annotations},
Status: api.ResourceQuotaStatus{
Hard: hardLimits,
Used: used,
},
}
dirty = dirty || !quota.Equals(usage.Status.Used, resourceQuota.Status.Used)
// there was a change observed by this controller that requires we update quota
if dirty {
v1Usage := &v1.ResourceQuota{}
if err := v1.Convert_api_ResourceQuota_To_v1_ResourceQuota(&usage, v1Usage, nil); err != nil {
return err
}
_, err = rq.kubeClient.Core().ResourceQuotas(usage.Namespace).UpdateStatus(v1Usage)
return err
}
return nil
}
func isLimitSynced(received, expected kapi.ResourceList) bool {
resourceNames := quota.ResourceNames(expected)
masked := quota.Mask(received, resourceNames)
if len(masked) != len(expected) {
return false
}
if le, _ := quota.LessThanOrEqual(masked, expected); !le {
return false
}
if le, _ := quota.LessThanOrEqual(expected, masked); !le {
return false
}
return true
}
// admitBlobWrite checks whether the blob does not exceed image quota, if set. Returns
// ErrAccessDenied error if the quota is exceeded.
func admitBlobWrite(ctx context.Context, repo *repository) error {
rqs, err := repo.quotaClient.ResourceQuotas(repo.namespace).List(kapi.ListOptions{})
if err != nil {
if kerrors.IsForbidden(err) {
context.GetLogger(ctx).Warnf("Cannot list resourcequotas because of outdated cluster roles: %v", err)
return nil
}
context.GetLogger(ctx).Errorf("Failed to list resourcequotas: %v", err)
return err
}
usage := kapi.ResourceList{
// we are about to tag a single image to an image stream
imageapi.ResourceImages: *resource.NewQuantity(1, resource.DecimalSI),
}
resources := quota.ResourceNames(usage)
for _, rq := range rqs.Items {
newUsage := quota.Add(usage, rq.Status.Used)
newUsage = quota.Mask(newUsage, resources)
requested := quota.Mask(rq.Spec.Hard, resources)
allowed, exceeded := quota.LessThanOrEqual(newUsage, requested)
if !allowed {
details := make([]string, len(exceeded))
by := quota.Subtract(newUsage, requested)
for i, r := range exceeded {
details[i] = fmt.Sprintf("%s limited to %s by %s", r, requested[r], by[r])
}
context.GetLogger(ctx).Error("Refusing to write blob exceeding quota: " + strings.Join(details, ", "))
return distribution.ErrAccessDenied
}
}
return nil
}
func isUsageSynced(received, expected kapi.ResourceList, expectedIsUpperLimit bool) bool {
resourceNames := quota.ResourceNames(expected)
masked := quota.Mask(received, resourceNames)
if len(masked) != len(expected) {
return false
}
if expectedIsUpperLimit {
if le, _ := quota.LessThanOrEqual(masked, expected); !le {
return false
}
} else {
if le, _ := quota.LessThanOrEqual(expected, masked); !le {
return false
}
}
return true
}
func TestImageStreamAdmissionEvaluatorUsage(t *testing.T) {
for _, tc := range []struct {
name string
oldSpec *imageapi.ImageStreamSpec
oldStatus *imageapi.ImageStreamStatus
newSpec *imageapi.ImageStreamSpec
newStatus *imageapi.ImageStreamStatus
expectedImages int64
}{
{
name: "empty image stream",
oldStatus: nil,
newStatus: &imageapi.ImageStreamStatus{},
expectedImages: 0,
},
{
name: "new image stream with one image",
oldStatus: nil,
newStatus: &imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
},
},
},
},
expectedImages: 1,
},
{
name: "no change",
oldSpec: &imageapi.ImageStreamSpec{
Tags: map[string]imageapi.TagReference{
"new": {
Name: "new",
From: &kapi.ObjectReference{
Kind: "ImageStreamImage",
Namespace: "shared",
Name: fmt.Sprintf("is@%s", miscImageDigest),
},
},
},
},
oldStatus: &imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
},
},
},
},
newSpec: &imageapi.ImageStreamSpec{
Tags: map[string]imageapi.TagReference{
"new": {
Name: "new",
From: &kapi.ObjectReference{
Kind: "ImageStreamImage",
Namespace: "shared",
Name: fmt.Sprintf("is@%s", miscImageDigest),
},
},
},
},
newStatus: &imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
},
},
},
},
// misc image is already present in common is
expectedImages: 1,
},
{
name: "adding two new tags",
oldStatus: &imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
},
},
},
},
newStatus: &imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
},
},
"foo": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/sharedlayer@%s", childImageWith2LayersDigest),
Image: childImageWith2LayersDigest,
},
},
},
"bar": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/sharedlayer@%s", baseImageWith2LayersDigest),
Image: baseImageWith2LayersDigest,
},
},
},
},
},
expectedImages: 3,
},
{
name: "adding an item and deleting the other tag",
oldStatus: &imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"foo": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/sharedlayer@%s", miscImageDigest),
Image: miscImageDigest,
},
},
},
"bar": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/sharedlayer@%s", baseImageWith2LayersDigest),
Image: baseImageWith2LayersDigest,
},
},
},
},
},
newStatus: &imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"foo": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/sharedlayer@%s", childImageWith3LayersDigest),
Image: childImageWith3LayersDigest,
},
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/sharedlayer@%s", miscImageDigest),
Image: miscImageDigest,
},
},
},
},
},
// misc image is already present in common is
expectedImages: 1,
},
{
name: "adding a tag to spec",
oldStatus: &imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
},
},
},
},
newSpec: &imageapi.ImageStreamSpec{
Tags: map[string]imageapi.TagReference{
"new": {
Name: "new",
From: &kapi.ObjectReference{
Kind: "ImageStreamImage",
Namespace: "shared",
Name: fmt.Sprintf("is@%s", baseImageWith2LayersDigest),
},
},
},
},
newStatus: &imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
},
},
},
},
expectedImages: 2,
},
{
name: "adding a tag to status already present in spec",
oldSpec: &imageapi.ImageStreamSpec{
Tags: map[string]imageapi.TagReference{
"latest": {
Name: "new",
From: &kapi.ObjectReference{
Kind: "ImageStreamImage",
Namespace: "shared",
Name: fmt.Sprintf("is@%s", childImageWith2LayersDigest),
},
},
},
},
newSpec: &imageapi.ImageStreamSpec{
Tags: map[string]imageapi.TagReference{
"latest": {
Name: "new",
From: &kapi.ObjectReference{
Kind: "ImageStreamImage",
Namespace: "shared",
Name: fmt.Sprintf("is@%s", childImageWith2LayersDigest),
},
},
},
},
newStatus: &imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is@%s", baseImageWith2LayersDigest),
Image: childImageWith2LayersDigest,
},
},
},
},
},
expectedImages: 1,
},
{
name: "refer to image in another namespace already present",
newSpec: &imageapi.ImageStreamSpec{
Tags: map[string]imageapi.TagReference{
"misc": {
Name: "misc",
From: &kapi.ObjectReference{
Kind: "ImageStreamImage",
Namespace: "shared",
Name: fmt.Sprintf("is@%s", miscImageDigest),
},
},
},
},
expectedImages: 0,
},
{
name: "refer to imagestreamimage in the same namespace",
newSpec: &imageapi.ImageStreamSpec{
Tags: map[string]imageapi.TagReference{
"commonisi": {
Name: "commonisi",
From: &kapi.ObjectReference{
Kind: "ImageStreamImage",
Name: fmt.Sprintf("common@%s", miscImageDigest),
},
},
},
},
expectedImages: 0,
},
{
name: "refer to imagestreamtag in the same namespace",
newSpec: &imageapi.ImageStreamSpec{
Tags: map[string]imageapi.TagReference{
"commonist": {
Name: "commonist",
From: &kapi.ObjectReference{
Kind: "ImageStreamTag",
Name: "common:misc",
},
},
},
},
expectedImages: 0,
},
} {
var newIS, oldIS *imageapi.ImageStream
if tc.oldStatus != nil || tc.oldSpec != nil {
oldIS = &imageapi.ImageStream{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is",
},
}
if tc.oldSpec != nil {
oldIS.Spec = *tc.oldSpec
}
if tc.oldStatus != nil {
oldIS.Status = *tc.oldStatus
}
}
if tc.newStatus != nil || tc.newSpec != nil {
newIS = &imageapi.ImageStream{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is",
},
}
if tc.newSpec != nil {
newIS.Spec = *tc.newSpec
}
if tc.newStatus != nil {
newIS.Status = *tc.newStatus
}
}
commonIS := imageapi.ImageStream{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "common",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"misc": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/common@%s", miscImageDigest),
Image: miscImageDigest,
},
},
},
},
},
}
iss := []imageapi.ImageStream{commonIS}
if oldIS != nil {
iss = append(iss, *oldIS)
}
fakeClient := &testclient.Fake{}
fakeClient.AddReactor("get", "imagestreams", getFakeImageStreamGetHandler(t, iss...))
fakeClient.AddReactor("list", "imagestreams", getFakeImageStreamListHandler(t, iss...))
fakeClient.AddReactor("get", "images", getFakeImageGetHandler(t, "test"))
evaluator := NewImageStreamAdmissionEvaluator(fakeClient)
usage := evaluator.Usage(newIS)
if len(usage) != len(expectedResources) {
t.Errorf("[%s]: got unexpected number of computed resources: %d != %d", tc.name, len(usage), len(expectedResources))
}
expectedUsage := kapi.ResourceList{
imageapi.ResourceImages: *resource.NewQuantity(tc.expectedImages, resource.DecimalSI),
}
masked := kquota.Mask(usage, expectedResources)
if len(masked) != len(expectedUsage) {
for k := range usage {
if _, exists := masked[k]; !exists {
t.Errorf("[%s]: got unexpected resource %q from Usage() method", tc.name, k)
}
}
for k := range expectedUsage {
if _, exists := masked[k]; !exists {
t.Errorf("[%s]: expected resource %q not computed", tc.name, k)
}
}
}
for rname, expectedValue := range expectedUsage {
if v, exists := masked[rname]; exists {
if v.Cmp(expectedValue) != 0 {
t.Errorf("[%s]: got unexpected usage for %q: %s != %s", tc.name, rname, v.String(), expectedValue.String())
}
}
}
}
}
func TestImageStreamEvaluatorUsageStats(t *testing.T) {
for _, tc := range []struct {
name string
iss []imageapi.ImageStream
namespace string
expectedImages int64
}{
{
"no image stream",
[]imageapi.ImageStream{},
"test",
0,
},
{
"one image stream with one tag",
[]imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "onetag",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
},
},
},
},
},
},
"test",
1,
},
{
"image stream with two references under one tag",
[]imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "sharedlayer",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/sharedlayer@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/sharedlayer@%s", baseImageWith2LayersDigest),
Image: baseImageWith2LayersDigest,
},
},
},
},
},
},
},
"test",
2,
},
{
"two images in two image streams",
[]imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is1",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is1@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
},
},
},
},
},
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is2",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is2@%s", baseImageWith2LayersDigest),
Image: baseImageWith2LayersDigest,
},
},
},
},
},
},
},
"test",
2,
},
{
"two image streams in different namespaces",
[]imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is1",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is1@%s", childImageWith2LayersDigest),
Image: childImageWith2LayersDigest,
},
},
},
},
},
},
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "other",
Name: "is2",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is2@%s", miscImageDigest),
Image: miscImageDigest,
},
},
},
},
},
},
},
"test",
1,
},
{
"same image in two different image streams",
[]imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is1",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is1@%s", childImageWith2LayersDigest),
Image: childImageWith2LayersDigest,
},
},
},
},
},
},
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is2",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is2@%s", miscImageDigest),
Image: miscImageDigest,
},
},
},
"foo": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is2@%s", childImageWith2LayersDigest),
Image: childImageWith2LayersDigest,
},
},
},
},
},
},
},
"test",
2,
},
} {
fakeClient := &testclient.Fake{}
fakeClient.AddReactor("list", "imagestreams", getFakeImageStreamListHandler(t, tc.iss...))
fakeClient.AddReactor("get", "imagestreamimages", getFakeImageStreamImageGetHandler(t, tc.iss...))
evaluator := NewImageStreamEvaluator(fakeClient)
stats, err := evaluator.UsageStats(kquota.UsageStatsOptions{Namespace: tc.namespace})
if err != nil {
t.Errorf("[%s]: could not get usage stats for namespace %q: %v", tc.name, tc.namespace, err)
continue
}
if len(stats.Used) != len(expectedResources) {
t.Errorf("[%s]: got unexpected number of computed resources: %d != %d", tc.name, len(stats.Used), len(expectedResources))
}
masked := kquota.Mask(stats.Used, expectedResources)
expectedUsage := kapi.ResourceList{
imageapi.ResourceImages: *resource.NewQuantity(tc.expectedImages, resource.DecimalSI),
}
if len(masked) != len(expectedResources) {
for k := range stats.Used {
if _, exists := masked[k]; !exists {
t.Errorf("[%s]: got unexpected resource %q from Usage() method", tc.name, k)
}
}
for _, k := range expectedResources {
if _, exists := masked[k]; !exists {
t.Errorf("[%s]: expected resource %q not computed", tc.name, k)
}
}
}
for rname, expectedValue := range expectedUsage {
if v, exists := masked[rname]; exists {
if v.Cmp(expectedValue) != 0 {
t.Errorf("[%s]: got unexpected usage for %q: %s != %s", tc.name, rname, v.String(), expectedValue.String())
}
}
}
}
}
func TestImageStreamImportEvaluatorUsage(t *testing.T) {
for _, tc := range []struct {
name string
iss []imageapi.ImageStream
isiSpec imageapi.ImageStreamImportSpec
expectedISCount int64
}{
{
name: "nothing to import",
isiSpec: imageapi.ImageStreamImportSpec{
Import: true,
},
},
{
name: "dry run",
isiSpec: imageapi.ImageStreamImportSpec{
Import: false,
Repository: &imageapi.RepositoryImportSpec{
From: kapi.ObjectReference{
Kind: "DockerImage",
Name: "docker.io/library/fedora",
},
},
},
},
{
name: "wrong from kind",
isiSpec: imageapi.ImageStreamImportSpec{
Import: true,
Repository: &imageapi.RepositoryImportSpec{
From: kapi.ObjectReference{
Kind: "ImageStreamImage",
Namespace: "test",
Name: imageapi.MakeImageStreamImageName("someis", imagetest.BaseImageWith1LayerDigest),
},
},
},
expectedISCount: 1,
},
{
name: "import from repository to empty project",
isiSpec: imageapi.ImageStreamImportSpec{
Import: true,
Repository: &imageapi.RepositoryImportSpec{
From: kapi.ObjectReference{
Kind: "DockerImage",
Name: "docker.io/fedora",
},
},
},
expectedISCount: 1,
},
{
name: "import from repository to existing image stream",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "havingtag",
},
},
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is",
},
},
},
isiSpec: imageapi.ImageStreamImportSpec{
Import: true,
Repository: &imageapi.RepositoryImportSpec{
From: kapi.ObjectReference{
Kind: "DockerImage",
Name: "docker.io/fedora",
},
},
},
// target image stream already exists
expectedISCount: 0,
},
{
name: "import from repository to non-empty project",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "spec",
},
},
},
isiSpec: imageapi.ImageStreamImportSpec{
Import: true,
Repository: &imageapi.RepositoryImportSpec{
From: kapi.ObjectReference{
Kind: "DockerImage",
Name: "docker.io/library/fedora",
},
},
},
expectedISCount: 1,
},
{
name: "import images",
isiSpec: imageapi.ImageStreamImportSpec{
Import: true,
Images: []imageapi.ImageImportSpec{
{
From: kapi.ObjectReference{
Kind: "DockerImage",
Name: "docker.io/library/fedora:f23",
},
},
},
},
expectedISCount: 1,
},
{
name: "import image and repository",
isiSpec: imageapi.ImageStreamImportSpec{
Import: true,
Images: []imageapi.ImageImportSpec{
{
From: kapi.ObjectReference{
Kind: "DockerImage",
Name: "docker.io/centos:latest",
},
},
},
Repository: &imageapi.RepositoryImportSpec{
From: kapi.ObjectReference{
Kind: "DockerImage",
Name: "docker.io/library/fedora",
},
},
},
expectedISCount: 1,
},
} {
fakeClient := &testclient.Fake{}
fakeClient.AddReactor("get", "imagestreams", imagetest.GetFakeImageStreamGetHandler(t, tc.iss...))
evaluator := NewImageStreamImportEvaluator(fakeClient)
isi := &imageapi.ImageStreamImport{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is",
},
Spec: tc.isiSpec,
}
usage := evaluator.Usage(isi)
expectedUsage := imagetest.ExpectedResourceListFor(tc.expectedISCount)
expectedResources := kquota.ResourceNames(expectedUsage)
if len(usage) != len(expectedResources) {
t.Errorf("[%s]: got unexpected number of computed resources: %d != %d", tc.name, len(usage), len(expectedResources))
}
masked := kquota.Mask(usage, expectedResources)
if len(masked) != len(expectedUsage) {
for k := range usage {
if _, exists := masked[k]; !exists {
t.Errorf("[%s]: got unexpected resource %q from Usage() method", tc.name, k)
}
}
for k := range expectedUsage {
if _, exists := masked[k]; !exists {
t.Errorf("[%s]: expected resource %q not computed", tc.name, k)
}
}
}
for rname, expectedValue := range expectedUsage {
if v, exists := masked[rname]; exists {
if v.Cmp(expectedValue) != 0 {
t.Errorf("[%s]: got unexpected usage for %q: %s != %s", tc.name, rname, v.String(), expectedValue.String())
}
}
}
}
}
// Admit makes admission decisions while enforcing quota
func (q *quotaAdmission) Admit(a admission.Attributes) (err error) {
// ignore all operations that correspond to sub-resource actions
if a.GetSubresource() != "" {
return nil
}
// if we do not know how to evaluate use for this kind, just ignore
evaluators := q.registry.Evaluators()
evaluator, found := evaluators[a.GetKind()]
if !found {
return nil
}
// for this kind, check if the operation could mutate any quota resources
// if no resources tracked by quota are impacted, then just return
op := a.GetOperation()
operationResources := evaluator.OperationResources(op)
if len(operationResources) == 0 {
return nil
}
// determine if there are any quotas in this namespace
// if there are no quotas, we don't need to do anything
namespace, name := a.GetNamespace(), a.GetName()
items, err := q.indexer.Index("namespace", &api.ResourceQuota{ObjectMeta: api.ObjectMeta{Namespace: namespace, Name: ""}})
if err != nil {
return admission.NewForbidden(a, fmt.Errorf("Error resolving quota."))
}
// if there are no items held in our indexer, check our live-lookup LRU, if that misses, do the live lookup to prime it.
if len(items) == 0 {
lruItemObj, ok := q.liveLookupCache.Get(a.GetNamespace())
if !ok || lruItemObj.(liveLookupEntry).expiry.Before(time.Now()) {
// TODO: If there are multiple operations at the same time and cache has just expired,
// this may cause multiple List operations being issued at the same time.
// If there is already in-flight List() for a given namespace, we should wait until
// it is finished and cache is updated instead of doing the same, also to avoid
// throttling - see #22422 for details.
liveList, err := q.client.Core().ResourceQuotas(namespace).List(api.ListOptions{})
if err != nil {
return admission.NewForbidden(a, err)
}
newEntry := liveLookupEntry{expiry: time.Now().Add(q.liveTTL)}
for i := range liveList.Items {
newEntry.items = append(newEntry.items, &liveList.Items[i])
}
q.liveLookupCache.Add(a.GetNamespace(), newEntry)
lruItemObj = newEntry
}
lruEntry := lruItemObj.(liveLookupEntry)
for i := range lruEntry.items {
items = append(items, lruEntry.items[i])
}
}
// if there are still no items, we can return
if len(items) == 0 {
return nil
}
// find the set of quotas that are pertinent to this request
// reject if we match the quota, but usage is not calculated yet
// reject if the input object does not satisfy quota constraints
// if there are no pertinent quotas, we can just return
inputObject := a.GetObject()
resourceQuotas := []*api.ResourceQuota{}
for i := range items {
resourceQuota := items[i].(*api.ResourceQuota)
match := evaluator.Matches(resourceQuota, inputObject)
if !match {
continue
}
hardResources := quota.ResourceNames(resourceQuota.Status.Hard)
evaluatorResources := evaluator.MatchesResources()
requiredResources := quota.Intersection(hardResources, evaluatorResources)
err := evaluator.Constraints(requiredResources, inputObject)
if err != nil {
return admission.NewForbidden(a, fmt.Errorf("Failed quota: %s: %v", resourceQuota.Name, err))
}
if !hasUsageStats(resourceQuota) {
return admission.NewForbidden(a, fmt.Errorf("Status unknown for quota: %s", resourceQuota.Name))
}
resourceQuotas = append(resourceQuotas, resourceQuota)
}
if len(resourceQuotas) == 0 {
return nil
}
// there is at least one quota that definitely matches our object
// as a result, we need to measure the usage of this object for quota
// on updates, we need to subtract the previous measured usage
// if usage shows no change, just return since it has no impact on quota
deltaUsage := evaluator.Usage(inputObject)
if admission.Update == op {
prevItem, err := evaluator.Get(namespace, name)
if err != nil {
return admission.NewForbidden(a, fmt.Errorf("Unable to get previous: %v", err))
}
prevUsage := evaluator.Usage(prevItem)
deltaUsage = quota.Subtract(deltaUsage, prevUsage)
}
if quota.IsZero(deltaUsage) {
return nil
}
// TODO: Move to a bucketing work queue
// If we guaranteed that we processed the request in order it was received to server, we would reduce quota conflicts.
// Until we have the bucketing work queue, we jitter requests and retry on conflict.
numRetries := 10
interval := time.Duration(rand.Int63n(90)+int64(10)) * time.Millisecond
// seed the retry loop with the initial set of quotas to process (should reduce each iteration)
resourceQuotasToProcess := resourceQuotas
for retry := 1; retry <= numRetries; retry++ {
// the list of quotas we will try again if there is a version conflict
tryAgain := []*api.ResourceQuota{}
// check that we pass all remaining quotas so we do not prematurely charge
// for each quota, mask the usage to the set of resources tracked by the quota
// if request + used > hard, return an error describing the failure
updatedUsage := map[string]api.ResourceList{}
for _, resourceQuota := range resourceQuotasToProcess {
hardResources := quota.ResourceNames(resourceQuota.Status.Hard)
requestedUsage := quota.Mask(deltaUsage, hardResources)
newUsage := quota.Add(resourceQuota.Status.Used, requestedUsage)
if allowed, exceeded := quota.LessThanOrEqual(newUsage, resourceQuota.Status.Hard); !allowed {
failedRequestedUsage := quota.Mask(requestedUsage, exceeded)
failedUsed := quota.Mask(resourceQuota.Status.Used, exceeded)
failedHard := quota.Mask(resourceQuota.Status.Hard, exceeded)
return admission.NewForbidden(a,
fmt.Errorf("Exceeded quota: %s, requested: %s, used: %s, limited: %s",
resourceQuota.Name,
prettyPrint(failedRequestedUsage),
prettyPrint(failedUsed),
prettyPrint(failedHard)))
}
updatedUsage[resourceQuota.Name] = newUsage
}
// update the status for each quota with its new usage
// if we get a conflict, get updated quota, and enqueue
for i, resourceQuota := range resourceQuotasToProcess {
newUsage := updatedUsage[resourceQuota.Name]
quotaToUpdate := &api.ResourceQuota{
ObjectMeta: api.ObjectMeta{
Name: resourceQuota.Name,
Namespace: resourceQuota.Namespace,
ResourceVersion: resourceQuota.ResourceVersion,
},
Status: api.ResourceQuotaStatus{
Hard: quota.Add(api.ResourceList{}, resourceQuota.Status.Hard),
Used: newUsage,
},
}
_, err = q.client.Core().ResourceQuotas(quotaToUpdate.Namespace).UpdateStatus(quotaToUpdate)
if err != nil {
if !errors.IsConflict(err) {
return admission.NewForbidden(a, fmt.Errorf("Unable to update quota status: %s %v", resourceQuota.Name, err))
}
// if we get a conflict, we get the latest copy of the quota documents that were not yet modified so we retry all with latest state.
for fetchIndex := i; fetchIndex < len(resourceQuotasToProcess); fetchIndex++ {
latestQuota, err := q.client.Core().ResourceQuotas(namespace).Get(resourceQuotasToProcess[fetchIndex].Name)
if err != nil {
return admission.NewForbidden(a, fmt.Errorf("Unable to get quota: %s %v", resourceQuotasToProcess[fetchIndex].Name, err))
}
tryAgain = append(tryAgain, latestQuota)
}
break
}
}
// all quotas were updated, so we can return
if len(tryAgain) == 0 {
return nil
}
// we have concurrent requests to update quota, so look to retry if needed
// next iteration, we need to process the items that have to try again
// pause the specified interval to encourage jitter
if retry == numRetries {
names := []string{}
for _, quota := range tryAgain {
names = append(names, quota.Name)
}
return admission.NewForbidden(a, fmt.Errorf("Unable to update status for quota: %s, ", strings.Join(names, ",")))
}
resourceQuotasToProcess = tryAgain
time.Sleep(interval)
}
return nil
}
func TestImageStreamEvaluatorUsageStats(t *testing.T) {
for _, tc := range []struct {
name string
iss []imageapi.ImageStream
namespace string
expectedISCount int64
}{
{
name: "no image stream",
iss: []imageapi.ImageStream{},
namespace: "test",
expectedISCount: 0,
},
{
name: "one image stream",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "onetag",
},
},
},
namespace: "test",
expectedISCount: 1,
},
{
name: "two image streams",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is1",
},
},
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is2",
},
},
},
namespace: "test",
expectedISCount: 2,
},
{
name: "two image streams in different namespaces",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is1",
},
},
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "other",
Name: "is2",
},
},
},
namespace: "test",
expectedISCount: 1,
},
} {
fakeClient := &testclient.Fake{}
fakeClient.AddReactor("list", "imagestreams", imagetest.GetFakeImageStreamListHandler(t, tc.iss...))
evaluator := NewImageStreamEvaluator(fakeClient)
stats, err := evaluator.UsageStats(kquota.UsageStatsOptions{Namespace: tc.namespace})
if err != nil {
t.Errorf("[%s]: could not get usage stats for namespace %q: %v", tc.name, tc.namespace, err)
continue
}
expectedUsage := imagetest.ExpectedResourceListFor(tc.expectedISCount)
expectedResources := kquota.ResourceNames(expectedUsage)
if len(stats.Used) != len(expectedResources) {
t.Errorf("[%s]: got unexpected number of computed resources: %d != %d", tc.name, len(stats.Used), len(expectedResources))
}
masked := kquota.Mask(stats.Used, expectedResources)
if len(masked) != len(expectedResources) {
for k := range stats.Used {
if _, exists := masked[k]; !exists {
t.Errorf("[%s]: got unexpected resource %q from Usage() method", tc.name, k)
}
}
for _, k := range expectedResources {
if _, exists := masked[k]; !exists {
t.Errorf("[%s]: expected resource %q not computed", tc.name, k)
}
}
}
for rname, expectedValue := range expectedUsage {
if v, exists := masked[rname]; exists {
if v.Cmp(expectedValue) != 0 {
t.Errorf("[%s]: got unexpected usage for %q: %s != %s", tc.name, rname, v.String(), expectedValue.String())
}
}
}
}
}
func TestImageStreamEvaluatorUsage(t *testing.T) {
for _, tc := range []struct {
name string
iss []imageapi.ImageStream
expectedISCount int64
}{
{
name: "new image stream",
expectedISCount: 1,
},
{
name: "image stream already exists",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is",
},
},
},
expectedISCount: 1,
},
{
name: "new image stream in non-empty project",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "existing",
},
},
},
expectedISCount: 1,
},
} {
newIS := &imageapi.ImageStream{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is",
},
}
fakeClient := &testclient.Fake{}
fakeClient.AddReactor("get", "imagestreams", imagetest.GetFakeImageStreamGetHandler(t, tc.iss...))
evaluator := NewImageStreamEvaluator(fakeClient)
usage := evaluator.Usage(newIS)
expectedUsage := imagetest.ExpectedResourceListFor(tc.expectedISCount)
expectedResources := kquota.ResourceNames(expectedUsage)
if len(usage) != len(expectedResources) {
t.Errorf("[%s]: got unexpected number of computed resources: %d != %d", tc.name, len(usage), len(expectedResources))
}
masked := kquota.Mask(usage, expectedResources)
if len(masked) != len(expectedUsage) {
for k := range usage {
if _, exists := masked[k]; !exists {
t.Errorf("[%s]: got unexpected resource %q from Usage() method", tc.name, k)
}
}
for k := range expectedUsage {
if _, exists := masked[k]; !exists {
t.Errorf("[%s]: expected resource %q not computed", tc.name, k)
}
}
}
for rname, expectedValue := range expectedUsage {
if v, exists := masked[rname]; exists {
if v.Cmp(expectedValue) != 0 {
t.Errorf("[%s]: got unexpected usage for %q: %s != %s", tc.name, rname, v.String(), expectedValue.String())
}
}
}
}
}
func TestImageStreamTagEvaluatorUsage(t *testing.T) {
for _, tc := range []struct {
name string
iss []imageapi.ImageStream
ist imageapi.ImageStreamTag
expectedImages int64
}{
{
name: "empty image stream",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is",
},
Status: imageapi.ImageStreamStatus{},
},
},
ist: imageapi.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is:dest",
},
Tag: &imageapi.TagReference{
Name: "dest",
From: &kapi.ObjectReference{
Kind: "ImageStreamImage",
Namespace: "shared",
Name: "is@" + miscImageDigest,
},
},
},
expectedImages: 1,
},
{
name: "no image stream",
ist: imageapi.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is:dest",
},
Tag: &imageapi.TagReference{
Name: "dest",
From: &kapi.ObjectReference{
Kind: "ImageStreamImage",
Namespace: "shared",
Name: "is@" + miscImageDigest,
},
},
},
expectedImages: 1,
},
{
name: "no image stream using image stream tag",
ist: imageapi.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is:dest",
},
Tag: &imageapi.TagReference{
Name: "dest",
From: &kapi.ObjectReference{
Kind: "ImageStreamTag",
Namespace: "shared",
Name: "is:latest",
},
},
},
expectedImages: 1,
},
{
name: "no tag given",
ist: imageapi.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is:dest",
},
Image: imageapi.Image{
ObjectMeta: kapi.ObjectMeta{
Name: miscImageDigest,
Annotations: map[string]string{imageapi.ManagedByOpenShiftAnnotation: "true"},
},
DockerImageReference: fmt.Sprintf("registry.example.org/%s/%s@%s", "shared", "is", miscImageDigest),
DockerImageManifest: miscImageDigest,
},
},
expectedImages: 0,
},
{
name: "missing from",
ist: imageapi.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is:dest",
},
Tag: &imageapi.TagReference{
Name: "dest",
},
Image: imageapi.Image{
ObjectMeta: kapi.ObjectMeta{
Name: miscImageDigest,
Annotations: map[string]string{imageapi.ManagedByOpenShiftAnnotation: "true"},
},
DockerImageReference: fmt.Sprintf("registry.example.org/%s/%s@%s", "test", "dest", miscImageDigest),
DockerImageManifest: miscImage,
},
},
expectedImages: 0,
},
{
name: "update existing tag",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "havingtag",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/havingtag@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
},
},
},
},
},
},
ist: imageapi.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "havingtag:latest",
},
Tag: &imageapi.TagReference{
Name: "latest",
From: &kapi.ObjectReference{
Kind: "ImageStreamImage",
Namespace: "shared",
Name: "is@" + childImageWith2LayersDigest,
},
},
},
expectedImages: 1,
},
{
name: "add a new tag with with 2 image streams",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "destis",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/destis@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is2@%s", miscImageDigest),
Image: miscImageDigest,
},
},
},
},
},
},
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "other",
Name: "is2",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is2@%s", baseImageWith2LayersDigest),
Image: baseImageWith2LayersDigest,
},
},
},
},
},
},
},
ist: imageapi.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "destis:latest",
},
Tag: &imageapi.TagReference{
Name: "latest",
From: &kapi.ObjectReference{
Kind: "ImageStreamTag",
Namespace: "other",
Name: "is2:latest",
},
},
},
expectedImages: 1,
},
{
name: "tag an image already present using image stream image",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "destis",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/destis@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is2@%s", miscImageDigest),
Image: miscImageDigest,
},
},
},
},
},
},
},
ist: imageapi.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "destis:latest",
},
Tag: &imageapi.TagReference{
Name: "latest",
From: &kapi.ObjectReference{
Kind: "ImageStreamImage",
Namespace: "shared",
Name: "is@" + baseImageWith1LayerDigest,
},
},
},
expectedImages: 0,
},
{
name: "tag an image already present using image stream tag",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "destis",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/destis@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is2@%s", miscImageDigest),
Image: miscImageDigest,
},
},
},
},
},
},
},
ist: imageapi.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "another:latest",
},
Tag: &imageapi.TagReference{
Name: "latest",
// shared is has name of baseImageWith1Layer at the first place in event list
From: &kapi.ObjectReference{
Kind: "ImageStreamTag",
Namespace: "shared",
Name: "is:latest",
},
},
},
expectedImages: 0,
},
} {
fakeClient := &testclient.Fake{}
fakeClient.AddReactor("get", "images", getFakeImageGetHandler(t, "ns"))
fakeClient.AddReactor("get", "imagestreams", getFakeImageStreamGetHandler(t, tc.iss...))
fakeClient.AddReactor("list", "imagestreams", getFakeImageStreamListHandler(t, tc.iss...))
fakeClient.AddReactor("get", "imagestreamimages", getFakeImageStreamImageGetHandler(t, tc.iss...))
evaluator := NewImageStreamTagEvaluator(fakeClient)
usage := evaluator.Usage(&tc.ist)
expectedUsage := kapi.ResourceList{
imageapi.ResourceImages: *resource.NewQuantity(tc.expectedImages, resource.DecimalSI),
}
if len(usage) != len(expectedUsage) {
t.Errorf("[%s]: got unexpected number of computed resources: %d != %d", tc.name, len(usage), len(expectedResources))
}
masked := kquota.Mask(usage, expectedResources)
if len(masked) != len(expectedUsage) {
for k := range usage {
if _, exists := masked[k]; !exists {
t.Errorf("[%s]: got unexpected resource %q from Usage() method", tc.name, k)
}
}
for k := range expectedUsage {
if _, exists := masked[k]; !exists {
t.Errorf("[%s]: expected resource %q not computed", tc.name, k)
}
}
}
for rname, expectedValue := range expectedUsage {
if v, exists := masked[rname]; exists {
if v.Cmp(expectedValue) != 0 {
t.Errorf("[%s]: got unexpected usage for %q: %s != %s", tc.name, rname, v.String(), expectedValue.String())
}
}
}
}
}
func TestImageStreamTagEvaluatorUsage(t *testing.T) {
for _, tc := range []struct {
name string
iss []imageapi.ImageStream
ist imageapi.ImageStreamTag
expectedISCount int64
}{
{
name: "empty image stream",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is",
},
Status: imageapi.ImageStreamStatus{},
},
},
ist: imageapi.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is:dest",
},
Tag: &imageapi.TagReference{
Name: "dest",
From: &kapi.ObjectReference{
Kind: "ImageStreamImage",
Namespace: "shared",
Name: "is@" + imagetest.MiscImageDigest,
},
},
},
expectedISCount: 0,
},
{
name: "no image stream",
ist: imageapi.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is:dest",
},
Tag: &imageapi.TagReference{
Name: "dest",
From: &kapi.ObjectReference{
Kind: "ImageStreamImage",
Namespace: "shared",
Name: "is@" + imagetest.MiscImageDigest,
},
},
},
expectedISCount: 1,
},
{
name: "no image stream using image stream tag",
ist: imageapi.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is:dest",
},
Tag: &imageapi.TagReference{
Name: "dest",
From: &kapi.ObjectReference{
Kind: "ImageStreamTag",
Namespace: "shared",
Name: "is:latest",
},
},
},
expectedISCount: 1,
},
{
name: "no tag given",
ist: imageapi.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is:dest",
},
Image: imageapi.Image{
ObjectMeta: kapi.ObjectMeta{
Name: imagetest.MiscImageDigest,
Annotations: map[string]string{imageapi.ManagedByOpenShiftAnnotation: "true"},
},
DockerImageReference: imagetest.MakeDockerImageReference("shared", "is", imagetest.MiscImageDigest),
DockerImageManifest: imagetest.MiscImageDigest,
},
},
expectedISCount: 1,
},
{
name: "missing from",
ist: imageapi.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is:dest",
},
Tag: &imageapi.TagReference{
Name: "dest",
},
Image: imageapi.Image{
ObjectMeta: kapi.ObjectMeta{
Name: imagetest.MiscImageDigest,
Annotations: map[string]string{imageapi.ManagedByOpenShiftAnnotation: "true"},
},
DockerImageReference: imagetest.MakeDockerImageReference("test", "dest", imagetest.MiscImageDigest),
DockerImageManifest: imagetest.MiscImage,
},
},
expectedISCount: 1,
},
{
name: "update existing tag",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "havingtag",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: imagetest.MakeDockerImageReference("test", "havingtag", imagetest.BaseImageWith1LayerDigest),
Image: imagetest.BaseImageWith1LayerDigest,
},
},
},
},
},
},
},
ist: imageapi.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "havingtag:latest",
},
Tag: &imageapi.TagReference{
Name: "latest",
From: &kapi.ObjectReference{
Kind: "ImageStreamImage",
Namespace: "shared",
Name: "is@" + imagetest.ChildImageWith2LayersDigest,
},
},
},
expectedISCount: 0,
},
{
name: "add a new tag with 2 image streams",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is",
},
},
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "other",
Name: "is2",
},
},
},
ist: imageapi.ImageStreamTag{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "destis:latest",
},
Tag: &imageapi.TagReference{
Name: "latest",
From: &kapi.ObjectReference{
Kind: "ImageStreamTag",
Namespace: "other",
Name: "is2:latest",
},
},
},
expectedISCount: 1,
},
} {
fakeClient := &testclient.Fake{}
fakeClient.AddReactor("get", "imagestreams", imagetest.GetFakeImageStreamGetHandler(t, tc.iss...))
evaluator := NewImageStreamTagEvaluator(fakeClient, fakeClient)
usage := evaluator.Usage(&tc.ist)
expectedUsage := imagetest.ExpectedResourceListFor(tc.expectedISCount)
expectedResources := kquota.ResourceNames(expectedUsage)
if len(usage) != len(expectedUsage) {
t.Errorf("[%s]: got unexpected number of computed resources: %d != %d", tc.name, len(usage), len(expectedResources))
}
masked := kquota.Mask(usage, expectedResources)
if len(masked) != len(expectedUsage) {
for k := range usage {
if _, exists := masked[k]; !exists {
t.Errorf("[%s]: got unexpected resource %q from Usage() method", tc.name, k)
}
}
for k := range expectedUsage {
if _, exists := masked[k]; !exists {
t.Errorf("[%s]: expected resource %q not computed", tc.name, k)
}
}
}
for rname, expectedValue := range expectedUsage {
if v, exists := masked[rname]; exists {
if v.Cmp(expectedValue) != 0 {
t.Errorf("[%s]: got unexpected usage for %q: %s != %s", tc.name, rname, v.String(), expectedValue.String())
}
}
}
}
}
// syncResourceQuota runs a complete sync of resource quota status across all known kinds
func (rq *ResourceQuotaController) syncResourceQuota(resourceQuota api.ResourceQuota) (err error) {
// quota is dirty if any part of spec hard limits differs from the status hard limits
dirty := !api.Semantic.DeepEqual(resourceQuota.Spec.Hard, resourceQuota.Status.Hard)
// dirty tracks if the usage status differs from the previous sync,
// if so, we send a new usage with latest status
// if this is our first sync, it will be dirty by default, since we need track usage
dirty = dirty || (resourceQuota.Status.Hard == nil || resourceQuota.Status.Used == nil)
// Create a usage object that is based on the quota resource version that will handle updates
// by default, we preserve the past usage observation, and set hard to the current spec
previousUsed := api.ResourceList{}
if resourceQuota.Status.Used != nil {
previousUsed = quota.Add(api.ResourceList{}, resourceQuota.Status.Used)
}
usage := api.ResourceQuota{
ObjectMeta: api.ObjectMeta{
Name: resourceQuota.Name,
Namespace: resourceQuota.Namespace,
ResourceVersion: resourceQuota.ResourceVersion,
Labels: resourceQuota.Labels,
Annotations: resourceQuota.Annotations},
Status: api.ResourceQuotaStatus{
Hard: quota.Add(api.ResourceList{}, resourceQuota.Spec.Hard),
Used: previousUsed,
},
}
// find the intersection between the hard resources on the quota
// and the resources this controller can track to know what we can
// look to measure updated usage stats for
hardResources := quota.ResourceNames(usage.Status.Hard)
potentialResources := []api.ResourceName{}
evaluators := rq.registry.Evaluators()
for _, evaluator := range evaluators {
potentialResources = append(potentialResources, evaluator.MatchesResources()...)
}
matchedResources := quota.Intersection(hardResources, potentialResources)
// sum the observed usage from each evaluator
newUsage := api.ResourceList{}
usageStatsOptions := quota.UsageStatsOptions{Namespace: resourceQuota.Namespace, Scopes: resourceQuota.Spec.Scopes}
for _, evaluator := range evaluators {
stats, err := evaluator.UsageStats(usageStatsOptions)
if err != nil {
return err
}
newUsage = quota.Add(newUsage, stats.Used)
}
// mask the observed usage to only the set of resources tracked by this quota
// merge our observed usage with the quota usage status
// if the new usage is different than the last usage, we will need to do an update
newUsage = quota.Mask(newUsage, matchedResources)
for key, value := range newUsage {
usage.Status.Used[key] = value
}
dirty = dirty || !quota.Equals(usage.Status.Used, resourceQuota.Status.Used)
// there was a change observed by this controller that requires we update quota
if dirty {
_, err = rq.kubeClient.Core().ResourceQuotas(usage.Namespace).UpdateStatus(&usage)
return err
}
return nil
}
func TestImageStreamEvaluatorUsage(t *testing.T) {
for _, tc := range []struct {
name string
is imageapi.ImageStream
expectedImages int64
}{
{
"empty image stream",
imageapi.ImageStream{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "empty",
},
Status: imageapi.ImageStreamStatus{},
},
0,
},
{
"image stream with one tag",
imageapi.ImageStream{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
},
},
},
},
},
1,
},
{
"image stream with spec filled",
imageapi.ImageStream{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is",
},
Spec: imageapi.ImageStreamSpec{
Tags: map[string]imageapi.TagReference{
"new": {
Name: "new",
From: &kapi.ObjectReference{
Kind: "ImageStreamImage",
Namespace: "shared",
Name: fmt.Sprintf("is@%s", miscImageDigest),
},
},
},
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
},
},
},
},
},
1, // spec isn't taken into account
},
{
"image stream with two images under one tag",
imageapi.ImageStream{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "sharedlayer",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/sharedlayer@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/sharedlayer@%s", baseImageWith2LayersDigest),
Image: baseImageWith2LayersDigest,
},
},
},
},
},
},
2,
},
{
"image stream with two tags",
imageapi.ImageStream{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "sharedlayer",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"foo": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/sharedlayer@%s", baseImageWith2LayersDigest),
Image: baseImageWith2LayersDigest,
},
},
},
"bar": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/sharedlayer@%s", childImageWith3LayersDigest),
Image: childImageWith3LayersDigest,
},
},
},
},
},
},
2,
},
{
"image stream with the same image under different tag",
imageapi.ImageStream{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "noshared",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/noshared@%s", childImageWith2LayersDigest),
Image: childImageWith2LayersDigest,
},
},
},
"foo": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/noshared@%s", childImageWith2LayersDigest),
Image: childImageWith2LayersDigest,
},
},
},
},
},
},
1,
},
} {
fakeClient := &testclient.Fake{}
fakeClient.AddReactor("get", "imagestreams", getFakeImageStreamGetHandler(t, tc.is))
fakeClient.AddReactor("get", "imagestreamimages", getFakeImageStreamImageGetHandler(t, tc.is))
evaluator := NewImageStreamEvaluator(fakeClient)
is, err := evaluator.Get(tc.is.Namespace, tc.is.Name)
if err != nil {
t.Errorf("[%s]: could not get image stream %q: %v", tc.name, tc.is.Name, err)
continue
}
usage := evaluator.Usage(is)
if len(usage) != len(expectedResources) {
t.Errorf("[%s]: got unexpected number of computed resources: %d != %d", tc.name, len(usage), len(expectedResources))
}
masked := kquota.Mask(usage, expectedResources)
expectedUsage := kapi.ResourceList{
imageapi.ResourceImages: *resource.NewQuantity(tc.expectedImages, resource.DecimalSI),
}
if len(masked) != len(expectedResources) {
for k := range usage {
if _, exists := masked[k]; !exists {
t.Errorf("[%s]: got unexpected resource %q from Usage() method", tc.name, k)
}
}
for _, k := range expectedResources {
if _, exists := masked[k]; !exists {
t.Errorf("[%s]: expected resource %q not computed", tc.name, k)
}
}
}
for rname, expectedValue := range expectedUsage {
if v, exists := masked[rname]; exists {
if v.Cmp(expectedValue) != 0 {
t.Errorf("[%s]: got unexpected usage for %q: %s != %s", tc.name, rname, v.String(), expectedValue.String())
}
}
}
}
}
// checkRequest verifies that the request does not exceed any quota constraint. it returns back a copy of quotas not yet persisted
// that capture what the usage would be if the request succeeded. It return an error if the is insufficient quota to satisfy the request
func (e *quotaEvaluator) checkRequest(quotas []api.ResourceQuota, a admission.Attributes) ([]api.ResourceQuota, error) {
namespace := a.GetNamespace()
evaluators := e.registry.Evaluators()
evaluator, found := evaluators[a.GetKind().GroupKind()]
if !found {
return quotas, nil
}
op := a.GetOperation()
operationResources := evaluator.OperationResources(op)
if len(operationResources) == 0 {
return quotas, nil
}
// find the set of quotas that are pertinent to this request
// reject if we match the quota, but usage is not calculated yet
// reject if the input object does not satisfy quota constraints
// if there are no pertinent quotas, we can just return
inputObject := a.GetObject()
interestingQuotaIndexes := []int{}
for i := range quotas {
resourceQuota := quotas[i]
match := evaluator.Matches(&resourceQuota, inputObject)
if !match {
continue
}
hardResources := quota.ResourceNames(resourceQuota.Status.Hard)
evaluatorResources := evaluator.MatchesResources()
requiredResources := quota.Intersection(hardResources, evaluatorResources)
err := evaluator.Constraints(requiredResources, inputObject)
if err != nil {
return nil, admission.NewForbidden(a, fmt.Errorf("Failed quota: %s: %v", resourceQuota.Name, err))
}
if !hasUsageStats(&resourceQuota) {
return nil, admission.NewForbidden(a, fmt.Errorf("Status unknown for quota: %s", resourceQuota.Name))
}
interestingQuotaIndexes = append(interestingQuotaIndexes, i)
}
if len(interestingQuotaIndexes) == 0 {
return quotas, nil
}
// Usage of some resources cannot be counted in isolation. For example when
// the resource represents a number of unique references to external
// resource. In such a case an evaluator needs to process other objects in
// the same namespace which needs to be known.
if accessor, err := meta.Accessor(inputObject); namespace != "" && err == nil {
if accessor.GetNamespace() == "" {
accessor.SetNamespace(namespace)
}
}
// there is at least one quota that definitely matches our object
// as a result, we need to measure the usage of this object for quota
// on updates, we need to subtract the previous measured usage
// if usage shows no change, just return since it has no impact on quota
deltaUsage := evaluator.Usage(inputObject)
if admission.Update == op {
prevItem := a.GetOldObject()
if prevItem == nil {
return nil, admission.NewForbidden(a, fmt.Errorf("Unable to get previous usage since prior version of object was not found"))
}
// if we can definitively determine that this is not a case of "create on update",
// then charge based on the delta. Otherwise, bill the maximum
metadata, err := meta.Accessor(prevItem)
if err == nil && len(metadata.GetResourceVersion()) > 0 {
prevUsage := evaluator.Usage(prevItem)
deltaUsage = quota.Subtract(deltaUsage, prevUsage)
}
}
if quota.IsZero(deltaUsage) {
return quotas, nil
}
for _, index := range interestingQuotaIndexes {
resourceQuota := quotas[index]
hardResources := quota.ResourceNames(resourceQuota.Status.Hard)
requestedUsage := quota.Mask(deltaUsage, hardResources)
newUsage := quota.Add(resourceQuota.Status.Used, requestedUsage)
maskedNewUsage := quota.Mask(newUsage, quota.ResourceNames(requestedUsage))
if allowed, exceeded := quota.LessThanOrEqual(maskedNewUsage, resourceQuota.Status.Hard); !allowed {
failedRequestedUsage := quota.Mask(requestedUsage, exceeded)
failedUsed := quota.Mask(resourceQuota.Status.Used, exceeded)
failedHard := quota.Mask(resourceQuota.Status.Hard, exceeded)
return nil, admission.NewForbidden(a,
fmt.Errorf("Exceeded quota: %s, requested: %s, used: %s, limited: %s",
resourceQuota.Name,
prettyPrint(failedRequestedUsage),
prettyPrint(failedUsed),
prettyPrint(failedHard)))
}
// update to the new usage number
quotas[index].Status.Used = newUsage
}
return quotas, nil
}
func TestImageStreamMappingEvaluatorUsage(t *testing.T) {
for _, tc := range []struct {
name string
iss []imageapi.ImageStream
imageName string
imageManifest string
imageAnnotations map[string]string
destISNamespace string
destISName string
expectedImages int64
}{
{
name: "empty image stream",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is",
},
Status: imageapi.ImageStreamStatus{},
},
},
imageName: miscImageDigest,
imageManifest: miscImage,
imageAnnotations: map[string]string{imageapi.ManagedByOpenShiftAnnotation: "true"},
destISNamespace: "test",
destISName: "is",
expectedImages: 1,
},
{
name: "no image stream",
imageName: miscImageDigest,
imageManifest: miscImage,
imageAnnotations: map[string]string{imageapi.ManagedByOpenShiftAnnotation: "true"},
destISNamespace: "test",
destISName: "is",
expectedImages: 1,
},
{
name: "missing image annotation",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "is",
},
Status: imageapi.ImageStreamStatus{},
},
},
imageName: miscImageDigest,
imageManifest: miscImage,
destISNamespace: "test",
destISName: "is",
expectedImages: 0,
},
{
name: "update existing tag",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "havingtag",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/havingtag@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
},
},
},
},
},
},
imageName: childImageWith2LayersDigest,
imageManifest: childImageWith2Layers,
imageAnnotations: map[string]string{imageapi.ManagedByOpenShiftAnnotation: "true"},
destISNamespace: "test",
destISName: "havingtag",
expectedImages: 1,
},
{
name: "add a new tag with 2 image streams",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "destis",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/destis@%s", baseImageWith1LayerDigest),
Image: baseImageWith1LayerDigest,
},
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is2@%s", miscImageDigest),
Image: miscImageDigest,
},
},
},
},
},
},
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "other",
Name: "is2",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is2@%s", baseImageWith2LayersDigest),
Image: baseImageWith2LayersDigest,
},
},
},
},
},
},
},
imageName: childImageWith3LayersDigest,
imageManifest: childImageWith3Layers,
imageAnnotations: map[string]string{imageapi.ManagedByOpenShiftAnnotation: "true"},
destISNamespace: "test",
destISName: "destis",
expectedImages: 1,
},
{
name: "add a new tag to a new image stream with image present in the other",
iss: []imageapi.ImageStream{
{
ObjectMeta: kapi.ObjectMeta{
Namespace: "test",
Name: "other",
},
Status: imageapi.ImageStreamStatus{
Tags: map[string]imageapi.TagEventList{
"latest": {
Items: []imageapi.TagEvent{
{
DockerImageReference: fmt.Sprintf("172.30.12.34:5000/test/is2@%s", baseImageWith2LayersDigest),
Image: baseImageWith2LayersDigest,
},
},
},
},
},
},
},
imageName: baseImageWith2LayersDigest,
imageManifest: baseImageWith2Layers,
imageAnnotations: map[string]string{imageapi.ManagedByOpenShiftAnnotation: "true"},
destISNamespace: "test",
destISName: "destis",
expectedImages: 0,
},
} {
fakeClient := &testclient.Fake{}
fakeClient.AddReactor("list", "imagestreams", getFakeImageStreamListHandler(t, tc.iss...))
fakeClient.AddReactor("get", "imagestreamimages", getFakeImageStreamImageGetHandler(t, tc.iss...))
evaluator := NewImageStreamMappingEvaluator(fakeClient)
ism := &imageapi.ImageStreamMapping{
ObjectMeta: kapi.ObjectMeta{
Namespace: tc.destISNamespace,
Name: tc.destISName,
},
Image: imageapi.Image{
ObjectMeta: kapi.ObjectMeta{
Name: tc.imageName,
Annotations: tc.imageAnnotations,
},
DockerImageReference: fmt.Sprintf("registry.example.org/%s/%s@%s", tc.destISNamespace, tc.destISName, tc.imageName),
DockerImageManifest: tc.imageManifest,
},
}
usage := evaluator.Usage(ism)
if len(usage) != len(expectedResources) {
t.Errorf("[%s]: got unexpected number of computed resources: %d != %d", tc.name, len(usage), len(expectedResources))
}
masked := kquota.Mask(usage, expectedResources)
expectedUsage := kapi.ResourceList{
imageapi.ResourceImages: *resource.NewQuantity(tc.expectedImages, resource.DecimalSI),
}
if len(masked) != len(expectedUsage) {
for k := range usage {
if _, exists := masked[k]; !exists {
t.Errorf("[%s]: got unexpected resource %q from Usage() method", tc.name, k)
}
}
for k := range expectedUsage {
if _, exists := masked[k]; !exists {
t.Errorf("[%s]: expected resource %q not computed", tc.name, k)
}
}
}
for rname, expectedValue := range expectedUsage {
if v, exists := masked[rname]; exists {
if v.Cmp(expectedValue) != 0 {
t.Errorf("[%s]: got unexpected usage for %q: %s != %s", tc.name, rname, v.String(), expectedValue.String())
}
}
}
}
}