func TestPullthroughServeBlob(t *testing.T) {
	ctx := context.Background()

	installFakeAccessController(t)

	testImage, err := registrytest.NewImageForManifest("user/app", registrytest.SampleImageManifestSchema1, false)
	if err != nil {
		t.Fatal(err)
	}
	client := &testclient.Fake{}
	client.AddReactor("get", "images", registrytest.GetFakeImageGetHandler(t, *testImage))

	// TODO: get rid of those nasty global vars
	backupRegistryClient := DefaultRegistryClient
	DefaultRegistryClient = makeFakeRegistryClient(client, fake.NewSimpleClientset())
	defer func() {
		// set it back once this test finishes to make other unit tests working again
		DefaultRegistryClient = backupRegistryClient
	}()

	// pullthrough middleware will attempt to pull from this registry instance
	remoteRegistryApp := handlers.NewApp(ctx, &configuration.Configuration{
		Loglevel: "debug",
		Auth: map[string]configuration.Parameters{
			fakeAuthorizerName: {"realm": fakeAuthorizerName},
		},
		Storage: configuration.Storage{
			"inmemory": configuration.Parameters{},
			"cache": configuration.Parameters{
				"blobdescriptor": "inmemory",
			},
			"delete": configuration.Parameters{
				"enabled": true,
			},
		},
		Middleware: map[string][]configuration.Middleware{
			"registry":   {{Name: "openshift"}},
			"repository": {{Name: "openshift", Options: configuration.Parameters{"pullthrough": false}}},
			"storage":    {{Name: "openshift"}},
		},
	})
	remoteRegistryServer := httptest.NewServer(remoteRegistryApp)
	defer remoteRegistryServer.Close()

	serverURL, err := url.Parse(remoteRegistryServer.URL)
	if err != nil {
		t.Fatalf("error parsing server url: %v", err)
	}
	os.Setenv("DOCKER_REGISTRY_URL", serverURL.Host)
	testImage.DockerImageReference = fmt.Sprintf("%s/%s@%s", serverURL.Host, "user/app", testImage.Name)

	testImageStream := registrytest.TestNewImageStreamObject("user", "app", "latest", testImage.Name, testImage.DockerImageReference)
	if testImageStream.Annotations == nil {
		testImageStream.Annotations = make(map[string]string)
	}
	testImageStream.Annotations[imageapi.InsecureRepositoryAnnotation] = "true"

	client.AddReactor("get", "imagestreams", imagetest.GetFakeImageStreamGetHandler(t, *testImageStream))

	blob1Desc, blob1Content, err := registrytest.UploadTestBlob(serverURL, nil, "user/app")
	if err != nil {
		t.Fatal(err)
	}
	blob2Desc, blob2Content, err := registrytest.UploadTestBlob(serverURL, nil, "user/app")
	if err != nil {
		t.Fatal(err)
	}

	blob1Storage := map[digest.Digest][]byte{blob1Desc.Digest: blob1Content}
	blob2Storage := map[digest.Digest][]byte{blob2Desc.Digest: blob2Content}

	for _, tc := range []struct {
		name                       string
		method                     string
		blobDigest                 digest.Digest
		localBlobs                 map[digest.Digest][]byte
		expectedStatError          error
		expectedContentLength      int64
		expectedBytesServed        int64
		expectedBytesServedLocally int64
		expectedLocalCalls         map[string]int
	}{
		{
			name:                  "stat local blob",
			method:                "HEAD",
			blobDigest:            blob1Desc.Digest,
			localBlobs:            blob1Storage,
			expectedContentLength: int64(len(blob1Content)),
			expectedLocalCalls: map[string]int{
				"Stat":      1,
				"ServeBlob": 1,
			},
		},

		{
			name:                       "serve local blob",
			method:                     "GET",
			blobDigest:                 blob1Desc.Digest,
			localBlobs:                 blob1Storage,
			expectedContentLength:      int64(len(blob1Content)),
			expectedBytesServed:        int64(len(blob1Content)),
			expectedBytesServedLocally: int64(len(blob1Content)),
			expectedLocalCalls: map[string]int{
				"Stat":      1,
				"ServeBlob": 1,
			},
		},

		{
			name:                  "stat remote blob",
			method:                "HEAD",
			blobDigest:            blob1Desc.Digest,
			localBlobs:            blob2Storage,
			expectedContentLength: int64(len(blob1Content)),
			expectedLocalCalls:    map[string]int{"Stat": 1},
		},

		{
			name:                  "serve remote blob",
			method:                "GET",
			blobDigest:            blob1Desc.Digest,
			expectedContentLength: int64(len(blob1Content)),
			expectedBytesServed:   int64(len(blob1Content)),
			expectedLocalCalls:    map[string]int{"Stat": 1},
		},

		{
			name:               "unknown blob digest",
			method:             "GET",
			blobDigest:         unknownBlobDigest,
			expectedStatError:  distribution.ErrBlobUnknown,
			expectedLocalCalls: map[string]int{"Stat": 1},
		},
	} {
		localBlobStore := newTestBlobStore(tc.localBlobs)

		cachedLayers, err := newDigestToRepositoryCache(10)
		if err != nil {
			t.Fatal(err)
		}
		ptbs := &pullthroughBlobStore{
			BlobStore: localBlobStore,
			repo: &repository{
				ctx:              ctx,
				namespace:        "user",
				name:             "app",
				pullthrough:      true,
				cachedLayers:     cachedLayers,
				registryOSClient: client,
			},
			digestToStore: make(map[string]distribution.BlobStore),
		}

		req, err := http.NewRequest(tc.method, fmt.Sprintf("http://example.org/v2/user/app/blobs/%s", tc.blobDigest), nil)
		if err != nil {
			t.Fatalf("[%s] failed to create http request: %v", tc.name, err)
		}
		w := httptest.NewRecorder()

		dgst := digest.Digest(tc.blobDigest)

		_, err = ptbs.Stat(ctx, dgst)
		if err != tc.expectedStatError {
			t.Errorf("[%s] Stat returned unexpected error: %#+v != %#+v", tc.name, err, tc.expectedStatError)
		}
		if err != nil || tc.expectedStatError != nil {
			continue
		}
		err = ptbs.ServeBlob(ctx, w, req, dgst)
		if err != nil {
			t.Errorf("[%s] unexpected ServeBlob error: %v", tc.name, err)
			continue
		}

		clstr := w.Header().Get("Content-Length")
		if cl, err := strconv.ParseInt(clstr, 10, 64); err != nil {
			t.Errorf(`[%s] unexpected Content-Length: %q != "%d"`, tc.name, clstr, tc.expectedContentLength)
		} else {
			if cl != tc.expectedContentLength {
				t.Errorf("[%s] Content-Length does not match expected size: %d != %d", tc.name, cl, tc.expectedContentLength)
			}
		}
		if w.Header().Get("Content-Type") != "application/octet-stream" {
			t.Errorf("[%s] Content-Type does not match expected: %q != %q", tc.name, w.Header().Get("Content-Type"), "application/octet-stream")
		}

		body := w.Body.Bytes()
		if int64(len(body)) != tc.expectedBytesServed {
			t.Errorf("[%s] unexpected size of body: %d != %d", tc.name, len(body), tc.expectedBytesServed)
		}

		for name, expCount := range tc.expectedLocalCalls {
			count := localBlobStore.calls[name]
			if count != expCount {
				t.Errorf("[%s] expected %d calls to method %s of local blob store, not %d", tc.name, expCount, name, count)
			}
		}
		for name, count := range localBlobStore.calls {
			if _, exists := tc.expectedLocalCalls[name]; !exists {
				t.Errorf("[%s] expected no calls to method %s of local blob store, got %d", tc.name, name, count)
			}
		}

		if localBlobStore.bytesServed != tc.expectedBytesServedLocally {
			t.Errorf("[%s] unexpected number of bytes served locally: %d != %d", tc.name, localBlobStore.bytesServed, tc.expectedBytesServed)
		}
	}
}
// TestBlobDescriptorServiceIsApplied ensures that blobDescriptorService middleware gets applied.
// It relies on the fact that blobDescriptorService requires higher levels to set repository object on given
// context. If the object isn't given, its method will err out.
func TestBlobDescriptorServiceIsApplied(t *testing.T) {
	ctx := context.Background()

	// don't do any authorization check
	installFakeAccessController(t)
	m := fakeBlobDescriptorService(t)
	// to make other unit tests working
	defer m.changeUnsetRepository(false)

	testImage, err := registrytest.NewImageForManifest("user/app", registrytest.SampleImageManifestSchema1, true)
	if err != nil {
		t.Fatal(err)
	}
	testImageStream := registrytest.TestNewImageStreamObject("user", "app", "latest", testImage.Name, "")
	client := &testclient.Fake{}
	client.AddReactor("get", "imagestreams", imagetest.GetFakeImageStreamGetHandler(t, *testImageStream))
	client.AddReactor("get", "images", registrytest.GetFakeImageGetHandler(t, *testImage))

	// TODO: get rid of those nasty global vars
	backupRegistryClient := DefaultRegistryClient
	DefaultRegistryClient = makeFakeRegistryClient(client, fake.NewSimpleClientset())
	defer func() {
		// set it back once this test finishes to make other unit tests working
		DefaultRegistryClient = backupRegistryClient
	}()

	app := handlers.NewApp(ctx, &configuration.Configuration{
		Loglevel: "debug",
		Auth: map[string]configuration.Parameters{
			fakeAuthorizerName: {"realm": fakeAuthorizerName},
		},
		Storage: configuration.Storage{
			"inmemory": configuration.Parameters{},
			"cache": configuration.Parameters{
				"blobdescriptor": "inmemory",
			},
			"delete": configuration.Parameters{
				"enabled": true,
			},
		},
		Middleware: map[string][]configuration.Middleware{
			"registry":   {{Name: "openshift"}},
			"repository": {{Name: "openshift"}},
			"storage":    {{Name: "openshift"}},
		},
	})
	server := httptest.NewServer(app)
	router := v2.Router()

	serverURL, err := url.Parse(server.URL)
	if err != nil {
		t.Fatalf("error parsing server url: %v", err)
	}
	os.Setenv("DOCKER_REGISTRY_URL", serverURL.Host)

	desc, _, err := registrytest.UploadTestBlob(serverURL, nil, "user/app")
	if err != nil {
		t.Fatal(err)
	}

	for _, tc := range []struct {
		name                      string
		method                    string
		endpoint                  string
		vars                      []string
		unsetRepository           bool
		expectedStatus            int
		expectedMethodInvocations map[string]int
	}{
		{
			name:     "get blob with repository unset",
			method:   http.MethodGet,
			endpoint: v2.RouteNameBlob,
			vars: []string{
				"name", "user/app",
				"digest", desc.Digest.String(),
			},
			unsetRepository:           true,
			expectedStatus:            http.StatusInternalServerError,
			expectedMethodInvocations: map[string]int{"Stat": 1},
		},

		{
			name:     "get blob",
			method:   http.MethodGet,
			endpoint: v2.RouteNameBlob,
			vars: []string{
				"name", "user/app",
				"digest", desc.Digest.String(),
			},
			expectedStatus: http.StatusOK,
			// 1st stat is invoked in (*distribution/registry/handlers.blobHandler).GetBlob() as a
			//   check of blob existence
			// 2nd stat happens in (*errorBlobStore).ServeBlob() invoked by the same GetBlob handler
			// 3rd stat is done by (*blobServiceListener).ServeBlob once the blob serving is finished;
			//     it may happen with a slight delay after the blob was served
			expectedMethodInvocations: map[string]int{"Stat": 3},
		},

		{
			name:     "stat blob with repository unset",
			method:   http.MethodHead,
			endpoint: v2.RouteNameBlob,
			vars: []string{
				"name", "user/app",
				"digest", desc.Digest.String(),
			},
			unsetRepository:           true,
			expectedStatus:            http.StatusInternalServerError,
			expectedMethodInvocations: map[string]int{"Stat": 1},
		},

		{
			name:     "stat blob",
			method:   http.MethodHead,
			endpoint: v2.RouteNameBlob,
			vars: []string{
				"name", "user/app",
				"digest", desc.Digest.String(),
			},
			expectedStatus: http.StatusOK,
			// 1st stat is invoked in (*distribution/registry/handlers.blobHandler).GetBlob() as a
			//   check of blob existence
			// 2nd stat happens in (*errorBlobStore).ServeBlob() invoked by the same GetBlob handler
			// 3rd stat is done by (*blobServiceListener).ServeBlob once the blob serving is finished;
			//     it may happen with a slight delay after the blob was served
			expectedMethodInvocations: map[string]int{"Stat": 3},
		},

		{
			name:     "delete blob with repository unset",
			method:   http.MethodDelete,
			endpoint: v2.RouteNameBlob,
			vars: []string{
				"name", "user/app",
				"digest", desc.Digest.String(),
			},
			unsetRepository:           true,
			expectedStatus:            http.StatusInternalServerError,
			expectedMethodInvocations: map[string]int{"Stat": 1},
		},

		{
			name:     "delete blob",
			method:   http.MethodDelete,
			endpoint: v2.RouteNameBlob,
			vars: []string{
				"name", "user/app",
				"digest", desc.Digest.String(),
			},
			expectedStatus:            http.StatusAccepted,
			expectedMethodInvocations: map[string]int{"Stat": 1, "Clear": 1},
		},

		{
			name:     "get manifest with repository unset",
			method:   http.MethodGet,
			endpoint: v2.RouteNameManifest,
			vars: []string{
				"name", "user/app",
				"reference", "latest",
			},
			unsetRepository: true,
			// failed because we trying to get manifest from storage driver first.
			expectedStatus: http.StatusNotFound,
			// manifest can't be retrieved from etcd
			expectedMethodInvocations: map[string]int{"Stat": 1},
		},

		{
			name:     "get manifest",
			method:   http.MethodGet,
			endpoint: v2.RouteNameManifest,
			vars: []string{
				"name", "user/app",
				"reference", "latest",
			},
			expectedStatus: http.StatusOK,
			// manifest is retrieved from etcd
			expectedMethodInvocations: map[string]int{"Stat": 3},
		},

		{
			name:     "delete manifest with repository unset",
			method:   http.MethodDelete,
			endpoint: v2.RouteNameManifest,
			vars: []string{
				"name", "user/app",
				"reference", testImage.Name,
			},
			unsetRepository: true,
			expectedStatus:  http.StatusInternalServerError,
			// we don't allow to delete manifests from etcd; in this case, we attempt to delete layer link
			expectedMethodInvocations: map[string]int{"Stat": 1},
		},

		{
			name:     "delete manifest",
			method:   http.MethodDelete,
			endpoint: v2.RouteNameManifest,
			vars: []string{
				"name", "user/app",
				"reference", testImage.Name,
			},
			expectedStatus: http.StatusNotFound,
			// we don't allow to delete manifests from etcd; in this case, we attempt to delete layer link
			expectedMethodInvocations: map[string]int{"Stat": 1},
		},
	} {
		m.clearStats()
		m.changeUnsetRepository(tc.unsetRepository)

		route := router.GetRoute(tc.endpoint).Host(serverURL.Host)
		u, err := route.URL(tc.vars...)
		if err != nil {
			t.Errorf("[%s] failed to build route: %v", tc.name, err)
			continue
		}

		req, err := http.NewRequest(tc.method, u.String(), nil)
		if err != nil {
			t.Errorf("[%s] failed to make request: %v", tc.name, err)
		}

		client := &http.Client{}
		resp, err := client.Do(req)
		if err != nil {
			t.Errorf("[%s] failed to do the request: %v", tc.name, err)
			continue
		}
		defer resp.Body.Close()

		if resp.StatusCode != tc.expectedStatus {
			t.Errorf("[%s] unexpected status code: %v != %v", tc.name, resp.StatusCode, tc.expectedStatus)
		}

		if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
			content, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				t.Errorf("[%s] failed to read body: %v", tc.name, err)
			} else if len(content) > 0 {
				errs := errcode.Errors{}
				err := errs.UnmarshalJSON(content)
				if err != nil {
					t.Logf("[%s] failed to parse body as error: %v", tc.name, err)
					t.Logf("[%s] received body: %v", tc.name, string(content))
				} else {
					t.Logf("[%s] received errors: %#+v", tc.name, errs)
				}
			}
		}

		stats, err := m.getStats(tc.expectedMethodInvocations, time.Second*5)
		if err != nil {
			t.Errorf("[%s] failed to get stats: %v", tc.name, err)
		}
		for method, exp := range tc.expectedMethodInvocations {
			invoked := stats[method]
			if invoked != exp {
				t.Errorf("[%s] unexpected number of invocations of method %q: %v != %v", tc.name, method, invoked, exp)
			}
		}
		for method, invoked := range stats {
			if _, ok := tc.expectedMethodInvocations[method]; !ok {
				t.Errorf("[%s] unexpected method %q invoked %d times", tc.name, method, invoked)
			}
		}
	}
}