// TestBlobDescriptorServiceIsApplied ensures that blobDescriptorService middleware gets applied.
// It relies on the fact that blobDescriptorService requires higher levels to set repository object on given
// context. If the object isn't given, its method will err out.
func TestBlobDescriptorServiceIsApplied(t *testing.T) {
ctx := context.Background()
// don't do any authorization check
installFakeAccessController(t)
m := fakeBlobDescriptorService(t)
// to make other unit tests working
defer m.changeUnsetRepository(false)
testImage, err := registrytest.NewImageForManifest("user/app", registrytest.SampleImageManifestSchema1, true)
if err != nil {
t.Fatal(err)
}
testImageStream := registrytest.TestNewImageStreamObject("user", "app", "latest", testImage.Name, "")
client := &testclient.Fake{}
client.AddReactor("get", "imagestreams", imagetest.GetFakeImageStreamGetHandler(t, *testImageStream))
client.AddReactor("get", "images", registrytest.GetFakeImageGetHandler(t, *testImage))
// TODO: get rid of those nasty global vars
backupRegistryClient := DefaultRegistryClient
DefaultRegistryClient = makeFakeRegistryClient(client, fake.NewSimpleClientset())
defer func() {
// set it back once this test finishes to make other unit tests working
DefaultRegistryClient = backupRegistryClient
}()
app := handlers.NewApp(ctx, &configuration.Configuration{
Loglevel: "debug",
Auth: map[string]configuration.Parameters{
fakeAuthorizerName: {"realm": fakeAuthorizerName},
},
Storage: configuration.Storage{
"inmemory": configuration.Parameters{},
"cache": configuration.Parameters{
"blobdescriptor": "inmemory",
},
"delete": configuration.Parameters{
"enabled": true,
},
},
Middleware: map[string][]configuration.Middleware{
"registry": {{Name: "openshift"}},
"repository": {{Name: "openshift"}},
"storage": {{Name: "openshift"}},
},
})
server := httptest.NewServer(app)
router := v2.Router()
serverURL, err := url.Parse(server.URL)
if err != nil {
t.Fatalf("error parsing server url: %v", err)
}
os.Setenv("DOCKER_REGISTRY_URL", serverURL.Host)
desc, _, err := registrytest.UploadTestBlob(serverURL, nil, "user/app")
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
name string
method string
endpoint string
vars []string
unsetRepository bool
expectedStatus int
expectedMethodInvocations map[string]int
}{
{
name: "get blob with repository unset",
method: http.MethodGet,
endpoint: v2.RouteNameBlob,
vars: []string{
"name", "user/app",
"digest", desc.Digest.String(),
},
unsetRepository: true,
expectedStatus: http.StatusInternalServerError,
expectedMethodInvocations: map[string]int{"Stat": 1},
},
{
name: "get blob",
method: http.MethodGet,
endpoint: v2.RouteNameBlob,
vars: []string{
"name", "user/app",
"digest", desc.Digest.String(),
},
expectedStatus: http.StatusOK,
// 1st stat is invoked in (*distribution/registry/handlers.blobHandler).GetBlob() as a
// check of blob existence
// 2nd stat happens in (*errorBlobStore).ServeBlob() invoked by the same GetBlob handler
// 3rd stat is done by (*blobServiceListener).ServeBlob once the blob serving is finished;
// it may happen with a slight delay after the blob was served
expectedMethodInvocations: map[string]int{"Stat": 3},
},
{
name: "stat blob with repository unset",
method: http.MethodHead,
endpoint: v2.RouteNameBlob,
vars: []string{
"name", "user/app",
"digest", desc.Digest.String(),
},
unsetRepository: true,
expectedStatus: http.StatusInternalServerError,
expectedMethodInvocations: map[string]int{"Stat": 1},
},
{
name: "stat blob",
method: http.MethodHead,
endpoint: v2.RouteNameBlob,
vars: []string{
"name", "user/app",
"digest", desc.Digest.String(),
},
expectedStatus: http.StatusOK,
// 1st stat is invoked in (*distribution/registry/handlers.blobHandler).GetBlob() as a
// check of blob existence
// 2nd stat happens in (*errorBlobStore).ServeBlob() invoked by the same GetBlob handler
// 3rd stat is done by (*blobServiceListener).ServeBlob once the blob serving is finished;
// it may happen with a slight delay after the blob was served
expectedMethodInvocations: map[string]int{"Stat": 3},
},
{
name: "delete blob with repository unset",
method: http.MethodDelete,
endpoint: v2.RouteNameBlob,
vars: []string{
"name", "user/app",
"digest", desc.Digest.String(),
},
unsetRepository: true,
expectedStatus: http.StatusInternalServerError,
expectedMethodInvocations: map[string]int{"Stat": 1},
},
{
name: "delete blob",
method: http.MethodDelete,
endpoint: v2.RouteNameBlob,
vars: []string{
"name", "user/app",
"digest", desc.Digest.String(),
},
expectedStatus: http.StatusAccepted,
expectedMethodInvocations: map[string]int{"Stat": 1, "Clear": 1},
},
{
name: "get manifest with repository unset",
method: http.MethodGet,
endpoint: v2.RouteNameManifest,
vars: []string{
"name", "user/app",
"reference", "latest",
},
unsetRepository: true,
// failed because we trying to get manifest from storage driver first.
expectedStatus: http.StatusNotFound,
// manifest can't be retrieved from etcd
expectedMethodInvocations: map[string]int{"Stat": 1},
},
{
name: "get manifest",
method: http.MethodGet,
endpoint: v2.RouteNameManifest,
vars: []string{
"name", "user/app",
"reference", "latest",
},
expectedStatus: http.StatusOK,
// manifest is retrieved from etcd
expectedMethodInvocations: map[string]int{"Stat": 3},
},
{
name: "delete manifest with repository unset",
method: http.MethodDelete,
endpoint: v2.RouteNameManifest,
vars: []string{
"name", "user/app",
"reference", testImage.Name,
},
unsetRepository: true,
expectedStatus: http.StatusInternalServerError,
// we don't allow to delete manifests from etcd; in this case, we attempt to delete layer link
expectedMethodInvocations: map[string]int{"Stat": 1},
},
{
name: "delete manifest",
method: http.MethodDelete,
endpoint: v2.RouteNameManifest,
vars: []string{
"name", "user/app",
"reference", testImage.Name,
},
expectedStatus: http.StatusNotFound,
// we don't allow to delete manifests from etcd; in this case, we attempt to delete layer link
expectedMethodInvocations: map[string]int{"Stat": 1},
},
} {
m.clearStats()
m.changeUnsetRepository(tc.unsetRepository)
route := router.GetRoute(tc.endpoint).Host(serverURL.Host)
u, err := route.URL(tc.vars...)
if err != nil {
t.Errorf("[%s] failed to build route: %v", tc.name, err)
continue
}
req, err := http.NewRequest(tc.method, u.String(), nil)
if err != nil {
t.Errorf("[%s] failed to make request: %v", tc.name, err)
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
t.Errorf("[%s] failed to do the request: %v", tc.name, err)
continue
}
defer resp.Body.Close()
if resp.StatusCode != tc.expectedStatus {
t.Errorf("[%s] unexpected status code: %v != %v", tc.name, resp.StatusCode, tc.expectedStatus)
}
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
content, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Errorf("[%s] failed to read body: %v", tc.name, err)
} else if len(content) > 0 {
errs := errcode.Errors{}
err := errs.UnmarshalJSON(content)
if err != nil {
t.Logf("[%s] failed to parse body as error: %v", tc.name, err)
t.Logf("[%s] received body: %v", tc.name, string(content))
} else {
t.Logf("[%s] received errors: %#+v", tc.name, errs)
}
}
}
stats, err := m.getStats(tc.expectedMethodInvocations, time.Second*5)
if err != nil {
t.Errorf("[%s] failed to get stats: %v", tc.name, err)
}
for method, exp := range tc.expectedMethodInvocations {
invoked := stats[method]
if invoked != exp {
t.Errorf("[%s] unexpected number of invocations of method %q: %v != %v", tc.name, method, invoked, exp)
}
}
for method, invoked := range stats {
if _, ok := tc.expectedMethodInvocations[method]; !ok {
t.Errorf("[%s] unexpected method %q invoked %d times", tc.name, method, invoked)
}
}
}
}
func TestPullthroughServeBlob(t *testing.T) {
ctx := context.Background()
installFakeAccessController(t)
testImage, err := registrytest.NewImageForManifest("user/app", registrytest.SampleImageManifestSchema1, false)
if err != nil {
t.Fatal(err)
}
client := &testclient.Fake{}
client.AddReactor("get", "images", registrytest.GetFakeImageGetHandler(t, *testImage))
// TODO: get rid of those nasty global vars
backupRegistryClient := DefaultRegistryClient
DefaultRegistryClient = makeFakeRegistryClient(client, fake.NewSimpleClientset())
defer func() {
// set it back once this test finishes to make other unit tests working again
DefaultRegistryClient = backupRegistryClient
}()
// pullthrough middleware will attempt to pull from this registry instance
remoteRegistryApp := handlers.NewApp(ctx, &configuration.Configuration{
Loglevel: "debug",
Auth: map[string]configuration.Parameters{
fakeAuthorizerName: {"realm": fakeAuthorizerName},
},
Storage: configuration.Storage{
"inmemory": configuration.Parameters{},
"cache": configuration.Parameters{
"blobdescriptor": "inmemory",
},
"delete": configuration.Parameters{
"enabled": true,
},
},
Middleware: map[string][]configuration.Middleware{
"registry": {{Name: "openshift"}},
"repository": {{Name: "openshift", Options: configuration.Parameters{"pullthrough": false}}},
"storage": {{Name: "openshift"}},
},
})
remoteRegistryServer := httptest.NewServer(remoteRegistryApp)
defer remoteRegistryServer.Close()
serverURL, err := url.Parse(remoteRegistryServer.URL)
if err != nil {
t.Fatalf("error parsing server url: %v", err)
}
os.Setenv("DOCKER_REGISTRY_URL", serverURL.Host)
testImage.DockerImageReference = fmt.Sprintf("%s/%s@%s", serverURL.Host, "user/app", testImage.Name)
testImageStream := registrytest.TestNewImageStreamObject("user", "app", "latest", testImage.Name, testImage.DockerImageReference)
if testImageStream.Annotations == nil {
testImageStream.Annotations = make(map[string]string)
}
testImageStream.Annotations[imageapi.InsecureRepositoryAnnotation] = "true"
client.AddReactor("get", "imagestreams", imagetest.GetFakeImageStreamGetHandler(t, *testImageStream))
blob1Desc, blob1Content, err := registrytest.UploadTestBlob(serverURL, nil, "user/app")
if err != nil {
t.Fatal(err)
}
blob2Desc, blob2Content, err := registrytest.UploadTestBlob(serverURL, nil, "user/app")
if err != nil {
t.Fatal(err)
}
blob1Storage := map[digest.Digest][]byte{blob1Desc.Digest: blob1Content}
blob2Storage := map[digest.Digest][]byte{blob2Desc.Digest: blob2Content}
for _, tc := range []struct {
name string
method string
blobDigest digest.Digest
localBlobs map[digest.Digest][]byte
expectedStatError error
expectedContentLength int64
expectedBytesServed int64
expectedBytesServedLocally int64
expectedLocalCalls map[string]int
}{
{
name: "stat local blob",
method: "HEAD",
blobDigest: blob1Desc.Digest,
localBlobs: blob1Storage,
expectedContentLength: int64(len(blob1Content)),
expectedLocalCalls: map[string]int{
"Stat": 1,
"ServeBlob": 1,
},
},
{
name: "serve local blob",
method: "GET",
blobDigest: blob1Desc.Digest,
localBlobs: blob1Storage,
expectedContentLength: int64(len(blob1Content)),
expectedBytesServed: int64(len(blob1Content)),
expectedBytesServedLocally: int64(len(blob1Content)),
expectedLocalCalls: map[string]int{
"Stat": 1,
"ServeBlob": 1,
},
},
{
name: "stat remote blob",
method: "HEAD",
blobDigest: blob1Desc.Digest,
localBlobs: blob2Storage,
expectedContentLength: int64(len(blob1Content)),
expectedLocalCalls: map[string]int{"Stat": 1},
},
{
name: "serve remote blob",
method: "GET",
blobDigest: blob1Desc.Digest,
expectedContentLength: int64(len(blob1Content)),
expectedBytesServed: int64(len(blob1Content)),
expectedLocalCalls: map[string]int{"Stat": 1},
},
{
name: "unknown blob digest",
method: "GET",
blobDigest: unknownBlobDigest,
expectedStatError: distribution.ErrBlobUnknown,
expectedLocalCalls: map[string]int{"Stat": 1},
},
} {
localBlobStore := newTestBlobStore(tc.localBlobs)
cachedLayers, err := newDigestToRepositoryCache(10)
if err != nil {
t.Fatal(err)
}
ptbs := &pullthroughBlobStore{
BlobStore: localBlobStore,
repo: &repository{
ctx: ctx,
namespace: "user",
name: "app",
pullthrough: true,
cachedLayers: cachedLayers,
registryOSClient: client,
},
digestToStore: make(map[string]distribution.BlobStore),
}
req, err := http.NewRequest(tc.method, fmt.Sprintf("http://example.org/v2/user/app/blobs/%s", tc.blobDigest), nil)
if err != nil {
t.Fatalf("[%s] failed to create http request: %v", tc.name, err)
}
w := httptest.NewRecorder()
dgst := digest.Digest(tc.blobDigest)
_, err = ptbs.Stat(ctx, dgst)
if err != tc.expectedStatError {
t.Errorf("[%s] Stat returned unexpected error: %#+v != %#+v", tc.name, err, tc.expectedStatError)
}
if err != nil || tc.expectedStatError != nil {
continue
}
err = ptbs.ServeBlob(ctx, w, req, dgst)
if err != nil {
t.Errorf("[%s] unexpected ServeBlob error: %v", tc.name, err)
continue
}
clstr := w.Header().Get("Content-Length")
if cl, err := strconv.ParseInt(clstr, 10, 64); err != nil {
t.Errorf(`[%s] unexpected Content-Length: %q != "%d"`, tc.name, clstr, tc.expectedContentLength)
} else {
if cl != tc.expectedContentLength {
t.Errorf("[%s] Content-Length does not match expected size: %d != %d", tc.name, cl, tc.expectedContentLength)
}
}
if w.Header().Get("Content-Type") != "application/octet-stream" {
t.Errorf("[%s] Content-Type does not match expected: %q != %q", tc.name, w.Header().Get("Content-Type"), "application/octet-stream")
}
body := w.Body.Bytes()
if int64(len(body)) != tc.expectedBytesServed {
t.Errorf("[%s] unexpected size of body: %d != %d", tc.name, len(body), tc.expectedBytesServed)
}
for name, expCount := range tc.expectedLocalCalls {
count := localBlobStore.calls[name]
if count != expCount {
t.Errorf("[%s] expected %d calls to method %s of local blob store, not %d", tc.name, expCount, name, count)
}
}
for name, count := range localBlobStore.calls {
if _, exists := tc.expectedLocalCalls[name]; !exists {
t.Errorf("[%s] expected no calls to method %s of local blob store, got %d", tc.name, name, count)
}
}
if localBlobStore.bytesServed != tc.expectedBytesServedLocally {
t.Errorf("[%s] unexpected number of bytes served locally: %d != %d", tc.name, localBlobStore.bytesServed, tc.expectedBytesServed)
}
}
}
func TestRepositoryBlobStatCacheEviction(t *testing.T) {
const blobRepoCacheTTL = time.Millisecond * 500
quotaEnforcing = "aEnforcingConfig{}
ctx := WithAuthPerformed(context.Background())
// this driver holds all the testing blobs in memory during the whole test run
driver := inmemory.New()
// generate two images and store their blobs in the driver
testImages, err := populateTestStorage(t, driver, true, 1, map[string]int{"nm/is:latest": 1}, nil)
if err != nil {
t.Fatal(err)
}
testImage := testImages["nm/is:latest"][0]
testImageStream := registrytest.TestNewImageStreamObject("nm", "is", "latest", testImage.Name, "")
blob1Desc := testNewDescriptorForLayer(testImage.DockerImageLayers[0])
blob1Dgst := blob1Desc.Digest
blob2Desc := testNewDescriptorForLayer(testImage.DockerImageLayers[1])
blob2Dgst := blob2Desc.Digest
// remove repo layer repo link of the image's second blob
alg, hex := blob2Dgst.Algorithm(), blob2Dgst.Hex()
err = driver.Delete(ctx, fmt.Sprintf("/docker/registry/v2/repositories/%s/_layers/%s/%s", "nm/is", alg, hex))
cachedLayers, err = newDigestToRepositoryCache(defaultDigestToRepositoryCacheSize)
if err != nil {
t.Fatal(err)
}
client := &testclient.Fake{}
client.AddReactor("get", "imagestreams", imagetest.GetFakeImageStreamGetHandler(t, *testImageStream))
client.AddReactor("get", "images", registrytest.GetFakeImageGetHandler(t, *testImage))
reg, err := newTestRegistry(ctx, client, driver, blobRepoCacheTTL, false, false)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
ref, err := reference.ParseNamed("nm/is")
if err != nil {
t.Errorf("failed to parse blob reference %q: %v", "nm/is", err)
}
repo, err := reg.Repository(ctx, ref)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// hit the layer repo link - cache the result
desc, err := repo.Blobs(ctx).Stat(ctx, blob1Dgst)
if err != nil {
t.Fatalf("got unexpected stat error: %v", err)
}
if !reflect.DeepEqual(desc, blob1Desc) {
t.Fatalf("got unexpected descriptor: %#+v != %#+v", desc, blob1Desc)
}
compareActions(t, "no actions expected", client.Actions(), []clientAction{})
// remove layer repo link, delete the association from cache as well
err = repo.Blobs(ctx).Delete(ctx, blob1Dgst)
if err != nil {
t.Fatalf("got unexpected error: %v", err)
}
// query etcd
desc, err = repo.Blobs(ctx).Stat(ctx, blob1Dgst)
if err != nil {
t.Fatalf("got unexpected stat error: %v", err)
}
if !reflect.DeepEqual(desc, blob1Desc) {
t.Fatalf("got unexpected descriptor: %#+v != %#+v", desc, blob1Desc)
}
expectedActions := []clientAction{{"get", "imagestreams"}, {"get", "images"}}
compareActions(t, "1st roundtrip to etcd", client.Actions(), expectedActions)
// remove the underlying blob
vacuum := storage.NewVacuum(ctx, driver)
err = vacuum.RemoveBlob(blob1Dgst.String())
if err != nil {
t.Fatalf("got unexpected error: %v", err)
}
// fail because the blob isn't stored locally
desc, err = repo.Blobs(ctx).Stat(ctx, blob1Dgst)
if err == nil {
t.Fatalf("got unexpected non error: %v", err)
}
if err != distribution.ErrBlobUnknown {
t.Fatalf("got unexpected error: %#+v", err)
}
// cache hit - don't query etcd
desc, err = repo.Blobs(ctx).Stat(ctx, blob2Dgst)
if err != nil {
t.Fatalf("got unexpected stat error: %v", err)
}
if !reflect.DeepEqual(desc, blob2Desc) {
t.Fatalf("got unexpected descriptor: %#+v != %#+v", desc, blob2Desc)
}
compareActions(t, "no etcd query", client.Actions(), expectedActions)
lastStatTimestamp := time.Now()
// hit the cache
desc, err = repo.Blobs(ctx).Stat(ctx, blob2Dgst)
if err != nil {
t.Fatalf("got unexpected stat error: %v", err)
}
if !reflect.DeepEqual(desc, blob2Desc) {
t.Fatalf("got unexpected descriptor: %#+v != %#+v", desc, blob2Desc)
}
// cache hit - no additional etcd query
compareActions(t, "no roundrip to etcd", client.Actions(), expectedActions)
t.Logf("sleeping %s while waiting for eviction of blob %q from cache", blobRepoCacheTTL.String(), blob2Dgst.String())
time.Sleep(blobRepoCacheTTL - (time.Now().Sub(lastStatTimestamp)))
desc, err = repo.Blobs(ctx).Stat(ctx, blob2Dgst)
if err != nil {
t.Fatalf("got unexpected stat error: %v", err)
}
if !reflect.DeepEqual(desc, blob2Desc) {
t.Fatalf("got unexpected descriptor: %#+v != %#+v", desc, blob2Desc)
}
expectedActions = append(expectedActions, []clientAction{{"get", "imagestreams"}, {"get", "images"}}...)
compareActions(t, "2nd roundtrip to etcd", client.Actions(), expectedActions)
err = vacuum.RemoveBlob(blob2Dgst.String())
if err != nil {
t.Fatalf("got unexpected error: %v", err)
}
// fail because the blob isn't stored locally
desc, err = repo.Blobs(ctx).Stat(ctx, blob2Dgst)
if err == nil {
t.Fatalf("got unexpected non error: %v", err)
}
if err != distribution.ErrBlobUnknown {
t.Fatalf("got unexpected error: %#+v", err)
}
}