func TestDeleteDisabled(t *testing.T) { env := newTestEnv(t, false) imageName := "foo/bar" // "build" our layer file layerFile, tarSumStr, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating random layer file: %v", err) } layerDigest := digest.Digest(tarSumStr) layerURL, err := env.builder.BuildBlobURL(imageName, layerDigest) if err != nil { t.Fatalf("Error building blob URL") } uploadURLBase, _ := startPushLayer(t, env.builder, imageName) pushLayer(t, env.builder, imageName, layerDigest, uploadURLBase, layerFile) resp, err := httpDelete(layerURL) if err != nil { t.Fatalf("unexpected error deleting layer: %v", err) } checkResponse(t, "deleting layer with delete disabled", resp, http.StatusMethodNotAllowed) }
func uploadRandomTarball(t *testing.T, ctx context.Context, bi distribution.BlobIngester) digest.Digest { randomDataReader, tarSumStr, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating random reader: %v", err) } dgst := digest.Digest(tarSumStr) if err != nil { t.Fatalf("error allocating upload store: %v", err) } randomLayerSize, err := seekerSize(randomDataReader) if err != nil { t.Fatalf("error getting seeker size for random layer: %v", err) } _, err = addBlob(ctx, bi, distribution.Descriptor{ Digest: dgst, MediaType: "application/octet-stream", Size: randomLayerSize, }, randomDataReader) if err != nil { t.Fatalf("failed to add blob: %v", err) } return dgst }
func TestDeleteReadOnly(t *testing.T) { env := newTestEnv(t, true) imageName := "foo/bar" // "build" our layer file layerFile, layerDigest, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating random layer file: %v", err) } layerURL, err := env.builder.BuildBlobURL(imageName, layerDigest) if err != nil { t.Fatalf("Error building blob URL") } uploadURLBase, _ := startPushLayer(t, env.builder, imageName) pushLayer(t, env.builder, imageName, layerDigest, uploadURLBase, layerFile) env.app.readOnly = true resp, err := httpDelete(layerURL) if err != nil { t.Fatalf("unexpected error deleting layer: %v", err) } checkResponse(t, "deleting layer in read-only mode", resp, http.StatusMethodNotAllowed) }
func createRepository(env *testEnv, t *testing.T, imageName string, tag string) digest.Digest { unsignedManifest := &schema1.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, }, Name: imageName, Tag: tag, FSLayers: []schema1.FSLayer{ { BlobSum: "asdf", }, }, History: []schema1.History{ { V1Compatibility: "", }, }, } // Push 2 random layers expectedLayers := make(map[digest.Digest]io.ReadSeeker) for i := range unsignedManifest.FSLayers { rs, dgstStr, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating random layer %d: %v", i, err) } dgst := digest.Digest(dgstStr) expectedLayers[dgst] = rs unsignedManifest.FSLayers[i].BlobSum = dgst uploadURLBase, _ := startPushLayer(t, env.builder, imageName) pushLayer(t, env.builder, imageName, dgst, uploadURLBase, rs) } signedManifest, err := schema1.Sign(unsignedManifest, env.pk) if err != nil { t.Fatalf("unexpected error signing manifest: %v", err) } dgst := digest.FromBytes(signedManifest.Canonical) // Create this repository by tag to ensure the tag mapping is made in the registry manifestDigestURL, err := env.builder.BuildManifestURL(imageName, tag) checkErr(t, err, "building manifest url") location, err := env.builder.BuildManifestURL(imageName, dgst.String()) checkErr(t, err, "building location URL") resp := putManifest(t, "putting signed manifest", manifestDigestURL, signedManifest) checkResponse(t, "putting signed manifest", resp, http.StatusCreated) checkHeaders(t, resp, http.Header{ "Location": []string{location}, "Docker-Content-Digest": []string{dgst.String()}, }) return dgst }
func TestDigestVerifier(t *testing.T) { p := make([]byte, 1<<20) rand.Read(p) digest, err := FromBytes(p) if err != nil { t.Fatalf("unexpected error digesting bytes: %#v", err) } verifier, err := NewDigestVerifier(digest) if err != nil { t.Fatalf("unexpected error getting digest verifier: %s", err) } io.Copy(verifier, bytes.NewReader(p)) if !verifier.Verified() { t.Fatalf("bytes not verified") } tf, tarSum, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating tarfile: %v", err) } digest, err = FromTarArchive(tf) if err != nil { t.Fatalf("error digesting tarsum: %v", err) } if digest.String() != tarSum { t.Fatalf("unexpected digest: %q != %q", digest.String(), tarSum) } expectedSize, _ := tf.Seek(0, os.SEEK_END) // Get tar file size tf.Seek(0, os.SEEK_SET) // seek back // This is the most relevant example for the registry application. It's // effectively a read through pipeline, where the final sink is the digest // verifier. verifier, err = NewDigestVerifier(digest) if err != nil { t.Fatalf("unexpected error getting digest verifier: %s", err) } lengthVerifier := NewLengthVerifier(expectedSize) rd := io.TeeReader(tf, lengthVerifier) io.Copy(verifier, rd) if !lengthVerifier.Verified() { t.Fatalf("verifier detected incorrect length") } if !verifier.Verified() { t.Fatalf("bytes not verified") } }
func createRepository(env *testEnv, t *testing.T, imageName string, tag string) { unsignedManifest := &schema1.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, }, Name: imageName, Tag: tag, FSLayers: []schema1.FSLayer{ { BlobSum: "asdf", }, { BlobSum: "qwer", }, }, } // Push 2 random layers expectedLayers := make(map[digest.Digest]io.ReadSeeker) for i := range unsignedManifest.FSLayers { rs, dgstStr, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating random layer %d: %v", i, err) } dgst := digest.Digest(dgstStr) expectedLayers[dgst] = rs unsignedManifest.FSLayers[i].BlobSum = dgst uploadURLBase, _ := startPushLayer(t, env.builder, imageName) pushLayer(t, env.builder, imageName, dgst, uploadURLBase, rs) } signedManifest, err := schema1.Sign(unsignedManifest, env.pk) if err != nil { t.Fatalf("unexpected error signing manifest: %v", err) } payload, err := signedManifest.Payload() checkErr(t, err, "getting manifest payload") dgst, err := digest.FromBytes(payload) checkErr(t, err, "digesting manifest") manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String()) checkErr(t, err, "building manifest url") resp := putManifest(t, "putting signed manifest", manifestDigestURL, signedManifest) checkResponse(t, "putting signed manifest", resp, http.StatusCreated) checkHeaders(t, resp, http.Header{ "Location": []string{manifestDigestURL}, "Docker-Content-Digest": []string{dgst.String()}, }) }
func populateRepo(t *testing.T, ctx context.Context, repository distribution.Repository, name, tag string) (digest.Digest, error) { m := schema1.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, }, Name: name, Tag: tag, } for i := 0; i < 2; i++ { wr, err := repository.Blobs(ctx).Create(ctx) if err != nil { t.Fatalf("unexpected error creating test upload: %v", err) } rs, ts, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("unexpected error generating test layer file") } dgst := digest.Digest(ts) if _, err := io.Copy(wr, rs); err != nil { t.Fatalf("unexpected error copying to upload: %v", err) } if _, err := wr.Commit(ctx, distribution.Descriptor{Digest: dgst}); err != nil { t.Fatalf("unexpected error finishing upload: %v", err) } } pk, err := libtrust.GenerateECP256PrivateKey() if err != nil { t.Fatalf("unexpected error generating private key: %v", err) } sm, err := schema1.Sign(&m, pk) if err != nil { t.Fatalf("error signing manifest: %v", err) } ms, err := repository.Manifests(ctx) if err != nil { t.Fatalf(err.Error()) } ms.Put(sm) if err != nil { t.Fatalf("unexpected errors putting manifest: %v", err) } pl, err := sm.Payload() if err != nil { t.Fatal(err) } return digest.FromBytes(pl) }
func makeBlobArgs(t *testing.T) blobArgs { layerFile, layerDigest, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating random layer file: %v", err) } args := blobArgs{ imageName: "foo/bar", layerFile: layerFile, layerDigest: layerDigest, } return args }
// writeRandomLayer creates a random layer under name and tarSum using driver // and pathMapper. An io.ReadSeeker with the data is returned, along with the // sha256 hex digest. func writeRandomLayer(driver storagedriver.StorageDriver, pathMapper *pathMapper, name string) (rs io.ReadSeeker, tarSum digest.Digest, sha256digest digest.Digest, err error) { reader, tarSumStr, err := testutil.CreateRandomTarFile() if err != nil { return nil, "", "", err } tarSum = digest.Digest(tarSumStr) // Now, actually create the layer. randomLayerDigest, err := writeTestLayer(driver, pathMapper, name, tarSum, ioutil.NopCloser(reader)) if _, err := reader.Seek(0, os.SEEK_SET); err != nil { return nil, "", "", err } return reader, tarSum, randomLayerDigest, err }
func TestLinkedBlobStoreCreateWithMountFrom(t *testing.T) { fooRepoName, _ := reference.ParseNamed("nm/foo") fooEnv := newManifestStoreTestEnv(t, fooRepoName, "thetag") ctx := context.Background() // Build up some test layers and add them to the manifest, saving the // readseekers for upload later. testLayers := map[digest.Digest]io.ReadSeeker{} for i := 0; i < 2; i++ { rs, ds, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("unexpected error generating test layer file") } dgst := digest.Digest(ds) testLayers[digest.Digest(dgst)] = rs } // upload the layers to foo/bar for dgst, rs := range testLayers { wr, err := fooEnv.repository.Blobs(fooEnv.ctx).Create(fooEnv.ctx) if err != nil { t.Fatalf("unexpected error creating test upload: %v", err) } if _, err := io.Copy(wr, rs); err != nil { t.Fatalf("unexpected error copying to upload: %v", err) } if _, err := wr.Commit(fooEnv.ctx, distribution.Descriptor{Digest: dgst}); err != nil { t.Fatalf("unexpected error finishing upload: %v", err) } } // create another repository nm/bar barRepoName, _ := reference.ParseNamed("nm/bar") barRepo, err := fooEnv.registry.Repository(ctx, barRepoName) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } // cross-repo mount the test layers into a nm/bar for dgst := range testLayers { fooCanonical, _ := reference.WithDigest(fooRepoName, dgst) option := WithMountFrom(fooCanonical) // ensure we can instrospect it createOpts := distribution.CreateOptions{} if err := option.Apply(&createOpts); err != nil { t.Fatalf("failed to apply MountFrom option: %v", err) } if !createOpts.Mount.ShouldMount || createOpts.Mount.From.String() != fooCanonical.String() { t.Fatalf("unexpected create options: %#+v", createOpts.Mount) } _, err := barRepo.Blobs(ctx).Create(ctx, WithMountFrom(fooCanonical)) if err == nil { t.Fatalf("unexpected non-error while mounting from %q: %v", fooRepoName.String(), err) } if _, ok := err.(distribution.ErrBlobMounted); !ok { t.Fatalf("expected ErrMountFrom error, not %T: %v", err, err) } } }
func testManifestStorage(t *testing.T, options ...RegistryOption) { repoName, _ := reference.ParseNamed("foo/bar") env := newManifestStoreTestEnv(t, repoName, "thetag", options...) ctx := context.Background() ms, err := env.repository.Manifests(ctx) if err != nil { t.Fatal(err) } equalSignatures := env.registry.(*registry).schema1SignaturesEnabled m := schema1.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, }, Name: env.name.Name(), Tag: env.tag, } // Build up some test layers and add them to the manifest, saving the // readseekers for upload later. testLayers := map[digest.Digest]io.ReadSeeker{} for i := 0; i < 2; i++ { rs, ds, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("unexpected error generating test layer file") } dgst := digest.Digest(ds) testLayers[digest.Digest(dgst)] = rs m.FSLayers = append(m.FSLayers, schema1.FSLayer{ BlobSum: dgst, }) m.History = append(m.History, schema1.History{ V1Compatibility: "", }) } pk, err := libtrust.GenerateECP256PrivateKey() if err != nil { t.Fatalf("unexpected error generating private key: %v", err) } sm, merr := schema1.Sign(&m, pk) if merr != nil { t.Fatalf("error signing manifest: %v", err) } _, err = ms.Put(ctx, sm) if err == nil { t.Fatalf("expected errors putting manifest with full verification") } switch err := err.(type) { case distribution.ErrManifestVerification: if len(err) != 2 { t.Fatalf("expected 2 verification errors: %#v", err) } for _, err := range err { if _, ok := err.(distribution.ErrManifestBlobUnknown); !ok { t.Fatalf("unexpected error type: %v", err) } } default: t.Fatalf("unexpected error verifying manifest: %v", err) } // Now, upload the layers that were missing! for dgst, rs := range testLayers { wr, err := env.repository.Blobs(env.ctx).Create(env.ctx) if err != nil { t.Fatalf("unexpected error creating test upload: %v", err) } if _, err := io.Copy(wr, rs); err != nil { t.Fatalf("unexpected error copying to upload: %v", err) } if _, err := wr.Commit(env.ctx, distribution.Descriptor{Digest: dgst}); err != nil { t.Fatalf("unexpected error finishing upload: %v", err) } } var manifestDigest digest.Digest if manifestDigest, err = ms.Put(ctx, sm); err != nil { t.Fatalf("unexpected error putting manifest: %v", err) } exists, err := ms.Exists(ctx, manifestDigest) if err != nil { t.Fatalf("unexpected error checking manifest existence: %#v", err) } if !exists { t.Fatalf("manifest should exist") } fromStore, err := ms.Get(ctx, manifestDigest) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } fetchedManifest, ok := fromStore.(*schema1.SignedManifest) if !ok { t.Fatalf("unexpected manifest type from signedstore") } if !bytes.Equal(fetchedManifest.Canonical, sm.Canonical) { t.Fatalf("fetched payload does not match original payload: %q != %q", fetchedManifest.Canonical, sm.Canonical) } if equalSignatures { if !reflect.DeepEqual(fetchedManifest, sm) { t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedManifest.Manifest, sm.Manifest) } } _, pl, err := fetchedManifest.Payload() if err != nil { t.Fatalf("error getting payload %#v", err) } fetchedJWS, err := libtrust.ParsePrettySignature(pl, "signatures") if err != nil { t.Fatalf("unexpected error parsing jws: %v", err) } payload, err := fetchedJWS.Payload() if err != nil { t.Fatalf("unexpected error extracting payload: %v", err) } // Now that we have a payload, take a moment to check that the manifest is // return by the payload digest. dgst := digest.FromBytes(payload) exists, err = ms.Exists(ctx, dgst) if err != nil { t.Fatalf("error checking manifest existence by digest: %v", err) } if !exists { t.Fatalf("manifest %s should exist", dgst) } fetchedByDigest, err := ms.Get(ctx, dgst) if err != nil { t.Fatalf("unexpected error fetching manifest by digest: %v", err) } byDigestManifest, ok := fetchedByDigest.(*schema1.SignedManifest) if !ok { t.Fatalf("unexpected manifest type from signedstore") } if !bytes.Equal(byDigestManifest.Canonical, fetchedManifest.Canonical) { t.Fatalf("fetched manifest not equal: %q != %q", byDigestManifest.Canonical, fetchedManifest.Canonical) } if equalSignatures { if !reflect.DeepEqual(fetchedByDigest, fetchedManifest) { t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedByDigest, fetchedManifest) } } sigs, err := fetchedJWS.Signatures() if err != nil { t.Fatalf("unable to extract signatures: %v", err) } if len(sigs) != 1 { t.Fatalf("unexpected number of signatures: %d != %d", len(sigs), 1) } // Now, push the same manifest with a different key pk2, err := libtrust.GenerateECP256PrivateKey() if err != nil { t.Fatalf("unexpected error generating private key: %v", err) } sm2, err := schema1.Sign(&m, pk2) if err != nil { t.Fatalf("unexpected error signing manifest: %v", err) } _, pl, err = sm2.Payload() if err != nil { t.Fatalf("error getting payload %#v", err) } jws2, err := libtrust.ParsePrettySignature(pl, "signatures") if err != nil { t.Fatalf("error parsing signature: %v", err) } sigs2, err := jws2.Signatures() if err != nil { t.Fatalf("unable to extract signatures: %v", err) } if len(sigs2) != 1 { t.Fatalf("unexpected number of signatures: %d != %d", len(sigs2), 1) } if manifestDigest, err = ms.Put(ctx, sm2); err != nil { t.Fatalf("unexpected error putting manifest: %v", err) } fromStore, err = ms.Get(ctx, manifestDigest) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } fetched, ok := fromStore.(*schema1.SignedManifest) if !ok { t.Fatalf("unexpected type from signed manifeststore : %T", fetched) } if _, err := schema1.Verify(fetched); err != nil { t.Fatalf("unexpected error verifying manifest: %v", err) } // Assemble our payload and two signatures to get what we expect! expectedJWS, err := libtrust.NewJSONSignature(payload, sigs[0], sigs2[0]) if err != nil { t.Fatalf("unexpected error merging jws: %v", err) } expectedSigs, err := expectedJWS.Signatures() if err != nil { t.Fatalf("unexpected error getting expected signatures: %v", err) } _, pl, err = fetched.Payload() if err != nil { t.Fatalf("error getting payload %#v", err) } receivedJWS, err := libtrust.ParsePrettySignature(pl, "signatures") if err != nil { t.Fatalf("unexpected error parsing jws: %v", err) } receivedPayload, err := receivedJWS.Payload() if err != nil { t.Fatalf("unexpected error extracting received payload: %v", err) } if !bytes.Equal(receivedPayload, payload) { t.Fatalf("payloads are not equal") } if equalSignatures { receivedSigs, err := receivedJWS.Signatures() if err != nil { t.Fatalf("error getting signatures: %v", err) } for i, sig := range receivedSigs { if !bytes.Equal(sig, expectedSigs[i]) { t.Fatalf("mismatched signatures from remote: %v != %v", string(sig), string(expectedSigs[i])) } } } // Test deleting manifests err = ms.Delete(ctx, dgst) if err != nil { t.Fatalf("unexpected an error deleting manifest by digest: %v", err) } exists, err = ms.Exists(ctx, dgst) if err != nil { t.Fatalf("Error querying manifest existence") } if exists { t.Errorf("Deleted manifest should not exist") } deletedManifest, err := ms.Get(ctx, dgst) if err == nil { t.Errorf("Unexpected success getting deleted manifest") } switch err.(type) { case distribution.ErrManifestUnknownRevision: break default: t.Errorf("Unexpected error getting deleted manifest: %s", reflect.ValueOf(err).Type()) } if deletedManifest != nil { t.Errorf("Deleted manifest get returned non-nil") } // Re-upload should restore manifest to a good state _, err = ms.Put(ctx, sm) if err != nil { t.Errorf("Error re-uploading deleted manifest") } exists, err = ms.Exists(ctx, dgst) if err != nil { t.Fatalf("Error querying manifest existence") } if !exists { t.Errorf("Restored manifest should exist") } deletedManifest, err = ms.Get(ctx, dgst) if err != nil { t.Errorf("Unexpected error getting manifest") } if deletedManifest == nil { t.Errorf("Deleted manifest get returned non-nil") } r, err := NewRegistry(ctx, env.driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableRedirect) if err != nil { t.Fatalf("error creating registry: %v", err) } repo, err := r.Repository(ctx, env.name) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } ms, err = repo.Manifests(ctx) if err != nil { t.Fatal(err) } err = ms.Delete(ctx, dgst) if err == nil { t.Errorf("Unexpected success deleting while disabled") } }
// TestBlobAPI conducts a full test of the of the blob api. func TestBlobAPI(t *testing.T) { // TODO(stevvooe): This test code is complete junk but it should cover the // complete flow. This must be broken down and checked against the // specification *before* we submit the final to docker core. env := newTestEnv(t) imageName := "foo/bar" // "build" our layer file layerFile, tarSumStr, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating random layer file: %v", err) } layerDigest := digest.Digest(tarSumStr) // ----------------------------------- // Test fetch for non-existent content layerURL, err := env.builder.BuildBlobURL(imageName, layerDigest) if err != nil { t.Fatalf("error building url: %v", err) } resp, err := http.Get(layerURL) if err != nil { t.Fatalf("unexpected error fetching non-existent layer: %v", err) } checkResponse(t, "fetching non-existent content", resp, http.StatusNotFound) // ------------------------------------------ // Test head request for non-existent content resp, err = http.Head(layerURL) if err != nil { t.Fatalf("unexpected error checking head on non-existent layer: %v", err) } checkResponse(t, "checking head on non-existent layer", resp, http.StatusNotFound) // ------------------------------------------ // Start an upload, check the status then cancel uploadURLBase, uploadUUID := startPushLayer(t, env.builder, imageName) // A status check should work resp, err = http.Get(uploadURLBase) if err != nil { t.Fatalf("unexpected error getting upload status: %v", err) } checkResponse(t, "status of deleted upload", resp, http.StatusNoContent) checkHeaders(t, resp, http.Header{ "Location": []string{"*"}, "Range": []string{"0-0"}, "Docker-Upload-UUID": []string{uploadUUID}, }) req, err := http.NewRequest("DELETE", uploadURLBase, nil) if err != nil { t.Fatalf("unexpected error creating delete request: %v", err) } resp, err = http.DefaultClient.Do(req) if err != nil { t.Fatalf("unexpected error sending delete request: %v", err) } checkResponse(t, "deleting upload", resp, http.StatusNoContent) // A status check should result in 404 resp, err = http.Get(uploadURLBase) if err != nil { t.Fatalf("unexpected error getting upload status: %v", err) } checkResponse(t, "status of deleted upload", resp, http.StatusNotFound) // ----------------------------------------- // Do layer push with an empty body and different digest uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName) resp, err = doPushLayer(t, env.builder, imageName, layerDigest, uploadURLBase, bytes.NewReader([]byte{})) if err != nil { t.Fatalf("unexpected error doing bad layer push: %v", err) } checkResponse(t, "bad layer push", resp, http.StatusBadRequest) checkBodyHasErrorCodes(t, "bad layer push", resp, v2.ErrorCodeDigestInvalid) // ----------------------------------------- // Do layer push with an empty body and correct digest zeroDigest, err := digest.FromTarArchive(bytes.NewReader([]byte{})) if err != nil { t.Fatalf("unexpected error digesting empty buffer: %v", err) } uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName) pushLayer(t, env.builder, imageName, zeroDigest, uploadURLBase, bytes.NewReader([]byte{})) // ----------------------------------------- // Do layer push with an empty body and correct digest // This is a valid but empty tarfile! emptyTar := bytes.Repeat([]byte("\x00"), 1024) emptyDigest, err := digest.FromTarArchive(bytes.NewReader(emptyTar)) if err != nil { t.Fatalf("unexpected error digesting empty tar: %v", err) } uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName) pushLayer(t, env.builder, imageName, emptyDigest, uploadURLBase, bytes.NewReader(emptyTar)) // ------------------------------------------ // Now, actually do successful upload. layerLength, _ := layerFile.Seek(0, os.SEEK_END) layerFile.Seek(0, os.SEEK_SET) uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName) pushLayer(t, env.builder, imageName, layerDigest, uploadURLBase, layerFile) // ------------------------------------------ // Now, push just a chunk layerFile.Seek(0, 0) canonicalDigester := digest.Canonical.New() if _, err := io.Copy(canonicalDigester.Hash(), layerFile); err != nil { t.Fatalf("error copying to digest: %v", err) } canonicalDigest := canonicalDigester.Digest() layerFile.Seek(0, 0) uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName) uploadURLBase, dgst := pushChunk(t, env.builder, imageName, uploadURLBase, layerFile, layerLength) finishUpload(t, env.builder, imageName, uploadURLBase, dgst) // ------------------------ // Use a head request to see if the layer exists. resp, err = http.Head(layerURL) if err != nil { t.Fatalf("unexpected error checking head on existing layer: %v", err) } checkResponse(t, "checking head on existing layer", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Content-Length": []string{fmt.Sprint(layerLength)}, "Docker-Content-Digest": []string{canonicalDigest.String()}, }) // ---------------- // Fetch the layer! resp, err = http.Get(layerURL) if err != nil { t.Fatalf("unexpected error fetching layer: %v", err) } checkResponse(t, "fetching layer", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Content-Length": []string{fmt.Sprint(layerLength)}, "Docker-Content-Digest": []string{canonicalDigest.String()}, }) // Verify the body verifier, err := digest.NewDigestVerifier(layerDigest) if err != nil { t.Fatalf("unexpected error getting digest verifier: %s", err) } io.Copy(verifier, resp.Body) if !verifier.Verified() { t.Fatalf("response body did not pass verification") } // ---------------- // Fetch the layer with an invalid digest badURL := strings.Replace(layerURL, "tarsum", "trsum", 1) resp, err = http.Get(badURL) if err != nil { t.Fatalf("unexpected error fetching layer: %v", err) } checkResponse(t, "fetching layer bad digest", resp, http.StatusBadRequest) // Cache headers resp, err = http.Get(layerURL) if err != nil { t.Fatalf("unexpected error fetching layer: %v", err) } checkResponse(t, "fetching layer", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Content-Length": []string{fmt.Sprint(layerLength)}, "Docker-Content-Digest": []string{canonicalDigest.String()}, "ETag": []string{canonicalDigest.String()}, "Cache-Control": []string{"max-age=31536000"}, }) // Matching etag, gives 304 etag := resp.Header.Get("Etag") req, err = http.NewRequest("GET", layerURL, nil) if err != nil { t.Fatalf("Error constructing request: %s", err) } req.Header.Set("If-None-Match", etag) resp, err = http.DefaultClient.Do(req) if err != nil { t.Fatalf("Error constructing request: %s", err) } checkResponse(t, "fetching layer with etag", resp, http.StatusNotModified) // Non-matching etag, gives 200 req, err = http.NewRequest("GET", layerURL, nil) if err != nil { t.Fatalf("Error constructing request: %s", err) } req.Header.Set("If-None-Match", "") resp, err = http.DefaultClient.Do(req) checkResponse(t, "fetching layer with invalid etag", resp, http.StatusOK) // Missing tests: // - Upload the same tarsum file under and different repository and // ensure the content remains uncorrupted. }
func TestManifestAPI(t *testing.T) { env := newTestEnv(t) imageName := "foo/bar" tag := "thetag" manifestURL, err := env.builder.BuildManifestURL(imageName, tag) if err != nil { t.Fatalf("unexpected error getting manifest url: %v", err) } // ----------------------------- // Attempt to fetch the manifest resp, err := http.Get(manifestURL) if err != nil { t.Fatalf("unexpected error getting manifest: %v", err) } defer resp.Body.Close() checkResponse(t, "getting non-existent manifest", resp, http.StatusNotFound) checkBodyHasErrorCodes(t, "getting non-existent manifest", resp, v2.ErrorCodeManifestUnknown) tagsURL, err := env.builder.BuildTagsURL(imageName) if err != nil { t.Fatalf("unexpected error building tags url: %v", err) } resp, err = http.Get(tagsURL) if err != nil { t.Fatalf("unexpected error getting unknown tags: %v", err) } defer resp.Body.Close() // Check that we get an unknown repository error when asking for tags checkResponse(t, "getting unknown manifest tags", resp, http.StatusNotFound) checkBodyHasErrorCodes(t, "getting unknown manifest tags", resp, v2.ErrorCodeNameUnknown) // -------------------------------- // Attempt to push unsigned manifest with missing layers unsignedManifest := &manifest.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, }, Name: imageName, Tag: tag, FSLayers: []manifest.FSLayer{ { BlobSum: "asdf", }, { BlobSum: "qwer", }, }, } resp = putManifest(t, "putting unsigned manifest", manifestURL, unsignedManifest) defer resp.Body.Close() checkResponse(t, "posting unsigned manifest", resp, http.StatusBadRequest) _, p, counts := checkBodyHasErrorCodes(t, "getting unknown manifest tags", resp, v2.ErrorCodeManifestUnverified, v2.ErrorCodeBlobUnknown, v2.ErrorCodeDigestInvalid) expectedCounts := map[errcode.ErrorCode]int{ v2.ErrorCodeManifestUnverified: 1, v2.ErrorCodeBlobUnknown: 2, v2.ErrorCodeDigestInvalid: 2, } if !reflect.DeepEqual(counts, expectedCounts) { t.Fatalf("unexpected number of error codes encountered: %v\n!=\n%v\n---\n%s", counts, expectedCounts, string(p)) } // TODO(stevvooe): Add a test case where we take a mostly valid registry, // tamper with the content and ensure that we get a unverified manifest // error. // Push 2 random layers expectedLayers := make(map[digest.Digest]io.ReadSeeker) for i := range unsignedManifest.FSLayers { rs, dgstStr, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating random layer %d: %v", i, err) } dgst := digest.Digest(dgstStr) expectedLayers[dgst] = rs unsignedManifest.FSLayers[i].BlobSum = dgst uploadURLBase, _ := startPushLayer(t, env.builder, imageName) pushLayer(t, env.builder, imageName, dgst, uploadURLBase, rs) } // ------------------- // Push the signed manifest with all layers pushed. signedManifest, err := manifest.Sign(unsignedManifest, env.pk) if err != nil { t.Fatalf("unexpected error signing manifest: %v", err) } payload, err := signedManifest.Payload() checkErr(t, err, "getting manifest payload") dgst, err := digest.FromBytes(payload) checkErr(t, err, "digesting manifest") manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String()) checkErr(t, err, "building manifest url") resp = putManifest(t, "putting signed manifest", manifestURL, signedManifest) checkResponse(t, "putting signed manifest", resp, http.StatusAccepted) checkHeaders(t, resp, http.Header{ "Location": []string{manifestDigestURL}, "Docker-Content-Digest": []string{dgst.String()}, }) // -------------------- // Push by digest -- should get same result resp = putManifest(t, "putting signed manifest", manifestDigestURL, signedManifest) checkResponse(t, "putting signed manifest", resp, http.StatusAccepted) checkHeaders(t, resp, http.Header{ "Location": []string{manifestDigestURL}, "Docker-Content-Digest": []string{dgst.String()}, }) // ------------------ // Fetch by tag name resp, err = http.Get(manifestURL) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } defer resp.Body.Close() checkResponse(t, "fetching uploaded manifest", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Docker-Content-Digest": []string{dgst.String()}, }) var fetchedManifest manifest.SignedManifest dec := json.NewDecoder(resp.Body) if err := dec.Decode(&fetchedManifest); err != nil { t.Fatalf("error decoding fetched manifest: %v", err) } if !bytes.Equal(fetchedManifest.Raw, signedManifest.Raw) { t.Fatalf("manifests do not match") } // --------------- // Fetch by digest resp, err = http.Get(manifestDigestURL) checkErr(t, err, "fetching manifest by digest") defer resp.Body.Close() checkResponse(t, "fetching uploaded manifest", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Docker-Content-Digest": []string{dgst.String()}, }) var fetchedManifestByDigest manifest.SignedManifest dec = json.NewDecoder(resp.Body) if err := dec.Decode(&fetchedManifestByDigest); err != nil { t.Fatalf("error decoding fetched manifest: %v", err) } if !bytes.Equal(fetchedManifestByDigest.Raw, signedManifest.Raw) { t.Fatalf("manifests do not match") } // Ensure that the tag is listed. resp, err = http.Get(tagsURL) if err != nil { t.Fatalf("unexpected error getting unknown tags: %v", err) } defer resp.Body.Close() // Check that we get an unknown repository error when asking for tags checkResponse(t, "getting unknown manifest tags", resp, http.StatusOK) dec = json.NewDecoder(resp.Body) var tagsResponse tagsAPIResponse if err := dec.Decode(&tagsResponse); err != nil { t.Fatalf("unexpected error decoding error response: %v", err) } if tagsResponse.Name != imageName { t.Fatalf("tags name should match image name: %v != %v", tagsResponse.Name, imageName) } if len(tagsResponse.Tags) != 1 { t.Fatalf("expected some tags in response: %v", tagsResponse.Tags) } if tagsResponse.Tags[0] != tag { t.Fatalf("tag not as expected: %q != %q", tagsResponse.Tags[0], tag) } }
// TestSimpleBlobUpload covers the blob upload process, exercising common // error paths that might be seen during an upload. func TestSimpleBlobUpload(t *testing.T) { randomDataReader, dgst, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating random reader: %v", err) } ctx := context.Background() imageName := "foo/bar" driver := inmemory.New() registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect) if err != nil { t.Fatalf("error creating registry: %v", err) } repository, err := registry.Repository(ctx, imageName) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } bs := repository.Blobs(ctx) h := sha256.New() rd := io.TeeReader(randomDataReader, h) blobUpload, err := bs.Create(ctx) if err != nil { t.Fatalf("unexpected error starting layer upload: %s", err) } // Cancel the upload then restart it if err := blobUpload.Cancel(ctx); err != nil { t.Fatalf("unexpected error during upload cancellation: %v", err) } // Do a resume, get unknown upload blobUpload, err = bs.Resume(ctx, blobUpload.ID()) if err != distribution.ErrBlobUploadUnknown { t.Fatalf("unexpected error resuming upload, should be unknown: %v", err) } // Restart! blobUpload, err = bs.Create(ctx) if err != nil { t.Fatalf("unexpected error starting layer upload: %s", err) } // Get the size of our random tarfile randomDataSize, err := seekerSize(randomDataReader) if err != nil { t.Fatalf("error getting seeker size of random data: %v", err) } nn, err := io.Copy(blobUpload, rd) if err != nil { t.Fatalf("unexpected error uploading layer data: %v", err) } if nn != randomDataSize { t.Fatalf("layer data write incomplete") } offset, err := blobUpload.Seek(0, os.SEEK_CUR) if err != nil { t.Fatalf("unexpected error seeking layer upload: %v", err) } if offset != nn { t.Fatalf("blobUpload not updated with correct offset: %v != %v", offset, nn) } blobUpload.Close() // Do a resume, for good fun blobUpload, err = bs.Resume(ctx, blobUpload.ID()) if err != nil { t.Fatalf("unexpected error resuming upload: %v", err) } sha256Digest := digest.NewDigest("sha256", h) desc, err := blobUpload.Commit(ctx, distribution.Descriptor{Digest: dgst}) if err != nil { t.Fatalf("unexpected error finishing layer upload: %v", err) } // After finishing an upload, it should no longer exist. if _, err := bs.Resume(ctx, blobUpload.ID()); err != distribution.ErrBlobUploadUnknown { t.Fatalf("expected layer upload to be unknown, got %v", err) } // Test for existence. statDesc, err := bs.Stat(ctx, desc.Digest) if err != nil { t.Fatalf("unexpected error checking for existence: %v, %#v", err, bs) } if statDesc != desc { t.Fatalf("descriptors not equal: %v != %v", statDesc, desc) } rc, err := bs.Open(ctx, desc.Digest) if err != nil { t.Fatalf("unexpected error opening blob for read: %v", err) } defer rc.Close() h.Reset() nn, err = io.Copy(h, rc) if err != nil { t.Fatalf("error reading layer: %v", err) } if nn != randomDataSize { t.Fatalf("incorrect read length") } if digest.NewDigest("sha256", h) != sha256Digest { t.Fatalf("unexpected digest from uploaded layer: %q != %q", digest.NewDigest("sha256", h), sha256Digest) } // Delete a blob err = bs.Delete(ctx, desc.Digest) if err != nil { t.Fatalf("Unexpected error deleting blob") } d, err := bs.Stat(ctx, desc.Digest) if err == nil { t.Fatalf("unexpected non-error stating deleted blob: %v", d) } switch err { case distribution.ErrBlobUnknown: break default: t.Errorf("Unexpected error type stat-ing deleted manifest: %#v", err) } _, err = bs.Open(ctx, desc.Digest) if err == nil { t.Fatalf("unexpected success opening deleted blob for read") } switch err { case distribution.ErrBlobUnknown: break default: t.Errorf("Unexpected error type getting deleted manifest: %#v", err) } // Re-upload the blob randomBlob, err := ioutil.ReadAll(randomDataReader) if err != nil { t.Fatalf("Error reading all of blob %s", err.Error()) } expectedDigest := digest.FromBytes(randomBlob) simpleUpload(t, bs, randomBlob, expectedDigest) d, err = bs.Stat(ctx, expectedDigest) if err != nil { t.Errorf("unexpected error stat-ing blob") } if d.Digest != expectedDigest { t.Errorf("Mismatching digest with restored blob") } _, err = bs.Open(ctx, expectedDigest) if err != nil { t.Errorf("Unexpected error opening blob") } // Reuse state to test delete with a delete-disabled registry registry, err = NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableRedirect) if err != nil { t.Fatalf("error creating registry: %v", err) } repository, err = registry.Repository(ctx, imageName) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } bs = repository.Blobs(ctx) err = bs.Delete(ctx, desc.Digest) if err == nil { t.Errorf("Unexpected success deleting while disabled") } }
// TestSimpleBlobRead just creates a simple blob file and ensures that basic // open, read, seek, read works. More specific edge cases should be covered in // other tests. func TestSimpleBlobRead(t *testing.T) { ctx := context.Background() imageName := "foo/bar" driver := inmemory.New() registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect) if err != nil { t.Fatalf("error creating registry: %v", err) } repository, err := registry.Repository(ctx, imageName) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } bs := repository.Blobs(ctx) randomLayerReader, dgst, err := testutil.CreateRandomTarFile() // TODO(stevvooe): Consider using just a random string. if err != nil { t.Fatalf("error creating random data: %v", err) } // Test for existence. desc, err := bs.Stat(ctx, dgst) if err != distribution.ErrBlobUnknown { t.Fatalf("expected not found error when testing for existence: %v", err) } rc, err := bs.Open(ctx, dgst) if err != distribution.ErrBlobUnknown { t.Fatalf("expected not found error when opening non-existent blob: %v", err) } randomLayerSize, err := seekerSize(randomLayerReader) if err != nil { t.Fatalf("error getting seeker size for random layer: %v", err) } descBefore := distribution.Descriptor{Digest: dgst, MediaType: "application/octet-stream", Size: randomLayerSize} t.Logf("desc: %v", descBefore) desc, err = addBlob(ctx, bs, descBefore, randomLayerReader) if err != nil { t.Fatalf("error adding blob to blobservice: %v", err) } if desc.Size != randomLayerSize { t.Fatalf("committed blob has incorrect length: %v != %v", desc.Size, randomLayerSize) } rc, err = bs.Open(ctx, desc.Digest) // note that we are opening with original digest. if err != nil { t.Fatalf("error opening blob with %v: %v", dgst, err) } defer rc.Close() // Now check the sha digest and ensure its the same h := sha256.New() nn, err := io.Copy(h, rc) if err != nil { t.Fatalf("unexpected error copying to hash: %v", err) } if nn != randomLayerSize { t.Fatalf("stored incorrect number of bytes in blob: %d != %d", nn, randomLayerSize) } sha256Digest := digest.NewDigest("sha256", h) if sha256Digest != desc.Digest { t.Fatalf("fetched digest does not match: %q != %q", sha256Digest, desc.Digest) } // Now seek back the blob, read the whole thing and check against randomLayerData offset, err := rc.Seek(0, os.SEEK_SET) if err != nil { t.Fatalf("error seeking blob: %v", err) } if offset != 0 { t.Fatalf("seek failed: expected 0 offset, got %d", offset) } p, err := ioutil.ReadAll(rc) if err != nil { t.Fatalf("error reading all of blob: %v", err) } if len(p) != int(randomLayerSize) { t.Fatalf("blob data read has different length: %v != %v", len(p), randomLayerSize) } // Reset the randomLayerReader and read back the buffer _, err = randomLayerReader.Seek(0, os.SEEK_SET) if err != nil { t.Fatalf("error resetting layer reader: %v", err) } randomLayerData, err := ioutil.ReadAll(randomLayerReader) if err != nil { t.Fatalf("random layer read failed: %v", err) } if !bytes.Equal(p, randomLayerData) { t.Fatalf("layer data not equal") } }
// TestBlobMount covers the blob mount process, exercising common // error paths that might be seen during a mount. func TestBlobMount(t *testing.T) { randomDataReader, dgst, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating random reader: %v", err) } ctx := context.Background() imageName, _ := reference.ParseNamed("foo/bar") sourceImageName, _ := reference.ParseNamed("foo/source") driver := inmemory.New() registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect) if err != nil { t.Fatalf("error creating registry: %v", err) } repository, err := registry.Repository(ctx, imageName) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } sourceRepository, err := registry.Repository(ctx, sourceImageName) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } sbs := sourceRepository.Blobs(ctx) blobUpload, err := sbs.Create(ctx) if err != nil { t.Fatalf("unexpected error starting layer upload: %s", err) } // Get the size of our random tarfile randomDataSize, err := seekerSize(randomDataReader) if err != nil { t.Fatalf("error getting seeker size of random data: %v", err) } nn, err := io.Copy(blobUpload, randomDataReader) if err != nil { t.Fatalf("unexpected error uploading layer data: %v", err) } desc, err := blobUpload.Commit(ctx, distribution.Descriptor{Digest: dgst}) if err != nil { t.Fatalf("unexpected error finishing layer upload: %v", err) } // Test for existence. statDesc, err := sbs.Stat(ctx, desc.Digest) if err != nil { t.Fatalf("unexpected error checking for existence: %v, %#v", err, sbs) } if statDesc != desc { t.Fatalf("descriptors not equal: %v != %v", statDesc, desc) } bs := repository.Blobs(ctx) // Test destination for existence. statDesc, err = bs.Stat(ctx, desc.Digest) if err == nil { t.Fatalf("unexpected non-error stating unmounted blob: %v", desc) } canonicalRef, err := reference.WithDigest(sourceRepository.Name(), desc.Digest) if err != nil { t.Fatal(err) } bw, err := bs.Create(ctx, WithMountFrom(canonicalRef)) if bw != nil { t.Fatal("unexpected blobwriter returned from Create call, should mount instead") } ebm, ok := err.(distribution.ErrBlobMounted) if !ok { t.Fatalf("unexpected error mounting layer: %v", err) } if ebm.Descriptor != desc { t.Fatalf("descriptors not equal: %v != %v", ebm.Descriptor, desc) } // Test for existence. statDesc, err = bs.Stat(ctx, desc.Digest) if err != nil { t.Fatalf("unexpected error checking for existence: %v, %#v", err, bs) } if statDesc != desc { t.Fatalf("descriptors not equal: %v != %v", statDesc, desc) } rc, err := bs.Open(ctx, desc.Digest) if err != nil { t.Fatalf("unexpected error opening blob for read: %v", err) } defer rc.Close() h := sha256.New() nn, err = io.Copy(h, rc) if err != nil { t.Fatalf("error reading layer: %v", err) } if nn != randomDataSize { t.Fatalf("incorrect read length") } if digest.NewDigest("sha256", h) != dgst { t.Fatalf("unexpected digest from uploaded layer: %q != %q", digest.NewDigest("sha256", h), dgst) } // Delete the blob from the source repo err = sbs.Delete(ctx, desc.Digest) if err != nil { t.Fatalf("Unexpected error deleting blob") } d, err := bs.Stat(ctx, desc.Digest) if err != nil { t.Fatalf("unexpected error stating blob deleted from source repository: %v", err) } d, err = sbs.Stat(ctx, desc.Digest) if err == nil { t.Fatalf("unexpected non-error stating deleted blob: %v", d) } switch err { case distribution.ErrBlobUnknown: break default: t.Errorf("Unexpected error type stat-ing deleted manifest: %#v", err) } // Delete the blob from the dest repo err = bs.Delete(ctx, desc.Digest) if err != nil { t.Fatalf("Unexpected error deleting blob") } d, err = bs.Stat(ctx, desc.Digest) if err == nil { t.Fatalf("unexpected non-error stating deleted blob: %v", d) } switch err { case distribution.ErrBlobUnknown: break default: t.Errorf("Unexpected error type stat-ing deleted manifest: %#v", err) } }
func TestManifestStorage(t *testing.T) { env := newManifestStoreTestEnv(t, "foo/bar", "thetag") ctx := context.Background() ms, err := env.repository.Manifests(ctx) if err != nil { t.Fatal(err) } exists, err := ms.ExistsByTag(env.tag) if err != nil { t.Fatalf("unexpected error checking manifest existence: %v", err) } if exists { t.Fatalf("manifest should not exist") } if _, err := ms.GetByTag(env.tag); true { switch err.(type) { case distribution.ErrManifestUnknown: break default: t.Fatalf("expected manifest unknown error: %#v", err) } } m := manifest.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, }, Name: env.name, Tag: env.tag, } // Build up some test layers and add them to the manifest, saving the // readseekers for upload later. testLayers := map[digest.Digest]io.ReadSeeker{} for i := 0; i < 2; i++ { rs, ds, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("unexpected error generating test layer file") } dgst := digest.Digest(ds) testLayers[digest.Digest(dgst)] = rs m.FSLayers = append(m.FSLayers, manifest.FSLayer{ BlobSum: dgst, }) } pk, err := libtrust.GenerateECP256PrivateKey() if err != nil { t.Fatalf("unexpected error generating private key: %v", err) } sm, merr := manifest.Sign(&m, pk) if merr != nil { t.Fatalf("error signing manifest: %v", err) } err = ms.Put(sm) if err == nil { t.Fatalf("expected errors putting manifest with full verification") } switch err := err.(type) { case distribution.ErrManifestVerification: if len(err) != 2 { t.Fatalf("expected 2 verification errors: %#v", err) } for _, err := range err { if _, ok := err.(distribution.ErrManifestBlobUnknown); !ok { t.Fatalf("unexpected error type: %v", err) } } default: t.Fatalf("unexpected error verifying manifest: %v", err) } // Now, upload the layers that were missing! for dgst, rs := range testLayers { wr, err := env.repository.Blobs(env.ctx).Create(env.ctx) if err != nil { t.Fatalf("unexpected error creating test upload: %v", err) } if _, err := io.Copy(wr, rs); err != nil { t.Fatalf("unexpected error copying to upload: %v", err) } if _, err := wr.Commit(env.ctx, distribution.Descriptor{Digest: dgst}); err != nil { t.Fatalf("unexpected error finishing upload: %v", err) } } if err = ms.Put(sm); err != nil { t.Fatalf("unexpected error putting manifest: %v", err) } exists, err = ms.ExistsByTag(env.tag) if err != nil { t.Fatalf("unexpected error checking manifest existence: %v", err) } if !exists { t.Fatalf("manifest should exist") } fetchedManifest, err := ms.GetByTag(env.tag) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } if !reflect.DeepEqual(fetchedManifest, sm) { t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedManifest, sm) } fetchedJWS, err := libtrust.ParsePrettySignature(fetchedManifest.Raw, "signatures") if err != nil { t.Fatalf("unexpected error parsing jws: %v", err) } payload, err := fetchedJWS.Payload() if err != nil { t.Fatalf("unexpected error extracting payload: %v", err) } // Now that we have a payload, take a moment to check that the manifest is // return by the payload digest. dgst, err := digest.FromBytes(payload) if err != nil { t.Fatalf("error getting manifest digest: %v", err) } exists, err = ms.Exists(dgst) if err != nil { t.Fatalf("error checking manifest existence by digest: %v", err) } if !exists { t.Fatalf("manifest %s should exist", dgst) } fetchedByDigest, err := ms.Get(dgst) if err != nil { t.Fatalf("unexpected error fetching manifest by digest: %v", err) } if !reflect.DeepEqual(fetchedByDigest, fetchedManifest) { t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedByDigest, fetchedManifest) } sigs, err := fetchedJWS.Signatures() if err != nil { t.Fatalf("unable to extract signatures: %v", err) } if len(sigs) != 1 { t.Fatalf("unexpected number of signatures: %d != %d", len(sigs), 1) } // Grabs the tags and check that this tagged manifest is present tags, err := ms.Tags() if err != nil { t.Fatalf("unexpected error fetching tags: %v", err) } if len(tags) != 1 { t.Fatalf("unexpected tags returned: %v", tags) } if tags[0] != env.tag { t.Fatalf("unexpected tag found in tags: %v != %v", tags, []string{env.tag}) } // Now, push the same manifest with a different key pk2, err := libtrust.GenerateECP256PrivateKey() if err != nil { t.Fatalf("unexpected error generating private key: %v", err) } sm2, err := manifest.Sign(&m, pk2) if err != nil { t.Fatalf("unexpected error signing manifest: %v", err) } jws2, err := libtrust.ParsePrettySignature(sm2.Raw, "signatures") if err != nil { t.Fatalf("error parsing signature: %v", err) } sigs2, err := jws2.Signatures() if err != nil { t.Fatalf("unable to extract signatures: %v", err) } if len(sigs2) != 1 { t.Fatalf("unexpected number of signatures: %d != %d", len(sigs2), 1) } if err = ms.Put(sm2); err != nil { t.Fatalf("unexpected error putting manifest: %v", err) } fetched, err := ms.GetByTag(env.tag) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } if _, err := manifest.Verify(fetched); err != nil { t.Fatalf("unexpected error verifying manifest: %v", err) } // Assemble our payload and two signatures to get what we expect! expectedJWS, err := libtrust.NewJSONSignature(payload, sigs[0], sigs2[0]) if err != nil { t.Fatalf("unexpected error merging jws: %v", err) } expectedSigs, err := expectedJWS.Signatures() if err != nil { t.Fatalf("unexpected error getting expected signatures: %v", err) } receivedJWS, err := libtrust.ParsePrettySignature(fetched.Raw, "signatures") if err != nil { t.Fatalf("unexpected error parsing jws: %v", err) } receivedPayload, err := receivedJWS.Payload() if err != nil { t.Fatalf("unexpected error extracting received payload: %v", err) } if !bytes.Equal(receivedPayload, payload) { t.Fatalf("payloads are not equal") } receivedSigs, err := receivedJWS.Signatures() if err != nil { t.Fatalf("error getting signatures: %v", err) } for i, sig := range receivedSigs { if !bytes.Equal(sig, expectedSigs[i]) { t.Fatalf("mismatched signatures from remote: %v != %v", string(sig), string(expectedSigs[i])) } } // Test deleting manifests err = ms.Delete(dgst) if err != nil { t.Fatalf("unexpected an error deleting manifest by digest: %v", err) } exists, err = ms.Exists(dgst) if err != nil { t.Fatalf("Error querying manifest existence") } if exists { t.Errorf("Deleted manifest should not exist") } deletedManifest, err := ms.Get(dgst) if err == nil { t.Errorf("Unexpected success getting deleted manifest") } switch err.(type) { case distribution.ErrManifestUnknownRevision: break default: t.Errorf("Unexpected error getting deleted manifest: %s", reflect.ValueOf(err).Type()) } if deletedManifest != nil { t.Errorf("Deleted manifest get returned non-nil") } // Re-upload should restore manifest to a good state err = ms.Put(sm) if err != nil { t.Errorf("Error re-uploading deleted manifest") } exists, err = ms.Exists(dgst) if err != nil { t.Fatalf("Error querying manifest existence") } if !exists { t.Errorf("Restored manifest should exist") } deletedManifest, err = ms.Get(dgst) if err != nil { t.Errorf("Unexpected error getting manifest") } if deletedManifest == nil { t.Errorf("Deleted manifest get returned non-nil") } r := NewRegistryWithDriver(ctx, env.driver, memory.NewInMemoryBlobDescriptorCacheProvider(), false) repo, err := r.Repository(ctx, env.name) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } ms, err = repo.Manifests(ctx) if err != nil { t.Fatal(err) } err = ms.Delete(dgst) if err == nil { t.Errorf("Unexpected success deleting while disabled") } }
func TestManifestStorage(t *testing.T) { env := newManifestStoreTestEnv(t, "foo/bar", "thetag") ms := env.repository.Manifests() exists, err := ms.ExistsByTag(env.ctx, env.tag) if err != nil { t.Fatalf("unexpected error checking manifest existence: %v", err) } if exists { t.Fatalf("manifest should not exist") } if _, err := ms.GetByTag(env.ctx, env.tag); true { switch err.(type) { case distribution.ErrManifestUnknown: break default: t.Fatalf("expected manifest unknown error: %#v", err) } } m := manifest.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, }, Name: env.name, Tag: env.tag, } // Build up some test layers and add them to the manifest, saving the // readseekers for upload later. testLayers := map[digest.Digest]io.ReadSeeker{} for i := 0; i < 2; i++ { rs, ds, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("unexpected error generating test layer file") } dgst := digest.Digest(ds) testLayers[digest.Digest(dgst)] = rs m.FSLayers = append(m.FSLayers, manifest.FSLayer{ BlobSum: dgst, }) } pk, err := libtrust.GenerateECP256PrivateKey() if err != nil { t.Fatalf("unexpected error generating private key: %v", err) } sm, err := manifest.Sign(&m, pk) if err != nil { t.Fatalf("error signing manifest: %v", err) } err = ms.Put(env.ctx, sm) if err == nil { t.Fatalf("expected errors putting manifest") } // TODO(stevvooe): We expect errors describing all of the missing layers. // Now, upload the layers that were missing! for dgst, rs := range testLayers { upload, err := env.repository.Layers().Upload() if err != nil { t.Fatalf("unexpected error creating test upload: %v", err) } if _, err := io.Copy(upload, rs); err != nil { t.Fatalf("unexpected error copying to upload: %v", err) } if _, err := upload.Finish(dgst); err != nil { t.Fatalf("unexpected error finishing upload: %v", err) } } if err = ms.Put(env.ctx, sm); err != nil { t.Fatalf("unexpected error putting manifest: %v", err) } exists, err = ms.ExistsByTag(env.ctx, env.tag) if err != nil { t.Fatalf("unexpected error checking manifest existence: %v", err) } if !exists { t.Fatalf("manifest should exist") } fetchedManifest, err := ms.GetByTag(env.ctx, env.tag) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } if !reflect.DeepEqual(fetchedManifest, sm) { t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedManifest, sm) } fetchedJWS, err := libtrust.ParsePrettySignature(fetchedManifest.Raw, "signatures") if err != nil { t.Fatalf("unexpected error parsing jws: %v", err) } payload, err := fetchedJWS.Payload() if err != nil { t.Fatalf("unexpected error extracting payload: %v", err) } // Now that we have a payload, take a moment to check that the manifest is // return by the payload digest. dgst, err := digest.FromBytes(payload) if err != nil { t.Fatalf("error getting manifest digest: %v", err) } exists, err = ms.Exists(env.ctx, dgst) if err != nil { t.Fatalf("error checking manifest existence by digest: %v", err) } if !exists { t.Fatalf("manifest %s should exist", dgst) } fetchedByDigest, err := ms.Get(env.ctx, dgst) if err != nil { t.Fatalf("unexpected error fetching manifest by digest: %v", err) } if !reflect.DeepEqual(fetchedByDigest, fetchedManifest) { t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedByDigest, fetchedManifest) } sigs, err := fetchedJWS.Signatures() if err != nil { t.Fatalf("unable to extract signatures: %v", err) } if len(sigs) != 1 { t.Fatalf("unexpected number of signatures: %d != %d", len(sigs), 1) } // Grabs the tags and check that this tagged manifest is present tags, err := ms.Tags(env.ctx) if err != nil { t.Fatalf("unexpected error fetching tags: %v", err) } if len(tags) != 1 { t.Fatalf("unexpected tags returned: %v", tags) } if tags[0] != env.tag { t.Fatalf("unexpected tag found in tags: %v != %v", tags, []string{env.tag}) } // Now, push the same manifest with a different key pk2, err := libtrust.GenerateECP256PrivateKey() if err != nil { t.Fatalf("unexpected error generating private key: %v", err) } sm2, err := manifest.Sign(&m, pk2) if err != nil { t.Fatalf("unexpected error signing manifest: %v", err) } jws2, err := libtrust.ParsePrettySignature(sm2.Raw, "signatures") if err != nil { t.Fatalf("error parsing signature: %v", err) } sigs2, err := jws2.Signatures() if err != nil { t.Fatalf("unable to extract signatures: %v", err) } if len(sigs2) != 1 { t.Fatalf("unexpected number of signatures: %d != %d", len(sigs2), 1) } if err = ms.Put(env.ctx, sm2); err != nil { t.Fatalf("unexpected error putting manifest: %v", err) } fetched, err := ms.GetByTag(env.ctx, env.tag) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } if _, err := manifest.Verify(fetched); err != nil { t.Fatalf("unexpected error verifying manifest: %v", err) } // Assemble our payload and two signatures to get what we expect! expectedJWS, err := libtrust.NewJSONSignature(payload, sigs[0], sigs2[0]) if err != nil { t.Fatalf("unexpected error merging jws: %v", err) } expectedSigs, err := expectedJWS.Signatures() if err != nil { t.Fatalf("unexpected error getting expected signatures: %v", err) } receivedJWS, err := libtrust.ParsePrettySignature(fetched.Raw, "signatures") if err != nil { t.Fatalf("unexpected error parsing jws: %v", err) } receivedPayload, err := receivedJWS.Payload() if err != nil { t.Fatalf("unexpected error extracting received payload: %v", err) } if !bytes.Equal(receivedPayload, payload) { t.Fatalf("payloads are not equal") } receivedSigs, err := receivedJWS.Signatures() if err != nil { t.Fatalf("error getting signatures: %v", err) } for i, sig := range receivedSigs { if !bytes.Equal(sig, expectedSigs[i]) { t.Fatalf("mismatched signatures from remote: %v != %v", string(sig), string(expectedSigs[i])) } } // TODO(stevvooe): Currently, deletes are not supported due to some // complexity around managing tag indexes. We'll add this support back in // when the manifest format has settled. For now, we expect an error for // all deletes. if err := ms.Delete(env.ctx, dgst); err == nil { t.Fatalf("unexpected an error deleting manifest by digest: %v", err) } }
func TestLinkedBlobStoreCreateWithMountFrom(t *testing.T) { fooRepoName, _ := reference.ParseNamed("nm/foo") fooEnv := newManifestStoreTestEnv(t, fooRepoName, "thetag") ctx := context.Background() stats, err := mockRegistry(t, fooEnv.registry) if err != nil { t.Fatal(err) } // Build up some test layers and add them to the manifest, saving the // readseekers for upload later. testLayers := map[digest.Digest]io.ReadSeeker{} for i := 0; i < 2; i++ { rs, ds, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("unexpected error generating test layer file") } dgst := digest.Digest(ds) testLayers[digest.Digest(dgst)] = rs } // upload the layers to foo/bar for dgst, rs := range testLayers { wr, err := fooEnv.repository.Blobs(fooEnv.ctx).Create(fooEnv.ctx) if err != nil { t.Fatalf("unexpected error creating test upload: %v", err) } if _, err := io.Copy(wr, rs); err != nil { t.Fatalf("unexpected error copying to upload: %v", err) } if _, err := wr.Commit(fooEnv.ctx, distribution.Descriptor{Digest: dgst}); err != nil { t.Fatalf("unexpected error finishing upload: %v", err) } } // create another repository nm/bar barRepoName, _ := reference.ParseNamed("nm/bar") barRepo, err := fooEnv.registry.Repository(ctx, barRepoName) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } // cross-repo mount the test layers into a nm/bar for dgst := range testLayers { fooCanonical, _ := reference.WithDigest(fooRepoName, dgst) option := WithMountFrom(fooCanonical) // ensure we can instrospect it createOpts := distribution.CreateOptions{} if err := option.Apply(&createOpts); err != nil { t.Fatalf("failed to apply MountFrom option: %v", err) } if !createOpts.Mount.ShouldMount || createOpts.Mount.From.String() != fooCanonical.String() { t.Fatalf("unexpected create options: %#+v", createOpts.Mount) } _, err := barRepo.Blobs(ctx).Create(ctx, WithMountFrom(fooCanonical)) if err == nil { t.Fatalf("unexpected non-error while mounting from %q: %v", fooRepoName.String(), err) } if _, ok := err.(distribution.ErrBlobMounted); !ok { t.Fatalf("expected ErrMountFrom error, not %T: %v", err, err) } } for dgst := range testLayers { fooCanonical, _ := reference.WithDigest(fooRepoName, dgst) count, exists := stats[fooCanonical.String()] if !exists { t.Errorf("expected entry %q not found among handled stat calls", fooCanonical.String()) } else if count != 1 { t.Errorf("expected exactly one stat call for entry %q, not %d", fooCanonical.String(), count) } } clearStats(stats) // create yet another repository nm/baz bazRepoName, _ := reference.ParseNamed("nm/baz") bazRepo, err := fooEnv.registry.Repository(ctx, bazRepoName) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } // cross-repo mount them into a nm/baz and provide a prepopulated blob descriptor for dgst := range testLayers { fooCanonical, _ := reference.WithDigest(fooRepoName, dgst) size, err := strconv.ParseInt("0x"+dgst.Hex()[:8], 0, 64) if err != nil { t.Fatal(err) } prepolutatedDescriptor := distribution.Descriptor{ Digest: dgst, Size: size, MediaType: "application/octet-stream", } _, err = bazRepo.Blobs(ctx).Create(ctx, WithMountFrom(fooCanonical), &statCrossMountCreateOption{ desc: prepolutatedDescriptor, }) blobMounted, ok := err.(distribution.ErrBlobMounted) if !ok { t.Errorf("expected ErrMountFrom error, not %T: %v", err, err) continue } if !reflect.DeepEqual(blobMounted.Descriptor, prepolutatedDescriptor) { t.Errorf("unexpected descriptor: %#+v != %#+v", blobMounted.Descriptor, prepolutatedDescriptor) } } // this time no stat calls will be made if len(stats) != 0 { t.Errorf("unexpected number of stats made: %d != %d", len(stats), len(testLayers)) } }
// checkExerciseRegistry takes the registry through all of its operations, // carrying out generic checks. func checkExerciseRepository(t *testing.T, repository distribution.Repository) { // TODO(stevvooe): This would be a nice testutil function. Basically, it // takes the registry through a common set of operations. This could be // used to make cross-cutting updates by changing internals that affect // update counts. Basically, it would make writing tests a lot easier. ctx := context.Background() tag := "thetag" // todo: change this to use Builder m := schema1.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, }, Name: repository.Named().Name(), Tag: tag, } var blobDigests []digest.Digest blobs := repository.Blobs(ctx) for i := 0; i < 2; i++ { rs, ds, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating test layer: %v", err) } dgst := digest.Digest(ds) blobDigests = append(blobDigests, dgst) wr, err := blobs.Create(ctx) if err != nil { t.Fatalf("error creating layer upload: %v", err) } // Use the resumes, as well! wr, err = blobs.Resume(ctx, wr.ID()) if err != nil { t.Fatalf("error resuming layer upload: %v", err) } io.Copy(wr, rs) if _, err := wr.Commit(ctx, distribution.Descriptor{Digest: dgst}); err != nil { t.Fatalf("unexpected error finishing upload: %v", err) } m.FSLayers = append(m.FSLayers, schema1.FSLayer{ BlobSum: dgst, }) m.History = append(m.History, schema1.History{ V1Compatibility: "", }) // Then fetch the blobs if rc, err := blobs.Open(ctx, dgst); err != nil { t.Fatalf("error fetching layer: %v", err) } else { defer rc.Close() } } pk, err := libtrust.GenerateECP256PrivateKey() if err != nil { t.Fatalf("unexpected error generating key: %v", err) } sm, err := schema1.Sign(&m, pk) if err != nil { t.Fatalf("unexpected error signing manifest: %v", err) } manifests, err := repository.Manifests(ctx) if err != nil { t.Fatal(err.Error()) } var digestPut digest.Digest if digestPut, err = manifests.Put(ctx, sm); err != nil { t.Fatalf("unexpected error putting the manifest: %v", err) } dgst := digest.FromBytes(sm.Canonical) if dgst != digestPut { t.Fatalf("mismatching digest from payload and put") } _, err = manifests.Get(ctx, dgst) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } err = manifests.Delete(ctx, dgst) if err != nil { t.Fatalf("unexpected error deleting blob: %v", err) } for _, d := range blobDigests { err = blobs.Delete(ctx, d) if err != nil { t.Fatalf("unexpected error deleting blob: %v", err) } } }
// TestSimpleLayerRead just creates a simple layer file and ensures that basic // open, read, seek, read works. More specific edge cases should be covered in // other tests. func TestSimpleLayerRead(t *testing.T) { ctx := context.Background() imageName := "foo/bar" driver := inmemory.New() registry := NewRegistryWithDriver(ctx, driver, cache.NewInMemoryLayerInfoCache()) repository, err := registry.Repository(ctx, imageName) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } ls := repository.Layers() randomLayerReader, tarSumStr, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating random data: %v", err) } dgst := digest.Digest(tarSumStr) // Test for existence. exists, err := ls.Exists(dgst) if err != nil { t.Fatalf("unexpected error checking for existence: %v", err) } if exists { t.Fatalf("layer should not exist") } // Try to get the layer and make sure we get a not found error layer, err := ls.Fetch(dgst) if err == nil { t.Fatalf("error expected fetching unknown layer") } switch err.(type) { case distribution.ErrUnknownLayer: err = nil default: t.Fatalf("unexpected error fetching non-existent layer: %v", err) } randomLayerDigest, err := writeTestLayer(driver, defaultPathMapper, imageName, dgst, randomLayerReader) if err != nil { t.Fatalf("unexpected error writing test layer: %v", err) } randomLayerSize, err := seekerSize(randomLayerReader) if err != nil { t.Fatalf("error getting seeker size for random layer: %v", err) } layer, err = ls.Fetch(dgst) if err != nil { t.Fatal(err) } defer layer.Close() // Now check the sha digest and ensure its the same h := sha256.New() nn, err := io.Copy(h, layer) if err != nil && err != io.EOF { t.Fatalf("unexpected error copying to hash: %v", err) } if nn != randomLayerSize { t.Fatalf("stored incorrect number of bytes in layer: %d != %d", nn, randomLayerSize) } sha256Digest := digest.NewDigest("sha256", h) if sha256Digest != randomLayerDigest { t.Fatalf("fetched digest does not match: %q != %q", sha256Digest, randomLayerDigest) } // Now seek back the layer, read the whole thing and check against randomLayerData offset, err := layer.Seek(0, os.SEEK_SET) if err != nil { t.Fatalf("error seeking layer: %v", err) } if offset != 0 { t.Fatalf("seek failed: expected 0 offset, got %d", offset) } p, err := ioutil.ReadAll(layer) if err != nil { t.Fatalf("error reading all of layer: %v", err) } if len(p) != int(randomLayerSize) { t.Fatalf("layer data read has different length: %v != %v", len(p), randomLayerSize) } // Reset the randomLayerReader and read back the buffer _, err = randomLayerReader.Seek(0, os.SEEK_SET) if err != nil { t.Fatalf("error resetting layer reader: %v", err) } randomLayerData, err := ioutil.ReadAll(randomLayerReader) if err != nil { t.Fatalf("random layer read failed: %v", err) } if !bytes.Equal(p, randomLayerData) { t.Fatalf("layer data not equal") } }
// TestSimpleLayerUpload covers the layer upload process, exercising common // error paths that might be seen during an upload. func TestSimpleLayerUpload(t *testing.T) { randomDataReader, tarSumStr, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating random reader: %v", err) } dgst := digest.Digest(tarSumStr) if err != nil { t.Fatalf("error allocating upload store: %v", err) } ctx := context.Background() imageName := "foo/bar" driver := inmemory.New() registry := NewRegistryWithDriver(ctx, driver, cache.NewInMemoryLayerInfoCache()) repository, err := registry.Repository(ctx, imageName) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } ls := repository.Layers() h := sha256.New() rd := io.TeeReader(randomDataReader, h) layerUpload, err := ls.Upload() if err != nil { t.Fatalf("unexpected error starting layer upload: %s", err) } // Cancel the upload then restart it if err := layerUpload.Cancel(); err != nil { t.Fatalf("unexpected error during upload cancellation: %v", err) } // Do a resume, get unknown upload layerUpload, err = ls.Resume(layerUpload.UUID()) if err != distribution.ErrLayerUploadUnknown { t.Fatalf("unexpected error resuming upload, should be unkown: %v", err) } // Restart! layerUpload, err = ls.Upload() if err != nil { t.Fatalf("unexpected error starting layer upload: %s", err) } // Get the size of our random tarfile randomDataSize, err := seekerSize(randomDataReader) if err != nil { t.Fatalf("error getting seeker size of random data: %v", err) } nn, err := io.Copy(layerUpload, rd) if err != nil { t.Fatalf("unexpected error uploading layer data: %v", err) } if nn != randomDataSize { t.Fatalf("layer data write incomplete") } offset, err := layerUpload.Seek(0, os.SEEK_CUR) if err != nil { t.Fatalf("unexpected error seeking layer upload: %v", err) } if offset != nn { t.Fatalf("layerUpload not updated with correct offset: %v != %v", offset, nn) } layerUpload.Close() // Do a resume, for good fun layerUpload, err = ls.Resume(layerUpload.UUID()) if err != nil { t.Fatalf("unexpected error resuming upload: %v", err) } sha256Digest := digest.NewDigest("sha256", h) layer, err := layerUpload.Finish(dgst) if err != nil { t.Fatalf("unexpected error finishing layer upload: %v", err) } // After finishing an upload, it should no longer exist. if _, err := ls.Resume(layerUpload.UUID()); err != distribution.ErrLayerUploadUnknown { t.Fatalf("expected layer upload to be unknown, got %v", err) } // Test for existence. exists, err := ls.Exists(layer.Digest()) if err != nil { t.Fatalf("unexpected error checking for existence: %v", err) } if !exists { t.Fatalf("layer should now exist") } h.Reset() nn, err = io.Copy(h, layer) if err != nil { t.Fatalf("error reading layer: %v", err) } if nn != randomDataSize { t.Fatalf("incorrect read length") } if digest.NewDigest("sha256", h) != sha256Digest { t.Fatalf("unexpected digest from uploaded layer: %q != %q", digest.NewDigest("sha256", h), sha256Digest) } }
// TestSimpleBlobUpload covers the blob upload process, exercising common // error paths that might be seen during an upload. func TestSimpleBlobUpload(t *testing.T) { randomDataReader, tarSumStr, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating random reader: %v", err) } dgst := digest.Digest(tarSumStr) if err != nil { t.Fatalf("error allocating upload store: %v", err) } ctx := context.Background() imageName := "foo/bar" driver := inmemory.New() registry := NewRegistryWithDriver(ctx, driver, memory.NewInMemoryBlobDescriptorCacheProvider()) repository, err := registry.Repository(ctx, imageName) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } bs := repository.Blobs(ctx) h := sha256.New() rd := io.TeeReader(randomDataReader, h) blobUpload, err := bs.Create(ctx) if err != nil { t.Fatalf("unexpected error starting layer upload: %s", err) } // Cancel the upload then restart it if err := blobUpload.Cancel(ctx); err != nil { t.Fatalf("unexpected error during upload cancellation: %v", err) } // Do a resume, get unknown upload blobUpload, err = bs.Resume(ctx, blobUpload.ID()) if err != distribution.ErrBlobUploadUnknown { t.Fatalf("unexpected error resuming upload, should be unknown: %v", err) } // Restart! blobUpload, err = bs.Create(ctx) if err != nil { t.Fatalf("unexpected error starting layer upload: %s", err) } // Get the size of our random tarfile randomDataSize, err := seekerSize(randomDataReader) if err != nil { t.Fatalf("error getting seeker size of random data: %v", err) } nn, err := io.Copy(blobUpload, rd) if err != nil { t.Fatalf("unexpected error uploading layer data: %v", err) } if nn != randomDataSize { t.Fatalf("layer data write incomplete") } offset, err := blobUpload.Seek(0, os.SEEK_CUR) if err != nil { t.Fatalf("unexpected error seeking layer upload: %v", err) } if offset != nn { t.Fatalf("blobUpload not updated with correct offset: %v != %v", offset, nn) } blobUpload.Close() // Do a resume, for good fun blobUpload, err = bs.Resume(ctx, blobUpload.ID()) if err != nil { t.Fatalf("unexpected error resuming upload: %v", err) } sha256Digest := digest.NewDigest("sha256", h) desc, err := blobUpload.Commit(ctx, distribution.Descriptor{Digest: dgst}) if err != nil { t.Fatalf("unexpected error finishing layer upload: %v", err) } // After finishing an upload, it should no longer exist. if _, err := bs.Resume(ctx, blobUpload.ID()); err != distribution.ErrBlobUploadUnknown { t.Fatalf("expected layer upload to be unknown, got %v", err) } // Test for existence. statDesc, err := bs.Stat(ctx, desc.Digest) if err != nil { t.Fatalf("unexpected error checking for existence: %v, %#v", err, bs) } if statDesc != desc { t.Fatalf("descriptors not equal: %v != %v", statDesc, desc) } rc, err := bs.Open(ctx, desc.Digest) if err != nil { t.Fatalf("unexpected error opening blob for read: %v", err) } defer rc.Close() h.Reset() nn, err = io.Copy(h, rc) if err != nil { t.Fatalf("error reading layer: %v", err) } if nn != randomDataSize { t.Fatalf("incorrect read length") } if digest.NewDigest("sha256", h) != sha256Digest { t.Fatalf("unexpected digest from uploaded layer: %q != %q", digest.NewDigest("sha256", h), sha256Digest) } }
// checkExerciseRegistry takes the registry through all of its operations, // carrying out generic checks. func checkExerciseRepository(t *testing.T, repository distribution.Repository) { // TODO(stevvooe): This would be a nice testutil function. Basically, it // takes the registry through a common set of operations. This could be // used to make cross-cutting updates by changing internals that affect // update counts. Basically, it would make writing tests a lot easier. ctx := context.Background() tag := "thetag" m := manifest.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, }, Name: repository.Name(), Tag: tag, } blobs := repository.Blobs(ctx) for i := 0; i < 2; i++ { rs, ds, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating test layer: %v", err) } dgst := digest.Digest(ds) wr, err := blobs.Create(ctx) if err != nil { t.Fatalf("error creating layer upload: %v", err) } // Use the resumes, as well! wr, err = blobs.Resume(ctx, wr.ID()) if err != nil { t.Fatalf("error resuming layer upload: %v", err) } io.Copy(wr, rs) if _, err := wr.Commit(ctx, distribution.Descriptor{Digest: dgst}); err != nil { t.Fatalf("unexpected error finishing upload: %v", err) } m.FSLayers = append(m.FSLayers, manifest.FSLayer{ BlobSum: dgst, }) // Then fetch the blobs if rc, err := blobs.Open(ctx, dgst); err != nil { t.Fatalf("error fetching layer: %v", err) } else { defer rc.Close() } } pk, err := libtrust.GenerateECP256PrivateKey() if err != nil { t.Fatalf("unexpected error generating key: %v", err) } sm, err := manifest.Sign(&m, pk) if err != nil { t.Fatalf("unexpected error signing manifest: %v", err) } manifests, err := repository.Manifests(ctx) if err != nil { t.Fatal(err.Error()) } if err = manifests.Put(sm); err != nil { t.Fatalf("unexpected error putting the manifest: %v", err) } p, err := sm.Payload() if err != nil { t.Fatalf("unexpected error getting manifest payload: %v", err) } dgst, err := digest.FromBytes(p) if err != nil { t.Fatalf("unexpected error digesting manifest payload: %v", err) } fetchedByManifest, err := manifests.Get(dgst) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } if fetchedByManifest.Tag != sm.Tag { t.Fatalf("retrieved unexpected manifest: %v", err) } fetched, err := manifests.GetByTag(tag) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } if fetched.Tag != fetchedByManifest.Tag { t.Fatalf("retrieved unexpected manifest: %v", err) } }
func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, manifestArgs) { imageName := args.imageName tag := "thetag" manifestURL, err := env.builder.BuildManifestURL(imageName, tag) if err != nil { t.Fatalf("unexpected error getting manifest url: %v", err) } // ----------------------------- // Attempt to fetch the manifest resp, err := http.Get(manifestURL) if err != nil { t.Fatalf("unexpected error getting manifest: %v", err) } defer resp.Body.Close() checkResponse(t, "getting non-existent manifest", resp, http.StatusNotFound) checkBodyHasErrorCodes(t, "getting non-existent manifest", resp, v2.ErrorCodeManifestUnknown) tagsURL, err := env.builder.BuildTagsURL(imageName) if err != nil { t.Fatalf("unexpected error building tags url: %v", err) } resp, err = http.Get(tagsURL) if err != nil { t.Fatalf("unexpected error getting unknown tags: %v", err) } defer resp.Body.Close() // Check that we get an unknown repository error when asking for tags checkResponse(t, "getting unknown manifest tags", resp, http.StatusNotFound) checkBodyHasErrorCodes(t, "getting unknown manifest tags", resp, v2.ErrorCodeNameUnknown) // -------------------------------- // Attempt to push unsigned manifest with missing layers unsignedManifest := &schema1.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, }, Name: imageName, Tag: tag, FSLayers: []schema1.FSLayer{ { BlobSum: "asdf", }, { BlobSum: "qwer", }, }, History: []schema1.History{ { V1Compatibility: "", }, { V1Compatibility: "", }, }, } resp = putManifest(t, "putting unsigned manifest", manifestURL, unsignedManifest) defer resp.Body.Close() checkResponse(t, "putting unsigned manifest", resp, http.StatusBadRequest) _, p, counts := checkBodyHasErrorCodes(t, "getting unknown manifest tags", resp, v2.ErrorCodeManifestInvalid) expectedCounts := map[errcode.ErrorCode]int{ v2.ErrorCodeManifestInvalid: 1, } if !reflect.DeepEqual(counts, expectedCounts) { t.Fatalf("unexpected number of error codes encountered: %v\n!=\n%v\n---\n%s", counts, expectedCounts, string(p)) } // sign the manifest and still get some interesting errors. sm, err := schema1.Sign(unsignedManifest, env.pk) if err != nil { t.Fatalf("error signing manifest: %v", err) } resp = putManifest(t, "putting signed manifest with errors", manifestURL, sm) defer resp.Body.Close() checkResponse(t, "putting signed manifest with errors", resp, http.StatusBadRequest) _, p, counts = checkBodyHasErrorCodes(t, "putting signed manifest with errors", resp, v2.ErrorCodeManifestBlobUnknown, v2.ErrorCodeDigestInvalid) expectedCounts = map[errcode.ErrorCode]int{ v2.ErrorCodeManifestBlobUnknown: 2, v2.ErrorCodeDigestInvalid: 2, } if !reflect.DeepEqual(counts, expectedCounts) { t.Fatalf("unexpected number of error codes encountered: %v\n!=\n%v\n---\n%s", counts, expectedCounts, string(p)) } // TODO(stevvooe): Add a test case where we take a mostly valid registry, // tamper with the content and ensure that we get a unverified manifest // error. // Push 2 random layers expectedLayers := make(map[digest.Digest]io.ReadSeeker) for i := range unsignedManifest.FSLayers { rs, dgstStr, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating random layer %d: %v", i, err) } dgst := digest.Digest(dgstStr) expectedLayers[dgst] = rs unsignedManifest.FSLayers[i].BlobSum = dgst uploadURLBase, _ := startPushLayer(t, env.builder, imageName) pushLayer(t, env.builder, imageName, dgst, uploadURLBase, rs) } // ------------------- // Push the signed manifest with all layers pushed. signedManifest, err := schema1.Sign(unsignedManifest, env.pk) if err != nil { t.Fatalf("unexpected error signing manifest: %v", err) } dgst := digest.FromBytes(signedManifest.Canonical) args.signedManifest = signedManifest args.dgst = dgst manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String()) checkErr(t, err, "building manifest url") resp = putManifest(t, "putting signed manifest no error", manifestURL, signedManifest) checkResponse(t, "putting signed manifest no error", resp, http.StatusCreated) checkHeaders(t, resp, http.Header{ "Location": []string{manifestDigestURL}, "Docker-Content-Digest": []string{dgst.String()}, }) // -------------------- // Push by digest -- should get same result resp = putManifest(t, "putting signed manifest", manifestDigestURL, signedManifest) checkResponse(t, "putting signed manifest", resp, http.StatusCreated) checkHeaders(t, resp, http.Header{ "Location": []string{manifestDigestURL}, "Docker-Content-Digest": []string{dgst.String()}, }) // ------------------ // Fetch by tag name resp, err = http.Get(manifestURL) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } defer resp.Body.Close() checkResponse(t, "fetching uploaded manifest", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Docker-Content-Digest": []string{dgst.String()}, "ETag": []string{fmt.Sprintf(`"%s"`, dgst)}, }) var fetchedManifest schema1.SignedManifest dec := json.NewDecoder(resp.Body) if err := dec.Decode(&fetchedManifest); err != nil { t.Fatalf("error decoding fetched manifest: %v", err) } if !bytes.Equal(fetchedManifest.Canonical, signedManifest.Canonical) { t.Fatalf("manifests do not match") } // --------------- // Fetch by digest resp, err = http.Get(manifestDigestURL) checkErr(t, err, "fetching manifest by digest") defer resp.Body.Close() checkResponse(t, "fetching uploaded manifest", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Docker-Content-Digest": []string{dgst.String()}, "ETag": []string{fmt.Sprintf(`"%s"`, dgst)}, }) var fetchedManifestByDigest schema1.SignedManifest dec = json.NewDecoder(resp.Body) if err := dec.Decode(&fetchedManifestByDigest); err != nil { t.Fatalf("error decoding fetched manifest: %v", err) } if !bytes.Equal(fetchedManifestByDigest.Canonical, signedManifest.Canonical) { t.Fatalf("manifests do not match") } // check signature was roundtripped signatures, err := fetchedManifestByDigest.Signatures() if err != nil { t.Fatal(err) } if len(signatures) != 1 { t.Fatalf("expected 1 signature from manifest, got: %d", len(signatures)) } // Re-sign, push and pull the same digest sm2, err := schema1.Sign(&fetchedManifestByDigest.Manifest, env.pk) if err != nil { t.Fatal(err) } resp = putManifest(t, "re-putting signed manifest", manifestDigestURL, sm2) checkResponse(t, "re-putting signed manifest", resp, http.StatusCreated) resp, err = http.Get(manifestDigestURL) checkErr(t, err, "re-fetching manifest by digest") defer resp.Body.Close() checkResponse(t, "re-fetching uploaded manifest", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Docker-Content-Digest": []string{dgst.String()}, "ETag": []string{fmt.Sprintf(`"%s"`, dgst)}, }) dec = json.NewDecoder(resp.Body) if err := dec.Decode(&fetchedManifestByDigest); err != nil { t.Fatalf("error decoding fetched manifest: %v", err) } // check two signatures were roundtripped signatures, err = fetchedManifestByDigest.Signatures() if err != nil { t.Fatal(err) } if len(signatures) != 2 { t.Fatalf("expected 2 signature from manifest, got: %d", len(signatures)) } // Get by name with etag, gives 304 etag := resp.Header.Get("Etag") req, err := http.NewRequest("GET", manifestURL, nil) if err != nil { t.Fatalf("Error constructing request: %s", err) } req.Header.Set("If-None-Match", etag) resp, err = http.DefaultClient.Do(req) if err != nil { t.Fatalf("Error constructing request: %s", err) } checkResponse(t, "fetching manifest by name with etag", resp, http.StatusNotModified) // Get by digest with etag, gives 304 req, err = http.NewRequest("GET", manifestDigestURL, nil) if err != nil { t.Fatalf("Error constructing request: %s", err) } req.Header.Set("If-None-Match", etag) resp, err = http.DefaultClient.Do(req) if err != nil { t.Fatalf("Error constructing request: %s", err) } checkResponse(t, "fetching manifest by dgst with etag", resp, http.StatusNotModified) // Ensure that the tag is listed. resp, err = http.Get(tagsURL) if err != nil { t.Fatalf("unexpected error getting unknown tags: %v", err) } defer resp.Body.Close() // Check that we get an unknown repository error when asking for tags checkResponse(t, "getting unknown manifest tags", resp, http.StatusOK) dec = json.NewDecoder(resp.Body) var tagsResponse tagsAPIResponse if err := dec.Decode(&tagsResponse); err != nil { t.Fatalf("unexpected error decoding error response: %v", err) } if tagsResponse.Name != imageName { t.Fatalf("tags name should match image name: %v != %v", tagsResponse.Name, imageName) } if len(tagsResponse.Tags) != 1 { t.Fatalf("expected some tags in response: %v", tagsResponse.Tags) } if tagsResponse.Tags[0] != tag { t.Fatalf("tag not as expected: %q != %q", tagsResponse.Tags[0], tag) } // Attempt to put a manifest with mismatching FSLayer and History array cardinalities unsignedManifest.History = append(unsignedManifest.History, schema1.History{ V1Compatibility: "", }) invalidSigned, err := schema1.Sign(unsignedManifest, env.pk) if err != nil { t.Fatalf("error signing manifest") } resp = putManifest(t, "putting invalid signed manifest", manifestDigestURL, invalidSigned) checkResponse(t, "putting invalid signed manifest", resp, http.StatusBadRequest) return env, args }