func TestDeleteDisabled(t *testing.T) {
env := newTestEnv(t, false)
imageName := "foo/bar"
// "build" our layer file
layerFile, tarSumStr, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random layer file: %v", err)
}
layerDigest := digest.Digest(tarSumStr)
layerURL, err := env.builder.BuildBlobURL(imageName, layerDigest)
if err != nil {
t.Fatalf("Error building blob URL")
}
uploadURLBase, _ := startPushLayer(t, env.builder, imageName)
pushLayer(t, env.builder, imageName, layerDigest, uploadURLBase, layerFile)
resp, err := httpDelete(layerURL)
if err != nil {
t.Fatalf("unexpected error deleting layer: %v", err)
}
checkResponse(t, "deleting layer with delete disabled", resp, http.StatusMethodNotAllowed)
}
func uploadRandomTarball(t *testing.T, ctx context.Context, bi distribution.BlobIngester) digest.Digest {
randomDataReader, tarSumStr, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random reader: %v", err)
}
dgst := digest.Digest(tarSumStr)
if err != nil {
t.Fatalf("error allocating upload store: %v", err)
}
randomLayerSize, err := seekerSize(randomDataReader)
if err != nil {
t.Fatalf("error getting seeker size for random layer: %v", err)
}
_, err = addBlob(ctx, bi, distribution.Descriptor{
Digest: dgst,
MediaType: "application/octet-stream",
Size: randomLayerSize,
}, randomDataReader)
if err != nil {
t.Fatalf("failed to add blob: %v", err)
}
return dgst
}
func TestDeleteReadOnly(t *testing.T) {
env := newTestEnv(t, true)
imageName := "foo/bar"
// "build" our layer file
layerFile, layerDigest, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random layer file: %v", err)
}
layerURL, err := env.builder.BuildBlobURL(imageName, layerDigest)
if err != nil {
t.Fatalf("Error building blob URL")
}
uploadURLBase, _ := startPushLayer(t, env.builder, imageName)
pushLayer(t, env.builder, imageName, layerDigest, uploadURLBase, layerFile)
env.app.readOnly = true
resp, err := httpDelete(layerURL)
if err != nil {
t.Fatalf("unexpected error deleting layer: %v", err)
}
checkResponse(t, "deleting layer in read-only mode", resp, http.StatusMethodNotAllowed)
}
func createRepository(env *testEnv, t *testing.T, imageName string, tag string) digest.Digest {
unsignedManifest := &schema1.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
},
Name: imageName,
Tag: tag,
FSLayers: []schema1.FSLayer{
{
BlobSum: "asdf",
},
},
History: []schema1.History{
{
V1Compatibility: "",
},
},
}
// Push 2 random layers
expectedLayers := make(map[digest.Digest]io.ReadSeeker)
for i := range unsignedManifest.FSLayers {
rs, dgstStr, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random layer %d: %v", i, err)
}
dgst := digest.Digest(dgstStr)
expectedLayers[dgst] = rs
unsignedManifest.FSLayers[i].BlobSum = dgst
uploadURLBase, _ := startPushLayer(t, env.builder, imageName)
pushLayer(t, env.builder, imageName, dgst, uploadURLBase, rs)
}
signedManifest, err := schema1.Sign(unsignedManifest, env.pk)
if err != nil {
t.Fatalf("unexpected error signing manifest: %v", err)
}
dgst := digest.FromBytes(signedManifest.Canonical)
// Create this repository by tag to ensure the tag mapping is made in the registry
manifestDigestURL, err := env.builder.BuildManifestURL(imageName, tag)
checkErr(t, err, "building manifest url")
location, err := env.builder.BuildManifestURL(imageName, dgst.String())
checkErr(t, err, "building location URL")
resp := putManifest(t, "putting signed manifest", manifestDigestURL, signedManifest)
checkResponse(t, "putting signed manifest", resp, http.StatusCreated)
checkHeaders(t, resp, http.Header{
"Location": []string{location},
"Docker-Content-Digest": []string{dgst.String()},
})
return dgst
}
func TestDigestVerifier(t *testing.T) {
p := make([]byte, 1<<20)
rand.Read(p)
digest, err := FromBytes(p)
if err != nil {
t.Fatalf("unexpected error digesting bytes: %#v", err)
}
verifier, err := NewDigestVerifier(digest)
if err != nil {
t.Fatalf("unexpected error getting digest verifier: %s", err)
}
io.Copy(verifier, bytes.NewReader(p))
if !verifier.Verified() {
t.Fatalf("bytes not verified")
}
tf, tarSum, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating tarfile: %v", err)
}
digest, err = FromTarArchive(tf)
if err != nil {
t.Fatalf("error digesting tarsum: %v", err)
}
if digest.String() != tarSum {
t.Fatalf("unexpected digest: %q != %q", digest.String(), tarSum)
}
expectedSize, _ := tf.Seek(0, os.SEEK_END) // Get tar file size
tf.Seek(0, os.SEEK_SET) // seek back
// This is the most relevant example for the registry application. It's
// effectively a read through pipeline, where the final sink is the digest
// verifier.
verifier, err = NewDigestVerifier(digest)
if err != nil {
t.Fatalf("unexpected error getting digest verifier: %s", err)
}
lengthVerifier := NewLengthVerifier(expectedSize)
rd := io.TeeReader(tf, lengthVerifier)
io.Copy(verifier, rd)
if !lengthVerifier.Verified() {
t.Fatalf("verifier detected incorrect length")
}
if !verifier.Verified() {
t.Fatalf("bytes not verified")
}
}
func createRepository(env *testEnv, t *testing.T, imageName string, tag string) {
unsignedManifest := &schema1.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
},
Name: imageName,
Tag: tag,
FSLayers: []schema1.FSLayer{
{
BlobSum: "asdf",
},
{
BlobSum: "qwer",
},
},
}
// Push 2 random layers
expectedLayers := make(map[digest.Digest]io.ReadSeeker)
for i := range unsignedManifest.FSLayers {
rs, dgstStr, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random layer %d: %v", i, err)
}
dgst := digest.Digest(dgstStr)
expectedLayers[dgst] = rs
unsignedManifest.FSLayers[i].BlobSum = dgst
uploadURLBase, _ := startPushLayer(t, env.builder, imageName)
pushLayer(t, env.builder, imageName, dgst, uploadURLBase, rs)
}
signedManifest, err := schema1.Sign(unsignedManifest, env.pk)
if err != nil {
t.Fatalf("unexpected error signing manifest: %v", err)
}
payload, err := signedManifest.Payload()
checkErr(t, err, "getting manifest payload")
dgst, err := digest.FromBytes(payload)
checkErr(t, err, "digesting manifest")
manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String())
checkErr(t, err, "building manifest url")
resp := putManifest(t, "putting signed manifest", manifestDigestURL, signedManifest)
checkResponse(t, "putting signed manifest", resp, http.StatusCreated)
checkHeaders(t, resp, http.Header{
"Location": []string{manifestDigestURL},
"Docker-Content-Digest": []string{dgst.String()},
})
}
func populateRepo(t *testing.T, ctx context.Context, repository distribution.Repository, name, tag string) (digest.Digest, error) {
m := schema1.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
},
Name: name,
Tag: tag,
}
for i := 0; i < 2; i++ {
wr, err := repository.Blobs(ctx).Create(ctx)
if err != nil {
t.Fatalf("unexpected error creating test upload: %v", err)
}
rs, ts, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("unexpected error generating test layer file")
}
dgst := digest.Digest(ts)
if _, err := io.Copy(wr, rs); err != nil {
t.Fatalf("unexpected error copying to upload: %v", err)
}
if _, err := wr.Commit(ctx, distribution.Descriptor{Digest: dgst}); err != nil {
t.Fatalf("unexpected error finishing upload: %v", err)
}
}
pk, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
t.Fatalf("unexpected error generating private key: %v", err)
}
sm, err := schema1.Sign(&m, pk)
if err != nil {
t.Fatalf("error signing manifest: %v", err)
}
ms, err := repository.Manifests(ctx)
if err != nil {
t.Fatalf(err.Error())
}
ms.Put(sm)
if err != nil {
t.Fatalf("unexpected errors putting manifest: %v", err)
}
pl, err := sm.Payload()
if err != nil {
t.Fatal(err)
}
return digest.FromBytes(pl)
}
func makeBlobArgs(t *testing.T) blobArgs {
layerFile, layerDigest, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random layer file: %v", err)
}
args := blobArgs{
imageName: "foo/bar",
layerFile: layerFile,
layerDigest: layerDigest,
}
return args
}
// writeRandomLayer creates a random layer under name and tarSum using driver
// and pathMapper. An io.ReadSeeker with the data is returned, along with the
// sha256 hex digest.
func writeRandomLayer(driver storagedriver.StorageDriver, pathMapper *pathMapper, name string) (rs io.ReadSeeker, tarSum digest.Digest, sha256digest digest.Digest, err error) {
reader, tarSumStr, err := testutil.CreateRandomTarFile()
if err != nil {
return nil, "", "", err
}
tarSum = digest.Digest(tarSumStr)
// Now, actually create the layer.
randomLayerDigest, err := writeTestLayer(driver, pathMapper, name, tarSum, ioutil.NopCloser(reader))
if _, err := reader.Seek(0, os.SEEK_SET); err != nil {
return nil, "", "", err
}
return reader, tarSum, randomLayerDigest, err
}
func TestLinkedBlobStoreCreateWithMountFrom(t *testing.T) {
fooRepoName, _ := reference.ParseNamed("nm/foo")
fooEnv := newManifestStoreTestEnv(t, fooRepoName, "thetag")
ctx := context.Background()
// Build up some test layers and add them to the manifest, saving the
// readseekers for upload later.
testLayers := map[digest.Digest]io.ReadSeeker{}
for i := 0; i < 2; i++ {
rs, ds, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("unexpected error generating test layer file")
}
dgst := digest.Digest(ds)
testLayers[digest.Digest(dgst)] = rs
}
// upload the layers to foo/bar
for dgst, rs := range testLayers {
wr, err := fooEnv.repository.Blobs(fooEnv.ctx).Create(fooEnv.ctx)
if err != nil {
t.Fatalf("unexpected error creating test upload: %v", err)
}
if _, err := io.Copy(wr, rs); err != nil {
t.Fatalf("unexpected error copying to upload: %v", err)
}
if _, err := wr.Commit(fooEnv.ctx, distribution.Descriptor{Digest: dgst}); err != nil {
t.Fatalf("unexpected error finishing upload: %v", err)
}
}
// create another repository nm/bar
barRepoName, _ := reference.ParseNamed("nm/bar")
barRepo, err := fooEnv.registry.Repository(ctx, barRepoName)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
// cross-repo mount the test layers into a nm/bar
for dgst := range testLayers {
fooCanonical, _ := reference.WithDigest(fooRepoName, dgst)
option := WithMountFrom(fooCanonical)
// ensure we can instrospect it
createOpts := distribution.CreateOptions{}
if err := option.Apply(&createOpts); err != nil {
t.Fatalf("failed to apply MountFrom option: %v", err)
}
if !createOpts.Mount.ShouldMount || createOpts.Mount.From.String() != fooCanonical.String() {
t.Fatalf("unexpected create options: %#+v", createOpts.Mount)
}
_, err := barRepo.Blobs(ctx).Create(ctx, WithMountFrom(fooCanonical))
if err == nil {
t.Fatalf("unexpected non-error while mounting from %q: %v", fooRepoName.String(), err)
}
if _, ok := err.(distribution.ErrBlobMounted); !ok {
t.Fatalf("expected ErrMountFrom error, not %T: %v", err, err)
}
}
}
func testManifestStorage(t *testing.T, options ...RegistryOption) {
repoName, _ := reference.ParseNamed("foo/bar")
env := newManifestStoreTestEnv(t, repoName, "thetag", options...)
ctx := context.Background()
ms, err := env.repository.Manifests(ctx)
if err != nil {
t.Fatal(err)
}
equalSignatures := env.registry.(*registry).schema1SignaturesEnabled
m := schema1.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
},
Name: env.name.Name(),
Tag: env.tag,
}
// Build up some test layers and add them to the manifest, saving the
// readseekers for upload later.
testLayers := map[digest.Digest]io.ReadSeeker{}
for i := 0; i < 2; i++ {
rs, ds, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("unexpected error generating test layer file")
}
dgst := digest.Digest(ds)
testLayers[digest.Digest(dgst)] = rs
m.FSLayers = append(m.FSLayers, schema1.FSLayer{
BlobSum: dgst,
})
m.History = append(m.History, schema1.History{
V1Compatibility: "",
})
}
pk, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
t.Fatalf("unexpected error generating private key: %v", err)
}
sm, merr := schema1.Sign(&m, pk)
if merr != nil {
t.Fatalf("error signing manifest: %v", err)
}
_, err = ms.Put(ctx, sm)
if err == nil {
t.Fatalf("expected errors putting manifest with full verification")
}
switch err := err.(type) {
case distribution.ErrManifestVerification:
if len(err) != 2 {
t.Fatalf("expected 2 verification errors: %#v", err)
}
for _, err := range err {
if _, ok := err.(distribution.ErrManifestBlobUnknown); !ok {
t.Fatalf("unexpected error type: %v", err)
}
}
default:
t.Fatalf("unexpected error verifying manifest: %v", err)
}
// Now, upload the layers that were missing!
for dgst, rs := range testLayers {
wr, err := env.repository.Blobs(env.ctx).Create(env.ctx)
if err != nil {
t.Fatalf("unexpected error creating test upload: %v", err)
}
if _, err := io.Copy(wr, rs); err != nil {
t.Fatalf("unexpected error copying to upload: %v", err)
}
if _, err := wr.Commit(env.ctx, distribution.Descriptor{Digest: dgst}); err != nil {
t.Fatalf("unexpected error finishing upload: %v", err)
}
}
var manifestDigest digest.Digest
if manifestDigest, err = ms.Put(ctx, sm); err != nil {
t.Fatalf("unexpected error putting manifest: %v", err)
}
exists, err := ms.Exists(ctx, manifestDigest)
if err != nil {
t.Fatalf("unexpected error checking manifest existence: %#v", err)
}
if !exists {
t.Fatalf("manifest should exist")
}
fromStore, err := ms.Get(ctx, manifestDigest)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
fetchedManifest, ok := fromStore.(*schema1.SignedManifest)
if !ok {
t.Fatalf("unexpected manifest type from signedstore")
}
if !bytes.Equal(fetchedManifest.Canonical, sm.Canonical) {
t.Fatalf("fetched payload does not match original payload: %q != %q", fetchedManifest.Canonical, sm.Canonical)
}
if equalSignatures {
if !reflect.DeepEqual(fetchedManifest, sm) {
t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedManifest.Manifest, sm.Manifest)
}
}
_, pl, err := fetchedManifest.Payload()
if err != nil {
t.Fatalf("error getting payload %#v", err)
}
fetchedJWS, err := libtrust.ParsePrettySignature(pl, "signatures")
if err != nil {
t.Fatalf("unexpected error parsing jws: %v", err)
}
payload, err := fetchedJWS.Payload()
if err != nil {
t.Fatalf("unexpected error extracting payload: %v", err)
}
// Now that we have a payload, take a moment to check that the manifest is
// return by the payload digest.
dgst := digest.FromBytes(payload)
exists, err = ms.Exists(ctx, dgst)
if err != nil {
t.Fatalf("error checking manifest existence by digest: %v", err)
}
if !exists {
t.Fatalf("manifest %s should exist", dgst)
}
fetchedByDigest, err := ms.Get(ctx, dgst)
if err != nil {
t.Fatalf("unexpected error fetching manifest by digest: %v", err)
}
byDigestManifest, ok := fetchedByDigest.(*schema1.SignedManifest)
if !ok {
t.Fatalf("unexpected manifest type from signedstore")
}
if !bytes.Equal(byDigestManifest.Canonical, fetchedManifest.Canonical) {
t.Fatalf("fetched manifest not equal: %q != %q", byDigestManifest.Canonical, fetchedManifest.Canonical)
}
if equalSignatures {
if !reflect.DeepEqual(fetchedByDigest, fetchedManifest) {
t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedByDigest, fetchedManifest)
}
}
sigs, err := fetchedJWS.Signatures()
if err != nil {
t.Fatalf("unable to extract signatures: %v", err)
}
if len(sigs) != 1 {
t.Fatalf("unexpected number of signatures: %d != %d", len(sigs), 1)
}
// Now, push the same manifest with a different key
pk2, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
t.Fatalf("unexpected error generating private key: %v", err)
}
sm2, err := schema1.Sign(&m, pk2)
if err != nil {
t.Fatalf("unexpected error signing manifest: %v", err)
}
_, pl, err = sm2.Payload()
if err != nil {
t.Fatalf("error getting payload %#v", err)
}
jws2, err := libtrust.ParsePrettySignature(pl, "signatures")
if err != nil {
t.Fatalf("error parsing signature: %v", err)
}
sigs2, err := jws2.Signatures()
if err != nil {
t.Fatalf("unable to extract signatures: %v", err)
}
if len(sigs2) != 1 {
t.Fatalf("unexpected number of signatures: %d != %d", len(sigs2), 1)
}
if manifestDigest, err = ms.Put(ctx, sm2); err != nil {
t.Fatalf("unexpected error putting manifest: %v", err)
}
fromStore, err = ms.Get(ctx, manifestDigest)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
fetched, ok := fromStore.(*schema1.SignedManifest)
if !ok {
t.Fatalf("unexpected type from signed manifeststore : %T", fetched)
}
if _, err := schema1.Verify(fetched); err != nil {
t.Fatalf("unexpected error verifying manifest: %v", err)
}
// Assemble our payload and two signatures to get what we expect!
expectedJWS, err := libtrust.NewJSONSignature(payload, sigs[0], sigs2[0])
if err != nil {
t.Fatalf("unexpected error merging jws: %v", err)
}
expectedSigs, err := expectedJWS.Signatures()
if err != nil {
t.Fatalf("unexpected error getting expected signatures: %v", err)
}
_, pl, err = fetched.Payload()
if err != nil {
t.Fatalf("error getting payload %#v", err)
}
receivedJWS, err := libtrust.ParsePrettySignature(pl, "signatures")
if err != nil {
t.Fatalf("unexpected error parsing jws: %v", err)
}
receivedPayload, err := receivedJWS.Payload()
if err != nil {
t.Fatalf("unexpected error extracting received payload: %v", err)
}
if !bytes.Equal(receivedPayload, payload) {
t.Fatalf("payloads are not equal")
}
if equalSignatures {
receivedSigs, err := receivedJWS.Signatures()
if err != nil {
t.Fatalf("error getting signatures: %v", err)
}
for i, sig := range receivedSigs {
if !bytes.Equal(sig, expectedSigs[i]) {
t.Fatalf("mismatched signatures from remote: %v != %v", string(sig), string(expectedSigs[i]))
}
}
}
// Test deleting manifests
err = ms.Delete(ctx, dgst)
if err != nil {
t.Fatalf("unexpected an error deleting manifest by digest: %v", err)
}
exists, err = ms.Exists(ctx, dgst)
if err != nil {
t.Fatalf("Error querying manifest existence")
}
if exists {
t.Errorf("Deleted manifest should not exist")
}
deletedManifest, err := ms.Get(ctx, dgst)
if err == nil {
t.Errorf("Unexpected success getting deleted manifest")
}
switch err.(type) {
case distribution.ErrManifestUnknownRevision:
break
default:
t.Errorf("Unexpected error getting deleted manifest: %s", reflect.ValueOf(err).Type())
}
if deletedManifest != nil {
t.Errorf("Deleted manifest get returned non-nil")
}
// Re-upload should restore manifest to a good state
_, err = ms.Put(ctx, sm)
if err != nil {
t.Errorf("Error re-uploading deleted manifest")
}
exists, err = ms.Exists(ctx, dgst)
if err != nil {
t.Fatalf("Error querying manifest existence")
}
if !exists {
t.Errorf("Restored manifest should exist")
}
deletedManifest, err = ms.Get(ctx, dgst)
if err != nil {
t.Errorf("Unexpected error getting manifest")
}
if deletedManifest == nil {
t.Errorf("Deleted manifest get returned non-nil")
}
r, err := NewRegistry(ctx, env.driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableRedirect)
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
repo, err := r.Repository(ctx, env.name)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
ms, err = repo.Manifests(ctx)
if err != nil {
t.Fatal(err)
}
err = ms.Delete(ctx, dgst)
if err == nil {
t.Errorf("Unexpected success deleting while disabled")
}
}
// TestBlobAPI conducts a full test of the of the blob api.
func TestBlobAPI(t *testing.T) {
// TODO(stevvooe): This test code is complete junk but it should cover the
// complete flow. This must be broken down and checked against the
// specification *before* we submit the final to docker core.
env := newTestEnv(t)
imageName := "foo/bar"
// "build" our layer file
layerFile, tarSumStr, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random layer file: %v", err)
}
layerDigest := digest.Digest(tarSumStr)
// -----------------------------------
// Test fetch for non-existent content
layerURL, err := env.builder.BuildBlobURL(imageName, layerDigest)
if err != nil {
t.Fatalf("error building url: %v", err)
}
resp, err := http.Get(layerURL)
if err != nil {
t.Fatalf("unexpected error fetching non-existent layer: %v", err)
}
checkResponse(t, "fetching non-existent content", resp, http.StatusNotFound)
// ------------------------------------------
// Test head request for non-existent content
resp, err = http.Head(layerURL)
if err != nil {
t.Fatalf("unexpected error checking head on non-existent layer: %v", err)
}
checkResponse(t, "checking head on non-existent layer", resp, http.StatusNotFound)
// ------------------------------------------
// Start an upload, check the status then cancel
uploadURLBase, uploadUUID := startPushLayer(t, env.builder, imageName)
// A status check should work
resp, err = http.Get(uploadURLBase)
if err != nil {
t.Fatalf("unexpected error getting upload status: %v", err)
}
checkResponse(t, "status of deleted upload", resp, http.StatusNoContent)
checkHeaders(t, resp, http.Header{
"Location": []string{"*"},
"Range": []string{"0-0"},
"Docker-Upload-UUID": []string{uploadUUID},
})
req, err := http.NewRequest("DELETE", uploadURLBase, nil)
if err != nil {
t.Fatalf("unexpected error creating delete request: %v", err)
}
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("unexpected error sending delete request: %v", err)
}
checkResponse(t, "deleting upload", resp, http.StatusNoContent)
// A status check should result in 404
resp, err = http.Get(uploadURLBase)
if err != nil {
t.Fatalf("unexpected error getting upload status: %v", err)
}
checkResponse(t, "status of deleted upload", resp, http.StatusNotFound)
// -----------------------------------------
// Do layer push with an empty body and different digest
uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName)
resp, err = doPushLayer(t, env.builder, imageName, layerDigest, uploadURLBase, bytes.NewReader([]byte{}))
if err != nil {
t.Fatalf("unexpected error doing bad layer push: %v", err)
}
checkResponse(t, "bad layer push", resp, http.StatusBadRequest)
checkBodyHasErrorCodes(t, "bad layer push", resp, v2.ErrorCodeDigestInvalid)
// -----------------------------------------
// Do layer push with an empty body and correct digest
zeroDigest, err := digest.FromTarArchive(bytes.NewReader([]byte{}))
if err != nil {
t.Fatalf("unexpected error digesting empty buffer: %v", err)
}
uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName)
pushLayer(t, env.builder, imageName, zeroDigest, uploadURLBase, bytes.NewReader([]byte{}))
// -----------------------------------------
// Do layer push with an empty body and correct digest
// This is a valid but empty tarfile!
emptyTar := bytes.Repeat([]byte("\x00"), 1024)
emptyDigest, err := digest.FromTarArchive(bytes.NewReader(emptyTar))
if err != nil {
t.Fatalf("unexpected error digesting empty tar: %v", err)
}
uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName)
pushLayer(t, env.builder, imageName, emptyDigest, uploadURLBase, bytes.NewReader(emptyTar))
// ------------------------------------------
// Now, actually do successful upload.
layerLength, _ := layerFile.Seek(0, os.SEEK_END)
layerFile.Seek(0, os.SEEK_SET)
uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName)
pushLayer(t, env.builder, imageName, layerDigest, uploadURLBase, layerFile)
// ------------------------------------------
// Now, push just a chunk
layerFile.Seek(0, 0)
canonicalDigester := digest.Canonical.New()
if _, err := io.Copy(canonicalDigester.Hash(), layerFile); err != nil {
t.Fatalf("error copying to digest: %v", err)
}
canonicalDigest := canonicalDigester.Digest()
layerFile.Seek(0, 0)
uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName)
uploadURLBase, dgst := pushChunk(t, env.builder, imageName, uploadURLBase, layerFile, layerLength)
finishUpload(t, env.builder, imageName, uploadURLBase, dgst)
// ------------------------
// Use a head request to see if the layer exists.
resp, err = http.Head(layerURL)
if err != nil {
t.Fatalf("unexpected error checking head on existing layer: %v", err)
}
checkResponse(t, "checking head on existing layer", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Content-Length": []string{fmt.Sprint(layerLength)},
"Docker-Content-Digest": []string{canonicalDigest.String()},
})
// ----------------
// Fetch the layer!
resp, err = http.Get(layerURL)
if err != nil {
t.Fatalf("unexpected error fetching layer: %v", err)
}
checkResponse(t, "fetching layer", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Content-Length": []string{fmt.Sprint(layerLength)},
"Docker-Content-Digest": []string{canonicalDigest.String()},
})
// Verify the body
verifier, err := digest.NewDigestVerifier(layerDigest)
if err != nil {
t.Fatalf("unexpected error getting digest verifier: %s", err)
}
io.Copy(verifier, resp.Body)
if !verifier.Verified() {
t.Fatalf("response body did not pass verification")
}
// ----------------
// Fetch the layer with an invalid digest
badURL := strings.Replace(layerURL, "tarsum", "trsum", 1)
resp, err = http.Get(badURL)
if err != nil {
t.Fatalf("unexpected error fetching layer: %v", err)
}
checkResponse(t, "fetching layer bad digest", resp, http.StatusBadRequest)
// Cache headers
resp, err = http.Get(layerURL)
if err != nil {
t.Fatalf("unexpected error fetching layer: %v", err)
}
checkResponse(t, "fetching layer", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Content-Length": []string{fmt.Sprint(layerLength)},
"Docker-Content-Digest": []string{canonicalDigest.String()},
"ETag": []string{canonicalDigest.String()},
"Cache-Control": []string{"max-age=31536000"},
})
// Matching etag, gives 304
etag := resp.Header.Get("Etag")
req, err = http.NewRequest("GET", layerURL, nil)
if err != nil {
t.Fatalf("Error constructing request: %s", err)
}
req.Header.Set("If-None-Match", etag)
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("Error constructing request: %s", err)
}
checkResponse(t, "fetching layer with etag", resp, http.StatusNotModified)
// Non-matching etag, gives 200
req, err = http.NewRequest("GET", layerURL, nil)
if err != nil {
t.Fatalf("Error constructing request: %s", err)
}
req.Header.Set("If-None-Match", "")
resp, err = http.DefaultClient.Do(req)
checkResponse(t, "fetching layer with invalid etag", resp, http.StatusOK)
// Missing tests:
// - Upload the same tarsum file under and different repository and
// ensure the content remains uncorrupted.
}
func TestManifestAPI(t *testing.T) {
env := newTestEnv(t)
imageName := "foo/bar"
tag := "thetag"
manifestURL, err := env.builder.BuildManifestURL(imageName, tag)
if err != nil {
t.Fatalf("unexpected error getting manifest url: %v", err)
}
// -----------------------------
// Attempt to fetch the manifest
resp, err := http.Get(manifestURL)
if err != nil {
t.Fatalf("unexpected error getting manifest: %v", err)
}
defer resp.Body.Close()
checkResponse(t, "getting non-existent manifest", resp, http.StatusNotFound)
checkBodyHasErrorCodes(t, "getting non-existent manifest", resp, v2.ErrorCodeManifestUnknown)
tagsURL, err := env.builder.BuildTagsURL(imageName)
if err != nil {
t.Fatalf("unexpected error building tags url: %v", err)
}
resp, err = http.Get(tagsURL)
if err != nil {
t.Fatalf("unexpected error getting unknown tags: %v", err)
}
defer resp.Body.Close()
// Check that we get an unknown repository error when asking for tags
checkResponse(t, "getting unknown manifest tags", resp, http.StatusNotFound)
checkBodyHasErrorCodes(t, "getting unknown manifest tags", resp, v2.ErrorCodeNameUnknown)
// --------------------------------
// Attempt to push unsigned manifest with missing layers
unsignedManifest := &manifest.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
},
Name: imageName,
Tag: tag,
FSLayers: []manifest.FSLayer{
{
BlobSum: "asdf",
},
{
BlobSum: "qwer",
},
},
}
resp = putManifest(t, "putting unsigned manifest", manifestURL, unsignedManifest)
defer resp.Body.Close()
checkResponse(t, "posting unsigned manifest", resp, http.StatusBadRequest)
_, p, counts := checkBodyHasErrorCodes(t, "getting unknown manifest tags", resp,
v2.ErrorCodeManifestUnverified, v2.ErrorCodeBlobUnknown, v2.ErrorCodeDigestInvalid)
expectedCounts := map[errcode.ErrorCode]int{
v2.ErrorCodeManifestUnverified: 1,
v2.ErrorCodeBlobUnknown: 2,
v2.ErrorCodeDigestInvalid: 2,
}
if !reflect.DeepEqual(counts, expectedCounts) {
t.Fatalf("unexpected number of error codes encountered: %v\n!=\n%v\n---\n%s", counts, expectedCounts, string(p))
}
// TODO(stevvooe): Add a test case where we take a mostly valid registry,
// tamper with the content and ensure that we get a unverified manifest
// error.
// Push 2 random layers
expectedLayers := make(map[digest.Digest]io.ReadSeeker)
for i := range unsignedManifest.FSLayers {
rs, dgstStr, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random layer %d: %v", i, err)
}
dgst := digest.Digest(dgstStr)
expectedLayers[dgst] = rs
unsignedManifest.FSLayers[i].BlobSum = dgst
uploadURLBase, _ := startPushLayer(t, env.builder, imageName)
pushLayer(t, env.builder, imageName, dgst, uploadURLBase, rs)
}
// -------------------
// Push the signed manifest with all layers pushed.
signedManifest, err := manifest.Sign(unsignedManifest, env.pk)
if err != nil {
t.Fatalf("unexpected error signing manifest: %v", err)
}
payload, err := signedManifest.Payload()
checkErr(t, err, "getting manifest payload")
dgst, err := digest.FromBytes(payload)
checkErr(t, err, "digesting manifest")
manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String())
checkErr(t, err, "building manifest url")
resp = putManifest(t, "putting signed manifest", manifestURL, signedManifest)
checkResponse(t, "putting signed manifest", resp, http.StatusAccepted)
checkHeaders(t, resp, http.Header{
"Location": []string{manifestDigestURL},
"Docker-Content-Digest": []string{dgst.String()},
})
// --------------------
// Push by digest -- should get same result
resp = putManifest(t, "putting signed manifest", manifestDigestURL, signedManifest)
checkResponse(t, "putting signed manifest", resp, http.StatusAccepted)
checkHeaders(t, resp, http.Header{
"Location": []string{manifestDigestURL},
"Docker-Content-Digest": []string{dgst.String()},
})
// ------------------
// Fetch by tag name
resp, err = http.Get(manifestURL)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
defer resp.Body.Close()
checkResponse(t, "fetching uploaded manifest", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Docker-Content-Digest": []string{dgst.String()},
})
var fetchedManifest manifest.SignedManifest
dec := json.NewDecoder(resp.Body)
if err := dec.Decode(&fetchedManifest); err != nil {
t.Fatalf("error decoding fetched manifest: %v", err)
}
if !bytes.Equal(fetchedManifest.Raw, signedManifest.Raw) {
t.Fatalf("manifests do not match")
}
// ---------------
// Fetch by digest
resp, err = http.Get(manifestDigestURL)
checkErr(t, err, "fetching manifest by digest")
defer resp.Body.Close()
checkResponse(t, "fetching uploaded manifest", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Docker-Content-Digest": []string{dgst.String()},
})
var fetchedManifestByDigest manifest.SignedManifest
dec = json.NewDecoder(resp.Body)
if err := dec.Decode(&fetchedManifestByDigest); err != nil {
t.Fatalf("error decoding fetched manifest: %v", err)
}
if !bytes.Equal(fetchedManifestByDigest.Raw, signedManifest.Raw) {
t.Fatalf("manifests do not match")
}
// Ensure that the tag is listed.
resp, err = http.Get(tagsURL)
if err != nil {
t.Fatalf("unexpected error getting unknown tags: %v", err)
}
defer resp.Body.Close()
// Check that we get an unknown repository error when asking for tags
checkResponse(t, "getting unknown manifest tags", resp, http.StatusOK)
dec = json.NewDecoder(resp.Body)
var tagsResponse tagsAPIResponse
if err := dec.Decode(&tagsResponse); err != nil {
t.Fatalf("unexpected error decoding error response: %v", err)
}
if tagsResponse.Name != imageName {
t.Fatalf("tags name should match image name: %v != %v", tagsResponse.Name, imageName)
}
if len(tagsResponse.Tags) != 1 {
t.Fatalf("expected some tags in response: %v", tagsResponse.Tags)
}
if tagsResponse.Tags[0] != tag {
t.Fatalf("tag not as expected: %q != %q", tagsResponse.Tags[0], tag)
}
}
// TestSimpleBlobUpload covers the blob upload process, exercising common
// error paths that might be seen during an upload.
func TestSimpleBlobUpload(t *testing.T) {
randomDataReader, dgst, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random reader: %v", err)
}
ctx := context.Background()
imageName := "foo/bar"
driver := inmemory.New()
registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect)
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
repository, err := registry.Repository(ctx, imageName)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
bs := repository.Blobs(ctx)
h := sha256.New()
rd := io.TeeReader(randomDataReader, h)
blobUpload, err := bs.Create(ctx)
if err != nil {
t.Fatalf("unexpected error starting layer upload: %s", err)
}
// Cancel the upload then restart it
if err := blobUpload.Cancel(ctx); err != nil {
t.Fatalf("unexpected error during upload cancellation: %v", err)
}
// Do a resume, get unknown upload
blobUpload, err = bs.Resume(ctx, blobUpload.ID())
if err != distribution.ErrBlobUploadUnknown {
t.Fatalf("unexpected error resuming upload, should be unknown: %v", err)
}
// Restart!
blobUpload, err = bs.Create(ctx)
if err != nil {
t.Fatalf("unexpected error starting layer upload: %s", err)
}
// Get the size of our random tarfile
randomDataSize, err := seekerSize(randomDataReader)
if err != nil {
t.Fatalf("error getting seeker size of random data: %v", err)
}
nn, err := io.Copy(blobUpload, rd)
if err != nil {
t.Fatalf("unexpected error uploading layer data: %v", err)
}
if nn != randomDataSize {
t.Fatalf("layer data write incomplete")
}
offset, err := blobUpload.Seek(0, os.SEEK_CUR)
if err != nil {
t.Fatalf("unexpected error seeking layer upload: %v", err)
}
if offset != nn {
t.Fatalf("blobUpload not updated with correct offset: %v != %v", offset, nn)
}
blobUpload.Close()
// Do a resume, for good fun
blobUpload, err = bs.Resume(ctx, blobUpload.ID())
if err != nil {
t.Fatalf("unexpected error resuming upload: %v", err)
}
sha256Digest := digest.NewDigest("sha256", h)
desc, err := blobUpload.Commit(ctx, distribution.Descriptor{Digest: dgst})
if err != nil {
t.Fatalf("unexpected error finishing layer upload: %v", err)
}
// After finishing an upload, it should no longer exist.
if _, err := bs.Resume(ctx, blobUpload.ID()); err != distribution.ErrBlobUploadUnknown {
t.Fatalf("expected layer upload to be unknown, got %v", err)
}
// Test for existence.
statDesc, err := bs.Stat(ctx, desc.Digest)
if err != nil {
t.Fatalf("unexpected error checking for existence: %v, %#v", err, bs)
}
if statDesc != desc {
t.Fatalf("descriptors not equal: %v != %v", statDesc, desc)
}
rc, err := bs.Open(ctx, desc.Digest)
if err != nil {
t.Fatalf("unexpected error opening blob for read: %v", err)
}
defer rc.Close()
h.Reset()
nn, err = io.Copy(h, rc)
if err != nil {
t.Fatalf("error reading layer: %v", err)
}
if nn != randomDataSize {
t.Fatalf("incorrect read length")
}
if digest.NewDigest("sha256", h) != sha256Digest {
t.Fatalf("unexpected digest from uploaded layer: %q != %q", digest.NewDigest("sha256", h), sha256Digest)
}
// Delete a blob
err = bs.Delete(ctx, desc.Digest)
if err != nil {
t.Fatalf("Unexpected error deleting blob")
}
d, err := bs.Stat(ctx, desc.Digest)
if err == nil {
t.Fatalf("unexpected non-error stating deleted blob: %v", d)
}
switch err {
case distribution.ErrBlobUnknown:
break
default:
t.Errorf("Unexpected error type stat-ing deleted manifest: %#v", err)
}
_, err = bs.Open(ctx, desc.Digest)
if err == nil {
t.Fatalf("unexpected success opening deleted blob for read")
}
switch err {
case distribution.ErrBlobUnknown:
break
default:
t.Errorf("Unexpected error type getting deleted manifest: %#v", err)
}
// Re-upload the blob
randomBlob, err := ioutil.ReadAll(randomDataReader)
if err != nil {
t.Fatalf("Error reading all of blob %s", err.Error())
}
expectedDigest := digest.FromBytes(randomBlob)
simpleUpload(t, bs, randomBlob, expectedDigest)
d, err = bs.Stat(ctx, expectedDigest)
if err != nil {
t.Errorf("unexpected error stat-ing blob")
}
if d.Digest != expectedDigest {
t.Errorf("Mismatching digest with restored blob")
}
_, err = bs.Open(ctx, expectedDigest)
if err != nil {
t.Errorf("Unexpected error opening blob")
}
// Reuse state to test delete with a delete-disabled registry
registry, err = NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableRedirect)
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
repository, err = registry.Repository(ctx, imageName)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
bs = repository.Blobs(ctx)
err = bs.Delete(ctx, desc.Digest)
if err == nil {
t.Errorf("Unexpected success deleting while disabled")
}
}
// TestSimpleBlobRead just creates a simple blob file and ensures that basic
// open, read, seek, read works. More specific edge cases should be covered in
// other tests.
func TestSimpleBlobRead(t *testing.T) {
ctx := context.Background()
imageName := "foo/bar"
driver := inmemory.New()
registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect)
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
repository, err := registry.Repository(ctx, imageName)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
bs := repository.Blobs(ctx)
randomLayerReader, dgst, err := testutil.CreateRandomTarFile() // TODO(stevvooe): Consider using just a random string.
if err != nil {
t.Fatalf("error creating random data: %v", err)
}
// Test for existence.
desc, err := bs.Stat(ctx, dgst)
if err != distribution.ErrBlobUnknown {
t.Fatalf("expected not found error when testing for existence: %v", err)
}
rc, err := bs.Open(ctx, dgst)
if err != distribution.ErrBlobUnknown {
t.Fatalf("expected not found error when opening non-existent blob: %v", err)
}
randomLayerSize, err := seekerSize(randomLayerReader)
if err != nil {
t.Fatalf("error getting seeker size for random layer: %v", err)
}
descBefore := distribution.Descriptor{Digest: dgst, MediaType: "application/octet-stream", Size: randomLayerSize}
t.Logf("desc: %v", descBefore)
desc, err = addBlob(ctx, bs, descBefore, randomLayerReader)
if err != nil {
t.Fatalf("error adding blob to blobservice: %v", err)
}
if desc.Size != randomLayerSize {
t.Fatalf("committed blob has incorrect length: %v != %v", desc.Size, randomLayerSize)
}
rc, err = bs.Open(ctx, desc.Digest) // note that we are opening with original digest.
if err != nil {
t.Fatalf("error opening blob with %v: %v", dgst, err)
}
defer rc.Close()
// Now check the sha digest and ensure its the same
h := sha256.New()
nn, err := io.Copy(h, rc)
if err != nil {
t.Fatalf("unexpected error copying to hash: %v", err)
}
if nn != randomLayerSize {
t.Fatalf("stored incorrect number of bytes in blob: %d != %d", nn, randomLayerSize)
}
sha256Digest := digest.NewDigest("sha256", h)
if sha256Digest != desc.Digest {
t.Fatalf("fetched digest does not match: %q != %q", sha256Digest, desc.Digest)
}
// Now seek back the blob, read the whole thing and check against randomLayerData
offset, err := rc.Seek(0, os.SEEK_SET)
if err != nil {
t.Fatalf("error seeking blob: %v", err)
}
if offset != 0 {
t.Fatalf("seek failed: expected 0 offset, got %d", offset)
}
p, err := ioutil.ReadAll(rc)
if err != nil {
t.Fatalf("error reading all of blob: %v", err)
}
if len(p) != int(randomLayerSize) {
t.Fatalf("blob data read has different length: %v != %v", len(p), randomLayerSize)
}
// Reset the randomLayerReader and read back the buffer
_, err = randomLayerReader.Seek(0, os.SEEK_SET)
if err != nil {
t.Fatalf("error resetting layer reader: %v", err)
}
randomLayerData, err := ioutil.ReadAll(randomLayerReader)
if err != nil {
t.Fatalf("random layer read failed: %v", err)
}
if !bytes.Equal(p, randomLayerData) {
t.Fatalf("layer data not equal")
}
}
// TestBlobMount covers the blob mount process, exercising common
// error paths that might be seen during a mount.
func TestBlobMount(t *testing.T) {
randomDataReader, dgst, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random reader: %v", err)
}
ctx := context.Background()
imageName, _ := reference.ParseNamed("foo/bar")
sourceImageName, _ := reference.ParseNamed("foo/source")
driver := inmemory.New()
registry, err := NewRegistry(ctx, driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect)
if err != nil {
t.Fatalf("error creating registry: %v", err)
}
repository, err := registry.Repository(ctx, imageName)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
sourceRepository, err := registry.Repository(ctx, sourceImageName)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
sbs := sourceRepository.Blobs(ctx)
blobUpload, err := sbs.Create(ctx)
if err != nil {
t.Fatalf("unexpected error starting layer upload: %s", err)
}
// Get the size of our random tarfile
randomDataSize, err := seekerSize(randomDataReader)
if err != nil {
t.Fatalf("error getting seeker size of random data: %v", err)
}
nn, err := io.Copy(blobUpload, randomDataReader)
if err != nil {
t.Fatalf("unexpected error uploading layer data: %v", err)
}
desc, err := blobUpload.Commit(ctx, distribution.Descriptor{Digest: dgst})
if err != nil {
t.Fatalf("unexpected error finishing layer upload: %v", err)
}
// Test for existence.
statDesc, err := sbs.Stat(ctx, desc.Digest)
if err != nil {
t.Fatalf("unexpected error checking for existence: %v, %#v", err, sbs)
}
if statDesc != desc {
t.Fatalf("descriptors not equal: %v != %v", statDesc, desc)
}
bs := repository.Blobs(ctx)
// Test destination for existence.
statDesc, err = bs.Stat(ctx, desc.Digest)
if err == nil {
t.Fatalf("unexpected non-error stating unmounted blob: %v", desc)
}
canonicalRef, err := reference.WithDigest(sourceRepository.Name(), desc.Digest)
if err != nil {
t.Fatal(err)
}
bw, err := bs.Create(ctx, WithMountFrom(canonicalRef))
if bw != nil {
t.Fatal("unexpected blobwriter returned from Create call, should mount instead")
}
ebm, ok := err.(distribution.ErrBlobMounted)
if !ok {
t.Fatalf("unexpected error mounting layer: %v", err)
}
if ebm.Descriptor != desc {
t.Fatalf("descriptors not equal: %v != %v", ebm.Descriptor, desc)
}
// Test for existence.
statDesc, err = bs.Stat(ctx, desc.Digest)
if err != nil {
t.Fatalf("unexpected error checking for existence: %v, %#v", err, bs)
}
if statDesc != desc {
t.Fatalf("descriptors not equal: %v != %v", statDesc, desc)
}
rc, err := bs.Open(ctx, desc.Digest)
if err != nil {
t.Fatalf("unexpected error opening blob for read: %v", err)
}
defer rc.Close()
h := sha256.New()
nn, err = io.Copy(h, rc)
if err != nil {
t.Fatalf("error reading layer: %v", err)
}
if nn != randomDataSize {
t.Fatalf("incorrect read length")
}
if digest.NewDigest("sha256", h) != dgst {
t.Fatalf("unexpected digest from uploaded layer: %q != %q", digest.NewDigest("sha256", h), dgst)
}
// Delete the blob from the source repo
err = sbs.Delete(ctx, desc.Digest)
if err != nil {
t.Fatalf("Unexpected error deleting blob")
}
d, err := bs.Stat(ctx, desc.Digest)
if err != nil {
t.Fatalf("unexpected error stating blob deleted from source repository: %v", err)
}
d, err = sbs.Stat(ctx, desc.Digest)
if err == nil {
t.Fatalf("unexpected non-error stating deleted blob: %v", d)
}
switch err {
case distribution.ErrBlobUnknown:
break
default:
t.Errorf("Unexpected error type stat-ing deleted manifest: %#v", err)
}
// Delete the blob from the dest repo
err = bs.Delete(ctx, desc.Digest)
if err != nil {
t.Fatalf("Unexpected error deleting blob")
}
d, err = bs.Stat(ctx, desc.Digest)
if err == nil {
t.Fatalf("unexpected non-error stating deleted blob: %v", d)
}
switch err {
case distribution.ErrBlobUnknown:
break
default:
t.Errorf("Unexpected error type stat-ing deleted manifest: %#v", err)
}
}
func TestManifestStorage(t *testing.T) {
env := newManifestStoreTestEnv(t, "foo/bar", "thetag")
ctx := context.Background()
ms, err := env.repository.Manifests(ctx)
if err != nil {
t.Fatal(err)
}
exists, err := ms.ExistsByTag(env.tag)
if err != nil {
t.Fatalf("unexpected error checking manifest existence: %v", err)
}
if exists {
t.Fatalf("manifest should not exist")
}
if _, err := ms.GetByTag(env.tag); true {
switch err.(type) {
case distribution.ErrManifestUnknown:
break
default:
t.Fatalf("expected manifest unknown error: %#v", err)
}
}
m := manifest.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
},
Name: env.name,
Tag: env.tag,
}
// Build up some test layers and add them to the manifest, saving the
// readseekers for upload later.
testLayers := map[digest.Digest]io.ReadSeeker{}
for i := 0; i < 2; i++ {
rs, ds, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("unexpected error generating test layer file")
}
dgst := digest.Digest(ds)
testLayers[digest.Digest(dgst)] = rs
m.FSLayers = append(m.FSLayers, manifest.FSLayer{
BlobSum: dgst,
})
}
pk, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
t.Fatalf("unexpected error generating private key: %v", err)
}
sm, merr := manifest.Sign(&m, pk)
if merr != nil {
t.Fatalf("error signing manifest: %v", err)
}
err = ms.Put(sm)
if err == nil {
t.Fatalf("expected errors putting manifest with full verification")
}
switch err := err.(type) {
case distribution.ErrManifestVerification:
if len(err) != 2 {
t.Fatalf("expected 2 verification errors: %#v", err)
}
for _, err := range err {
if _, ok := err.(distribution.ErrManifestBlobUnknown); !ok {
t.Fatalf("unexpected error type: %v", err)
}
}
default:
t.Fatalf("unexpected error verifying manifest: %v", err)
}
// Now, upload the layers that were missing!
for dgst, rs := range testLayers {
wr, err := env.repository.Blobs(env.ctx).Create(env.ctx)
if err != nil {
t.Fatalf("unexpected error creating test upload: %v", err)
}
if _, err := io.Copy(wr, rs); err != nil {
t.Fatalf("unexpected error copying to upload: %v", err)
}
if _, err := wr.Commit(env.ctx, distribution.Descriptor{Digest: dgst}); err != nil {
t.Fatalf("unexpected error finishing upload: %v", err)
}
}
if err = ms.Put(sm); err != nil {
t.Fatalf("unexpected error putting manifest: %v", err)
}
exists, err = ms.ExistsByTag(env.tag)
if err != nil {
t.Fatalf("unexpected error checking manifest existence: %v", err)
}
if !exists {
t.Fatalf("manifest should exist")
}
fetchedManifest, err := ms.GetByTag(env.tag)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
if !reflect.DeepEqual(fetchedManifest, sm) {
t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedManifest, sm)
}
fetchedJWS, err := libtrust.ParsePrettySignature(fetchedManifest.Raw, "signatures")
if err != nil {
t.Fatalf("unexpected error parsing jws: %v", err)
}
payload, err := fetchedJWS.Payload()
if err != nil {
t.Fatalf("unexpected error extracting payload: %v", err)
}
// Now that we have a payload, take a moment to check that the manifest is
// return by the payload digest.
dgst, err := digest.FromBytes(payload)
if err != nil {
t.Fatalf("error getting manifest digest: %v", err)
}
exists, err = ms.Exists(dgst)
if err != nil {
t.Fatalf("error checking manifest existence by digest: %v", err)
}
if !exists {
t.Fatalf("manifest %s should exist", dgst)
}
fetchedByDigest, err := ms.Get(dgst)
if err != nil {
t.Fatalf("unexpected error fetching manifest by digest: %v", err)
}
if !reflect.DeepEqual(fetchedByDigest, fetchedManifest) {
t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedByDigest, fetchedManifest)
}
sigs, err := fetchedJWS.Signatures()
if err != nil {
t.Fatalf("unable to extract signatures: %v", err)
}
if len(sigs) != 1 {
t.Fatalf("unexpected number of signatures: %d != %d", len(sigs), 1)
}
// Grabs the tags and check that this tagged manifest is present
tags, err := ms.Tags()
if err != nil {
t.Fatalf("unexpected error fetching tags: %v", err)
}
if len(tags) != 1 {
t.Fatalf("unexpected tags returned: %v", tags)
}
if tags[0] != env.tag {
t.Fatalf("unexpected tag found in tags: %v != %v", tags, []string{env.tag})
}
// Now, push the same manifest with a different key
pk2, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
t.Fatalf("unexpected error generating private key: %v", err)
}
sm2, err := manifest.Sign(&m, pk2)
if err != nil {
t.Fatalf("unexpected error signing manifest: %v", err)
}
jws2, err := libtrust.ParsePrettySignature(sm2.Raw, "signatures")
if err != nil {
t.Fatalf("error parsing signature: %v", err)
}
sigs2, err := jws2.Signatures()
if err != nil {
t.Fatalf("unable to extract signatures: %v", err)
}
if len(sigs2) != 1 {
t.Fatalf("unexpected number of signatures: %d != %d", len(sigs2), 1)
}
if err = ms.Put(sm2); err != nil {
t.Fatalf("unexpected error putting manifest: %v", err)
}
fetched, err := ms.GetByTag(env.tag)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
if _, err := manifest.Verify(fetched); err != nil {
t.Fatalf("unexpected error verifying manifest: %v", err)
}
// Assemble our payload and two signatures to get what we expect!
expectedJWS, err := libtrust.NewJSONSignature(payload, sigs[0], sigs2[0])
if err != nil {
t.Fatalf("unexpected error merging jws: %v", err)
}
expectedSigs, err := expectedJWS.Signatures()
if err != nil {
t.Fatalf("unexpected error getting expected signatures: %v", err)
}
receivedJWS, err := libtrust.ParsePrettySignature(fetched.Raw, "signatures")
if err != nil {
t.Fatalf("unexpected error parsing jws: %v", err)
}
receivedPayload, err := receivedJWS.Payload()
if err != nil {
t.Fatalf("unexpected error extracting received payload: %v", err)
}
if !bytes.Equal(receivedPayload, payload) {
t.Fatalf("payloads are not equal")
}
receivedSigs, err := receivedJWS.Signatures()
if err != nil {
t.Fatalf("error getting signatures: %v", err)
}
for i, sig := range receivedSigs {
if !bytes.Equal(sig, expectedSigs[i]) {
t.Fatalf("mismatched signatures from remote: %v != %v", string(sig), string(expectedSigs[i]))
}
}
// Test deleting manifests
err = ms.Delete(dgst)
if err != nil {
t.Fatalf("unexpected an error deleting manifest by digest: %v", err)
}
exists, err = ms.Exists(dgst)
if err != nil {
t.Fatalf("Error querying manifest existence")
}
if exists {
t.Errorf("Deleted manifest should not exist")
}
deletedManifest, err := ms.Get(dgst)
if err == nil {
t.Errorf("Unexpected success getting deleted manifest")
}
switch err.(type) {
case distribution.ErrManifestUnknownRevision:
break
default:
t.Errorf("Unexpected error getting deleted manifest: %s", reflect.ValueOf(err).Type())
}
if deletedManifest != nil {
t.Errorf("Deleted manifest get returned non-nil")
}
// Re-upload should restore manifest to a good state
err = ms.Put(sm)
if err != nil {
t.Errorf("Error re-uploading deleted manifest")
}
exists, err = ms.Exists(dgst)
if err != nil {
t.Fatalf("Error querying manifest existence")
}
if !exists {
t.Errorf("Restored manifest should exist")
}
deletedManifest, err = ms.Get(dgst)
if err != nil {
t.Errorf("Unexpected error getting manifest")
}
if deletedManifest == nil {
t.Errorf("Deleted manifest get returned non-nil")
}
r := NewRegistryWithDriver(ctx, env.driver, memory.NewInMemoryBlobDescriptorCacheProvider(), false)
repo, err := r.Repository(ctx, env.name)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
ms, err = repo.Manifests(ctx)
if err != nil {
t.Fatal(err)
}
err = ms.Delete(dgst)
if err == nil {
t.Errorf("Unexpected success deleting while disabled")
}
}
func TestManifestStorage(t *testing.T) {
env := newManifestStoreTestEnv(t, "foo/bar", "thetag")
ms := env.repository.Manifests()
exists, err := ms.ExistsByTag(env.ctx, env.tag)
if err != nil {
t.Fatalf("unexpected error checking manifest existence: %v", err)
}
if exists {
t.Fatalf("manifest should not exist")
}
if _, err := ms.GetByTag(env.ctx, env.tag); true {
switch err.(type) {
case distribution.ErrManifestUnknown:
break
default:
t.Fatalf("expected manifest unknown error: %#v", err)
}
}
m := manifest.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
},
Name: env.name,
Tag: env.tag,
}
// Build up some test layers and add them to the manifest, saving the
// readseekers for upload later.
testLayers := map[digest.Digest]io.ReadSeeker{}
for i := 0; i < 2; i++ {
rs, ds, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("unexpected error generating test layer file")
}
dgst := digest.Digest(ds)
testLayers[digest.Digest(dgst)] = rs
m.FSLayers = append(m.FSLayers, manifest.FSLayer{
BlobSum: dgst,
})
}
pk, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
t.Fatalf("unexpected error generating private key: %v", err)
}
sm, err := manifest.Sign(&m, pk)
if err != nil {
t.Fatalf("error signing manifest: %v", err)
}
err = ms.Put(env.ctx, sm)
if err == nil {
t.Fatalf("expected errors putting manifest")
}
// TODO(stevvooe): We expect errors describing all of the missing layers.
// Now, upload the layers that were missing!
for dgst, rs := range testLayers {
upload, err := env.repository.Layers().Upload()
if err != nil {
t.Fatalf("unexpected error creating test upload: %v", err)
}
if _, err := io.Copy(upload, rs); err != nil {
t.Fatalf("unexpected error copying to upload: %v", err)
}
if _, err := upload.Finish(dgst); err != nil {
t.Fatalf("unexpected error finishing upload: %v", err)
}
}
if err = ms.Put(env.ctx, sm); err != nil {
t.Fatalf("unexpected error putting manifest: %v", err)
}
exists, err = ms.ExistsByTag(env.ctx, env.tag)
if err != nil {
t.Fatalf("unexpected error checking manifest existence: %v", err)
}
if !exists {
t.Fatalf("manifest should exist")
}
fetchedManifest, err := ms.GetByTag(env.ctx, env.tag)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
if !reflect.DeepEqual(fetchedManifest, sm) {
t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedManifest, sm)
}
fetchedJWS, err := libtrust.ParsePrettySignature(fetchedManifest.Raw, "signatures")
if err != nil {
t.Fatalf("unexpected error parsing jws: %v", err)
}
payload, err := fetchedJWS.Payload()
if err != nil {
t.Fatalf("unexpected error extracting payload: %v", err)
}
// Now that we have a payload, take a moment to check that the manifest is
// return by the payload digest.
dgst, err := digest.FromBytes(payload)
if err != nil {
t.Fatalf("error getting manifest digest: %v", err)
}
exists, err = ms.Exists(env.ctx, dgst)
if err != nil {
t.Fatalf("error checking manifest existence by digest: %v", err)
}
if !exists {
t.Fatalf("manifest %s should exist", dgst)
}
fetchedByDigest, err := ms.Get(env.ctx, dgst)
if err != nil {
t.Fatalf("unexpected error fetching manifest by digest: %v", err)
}
if !reflect.DeepEqual(fetchedByDigest, fetchedManifest) {
t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedByDigest, fetchedManifest)
}
sigs, err := fetchedJWS.Signatures()
if err != nil {
t.Fatalf("unable to extract signatures: %v", err)
}
if len(sigs) != 1 {
t.Fatalf("unexpected number of signatures: %d != %d", len(sigs), 1)
}
// Grabs the tags and check that this tagged manifest is present
tags, err := ms.Tags(env.ctx)
if err != nil {
t.Fatalf("unexpected error fetching tags: %v", err)
}
if len(tags) != 1 {
t.Fatalf("unexpected tags returned: %v", tags)
}
if tags[0] != env.tag {
t.Fatalf("unexpected tag found in tags: %v != %v", tags, []string{env.tag})
}
// Now, push the same manifest with a different key
pk2, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
t.Fatalf("unexpected error generating private key: %v", err)
}
sm2, err := manifest.Sign(&m, pk2)
if err != nil {
t.Fatalf("unexpected error signing manifest: %v", err)
}
jws2, err := libtrust.ParsePrettySignature(sm2.Raw, "signatures")
if err != nil {
t.Fatalf("error parsing signature: %v", err)
}
sigs2, err := jws2.Signatures()
if err != nil {
t.Fatalf("unable to extract signatures: %v", err)
}
if len(sigs2) != 1 {
t.Fatalf("unexpected number of signatures: %d != %d", len(sigs2), 1)
}
if err = ms.Put(env.ctx, sm2); err != nil {
t.Fatalf("unexpected error putting manifest: %v", err)
}
fetched, err := ms.GetByTag(env.ctx, env.tag)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
if _, err := manifest.Verify(fetched); err != nil {
t.Fatalf("unexpected error verifying manifest: %v", err)
}
// Assemble our payload and two signatures to get what we expect!
expectedJWS, err := libtrust.NewJSONSignature(payload, sigs[0], sigs2[0])
if err != nil {
t.Fatalf("unexpected error merging jws: %v", err)
}
expectedSigs, err := expectedJWS.Signatures()
if err != nil {
t.Fatalf("unexpected error getting expected signatures: %v", err)
}
receivedJWS, err := libtrust.ParsePrettySignature(fetched.Raw, "signatures")
if err != nil {
t.Fatalf("unexpected error parsing jws: %v", err)
}
receivedPayload, err := receivedJWS.Payload()
if err != nil {
t.Fatalf("unexpected error extracting received payload: %v", err)
}
if !bytes.Equal(receivedPayload, payload) {
t.Fatalf("payloads are not equal")
}
receivedSigs, err := receivedJWS.Signatures()
if err != nil {
t.Fatalf("error getting signatures: %v", err)
}
for i, sig := range receivedSigs {
if !bytes.Equal(sig, expectedSigs[i]) {
t.Fatalf("mismatched signatures from remote: %v != %v", string(sig), string(expectedSigs[i]))
}
}
// TODO(stevvooe): Currently, deletes are not supported due to some
// complexity around managing tag indexes. We'll add this support back in
// when the manifest format has settled. For now, we expect an error for
// all deletes.
if err := ms.Delete(env.ctx, dgst); err == nil {
t.Fatalf("unexpected an error deleting manifest by digest: %v", err)
}
}
func TestLinkedBlobStoreCreateWithMountFrom(t *testing.T) {
fooRepoName, _ := reference.ParseNamed("nm/foo")
fooEnv := newManifestStoreTestEnv(t, fooRepoName, "thetag")
ctx := context.Background()
stats, err := mockRegistry(t, fooEnv.registry)
if err != nil {
t.Fatal(err)
}
// Build up some test layers and add them to the manifest, saving the
// readseekers for upload later.
testLayers := map[digest.Digest]io.ReadSeeker{}
for i := 0; i < 2; i++ {
rs, ds, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("unexpected error generating test layer file")
}
dgst := digest.Digest(ds)
testLayers[digest.Digest(dgst)] = rs
}
// upload the layers to foo/bar
for dgst, rs := range testLayers {
wr, err := fooEnv.repository.Blobs(fooEnv.ctx).Create(fooEnv.ctx)
if err != nil {
t.Fatalf("unexpected error creating test upload: %v", err)
}
if _, err := io.Copy(wr, rs); err != nil {
t.Fatalf("unexpected error copying to upload: %v", err)
}
if _, err := wr.Commit(fooEnv.ctx, distribution.Descriptor{Digest: dgst}); err != nil {
t.Fatalf("unexpected error finishing upload: %v", err)
}
}
// create another repository nm/bar
barRepoName, _ := reference.ParseNamed("nm/bar")
barRepo, err := fooEnv.registry.Repository(ctx, barRepoName)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
// cross-repo mount the test layers into a nm/bar
for dgst := range testLayers {
fooCanonical, _ := reference.WithDigest(fooRepoName, dgst)
option := WithMountFrom(fooCanonical)
// ensure we can instrospect it
createOpts := distribution.CreateOptions{}
if err := option.Apply(&createOpts); err != nil {
t.Fatalf("failed to apply MountFrom option: %v", err)
}
if !createOpts.Mount.ShouldMount || createOpts.Mount.From.String() != fooCanonical.String() {
t.Fatalf("unexpected create options: %#+v", createOpts.Mount)
}
_, err := barRepo.Blobs(ctx).Create(ctx, WithMountFrom(fooCanonical))
if err == nil {
t.Fatalf("unexpected non-error while mounting from %q: %v", fooRepoName.String(), err)
}
if _, ok := err.(distribution.ErrBlobMounted); !ok {
t.Fatalf("expected ErrMountFrom error, not %T: %v", err, err)
}
}
for dgst := range testLayers {
fooCanonical, _ := reference.WithDigest(fooRepoName, dgst)
count, exists := stats[fooCanonical.String()]
if !exists {
t.Errorf("expected entry %q not found among handled stat calls", fooCanonical.String())
} else if count != 1 {
t.Errorf("expected exactly one stat call for entry %q, not %d", fooCanonical.String(), count)
}
}
clearStats(stats)
// create yet another repository nm/baz
bazRepoName, _ := reference.ParseNamed("nm/baz")
bazRepo, err := fooEnv.registry.Repository(ctx, bazRepoName)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
// cross-repo mount them into a nm/baz and provide a prepopulated blob descriptor
for dgst := range testLayers {
fooCanonical, _ := reference.WithDigest(fooRepoName, dgst)
size, err := strconv.ParseInt("0x"+dgst.Hex()[:8], 0, 64)
if err != nil {
t.Fatal(err)
}
prepolutatedDescriptor := distribution.Descriptor{
Digest: dgst,
Size: size,
MediaType: "application/octet-stream",
}
_, err = bazRepo.Blobs(ctx).Create(ctx, WithMountFrom(fooCanonical), &statCrossMountCreateOption{
desc: prepolutatedDescriptor,
})
blobMounted, ok := err.(distribution.ErrBlobMounted)
if !ok {
t.Errorf("expected ErrMountFrom error, not %T: %v", err, err)
continue
}
if !reflect.DeepEqual(blobMounted.Descriptor, prepolutatedDescriptor) {
t.Errorf("unexpected descriptor: %#+v != %#+v", blobMounted.Descriptor, prepolutatedDescriptor)
}
}
// this time no stat calls will be made
if len(stats) != 0 {
t.Errorf("unexpected number of stats made: %d != %d", len(stats), len(testLayers))
}
}
// checkExerciseRegistry takes the registry through all of its operations,
// carrying out generic checks.
func checkExerciseRepository(t *testing.T, repository distribution.Repository) {
// TODO(stevvooe): This would be a nice testutil function. Basically, it
// takes the registry through a common set of operations. This could be
// used to make cross-cutting updates by changing internals that affect
// update counts. Basically, it would make writing tests a lot easier.
ctx := context.Background()
tag := "thetag"
// todo: change this to use Builder
m := schema1.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
},
Name: repository.Named().Name(),
Tag: tag,
}
var blobDigests []digest.Digest
blobs := repository.Blobs(ctx)
for i := 0; i < 2; i++ {
rs, ds, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating test layer: %v", err)
}
dgst := digest.Digest(ds)
blobDigests = append(blobDigests, dgst)
wr, err := blobs.Create(ctx)
if err != nil {
t.Fatalf("error creating layer upload: %v", err)
}
// Use the resumes, as well!
wr, err = blobs.Resume(ctx, wr.ID())
if err != nil {
t.Fatalf("error resuming layer upload: %v", err)
}
io.Copy(wr, rs)
if _, err := wr.Commit(ctx, distribution.Descriptor{Digest: dgst}); err != nil {
t.Fatalf("unexpected error finishing upload: %v", err)
}
m.FSLayers = append(m.FSLayers, schema1.FSLayer{
BlobSum: dgst,
})
m.History = append(m.History, schema1.History{
V1Compatibility: "",
})
// Then fetch the blobs
if rc, err := blobs.Open(ctx, dgst); err != nil {
t.Fatalf("error fetching layer: %v", err)
} else {
defer rc.Close()
}
}
pk, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
t.Fatalf("unexpected error generating key: %v", err)
}
sm, err := schema1.Sign(&m, pk)
if err != nil {
t.Fatalf("unexpected error signing manifest: %v", err)
}
manifests, err := repository.Manifests(ctx)
if err != nil {
t.Fatal(err.Error())
}
var digestPut digest.Digest
if digestPut, err = manifests.Put(ctx, sm); err != nil {
t.Fatalf("unexpected error putting the manifest: %v", err)
}
dgst := digest.FromBytes(sm.Canonical)
if dgst != digestPut {
t.Fatalf("mismatching digest from payload and put")
}
_, err = manifests.Get(ctx, dgst)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
err = manifests.Delete(ctx, dgst)
if err != nil {
t.Fatalf("unexpected error deleting blob: %v", err)
}
for _, d := range blobDigests {
err = blobs.Delete(ctx, d)
if err != nil {
t.Fatalf("unexpected error deleting blob: %v", err)
}
}
}
// TestSimpleLayerRead just creates a simple layer file and ensures that basic
// open, read, seek, read works. More specific edge cases should be covered in
// other tests.
func TestSimpleLayerRead(t *testing.T) {
ctx := context.Background()
imageName := "foo/bar"
driver := inmemory.New()
registry := NewRegistryWithDriver(ctx, driver, cache.NewInMemoryLayerInfoCache())
repository, err := registry.Repository(ctx, imageName)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
ls := repository.Layers()
randomLayerReader, tarSumStr, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random data: %v", err)
}
dgst := digest.Digest(tarSumStr)
// Test for existence.
exists, err := ls.Exists(dgst)
if err != nil {
t.Fatalf("unexpected error checking for existence: %v", err)
}
if exists {
t.Fatalf("layer should not exist")
}
// Try to get the layer and make sure we get a not found error
layer, err := ls.Fetch(dgst)
if err == nil {
t.Fatalf("error expected fetching unknown layer")
}
switch err.(type) {
case distribution.ErrUnknownLayer:
err = nil
default:
t.Fatalf("unexpected error fetching non-existent layer: %v", err)
}
randomLayerDigest, err := writeTestLayer(driver, defaultPathMapper, imageName, dgst, randomLayerReader)
if err != nil {
t.Fatalf("unexpected error writing test layer: %v", err)
}
randomLayerSize, err := seekerSize(randomLayerReader)
if err != nil {
t.Fatalf("error getting seeker size for random layer: %v", err)
}
layer, err = ls.Fetch(dgst)
if err != nil {
t.Fatal(err)
}
defer layer.Close()
// Now check the sha digest and ensure its the same
h := sha256.New()
nn, err := io.Copy(h, layer)
if err != nil && err != io.EOF {
t.Fatalf("unexpected error copying to hash: %v", err)
}
if nn != randomLayerSize {
t.Fatalf("stored incorrect number of bytes in layer: %d != %d", nn, randomLayerSize)
}
sha256Digest := digest.NewDigest("sha256", h)
if sha256Digest != randomLayerDigest {
t.Fatalf("fetched digest does not match: %q != %q", sha256Digest, randomLayerDigest)
}
// Now seek back the layer, read the whole thing and check against randomLayerData
offset, err := layer.Seek(0, os.SEEK_SET)
if err != nil {
t.Fatalf("error seeking layer: %v", err)
}
if offset != 0 {
t.Fatalf("seek failed: expected 0 offset, got %d", offset)
}
p, err := ioutil.ReadAll(layer)
if err != nil {
t.Fatalf("error reading all of layer: %v", err)
}
if len(p) != int(randomLayerSize) {
t.Fatalf("layer data read has different length: %v != %v", len(p), randomLayerSize)
}
// Reset the randomLayerReader and read back the buffer
_, err = randomLayerReader.Seek(0, os.SEEK_SET)
if err != nil {
t.Fatalf("error resetting layer reader: %v", err)
}
randomLayerData, err := ioutil.ReadAll(randomLayerReader)
if err != nil {
t.Fatalf("random layer read failed: %v", err)
}
if !bytes.Equal(p, randomLayerData) {
t.Fatalf("layer data not equal")
}
}
// TestSimpleLayerUpload covers the layer upload process, exercising common
// error paths that might be seen during an upload.
func TestSimpleLayerUpload(t *testing.T) {
randomDataReader, tarSumStr, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random reader: %v", err)
}
dgst := digest.Digest(tarSumStr)
if err != nil {
t.Fatalf("error allocating upload store: %v", err)
}
ctx := context.Background()
imageName := "foo/bar"
driver := inmemory.New()
registry := NewRegistryWithDriver(ctx, driver, cache.NewInMemoryLayerInfoCache())
repository, err := registry.Repository(ctx, imageName)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
ls := repository.Layers()
h := sha256.New()
rd := io.TeeReader(randomDataReader, h)
layerUpload, err := ls.Upload()
if err != nil {
t.Fatalf("unexpected error starting layer upload: %s", err)
}
// Cancel the upload then restart it
if err := layerUpload.Cancel(); err != nil {
t.Fatalf("unexpected error during upload cancellation: %v", err)
}
// Do a resume, get unknown upload
layerUpload, err = ls.Resume(layerUpload.UUID())
if err != distribution.ErrLayerUploadUnknown {
t.Fatalf("unexpected error resuming upload, should be unkown: %v", err)
}
// Restart!
layerUpload, err = ls.Upload()
if err != nil {
t.Fatalf("unexpected error starting layer upload: %s", err)
}
// Get the size of our random tarfile
randomDataSize, err := seekerSize(randomDataReader)
if err != nil {
t.Fatalf("error getting seeker size of random data: %v", err)
}
nn, err := io.Copy(layerUpload, rd)
if err != nil {
t.Fatalf("unexpected error uploading layer data: %v", err)
}
if nn != randomDataSize {
t.Fatalf("layer data write incomplete")
}
offset, err := layerUpload.Seek(0, os.SEEK_CUR)
if err != nil {
t.Fatalf("unexpected error seeking layer upload: %v", err)
}
if offset != nn {
t.Fatalf("layerUpload not updated with correct offset: %v != %v", offset, nn)
}
layerUpload.Close()
// Do a resume, for good fun
layerUpload, err = ls.Resume(layerUpload.UUID())
if err != nil {
t.Fatalf("unexpected error resuming upload: %v", err)
}
sha256Digest := digest.NewDigest("sha256", h)
layer, err := layerUpload.Finish(dgst)
if err != nil {
t.Fatalf("unexpected error finishing layer upload: %v", err)
}
// After finishing an upload, it should no longer exist.
if _, err := ls.Resume(layerUpload.UUID()); err != distribution.ErrLayerUploadUnknown {
t.Fatalf("expected layer upload to be unknown, got %v", err)
}
// Test for existence.
exists, err := ls.Exists(layer.Digest())
if err != nil {
t.Fatalf("unexpected error checking for existence: %v", err)
}
if !exists {
t.Fatalf("layer should now exist")
}
h.Reset()
nn, err = io.Copy(h, layer)
if err != nil {
t.Fatalf("error reading layer: %v", err)
}
if nn != randomDataSize {
t.Fatalf("incorrect read length")
}
if digest.NewDigest("sha256", h) != sha256Digest {
t.Fatalf("unexpected digest from uploaded layer: %q != %q", digest.NewDigest("sha256", h), sha256Digest)
}
}
// TestSimpleBlobUpload covers the blob upload process, exercising common
// error paths that might be seen during an upload.
func TestSimpleBlobUpload(t *testing.T) {
randomDataReader, tarSumStr, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random reader: %v", err)
}
dgst := digest.Digest(tarSumStr)
if err != nil {
t.Fatalf("error allocating upload store: %v", err)
}
ctx := context.Background()
imageName := "foo/bar"
driver := inmemory.New()
registry := NewRegistryWithDriver(ctx, driver, memory.NewInMemoryBlobDescriptorCacheProvider())
repository, err := registry.Repository(ctx, imageName)
if err != nil {
t.Fatalf("unexpected error getting repo: %v", err)
}
bs := repository.Blobs(ctx)
h := sha256.New()
rd := io.TeeReader(randomDataReader, h)
blobUpload, err := bs.Create(ctx)
if err != nil {
t.Fatalf("unexpected error starting layer upload: %s", err)
}
// Cancel the upload then restart it
if err := blobUpload.Cancel(ctx); err != nil {
t.Fatalf("unexpected error during upload cancellation: %v", err)
}
// Do a resume, get unknown upload
blobUpload, err = bs.Resume(ctx, blobUpload.ID())
if err != distribution.ErrBlobUploadUnknown {
t.Fatalf("unexpected error resuming upload, should be unknown: %v", err)
}
// Restart!
blobUpload, err = bs.Create(ctx)
if err != nil {
t.Fatalf("unexpected error starting layer upload: %s", err)
}
// Get the size of our random tarfile
randomDataSize, err := seekerSize(randomDataReader)
if err != nil {
t.Fatalf("error getting seeker size of random data: %v", err)
}
nn, err := io.Copy(blobUpload, rd)
if err != nil {
t.Fatalf("unexpected error uploading layer data: %v", err)
}
if nn != randomDataSize {
t.Fatalf("layer data write incomplete")
}
offset, err := blobUpload.Seek(0, os.SEEK_CUR)
if err != nil {
t.Fatalf("unexpected error seeking layer upload: %v", err)
}
if offset != nn {
t.Fatalf("blobUpload not updated with correct offset: %v != %v", offset, nn)
}
blobUpload.Close()
// Do a resume, for good fun
blobUpload, err = bs.Resume(ctx, blobUpload.ID())
if err != nil {
t.Fatalf("unexpected error resuming upload: %v", err)
}
sha256Digest := digest.NewDigest("sha256", h)
desc, err := blobUpload.Commit(ctx, distribution.Descriptor{Digest: dgst})
if err != nil {
t.Fatalf("unexpected error finishing layer upload: %v", err)
}
// After finishing an upload, it should no longer exist.
if _, err := bs.Resume(ctx, blobUpload.ID()); err != distribution.ErrBlobUploadUnknown {
t.Fatalf("expected layer upload to be unknown, got %v", err)
}
// Test for existence.
statDesc, err := bs.Stat(ctx, desc.Digest)
if err != nil {
t.Fatalf("unexpected error checking for existence: %v, %#v", err, bs)
}
if statDesc != desc {
t.Fatalf("descriptors not equal: %v != %v", statDesc, desc)
}
rc, err := bs.Open(ctx, desc.Digest)
if err != nil {
t.Fatalf("unexpected error opening blob for read: %v", err)
}
defer rc.Close()
h.Reset()
nn, err = io.Copy(h, rc)
if err != nil {
t.Fatalf("error reading layer: %v", err)
}
if nn != randomDataSize {
t.Fatalf("incorrect read length")
}
if digest.NewDigest("sha256", h) != sha256Digest {
t.Fatalf("unexpected digest from uploaded layer: %q != %q", digest.NewDigest("sha256", h), sha256Digest)
}
}
// checkExerciseRegistry takes the registry through all of its operations,
// carrying out generic checks.
func checkExerciseRepository(t *testing.T, repository distribution.Repository) {
// TODO(stevvooe): This would be a nice testutil function. Basically, it
// takes the registry through a common set of operations. This could be
// used to make cross-cutting updates by changing internals that affect
// update counts. Basically, it would make writing tests a lot easier.
ctx := context.Background()
tag := "thetag"
m := manifest.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
},
Name: repository.Name(),
Tag: tag,
}
blobs := repository.Blobs(ctx)
for i := 0; i < 2; i++ {
rs, ds, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating test layer: %v", err)
}
dgst := digest.Digest(ds)
wr, err := blobs.Create(ctx)
if err != nil {
t.Fatalf("error creating layer upload: %v", err)
}
// Use the resumes, as well!
wr, err = blobs.Resume(ctx, wr.ID())
if err != nil {
t.Fatalf("error resuming layer upload: %v", err)
}
io.Copy(wr, rs)
if _, err := wr.Commit(ctx, distribution.Descriptor{Digest: dgst}); err != nil {
t.Fatalf("unexpected error finishing upload: %v", err)
}
m.FSLayers = append(m.FSLayers, manifest.FSLayer{
BlobSum: dgst,
})
// Then fetch the blobs
if rc, err := blobs.Open(ctx, dgst); err != nil {
t.Fatalf("error fetching layer: %v", err)
} else {
defer rc.Close()
}
}
pk, err := libtrust.GenerateECP256PrivateKey()
if err != nil {
t.Fatalf("unexpected error generating key: %v", err)
}
sm, err := manifest.Sign(&m, pk)
if err != nil {
t.Fatalf("unexpected error signing manifest: %v", err)
}
manifests, err := repository.Manifests(ctx)
if err != nil {
t.Fatal(err.Error())
}
if err = manifests.Put(sm); err != nil {
t.Fatalf("unexpected error putting the manifest: %v", err)
}
p, err := sm.Payload()
if err != nil {
t.Fatalf("unexpected error getting manifest payload: %v", err)
}
dgst, err := digest.FromBytes(p)
if err != nil {
t.Fatalf("unexpected error digesting manifest payload: %v", err)
}
fetchedByManifest, err := manifests.Get(dgst)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
if fetchedByManifest.Tag != sm.Tag {
t.Fatalf("retrieved unexpected manifest: %v", err)
}
fetched, err := manifests.GetByTag(tag)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
if fetched.Tag != fetchedByManifest.Tag {
t.Fatalf("retrieved unexpected manifest: %v", err)
}
}
func testManifestAPI(t *testing.T, env *testEnv, args manifestArgs) (*testEnv, manifestArgs) {
imageName := args.imageName
tag := "thetag"
manifestURL, err := env.builder.BuildManifestURL(imageName, tag)
if err != nil {
t.Fatalf("unexpected error getting manifest url: %v", err)
}
// -----------------------------
// Attempt to fetch the manifest
resp, err := http.Get(manifestURL)
if err != nil {
t.Fatalf("unexpected error getting manifest: %v", err)
}
defer resp.Body.Close()
checkResponse(t, "getting non-existent manifest", resp, http.StatusNotFound)
checkBodyHasErrorCodes(t, "getting non-existent manifest", resp, v2.ErrorCodeManifestUnknown)
tagsURL, err := env.builder.BuildTagsURL(imageName)
if err != nil {
t.Fatalf("unexpected error building tags url: %v", err)
}
resp, err = http.Get(tagsURL)
if err != nil {
t.Fatalf("unexpected error getting unknown tags: %v", err)
}
defer resp.Body.Close()
// Check that we get an unknown repository error when asking for tags
checkResponse(t, "getting unknown manifest tags", resp, http.StatusNotFound)
checkBodyHasErrorCodes(t, "getting unknown manifest tags", resp, v2.ErrorCodeNameUnknown)
// --------------------------------
// Attempt to push unsigned manifest with missing layers
unsignedManifest := &schema1.Manifest{
Versioned: manifest.Versioned{
SchemaVersion: 1,
},
Name: imageName,
Tag: tag,
FSLayers: []schema1.FSLayer{
{
BlobSum: "asdf",
},
{
BlobSum: "qwer",
},
},
History: []schema1.History{
{
V1Compatibility: "",
},
{
V1Compatibility: "",
},
},
}
resp = putManifest(t, "putting unsigned manifest", manifestURL, unsignedManifest)
defer resp.Body.Close()
checkResponse(t, "putting unsigned manifest", resp, http.StatusBadRequest)
_, p, counts := checkBodyHasErrorCodes(t, "getting unknown manifest tags", resp, v2.ErrorCodeManifestInvalid)
expectedCounts := map[errcode.ErrorCode]int{
v2.ErrorCodeManifestInvalid: 1,
}
if !reflect.DeepEqual(counts, expectedCounts) {
t.Fatalf("unexpected number of error codes encountered: %v\n!=\n%v\n---\n%s", counts, expectedCounts, string(p))
}
// sign the manifest and still get some interesting errors.
sm, err := schema1.Sign(unsignedManifest, env.pk)
if err != nil {
t.Fatalf("error signing manifest: %v", err)
}
resp = putManifest(t, "putting signed manifest with errors", manifestURL, sm)
defer resp.Body.Close()
checkResponse(t, "putting signed manifest with errors", resp, http.StatusBadRequest)
_, p, counts = checkBodyHasErrorCodes(t, "putting signed manifest with errors", resp,
v2.ErrorCodeManifestBlobUnknown, v2.ErrorCodeDigestInvalid)
expectedCounts = map[errcode.ErrorCode]int{
v2.ErrorCodeManifestBlobUnknown: 2,
v2.ErrorCodeDigestInvalid: 2,
}
if !reflect.DeepEqual(counts, expectedCounts) {
t.Fatalf("unexpected number of error codes encountered: %v\n!=\n%v\n---\n%s", counts, expectedCounts, string(p))
}
// TODO(stevvooe): Add a test case where we take a mostly valid registry,
// tamper with the content and ensure that we get a unverified manifest
// error.
// Push 2 random layers
expectedLayers := make(map[digest.Digest]io.ReadSeeker)
for i := range unsignedManifest.FSLayers {
rs, dgstStr, err := testutil.CreateRandomTarFile()
if err != nil {
t.Fatalf("error creating random layer %d: %v", i, err)
}
dgst := digest.Digest(dgstStr)
expectedLayers[dgst] = rs
unsignedManifest.FSLayers[i].BlobSum = dgst
uploadURLBase, _ := startPushLayer(t, env.builder, imageName)
pushLayer(t, env.builder, imageName, dgst, uploadURLBase, rs)
}
// -------------------
// Push the signed manifest with all layers pushed.
signedManifest, err := schema1.Sign(unsignedManifest, env.pk)
if err != nil {
t.Fatalf("unexpected error signing manifest: %v", err)
}
dgst := digest.FromBytes(signedManifest.Canonical)
args.signedManifest = signedManifest
args.dgst = dgst
manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String())
checkErr(t, err, "building manifest url")
resp = putManifest(t, "putting signed manifest no error", manifestURL, signedManifest)
checkResponse(t, "putting signed manifest no error", resp, http.StatusCreated)
checkHeaders(t, resp, http.Header{
"Location": []string{manifestDigestURL},
"Docker-Content-Digest": []string{dgst.String()},
})
// --------------------
// Push by digest -- should get same result
resp = putManifest(t, "putting signed manifest", manifestDigestURL, signedManifest)
checkResponse(t, "putting signed manifest", resp, http.StatusCreated)
checkHeaders(t, resp, http.Header{
"Location": []string{manifestDigestURL},
"Docker-Content-Digest": []string{dgst.String()},
})
// ------------------
// Fetch by tag name
resp, err = http.Get(manifestURL)
if err != nil {
t.Fatalf("unexpected error fetching manifest: %v", err)
}
defer resp.Body.Close()
checkResponse(t, "fetching uploaded manifest", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Docker-Content-Digest": []string{dgst.String()},
"ETag": []string{fmt.Sprintf(`"%s"`, dgst)},
})
var fetchedManifest schema1.SignedManifest
dec := json.NewDecoder(resp.Body)
if err := dec.Decode(&fetchedManifest); err != nil {
t.Fatalf("error decoding fetched manifest: %v", err)
}
if !bytes.Equal(fetchedManifest.Canonical, signedManifest.Canonical) {
t.Fatalf("manifests do not match")
}
// ---------------
// Fetch by digest
resp, err = http.Get(manifestDigestURL)
checkErr(t, err, "fetching manifest by digest")
defer resp.Body.Close()
checkResponse(t, "fetching uploaded manifest", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Docker-Content-Digest": []string{dgst.String()},
"ETag": []string{fmt.Sprintf(`"%s"`, dgst)},
})
var fetchedManifestByDigest schema1.SignedManifest
dec = json.NewDecoder(resp.Body)
if err := dec.Decode(&fetchedManifestByDigest); err != nil {
t.Fatalf("error decoding fetched manifest: %v", err)
}
if !bytes.Equal(fetchedManifestByDigest.Canonical, signedManifest.Canonical) {
t.Fatalf("manifests do not match")
}
// check signature was roundtripped
signatures, err := fetchedManifestByDigest.Signatures()
if err != nil {
t.Fatal(err)
}
if len(signatures) != 1 {
t.Fatalf("expected 1 signature from manifest, got: %d", len(signatures))
}
// Re-sign, push and pull the same digest
sm2, err := schema1.Sign(&fetchedManifestByDigest.Manifest, env.pk)
if err != nil {
t.Fatal(err)
}
resp = putManifest(t, "re-putting signed manifest", manifestDigestURL, sm2)
checkResponse(t, "re-putting signed manifest", resp, http.StatusCreated)
resp, err = http.Get(manifestDigestURL)
checkErr(t, err, "re-fetching manifest by digest")
defer resp.Body.Close()
checkResponse(t, "re-fetching uploaded manifest", resp, http.StatusOK)
checkHeaders(t, resp, http.Header{
"Docker-Content-Digest": []string{dgst.String()},
"ETag": []string{fmt.Sprintf(`"%s"`, dgst)},
})
dec = json.NewDecoder(resp.Body)
if err := dec.Decode(&fetchedManifestByDigest); err != nil {
t.Fatalf("error decoding fetched manifest: %v", err)
}
// check two signatures were roundtripped
signatures, err = fetchedManifestByDigest.Signatures()
if err != nil {
t.Fatal(err)
}
if len(signatures) != 2 {
t.Fatalf("expected 2 signature from manifest, got: %d", len(signatures))
}
// Get by name with etag, gives 304
etag := resp.Header.Get("Etag")
req, err := http.NewRequest("GET", manifestURL, nil)
if err != nil {
t.Fatalf("Error constructing request: %s", err)
}
req.Header.Set("If-None-Match", etag)
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("Error constructing request: %s", err)
}
checkResponse(t, "fetching manifest by name with etag", resp, http.StatusNotModified)
// Get by digest with etag, gives 304
req, err = http.NewRequest("GET", manifestDigestURL, nil)
if err != nil {
t.Fatalf("Error constructing request: %s", err)
}
req.Header.Set("If-None-Match", etag)
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("Error constructing request: %s", err)
}
checkResponse(t, "fetching manifest by dgst with etag", resp, http.StatusNotModified)
// Ensure that the tag is listed.
resp, err = http.Get(tagsURL)
if err != nil {
t.Fatalf("unexpected error getting unknown tags: %v", err)
}
defer resp.Body.Close()
// Check that we get an unknown repository error when asking for tags
checkResponse(t, "getting unknown manifest tags", resp, http.StatusOK)
dec = json.NewDecoder(resp.Body)
var tagsResponse tagsAPIResponse
if err := dec.Decode(&tagsResponse); err != nil {
t.Fatalf("unexpected error decoding error response: %v", err)
}
if tagsResponse.Name != imageName {
t.Fatalf("tags name should match image name: %v != %v", tagsResponse.Name, imageName)
}
if len(tagsResponse.Tags) != 1 {
t.Fatalf("expected some tags in response: %v", tagsResponse.Tags)
}
if tagsResponse.Tags[0] != tag {
t.Fatalf("tag not as expected: %q != %q", tagsResponse.Tags[0], tag)
}
// Attempt to put a manifest with mismatching FSLayer and History array cardinalities
unsignedManifest.History = append(unsignedManifest.History, schema1.History{
V1Compatibility: "",
})
invalidSigned, err := schema1.Sign(unsignedManifest, env.pk)
if err != nil {
t.Fatalf("error signing manifest")
}
resp = putManifest(t, "putting invalid signed manifest", manifestDigestURL, invalidSigned)
checkResponse(t, "putting invalid signed manifest", resp, http.StatusBadRequest)
return env, args
}