func (c *checksums) checksumForGraphIDNoTarsplit(id, parent, newTarDataPath string) (diffID layer.DiffID, size int64, err error) { rawarchive, err := c.driver.TarStream(id, parent) if err != nil { return } defer rawarchive.Close() f, err := os.Create(newTarDataPath) if err != nil { return } defer f.Close() mfz := gzip.NewWriter(f) defer mfz.Close() metaPacker := storage.NewJSONPacker(mfz) packerCounter := &packSizeCounter{metaPacker, &size} archive, err := asm.NewInputTarStream(rawarchive, packerCounter, nil) if err != nil { return } dgst, err := digest.FromReader(archive) if err != nil { return } diffID = layer.DiffID(dgst) return }
func simpleUpload(t *testing.T, bs distribution.BlobIngester, blob []byte, expectedDigest digest.Digest) { ctx := context.Background() wr, err := bs.Create(ctx) if err != nil { t.Fatalf("unexpected error starting upload: %v", err) } nn, err := io.Copy(wr, bytes.NewReader(blob)) if err != nil { t.Fatalf("error copying into blob writer: %v", err) } if nn != 0 { t.Fatalf("unexpected number of bytes copied: %v > 0", nn) } dgst, err := digest.FromReader(bytes.NewReader(blob)) if err != nil { t.Fatalf("error getting digest: %v", err) } if dgst != expectedDigest { // sanity check on zero digest t.Fatalf("digest not as expected: %v != %v", dgst, expectedDigest) } desc, err := wr.Commit(ctx, distribution.Descriptor{Digest: dgst}) if err != nil { t.Fatalf("unexpected error committing write: %v", err) } if desc.Digest != dgst { t.Fatalf("unexpected digest: %v != %v", desc.Digest, dgst) } }
func binaryDigest(source string) (digest.Digest, error) { f, err := os.Open(source) if err != nil { return "", err } defer f.Close() return digest.FromReader(f) }
// TestLayerUploadZeroLength uploads zero-length func TestLayerUploadZeroLength(t *testing.T) { ctx := context.Background() imageName := "foo/bar" driver := inmemory.New() registry := NewRegistryWithDriver(ctx, driver, memory.NewInMemoryBlobDescriptorCacheProvider()) repository, err := registry.Repository(ctx, imageName) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } bs := repository.Blobs(ctx) wr, err := bs.Create(ctx) if err != nil { t.Fatalf("unexpected error starting upload: %v", err) } nn, err := io.Copy(wr, bytes.NewReader([]byte{})) if err != nil { t.Fatalf("error copying into blob writer: %v", err) } if nn != 0 { t.Fatalf("unexpected number of bytes copied: %v > 0", nn) } dgst, err := digest.FromReader(bytes.NewReader([]byte{})) if err != nil { t.Fatalf("error getting zero digest: %v", err) } if dgst != digest.DigestSha256EmptyTar { // sanity check on zero digest t.Fatalf("digest not as expected: %v != %v", dgst, digest.DigestTarSumV1EmptyTar) } desc, err := wr.Commit(ctx, distribution.Descriptor{Digest: dgst}) if err != nil { t.Fatalf("unexpected error committing write: %v", err) } if desc.Digest != dgst { t.Fatalf("unexpected digest: %v != %v", desc.Digest, dgst) } }
func TestSimpleRead(t *testing.T) { ctx := context.Background() content := make([]byte, 1<<20) n, err := mrand.Read(content) if err != nil { t.Fatalf("unexpected error building random data: %v", err) } if n != len(content) { t.Fatalf("random read didn't fill buffer") } dgst, err := digest.FromReader(bytes.NewReader(content)) if err != nil { t.Fatalf("unexpected error digesting random content: %v", err) } driver := inmemory.New() path := "/random" if err := driver.PutContent(ctx, path, content); err != nil { t.Fatalf("error putting patterned content: %v", err) } fr, err := newFileReader(ctx, driver, path, int64(len(content))) if err != nil { t.Fatalf("error allocating file reader: %v", err) } verifier, err := digest.NewDigestVerifier(dgst) if err != nil { t.Fatalf("error getting digest verifier: %s", err) } io.Copy(verifier, fr) if !verifier.Verified() { t.Fatalf("unable to verify read data") } }
// TestLayerUploadZeroLength uploads zero-length func TestLayerUploadZeroLength(t *testing.T) { ctx := context.Background() imageName := "foo/bar" driver := inmemory.New() registry := NewRegistryWithDriver(ctx, driver, cache.NewInMemoryLayerInfoCache()) repository, err := registry.Repository(ctx, imageName) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } ls := repository.Layers() upload, err := ls.Upload() if err != nil { t.Fatalf("unexpected error starting upload: %v", err) } io.Copy(upload, bytes.NewReader([]byte{})) dgst, err := digest.FromReader(bytes.NewReader([]byte{})) if err != nil { t.Fatalf("error getting zero digest: %v", err) } if dgst != digest.DigestSha256EmptyTar { // sanity check on zero digest t.Fatalf("digest not as expected: %v != %v", dgst, digest.DigestTarSumV1EmptyTar) } layer, err := upload.Finish(dgst) if err != nil { t.Fatalf("unexpected error finishing upload: %v", err) } if layer.Digest() != dgst { t.Fatalf("unexpected digest: %v != %v", layer.Digest(), dgst) } }
func testBlobAPI(t *testing.T, env *testEnv, args blobArgs) *testEnv { // TODO(stevvooe): This test code is complete junk but it should cover the // complete flow. This must be broken down and checked against the // specification *before* we submit the final to docker core. imageName := args.imageName layerFile := args.layerFile layerDigest := args.layerDigest // ----------------------------------- // Test fetch for non-existent content layerURL, err := env.builder.BuildBlobURL(imageName, layerDigest) if err != nil { t.Fatalf("error building url: %v", err) } resp, err := http.Get(layerURL) if err != nil { t.Fatalf("unexpected error fetching non-existent layer: %v", err) } checkResponse(t, "fetching non-existent content", resp, http.StatusNotFound) // ------------------------------------------ // Test head request for non-existent content resp, err = http.Head(layerURL) if err != nil { t.Fatalf("unexpected error checking head on non-existent layer: %v", err) } checkResponse(t, "checking head on non-existent layer", resp, http.StatusNotFound) // ------------------------------------------ // Start an upload, check the status then cancel uploadURLBase, uploadUUID := startPushLayer(t, env.builder, imageName) // A status check should work resp, err = http.Get(uploadURLBase) if err != nil { t.Fatalf("unexpected error getting upload status: %v", err) } checkResponse(t, "status of deleted upload", resp, http.StatusNoContent) checkHeaders(t, resp, http.Header{ "Location": []string{"*"}, "Range": []string{"0-0"}, "Docker-Upload-UUID": []string{uploadUUID}, }) req, err := http.NewRequest("DELETE", uploadURLBase, nil) if err != nil { t.Fatalf("unexpected error creating delete request: %v", err) } resp, err = http.DefaultClient.Do(req) if err != nil { t.Fatalf("unexpected error sending delete request: %v", err) } checkResponse(t, "deleting upload", resp, http.StatusNoContent) // A status check should result in 404 resp, err = http.Get(uploadURLBase) if err != nil { t.Fatalf("unexpected error getting upload status: %v", err) } checkResponse(t, "status of deleted upload", resp, http.StatusNotFound) // ----------------------------------------- // Do layer push with an empty body and different digest uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName) resp, err = doPushLayer(t, env.builder, imageName, layerDigest, uploadURLBase, bytes.NewReader([]byte{})) if err != nil { t.Fatalf("unexpected error doing bad layer push: %v", err) } checkResponse(t, "bad layer push", resp, http.StatusBadRequest) checkBodyHasErrorCodes(t, "bad layer push", resp, v2.ErrorCodeDigestInvalid) // ----------------------------------------- // Do layer push with an empty body and correct digest zeroDigest, err := digest.FromReader(bytes.NewReader([]byte{})) if err != nil { t.Fatalf("unexpected error digesting empty buffer: %v", err) } uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName) pushLayer(t, env.builder, imageName, zeroDigest, uploadURLBase, bytes.NewReader([]byte{})) // ----------------------------------------- // Do layer push with an empty body and correct digest // This is a valid but empty tarfile! emptyTar := bytes.Repeat([]byte("\x00"), 1024) emptyDigest, err := digest.FromReader(bytes.NewReader(emptyTar)) if err != nil { t.Fatalf("unexpected error digesting empty tar: %v", err) } uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName) pushLayer(t, env.builder, imageName, emptyDigest, uploadURLBase, bytes.NewReader(emptyTar)) // ------------------------------------------ // Now, actually do successful upload. layerLength, _ := layerFile.Seek(0, os.SEEK_END) layerFile.Seek(0, os.SEEK_SET) uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName) pushLayer(t, env.builder, imageName, layerDigest, uploadURLBase, layerFile) // ------------------------------------------ // Now, push just a chunk layerFile.Seek(0, 0) canonicalDigester := digest.Canonical.New() if _, err := io.Copy(canonicalDigester.Hash(), layerFile); err != nil { t.Fatalf("error copying to digest: %v", err) } canonicalDigest := canonicalDigester.Digest() layerFile.Seek(0, 0) uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName) uploadURLBase, dgst := pushChunk(t, env.builder, imageName, uploadURLBase, layerFile, layerLength) finishUpload(t, env.builder, imageName, uploadURLBase, dgst) // ------------------------ // Use a head request to see if the layer exists. resp, err = http.Head(layerURL) if err != nil { t.Fatalf("unexpected error checking head on existing layer: %v", err) } checkResponse(t, "checking head on existing layer", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Content-Length": []string{fmt.Sprint(layerLength)}, "Docker-Content-Digest": []string{canonicalDigest.String()}, }) // ---------------- // Fetch the layer! resp, err = http.Get(layerURL) if err != nil { t.Fatalf("unexpected error fetching layer: %v", err) } checkResponse(t, "fetching layer", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Content-Length": []string{fmt.Sprint(layerLength)}, "Docker-Content-Digest": []string{canonicalDigest.String()}, }) // Verify the body verifier, err := digest.NewDigestVerifier(layerDigest) if err != nil { t.Fatalf("unexpected error getting digest verifier: %s", err) } io.Copy(verifier, resp.Body) if !verifier.Verified() { t.Fatalf("response body did not pass verification") } // ---------------- // Fetch the layer with an invalid digest badURL := strings.Replace(layerURL, "sha256", "sha257", 1) resp, err = http.Get(badURL) if err != nil { t.Fatalf("unexpected error fetching layer: %v", err) } checkResponse(t, "fetching layer bad digest", resp, http.StatusBadRequest) // Cache headers resp, err = http.Get(layerURL) if err != nil { t.Fatalf("unexpected error fetching layer: %v", err) } checkResponse(t, "fetching layer", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Content-Length": []string{fmt.Sprint(layerLength)}, "Docker-Content-Digest": []string{canonicalDigest.String()}, "ETag": []string{fmt.Sprintf(`"%s"`, canonicalDigest)}, "Cache-Control": []string{"max-age=31536000"}, }) // Matching etag, gives 304 etag := resp.Header.Get("Etag") req, err = http.NewRequest("GET", layerURL, nil) if err != nil { t.Fatalf("Error constructing request: %s", err) } req.Header.Set("If-None-Match", etag) resp, err = http.DefaultClient.Do(req) if err != nil { t.Fatalf("Error constructing request: %s", err) } checkResponse(t, "fetching layer with etag", resp, http.StatusNotModified) // Non-matching etag, gives 200 req, err = http.NewRequest("GET", layerURL, nil) if err != nil { t.Fatalf("Error constructing request: %s", err) } req.Header.Set("If-None-Match", "") resp, err = http.DefaultClient.Do(req) checkResponse(t, "fetching layer with invalid etag", resp, http.StatusOK) // Missing tests: // - Upload the same tar file under and different repository and // ensure the content remains uncorrupted. return env }
// TestSimpleWrite takes the fileWriter through common write operations // ensuring data integrity. func TestSimpleWrite(t *testing.T) { content := make([]byte, 1<<20) n, err := rand.Read(content) if err != nil { t.Fatalf("unexpected error building random data: %v", err) } if n != len(content) { t.Fatalf("random read did't fill buffer") } dgst, err := digest.FromReader(bytes.NewReader(content)) if err != nil { t.Fatalf("unexpected error digesting random content: %v", err) } driver := inmemory.New() path := "/random" fw, err := newFileWriter(driver, path) if err != nil { t.Fatalf("unexpected error creating fileWriter: %v", err) } defer fw.Close() n, err = fw.Write(content) if err != nil { t.Fatalf("unexpected error writing content: %v", err) } fw.Flush() if n != len(content) { t.Fatalf("unexpected write length: %d != %d", n, len(content)) } fr, err := newFileReader(driver, path) if err != nil { t.Fatalf("unexpected error creating fileReader: %v", err) } defer fr.Close() verifier, err := digest.NewDigestVerifier(dgst) if err != nil { t.Fatalf("unexpected error getting digest verifier: %s", err) } io.Copy(verifier, fr) if !verifier.Verified() { t.Fatalf("unable to verify write data") } // Check the seek position is equal to the content length end, err := fw.Seek(0, os.SEEK_END) if err != nil { t.Fatalf("unexpected error seeking: %v", err) } if end != int64(len(content)) { t.Fatalf("write did not advance offset: %d != %d", end, len(content)) } // Double the content, but use the WriteAt method doubled := append(content, content...) doubledgst, err := digest.FromReader(bytes.NewReader(doubled)) if err != nil { t.Fatalf("unexpected error digesting doubled content: %v", err) } n, err = fw.WriteAt(content, end) if err != nil { t.Fatalf("unexpected error writing content at %d: %v", end, err) } if n != len(content) { t.Fatalf("writeat was short: %d != %d", n, len(content)) } fr, err = newFileReader(driver, path) if err != nil { t.Fatalf("unexpected error creating fileReader: %v", err) } defer fr.Close() verifier, err = digest.NewDigestVerifier(doubledgst) if err != nil { t.Fatalf("unexpected error getting digest verifier: %s", err) } io.Copy(verifier, fr) if !verifier.Verified() { t.Fatalf("unable to verify write data") } // Check that WriteAt didn't update the offset. end, err = fw.Seek(0, os.SEEK_END) if err != nil { t.Fatalf("unexpected error seeking: %v", err) } if end != int64(len(content)) { t.Fatalf("write did not advance offset: %d != %d", end, len(content)) } // Now, we copy from one path to another, running the data through the // fileReader to fileWriter, rather than the driver.Move command to ensure // everything is working correctly. fr, err = newFileReader(driver, path) if err != nil { t.Fatalf("unexpected error creating fileReader: %v", err) } defer fr.Close() fw, err = newFileWriter(driver, "/copied") if err != nil { t.Fatalf("unexpected error creating fileWriter: %v", err) } defer fw.Close() nn, err := io.Copy(fw, fr) if err != nil { t.Fatalf("unexpected error copying data: %v", err) } if nn != int64(len(doubled)) { t.Fatalf("unexpected copy length: %d != %d", nn, len(doubled)) } fr, err = newFileReader(driver, "/copied") if err != nil { t.Fatalf("unexpected error creating fileReader: %v", err) } defer fr.Close() verifier, err = digest.NewDigestVerifier(doubledgst) if err != nil { t.Fatalf("unexpected error getting digest verifier: %s", err) } io.Copy(verifier, fr) if !verifier.Verified() { t.Fatalf("unable to verify write data") } }
func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error) { manifest := ®istry.ManifestData{ Name: remoteName, Tag: tag, SchemaVersion: 1, } localRepo, err := s.Get(localName) if err != nil { return nil, err } if localRepo == nil { return nil, fmt.Errorf("Repo does not exist: %s", localName) } // Get the top-most layer id which the tag points to layerId, exists := localRepo[tag] if !exists { return nil, fmt.Errorf("Tag does not exist for %s: %s", localName, tag) } layersSeen := make(map[string]bool) layer, err := s.graph.Get(layerId) if err != nil { return nil, err } manifest.Architecture = layer.Architecture manifest.FSLayers = make([]*registry.FSLayer, 0, 4) manifest.History = make([]*registry.ManifestHistory, 0, 4) var metadata runconfig.Config if layer.Config != nil { metadata = *layer.Config } for ; layer != nil; layer, err = s.graph.GetParent(layer) { if err != nil { return nil, err } if layersSeen[layer.ID] { break } if layer.Config != nil && metadata.Image != layer.ID { err = runconfig.Merge(&metadata, layer.Config) if err != nil { return nil, err } } dgst, err := s.graph.GetDigest(layer.ID) if err == ErrDigestNotSet { archive, err := s.graph.TarLayer(layer) if err != nil { return nil, err } defer archive.Close() dgst, err = digest.FromReader(archive) if err != nil { return nil, err } // Save checksum value if err := s.graph.SetDigest(layer.ID, dgst); err != nil { return nil, err } } else if err != nil { return nil, fmt.Errorf("Error getting image checksum: %s", err) } jsonData, err := s.graph.RawJSON(layer.ID) if err != nil { return nil, fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err) } manifest.FSLayers = append(manifest.FSLayers, ®istry.FSLayer{BlobSum: dgst.String()}) layersSeen[layer.ID] = true manifest.History = append(manifest.History, ®istry.ManifestHistory{V1Compatibility: string(jsonData)}) } manifestBytes, err := json.MarshalIndent(manifest, "", " ") if err != nil { return nil, err } return manifestBytes, nil }