func TestContainerApiGetExport(t *testing.T) { name := "exportcontainer" runCmd := exec.Command(dockerBinary, "run", "--name", name, "busybox", "touch", "/test") out, _, err := runCommandWithOutput(runCmd) if err != nil { t.Fatalf("Error on container creation: %v, output: %q", err, out) } body, err := sockRequest("GET", "/containers/"+name+"/export", nil) if err != nil { t.Fatalf("GET containers/export sockRequest failed: %v", err) } found := false for tarReader := tar.NewReader(bytes.NewReader(body)); ; { h, err := tarReader.Next() if err != nil { if err == io.EOF { break } t.Fatal(err) } if h.Name == "test" { found = true break } } if !found { t.Fatalf("The created test file has not been found in the exported image") } deleteAllContainers() logDone("container REST API - check GET containers/export") }
// Some tar have both GNU specific (huge uid) and Ustar specific (long name) things. // Not supposed to happen (should use PAX instead of Ustar for long name) but it does and it should still work. func TestUntarUstarGnuConflict(t *testing.T) { f, err := os.Open("testdata/broken.tar") if err != nil { t.Fatal(err) } found := false tr := tar.NewReader(f) // Iterate through the files in the archive. for { hdr, err := tr.Next() if err == io.EOF { // end of tar archive break } if err != nil { t.Fatal(err) } if hdr.Name == "root/.cpanm/work/1395823785.24209/Plack-1.0030/blib/man3/Plack::Middleware::LighttpdScriptNameFix.3pm" { found = true break } } if !found { t.Fatalf("%s not found in the archive", "root/.cpanm/work/1395823785.24209/Plack-1.0030/blib/man3/Plack::Middleware::LighttpdScriptNameFix.3pm") } }
// SetTar adds data to db from a tar strema decoded from `src`. // Raw data is stored at the key `_fs_data/', and metadata in a // separate key '_fs_metadata'. func (db *DB) SetTar(src io.Reader) error { tr := tar.NewReader(src) for { hdr, err := tr.Next() if err == io.EOF { break } if err != nil { return err } fmt.Printf("[META] %s\n", hdr.Name) metaBlob, err := headerReader(hdr) if err != nil { return err } fmt.Printf(" ---> storing metadata in %s\n", metaPath(hdr.Name)) if err := db.SetStream(metaPath(hdr.Name), metaBlob); err != nil { continue } // FIXME: git can carry symlinks as well if hdr.Typeflag == tar.TypeReg { fmt.Printf("[DATA] %s %d bytes\n", hdr.Name, hdr.Size) if err := db.SetStream(path.Join("_fs_data", hdr.Name), tr); err != nil { continue } } } return nil }
// GetTar generates a tar stream frmo the contents of db, and streams // it to `dst`. func (db *DB) GetTar(dst io.Writer) error { tw := tar.NewWriter(dst) defer tw.Close() // Walk the data tree return db.Walk(DataTree, func(name string, obj git.Object) error { fmt.Fprintf(os.Stderr, "Generating tar entry for '%s'...\n", name) metaBlob, err := db.Get(metaPath(name)) if err != nil { return err } tr := tar.NewReader(bytes.NewReader([]byte(metaBlob))) hdr, err := tr.Next() if err != nil { return err } // Write the reconstituted tar header+content if err := tw.WriteHeader(hdr); err != nil { return err } if blob, isBlob := obj.(*git.Blob); isBlob { fmt.Fprintf(os.Stderr, "--> writing %d bytes for blob %s\n", hdr.Size, hdr.Name) if _, err := tw.Write(blob.Contents()[:hdr.Size]); err != nil { return err } } return nil }) return nil }
func renderSumForHeader(v Version, h *tar.Header, data []byte) (string, error) { buf := bytes.NewBuffer(nil) // first build our test tar tw := tar.NewWriter(buf) if err := tw.WriteHeader(h); err != nil { return "", err } if _, err := tw.Write(data); err != nil { return "", err } tw.Close() ts, err := NewTarSum(buf, true, v) if err != nil { return "", err } tr := tar.NewReader(ts) for { hdr, err := tr.Next() if hdr == nil || err == io.EOF { // Signals the end of the archive. break } if err != nil { return "", err } if _, err = io.Copy(ioutil.Discard, tr); err != nil { return "", err } } return ts.Sum(nil), nil }
func IsArchive(header []byte) bool { compression := DetectCompression(header) if compression != Uncompressed { return true } r := tar.NewReader(bytes.NewBuffer(header)) _, err := r.Next() return err == nil }
func TestPostContainersCopy(t *testing.T) { eng := NewTestEngine(t) defer mkDaemonFromEngine(eng, t).Nuke() // Create a container and remove a file containerID := createTestContainer(eng, &runconfig.Config{ Image: unitTestImageID, Cmd: []string{"touch", "/test.txt"}, }, t, ) containerRun(eng, containerID, t) r := httptest.NewRecorder() var copyData engine.Env copyData.Set("Resource", "/test.txt") copyData.Set("HostPath", ".") jsonData := bytes.NewBuffer(nil) if err := copyData.Encode(jsonData); err != nil { t.Fatal(err) } req, err := http.NewRequest("POST", "/containers/"+containerID+"/copy", jsonData) if err != nil { t.Fatal(err) } req.Header.Add("Content-Type", "application/json") if err := server.ServeRequest(eng, api.APIVERSION, r, req); err != nil { t.Fatal(err) } assertHttpNotError(r, t) if r.Code != http.StatusOK { t.Fatalf("%d OK expected, received %d\n", http.StatusOK, r.Code) } found := false for tarReader := tar.NewReader(r.Body); ; { h, err := tarReader.Next() if err != nil { if err == io.EOF { break } t.Fatal(err) } if h.Name == "test.txt" { found = true break } } if !found { t.Fatalf("The created test file has not been found in the copied output") } }
func ListTar(f io.Reader) ([]string, error) { tr := tar.NewReader(f) var entries []string for { th, err := tr.Next() if err == io.EOF { // end of tar archive return entries, nil } if err != nil { return entries, err } entries = append(entries, th.Name) } }
func TestGetContainersExport(t *testing.T) { eng := NewTestEngine(t) defer mkDaemonFromEngine(eng, t).Nuke() // Create a container and remove a file containerID := createTestContainer(eng, &runconfig.Config{ Image: unitTestImageID, Cmd: []string{"touch", "/test"}, }, t, ) containerRun(eng, containerID, t) r := httptest.NewRecorder() req, err := http.NewRequest("GET", "/containers/"+containerID+"/export", nil) if err != nil { t.Fatal(err) } if err := server.ServeRequest(eng, api.APIVERSION, r, req); err != nil { t.Fatal(err) } assertHttpNotError(r, t) if r.Code != http.StatusOK { t.Fatalf("%d OK expected, received %d\n", http.StatusOK, r.Code) } found := false for tarReader := tar.NewReader(r.Body); ; { h, err := tarReader.Next() if err != nil { if err == io.EOF { break } t.Fatal(err) } if h.Name == "test" { found = true break } } if !found { t.Fatalf("The created test file has not been found in the exported image") } }
func (ts *tarSum) initTarSum() error { ts.bufTar = bytes.NewBuffer([]byte{}) ts.bufWriter = bytes.NewBuffer([]byte{}) ts.tarR = tar.NewReader(ts.Reader) ts.tarW = tar.NewWriter(ts.bufTar) if !ts.DisableCompression { ts.writer = gzip.NewWriter(ts.bufWriter) } else { ts.writer = &nopCloseFlusher{Writer: ts.bufWriter} } if ts.tHash == nil { ts.tHash = DefaultTHash } ts.h = ts.tHash.Hash() ts.h.Reset() ts.first = true ts.sums = FileInfoSums{} return nil }
func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) error { tr := tar.NewReader(decompressedArchive) trBuf := pools.BufioReader32KPool.Get(nil) defer pools.BufioReader32KPool.Put(trBuf) var dirs []*tar.Header // Iterate through the files in the archive. loop: for { hdr, err := tr.Next() if err == io.EOF { // end of tar archive break } if err != nil { return err } // Normalize name, for safety and for a simple is-root check // This keeps "../" as-is, but normalizes "/../" to "/" hdr.Name = filepath.Clean(hdr.Name) for _, exclude := range options.Excludes { if strings.HasPrefix(hdr.Name, exclude) { continue loop } } if !strings.HasSuffix(hdr.Name, "/") { // Not the root directory, ensure that the parent directory exists parent := filepath.Dir(hdr.Name) parentPath := filepath.Join(dest, parent) if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { err = os.MkdirAll(parentPath, 0777) if err != nil { return err } } } path := filepath.Join(dest, hdr.Name) rel, err := filepath.Rel(dest, path) if err != nil { return err } if strings.HasPrefix(rel, "..") { return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest)) } // If path exits we almost always just want to remove and replace it // The only exception is when it is a directory *and* the file from // the layer is also a directory. Then we want to merge them (i.e. // just apply the metadata from the layer). if fi, err := os.Lstat(path); err == nil { if fi.IsDir() && hdr.Name == "." { continue } if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) { if err := os.RemoveAll(path); err != nil { return err } } } trBuf.Reset(tr) if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown); err != nil { return err } // Directory mtimes must be handled at the end to avoid further // file creation in them to modify the directory mtime if hdr.Typeflag == tar.TypeDir { dirs = append(dirs, hdr) } } for _, hdr := range dirs { path := filepath.Join(dest, hdr.Name) ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)} if err := syscall.UtimesNano(path, ts); err != nil { return err } } return nil }
// ApplyLayer parses a diff in the standard layer format from `layer`, and // applies it to the directory `dest`. func ApplyLayer(dest string, layer ArchiveReader) error { dest = filepath.Clean(dest) // We need to be able to set any perms oldmask := syscall.Umask(0) defer syscall.Umask(oldmask) layer, err := DecompressStream(layer) if err != nil { return err } tr := tar.NewReader(layer) trBuf := pools.BufioReader32KPool.Get(tr) defer pools.BufioReader32KPool.Put(trBuf) var dirs []*tar.Header aufsTempdir := "" aufsHardlinks := make(map[string]*tar.Header) // Iterate through the files in the archive. for { hdr, err := tr.Next() if err == io.EOF { // end of tar archive break } if err != nil { return err } // Normalize name, for safety and for a simple is-root check hdr.Name = filepath.Clean(hdr.Name) if !strings.HasSuffix(hdr.Name, "/") { // Not the root directory, ensure that the parent directory exists. // This happened in some tests where an image had a tarfile without any // parent directories. parent := filepath.Dir(hdr.Name) parentPath := filepath.Join(dest, parent) if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { err = os.MkdirAll(parentPath, 0600) if err != nil { return err } } } // Skip AUFS metadata dirs if strings.HasPrefix(hdr.Name, ".wh..wh.") { // Regular files inside /.wh..wh.plnk can be used as hardlink targets // We don't want this directory, but we need the files in them so that // such hardlinks can be resolved. if strings.HasPrefix(hdr.Name, ".wh..wh.plnk") && hdr.Typeflag == tar.TypeReg { basename := filepath.Base(hdr.Name) aufsHardlinks[basename] = hdr if aufsTempdir == "" { if aufsTempdir, err = ioutil.TempDir("", "dockerplnk"); err != nil { return err } defer os.RemoveAll(aufsTempdir) } if err := createTarFile(filepath.Join(aufsTempdir, basename), dest, hdr, tr, true); err != nil { return err } } continue } path := filepath.Join(dest, hdr.Name) base := filepath.Base(path) // Prevent symlink breakout if !strings.HasPrefix(path, dest) { return breakoutError(fmt.Errorf("%q is outside of %q", path, dest)) } if strings.HasPrefix(base, ".wh.") { originalBase := base[len(".wh."):] originalPath := filepath.Join(filepath.Dir(path), originalBase) if err := os.RemoveAll(originalPath); err != nil { return err } } else { // If path exits we almost always just want to remove and replace it. // The only exception is when it is a directory *and* the file from // the layer is also a directory. Then we want to merge them (i.e. // just apply the metadata from the layer). if fi, err := os.Lstat(path); err == nil { if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) { if err := os.RemoveAll(path); err != nil { return err } } } trBuf.Reset(tr) srcData := io.Reader(trBuf) srcHdr := hdr // Hard links into /.wh..wh.plnk don't work, as we don't extract that directory, so // we manually retarget these into the temporary files we extracted them into if hdr.Typeflag == tar.TypeLink && strings.HasPrefix(filepath.Clean(hdr.Linkname), ".wh..wh.plnk") { linkBasename := filepath.Base(hdr.Linkname) srcHdr = aufsHardlinks[linkBasename] if srcHdr == nil { return fmt.Errorf("Invalid aufs hardlink") } tmpFile, err := os.Open(filepath.Join(aufsTempdir, linkBasename)) if err != nil { return err } defer tmpFile.Close() srcData = tmpFile } if err := createTarFile(path, dest, srcHdr, srcData, true); err != nil { return err } // Directory mtimes must be handled at the end to avoid further // file creation in them to modify the directory mtime if hdr.Typeflag == tar.TypeDir { dirs = append(dirs, hdr) } } } for _, hdr := range dirs { path := filepath.Join(dest, hdr.Name) ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)} if err := syscall.UtimesNano(path, ts); err != nil { return err } } return nil }
func (ts *tarSum) Read(buf []byte) (int, error) { if ts.gz == nil { ts.bufTar = bytes.NewBuffer([]byte{}) ts.bufGz = bytes.NewBuffer([]byte{}) ts.tarR = tar.NewReader(ts.Reader) ts.tarW = tar.NewWriter(ts.bufTar) if !ts.DisableCompression { ts.gz = gzip.NewWriter(ts.bufGz) } else { ts.gz = &nopCloseFlusher{Writer: ts.bufGz} } ts.h = sha256.New() ts.h.Reset() ts.first = true ts.sums = make(map[string]string) } if ts.finished { return ts.bufGz.Read(buf) } if ts.bufData == nil { switch { case len(buf) <= buf8K: ts.bufData = make([]byte, buf8K) case len(buf) <= buf16K: ts.bufData = make([]byte, buf16K) case len(buf) <= buf32K: ts.bufData = make([]byte, buf32K) default: ts.bufData = make([]byte, len(buf)) } } buf2 := ts.bufData[:len(buf)-1] n, err := ts.tarR.Read(buf2) if err != nil { if err == io.EOF { if _, err := ts.h.Write(buf2[:n]); err != nil { return 0, err } if !ts.first { ts.sums[ts.currentFile] = hex.EncodeToString(ts.h.Sum(nil)) ts.h.Reset() } else { ts.first = false } currentHeader, err := ts.tarR.Next() if err != nil { if err == io.EOF { if err := ts.gz.Close(); err != nil { return 0, err } ts.finished = true return n, nil } return n, err } ts.currentFile = strings.TrimSuffix(strings.TrimPrefix(currentHeader.Name, "./"), "/") if err := ts.encodeHeader(currentHeader); err != nil { return 0, err } if err := ts.tarW.WriteHeader(currentHeader); err != nil { return 0, err } if _, err := ts.tarW.Write(buf2[:n]); err != nil { return 0, err } ts.tarW.Flush() if _, err := io.Copy(ts.gz, ts.bufTar); err != nil { return 0, err } ts.gz.Flush() return ts.bufGz.Read(buf) } return n, err } // Filling the hash buffer if _, err = ts.h.Write(buf2[:n]); err != nil { return 0, err } // Filling the tar writter if _, err = ts.tarW.Write(buf2[:n]); err != nil { return 0, err } ts.tarW.Flush() // Filling the gz writter if _, err = io.Copy(ts.gz, ts.bufTar); err != nil { return 0, err } ts.gz.Flush() return ts.bufGz.Read(buf) }
// Untar reads a stream of bytes from `archive`, parses it as a tar archive, // and unpacks it into the directory at `path`. // The archive may be compressed with one of the following algorithms: // identity (uncompressed), gzip, bzip2, xz. // FIXME: specify behavior when target path exists vs. doesn't exist. func Untar(archive io.Reader, dest string, options *TarOptions) error { if archive == nil { return fmt.Errorf("Empty archive") } decompressedArchive, err := DecompressStream(archive) if err != nil { return err } defer decompressedArchive.Close() tr := tar.NewReader(decompressedArchive) trBuf := bufio.NewReaderSize(nil, trBufSize) var dirs []*tar.Header // Iterate through the files in the archive. for { hdr, err := tr.Next() if err == io.EOF { // end of tar archive break } if err != nil { return err } // Normalize name, for safety and for a simple is-root check hdr.Name = filepath.Clean(hdr.Name) if !strings.HasSuffix(hdr.Name, "/") { // Not the root directory, ensure that the parent directory exists parent := filepath.Dir(hdr.Name) parentPath := filepath.Join(dest, parent) if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { err = os.MkdirAll(parentPath, 0777) if err != nil { return err } } } path := filepath.Join(dest, hdr.Name) // If path exits we almost always just want to remove and replace it // The only exception is when it is a directory *and* the file from // the layer is also a directory. Then we want to merge them (i.e. // just apply the metadata from the layer). if fi, err := os.Lstat(path); err == nil { if fi.IsDir() && hdr.Name == "." { continue } if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) { if err := os.RemoveAll(path); err != nil { return err } } } trBuf.Reset(tr) if err := createTarFile(path, dest, hdr, trBuf, options == nil || !options.NoLchown); err != nil { return err } // Directory mtimes must be handled at the end to avoid further // file creation in them to modify the directory mtime if hdr.Typeflag == tar.TypeDir { dirs = append(dirs, hdr) } } for _, hdr := range dirs { path := filepath.Join(dest, hdr.Name) ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)} if err := syscall.UtimesNano(path, ts); err != nil { return err } } return nil }