func TestChrootApplyDotDotFile(t *testing.T) { tmpdir, err := ioutil.TempDir("", "docker-TestChrootApplyDotDotFile") if err != nil { t.Fatal(err) } defer os.RemoveAll(tmpdir) src := filepath.Join(tmpdir, "src") if err := system.MkdirAll(src, 0700); err != nil { t.Fatal(err) } if err := ioutil.WriteFile(filepath.Join(src, "..gitme"), []byte(""), 0644); err != nil { t.Fatal(err) } stream, err := archive.Tar(src, archive.Uncompressed) if err != nil { t.Fatal(err) } dest := filepath.Join(tmpdir, "dest") if err := system.MkdirAll(dest, 0700); err != nil { t.Fatal(err) } if _, err := ApplyLayer(dest, stream); err != nil { t.Fatal(err) } }
// gh#10426: Verify the fix for having a huge excludes list (like on `docker load` with large # of // local images) func TestChrootUntarWithHugeExcludesList(t *testing.T) { tmpdir, err := ioutil.TempDir("", "docker-TestChrootUntarHugeExcludes") if err != nil { t.Fatal(err) } defer os.RemoveAll(tmpdir) src := filepath.Join(tmpdir, "src") if err := system.MkdirAll(src, 0700); err != nil { t.Fatal(err) } if err := ioutil.WriteFile(filepath.Join(src, "toto"), []byte("hello toto"), 0644); err != nil { t.Fatal(err) } stream, err := archive.Tar(src, archive.Uncompressed) if err != nil { t.Fatal(err) } dest := filepath.Join(tmpdir, "dest") if err := system.MkdirAll(dest, 0700); err != nil { t.Fatal(err) } options := &archive.TarOptions{} //65534 entries of 64-byte strings ~= 4MB of environment space which should overflow //on most systems when passed via environment or command line arguments excludes := make([]string, 65534, 65534) for i := 0; i < 65534; i++ { excludes[i] = strings.Repeat(string(i), 64) } options.ExcludePatterns = excludes if err := Untar(stream, dest, options); err != nil { t.Fatal(err) } }
func TestChrootTarUntar(t *testing.T) { tmpdir, err := ioutil.TempDir("", "docker-TestChrootTarUntar") if err != nil { t.Fatal(err) } defer os.RemoveAll(tmpdir) src := filepath.Join(tmpdir, "src") if err := system.MkdirAll(src, 0700); err != nil { t.Fatal(err) } if err := ioutil.WriteFile(filepath.Join(src, "toto"), []byte("hello toto"), 0644); err != nil { t.Fatal(err) } if err := ioutil.WriteFile(filepath.Join(src, "lolo"), []byte("hello lolo"), 0644); err != nil { t.Fatal(err) } stream, err := archive.Tar(src, archive.Uncompressed) if err != nil { t.Fatal(err) } dest := filepath.Join(tmpdir, "src") if err := system.MkdirAll(dest, 0700); err != nil { t.Fatal(err) } if err := Untar(stream, dest, &archive.TarOptions{ExcludePatterns: []string{"lolo"}}); err != nil { t.Fatal(err) } }
// Untar reads a stream of bytes from `archive`, parses it as a tar archive, // and unpacks it into the directory at `dest`. // The archive may be compressed with one of the following algorithms: // identity (uncompressed), gzip, bzip2, xz. func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error { if tarArchive == nil { return fmt.Errorf("Empty archive") } if options == nil { options = &archive.TarOptions{} } if options.ExcludePatterns == nil { options.ExcludePatterns = []string{} } dest = filepath.Clean(dest) if _, err := os.Stat(dest); os.IsNotExist(err) { if err := system.MkdirAll(dest, 0777); err != nil { glog.Error(err.Error()) return err } } decompressedArchive, err := archive.DecompressStream(tarArchive) if err != nil { glog.Error(err.Error()) return err } defer decompressedArchive.Close() return invokeUnpack(decompressedArchive, dest, options) }
func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) { glog.V(1).Infof("CopyFileWithTar(%s, %s)", src, dst) srcSt, err := os.Stat(src) if err != nil { return err } if srcSt.IsDir() { return fmt.Errorf("Can't copy a directory") } // Clean up the trailing slash if dst[len(dst)-1] == os.PathSeparator { dst = filepath.Join(dst, filepath.Base(src)) } // Create the holding directory if necessary if err := system.MkdirAll(filepath.Dir(dst), 0700); err != nil && !os.IsExist(err) { return err } r, w := io.Pipe() errC := promise.Go(func() error { defer w.Close() srcF, err := os.Open(src) if err != nil { return err } defer srcF.Close() hdr, err := tar.FileInfoHeader(srcSt, "") if err != nil { return err } hdr.Name = filepath.Base(dst) hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode))) tw := tar.NewWriter(w) defer tw.Close() if err := tw.WriteHeader(hdr); err != nil { return err } if _, err := io.Copy(tw, srcF); err != nil { return err } return nil }) defer func() { if er := <-errC; err != nil { err = er } }() return archiver.Untar(r, filepath.Dir(dst), nil) }
func TestChrootApplyEmptyArchiveFromSlowReader(t *testing.T) { tmpdir, err := ioutil.TempDir("", "docker-TestChrootApplyEmptyArchiveFromSlowReader") if err != nil { t.Fatal(err) } defer os.RemoveAll(tmpdir) dest := filepath.Join(tmpdir, "dest") if err := system.MkdirAll(dest, 0700); err != nil { t.Fatal(err) } stream := &slowEmptyTarReader{size: 10240, chunkSize: 1024} if _, err := ApplyLayer(dest, stream); err != nil { t.Fatal(err) } }
func (archiver *Archiver) CopyWithTar(src, dst string) error { srcSt, err := os.Stat(src) if err != nil { return err } if !srcSt.IsDir() { return archiver.CopyFileWithTar(src, dst) } // Create dst, copy src's content into it glog.V(1).Infof("Creating dest directory: %s", dst) if err := system.MkdirAll(dst, 0755); err != nil && !os.IsExist(err) { return err } glog.V(1).Infof("Calling TarUntar(%s, %s)", src, dst) return archiver.TarUntar(src, dst) }
func TestChrootCopyWithTar(t *testing.T) { tmpdir, err := ioutil.TempDir("", "docker-TestChrootCopyWithTar") if err != nil { t.Fatal(err) } defer os.RemoveAll(tmpdir) src := filepath.Join(tmpdir, "src") if err := system.MkdirAll(src, 0700); err != nil { t.Fatal(err) } if _, err := prepareSourceDirectory(10, src, true); err != nil { t.Fatal(err) } // Copy directory dest := filepath.Join(tmpdir, "dest") if err := CopyWithTar(src, dest); err != nil { t.Fatal(err) } if err := compareDirectories(src, dest); err != nil { t.Fatal(err) } // Copy file srcfile := filepath.Join(src, "file-1") dest = filepath.Join(tmpdir, "destFile") destfile := filepath.Join(dest, "file-1") if err := CopyWithTar(srcfile, destfile); err != nil { t.Fatal(err) } if err := compareFiles(srcfile, destfile); err != nil { t.Fatal(err) } // Copy symbolic link srcLinkfile := filepath.Join(src, "file-1-link") dest = filepath.Join(tmpdir, "destSymlink") destLinkfile := filepath.Join(dest, "file-1-link") if err := CopyWithTar(srcLinkfile, destLinkfile); err != nil { t.Fatal(err) } if err := compareFiles(srcLinkfile, destLinkfile); err != nil { t.Fatal(err) } }
func TestChrootTarUntarWithSymlink(t *testing.T) { tmpdir, err := ioutil.TempDir("", "docker-TestChrootTarUntarWithSymlink") if err != nil { t.Fatal(err) } defer os.RemoveAll(tmpdir) src := filepath.Join(tmpdir, "src") if err := system.MkdirAll(src, 0700); err != nil { t.Fatal(err) } if _, err := prepareSourceDirectory(10, src, true); err != nil { t.Fatal(err) } dest := filepath.Join(tmpdir, "dest") if err := TarUntar(src, dest); err != nil { t.Fatal(err) } if err := compareDirectories(src, dest); err != nil { t.Fatal(err) } }
func TestChrootUntarPath(t *testing.T) { tmpdir, err := ioutil.TempDir("", "docker-TestChrootUntarPath") if err != nil { t.Fatal(err) } defer os.RemoveAll(tmpdir) src := filepath.Join(tmpdir, "src") if err := system.MkdirAll(src, 0700); err != nil { t.Fatal(err) } if _, err := prepareSourceDirectory(10, src, true); err != nil { t.Fatal(err) } dest := filepath.Join(tmpdir, "dest") // Untar a directory if err := UntarPath(src, dest); err == nil { t.Fatal("Expected error on untaring a directory") } // Untar a tar file stream, err := archive.Tar(src, archive.Uncompressed) if err != nil { t.Fatal(err) } buf := new(bytes.Buffer) buf.ReadFrom(stream) tarfile := filepath.Join(tmpdir, "src.tar") if err := ioutil.WriteFile(tarfile, buf.Bytes(), 0644); err != nil { t.Fatal(err) } if err := UntarPath(tarfile, dest); err != nil { t.Fatal(err) } if err := compareDirectories(src, dest); err != nil { t.Fatal(err) } }
func (b *Builder) addContext(container *daemon.Container, orig, dest string, decompress bool) error { var ( err error destExists = true origPath = filepath.Join(b.contextPath, orig) destPath string ) destPath = dest destStat, err := os.Stat(destPath) if err != nil { if !os.IsNotExist(err) { glog.Error(err.Error()) return err } destExists = false } fi, err := os.Stat(origPath) if err != nil { if os.IsNotExist(err) { return fmt.Errorf("%s: no such file or directory", orig) } glog.Error(err.Error()) return err } if fi.IsDir() { err := copyAsDirectory(origPath, destPath, destExists) if err != nil { glog.Error(err.Error()) } return err } // If we are adding a remote file (or we've been told not to decompress), do not try to untar it if decompress { // First try to unpack the source as an archive // to support the untar feature we need to clean up the path a little bit // because tar is very forgiving. First we need to strip off the archive's // filename from the path but this is only added if it does not end in / . tarDest := destPath if strings.HasSuffix(tarDest, "/") { tarDest = filepath.Dir(destPath) } // try to successfully untar the orig if err := chrootarchive.UntarPath(origPath, tarDest); err == nil { return nil } else if err != io.EOF { glog.Infof("Couldn't untar %s to %s: %s", origPath, tarDest, err) } } if err := system.MkdirAll(filepath.Dir(destPath), 0755); err != nil { glog.Error(err.Error()) return err } if err := chrootarchive.CopyWithTar(origPath, destPath); err != nil { glog.Error(err.Error()) return err } resPath := destPath if destExists && destStat.IsDir() { resPath = filepath.Join(destPath, filepath.Base(origPath)) } _ = resPath /* if err := fixPermissions(origPath, resPath, 0, 0, destExists); err != nil { glog.Error(err.Error()) return err } */ return nil }
func UnpackLayer(dest string, layer ArchiveReader) (size int64, err error) { tr := tar.NewReader(layer) trBuf := pools.BufioReader32KPool.Get(tr) defer pools.BufioReader32KPool.Put(trBuf) var dirs []*tar.Header aufsTempdir := "" aufsHardlinks := make(map[string]*tar.Header) // Iterate through the files in the archive. for { hdr, err := tr.Next() if err == io.EOF { // end of tar archive break } if err != nil { return 0, err } size += hdr.Size // Normalize name, for safety and for a simple is-root check hdr.Name = filepath.Clean(hdr.Name) if !strings.HasSuffix(hdr.Name, "/") { // Not the root directory, ensure that the parent directory exists. // This happened in some tests where an image had a tarfile without any // parent directories. parent := filepath.Dir(hdr.Name) parentPath := filepath.Join(dest, parent) if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { err = system.MkdirAll(parentPath, 0600) if err != nil { return 0, err } } } // Skip AUFS metadata dirs if strings.HasPrefix(hdr.Name, ".wh..wh.") { // Regular files inside /.wh..wh.plnk can be used as hardlink targets // We don't want this directory, but we need the files in them so that // such hardlinks can be resolved. if strings.HasPrefix(hdr.Name, ".wh..wh.plnk") && hdr.Typeflag == tar.TypeReg { basename := filepath.Base(hdr.Name) aufsHardlinks[basename] = hdr if aufsTempdir == "" { if aufsTempdir, err = ioutil.TempDir("", "dockerplnk"); err != nil { return 0, err } defer os.RemoveAll(aufsTempdir) } if err := createTarFile(filepath.Join(aufsTempdir, basename), dest, hdr, tr, true); err != nil { return 0, err } } continue } path := filepath.Join(dest, hdr.Name) rel, err := filepath.Rel(dest, path) if err != nil { return 0, err } if strings.HasPrefix(rel, "../") { return 0, breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest)) } base := filepath.Base(path) if strings.HasPrefix(base, ".wh.") { originalBase := base[len(".wh."):] originalPath := filepath.Join(filepath.Dir(path), originalBase) if err := os.RemoveAll(originalPath); err != nil { return 0, err } } else { // If path exits we almost always just want to remove and replace it. // The only exception is when it is a directory *and* the file from // the layer is also a directory. Then we want to merge them (i.e. // just apply the metadata from the layer). if fi, err := os.Lstat(path); err == nil { if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) { if err := os.RemoveAll(path); err != nil { return 0, err } } } trBuf.Reset(tr) srcData := io.Reader(trBuf) srcHdr := hdr // Hard links into /.wh..wh.plnk don't work, as we don't extract that directory, so // we manually retarget these into the temporary files we extracted them into if hdr.Typeflag == tar.TypeLink && strings.HasPrefix(filepath.Clean(hdr.Linkname), ".wh..wh.plnk") { linkBasename := filepath.Base(hdr.Linkname) srcHdr = aufsHardlinks[linkBasename] if srcHdr == nil { return 0, fmt.Errorf("Invalid aufs hardlink") } tmpFile, err := os.Open(filepath.Join(aufsTempdir, linkBasename)) if err != nil { return 0, err } defer tmpFile.Close() srcData = tmpFile } if err := createTarFile(path, dest, srcHdr, srcData, true); err != nil { return 0, err } // Directory mtimes must be handled at the end to avoid further // file creation in them to modify the directory mtime if hdr.Typeflag == tar.TypeDir { dirs = append(dirs, hdr) } } } for _, hdr := range dirs { path := filepath.Join(dest, hdr.Name) ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)} if err := syscall.UtimesNano(path, ts); err != nil { return 0, err } } return size, nil }
func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) error { tr := tar.NewReader(decompressedArchive) trBuf := pools.BufioReader32KPool.Get(nil) defer pools.BufioReader32KPool.Put(trBuf) var dirs []*tar.Header // Iterate through the files in the archive. loop: for { hdr, err := tr.Next() if err == io.EOF { // end of tar archive break } if err != nil { return err } // Normalize name, for safety and for a simple is-root check // This keeps "../" as-is, but normalizes "/../" to "/" hdr.Name = filepath.Clean(hdr.Name) for _, exclude := range options.ExcludePatterns { if strings.HasPrefix(hdr.Name, exclude) { continue loop } } if !strings.HasSuffix(hdr.Name, "/") { // Not the root directory, ensure that the parent directory exists parent := filepath.Dir(hdr.Name) parentPath := filepath.Join(dest, parent) if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { err = system.MkdirAll(parentPath, 0777) if err != nil { return err } } } path := filepath.Join(dest, hdr.Name) rel, err := filepath.Rel(dest, path) if err != nil { return err } if strings.HasPrefix(rel, "../") { return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest)) } // If path exits we almost always just want to remove and replace it // The only exception is when it is a directory *and* the file from // the layer is also a directory. Then we want to merge them (i.e. // just apply the metadata from the layer). if fi, err := os.Lstat(path); err == nil { if fi.IsDir() && hdr.Name == "." { continue } if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) { if err := os.RemoveAll(path); err != nil { return err } } } trBuf.Reset(tr) if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown); err != nil { return err } // Directory mtimes must be handled at the end to avoid further // file creation in them to modify the directory mtime if hdr.Typeflag == tar.TypeDir { dirs = append(dirs, hdr) } } for _, hdr := range dirs { path := filepath.Join(dest, hdr.Name) ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)} if err := syscall.UtimesNano(path, ts); err != nil { return err } } return nil }