func TestContainerApiGetExport(t *testing.T) {
name := "exportcontainer"
runCmd := exec.Command(dockerBinary, "run", "--name", name, "busybox", "touch", "/test")
out, _, err := runCommandWithOutput(runCmd)
if err != nil {
t.Fatalf("Error on container creation: %v, output: %q", err, out)
}
body, err := sockRequest("GET", "/containers/"+name+"/export", nil)
if err != nil {
t.Fatalf("GET containers/export sockRequest failed: %v", err)
}
found := false
for tarReader := tar.NewReader(bytes.NewReader(body)); ; {
h, err := tarReader.Next()
if err != nil {
if err == io.EOF {
break
}
t.Fatal(err)
}
if h.Name == "test" {
found = true
break
}
}
if !found {
t.Fatalf("The created test file has not been found in the exported image")
}
deleteAllContainers()
logDone("container REST API - check GET containers/export")
}
// Some tar have both GNU specific (huge uid) and Ustar specific (long name) things.
// Not supposed to happen (should use PAX instead of Ustar for long name) but it does and it should still work.
func TestUntarUstarGnuConflict(t *testing.T) {
f, err := os.Open("testdata/broken.tar")
if err != nil {
t.Fatal(err)
}
found := false
tr := tar.NewReader(f)
// Iterate through the files in the archive.
for {
hdr, err := tr.Next()
if err == io.EOF {
// end of tar archive
break
}
if err != nil {
t.Fatal(err)
}
if hdr.Name == "root/.cpanm/work/1395823785.24209/Plack-1.0030/blib/man3/Plack::Middleware::LighttpdScriptNameFix.3pm" {
found = true
break
}
}
if !found {
t.Fatalf("%s not found in the archive", "root/.cpanm/work/1395823785.24209/Plack-1.0030/blib/man3/Plack::Middleware::LighttpdScriptNameFix.3pm")
}
}
// SetTar adds data to db from a tar strema decoded from `src`.
// Raw data is stored at the key `_fs_data/', and metadata in a
// separate key '_fs_metadata'.
func (db *DB) SetTar(src io.Reader) error {
tr := tar.NewReader(src)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
fmt.Printf("[META] %s\n", hdr.Name)
metaBlob, err := headerReader(hdr)
if err != nil {
return err
}
fmt.Printf(" ---> storing metadata in %s\n", metaPath(hdr.Name))
if err := db.SetStream(metaPath(hdr.Name), metaBlob); err != nil {
continue
}
// FIXME: git can carry symlinks as well
if hdr.Typeflag == tar.TypeReg {
fmt.Printf("[DATA] %s %d bytes\n", hdr.Name, hdr.Size)
if err := db.SetStream(path.Join("_fs_data", hdr.Name), tr); err != nil {
continue
}
}
}
return nil
}
// GetTar generates a tar stream frmo the contents of db, and streams
// it to `dst`.
func (db *DB) GetTar(dst io.Writer) error {
tw := tar.NewWriter(dst)
defer tw.Close()
// Walk the data tree
return db.Walk(DataTree, func(name string, obj git.Object) error {
fmt.Fprintf(os.Stderr, "Generating tar entry for '%s'...\n", name)
metaBlob, err := db.Get(metaPath(name))
if err != nil {
return err
}
tr := tar.NewReader(bytes.NewReader([]byte(metaBlob)))
hdr, err := tr.Next()
if err != nil {
return err
}
// Write the reconstituted tar header+content
if err := tw.WriteHeader(hdr); err != nil {
return err
}
if blob, isBlob := obj.(*git.Blob); isBlob {
fmt.Fprintf(os.Stderr, "--> writing %d bytes for blob %s\n", hdr.Size, hdr.Name)
if _, err := tw.Write(blob.Contents()[:hdr.Size]); err != nil {
return err
}
}
return nil
})
return nil
}
func renderSumForHeader(v Version, h *tar.Header, data []byte) (string, error) {
buf := bytes.NewBuffer(nil)
// first build our test tar
tw := tar.NewWriter(buf)
if err := tw.WriteHeader(h); err != nil {
return "", err
}
if _, err := tw.Write(data); err != nil {
return "", err
}
tw.Close()
ts, err := NewTarSum(buf, true, v)
if err != nil {
return "", err
}
tr := tar.NewReader(ts)
for {
hdr, err := tr.Next()
if hdr == nil || err == io.EOF {
// Signals the end of the archive.
break
}
if err != nil {
return "", err
}
if _, err = io.Copy(ioutil.Discard, tr); err != nil {
return "", err
}
}
return ts.Sum(nil), nil
}
func IsArchive(header []byte) bool {
compression := DetectCompression(header)
if compression != Uncompressed {
return true
}
r := tar.NewReader(bytes.NewBuffer(header))
_, err := r.Next()
return err == nil
}
func TestPostContainersCopy(t *testing.T) {
eng := NewTestEngine(t)
defer mkDaemonFromEngine(eng, t).Nuke()
// Create a container and remove a file
containerID := createTestContainer(eng,
&runconfig.Config{
Image: unitTestImageID,
Cmd: []string{"touch", "/test.txt"},
},
t,
)
containerRun(eng, containerID, t)
r := httptest.NewRecorder()
var copyData engine.Env
copyData.Set("Resource", "/test.txt")
copyData.Set("HostPath", ".")
jsonData := bytes.NewBuffer(nil)
if err := copyData.Encode(jsonData); err != nil {
t.Fatal(err)
}
req, err := http.NewRequest("POST", "/containers/"+containerID+"/copy", jsonData)
if err != nil {
t.Fatal(err)
}
req.Header.Add("Content-Type", "application/json")
if err := server.ServeRequest(eng, api.APIVERSION, r, req); err != nil {
t.Fatal(err)
}
assertHttpNotError(r, t)
if r.Code != http.StatusOK {
t.Fatalf("%d OK expected, received %d\n", http.StatusOK, r.Code)
}
found := false
for tarReader := tar.NewReader(r.Body); ; {
h, err := tarReader.Next()
if err != nil {
if err == io.EOF {
break
}
t.Fatal(err)
}
if h.Name == "test.txt" {
found = true
break
}
}
if !found {
t.Fatalf("The created test file has not been found in the copied output")
}
}
func ListTar(f io.Reader) ([]string, error) {
tr := tar.NewReader(f)
var entries []string
for {
th, err := tr.Next()
if err == io.EOF {
// end of tar archive
return entries, nil
}
if err != nil {
return entries, err
}
entries = append(entries, th.Name)
}
}
func TestGetContainersExport(t *testing.T) {
eng := NewTestEngine(t)
defer mkDaemonFromEngine(eng, t).Nuke()
// Create a container and remove a file
containerID := createTestContainer(eng,
&runconfig.Config{
Image: unitTestImageID,
Cmd: []string{"touch", "/test"},
},
t,
)
containerRun(eng, containerID, t)
r := httptest.NewRecorder()
req, err := http.NewRequest("GET", "/containers/"+containerID+"/export", nil)
if err != nil {
t.Fatal(err)
}
if err := server.ServeRequest(eng, api.APIVERSION, r, req); err != nil {
t.Fatal(err)
}
assertHttpNotError(r, t)
if r.Code != http.StatusOK {
t.Fatalf("%d OK expected, received %d\n", http.StatusOK, r.Code)
}
found := false
for tarReader := tar.NewReader(r.Body); ; {
h, err := tarReader.Next()
if err != nil {
if err == io.EOF {
break
}
t.Fatal(err)
}
if h.Name == "test" {
found = true
break
}
}
if !found {
t.Fatalf("The created test file has not been found in the exported image")
}
}
func (ts *tarSum) initTarSum() error {
ts.bufTar = bytes.NewBuffer([]byte{})
ts.bufWriter = bytes.NewBuffer([]byte{})
ts.tarR = tar.NewReader(ts.Reader)
ts.tarW = tar.NewWriter(ts.bufTar)
if !ts.DisableCompression {
ts.writer = gzip.NewWriter(ts.bufWriter)
} else {
ts.writer = &nopCloseFlusher{Writer: ts.bufWriter}
}
if ts.tHash == nil {
ts.tHash = DefaultTHash
}
ts.h = ts.tHash.Hash()
ts.h.Reset()
ts.first = true
ts.sums = FileInfoSums{}
return nil
}
func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) error {
tr := tar.NewReader(decompressedArchive)
trBuf := pools.BufioReader32KPool.Get(nil)
defer pools.BufioReader32KPool.Put(trBuf)
var dirs []*tar.Header
// Iterate through the files in the archive.
loop:
for {
hdr, err := tr.Next()
if err == io.EOF {
// end of tar archive
break
}
if err != nil {
return err
}
// Normalize name, for safety and for a simple is-root check
// This keeps "../" as-is, but normalizes "/../" to "/"
hdr.Name = filepath.Clean(hdr.Name)
for _, exclude := range options.Excludes {
if strings.HasPrefix(hdr.Name, exclude) {
continue loop
}
}
if !strings.HasSuffix(hdr.Name, "/") {
// Not the root directory, ensure that the parent directory exists
parent := filepath.Dir(hdr.Name)
parentPath := filepath.Join(dest, parent)
if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {
err = os.MkdirAll(parentPath, 0777)
if err != nil {
return err
}
}
}
path := filepath.Join(dest, hdr.Name)
rel, err := filepath.Rel(dest, path)
if err != nil {
return err
}
if strings.HasPrefix(rel, "..") {
return breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest))
}
// If path exits we almost always just want to remove and replace it
// The only exception is when it is a directory *and* the file from
// the layer is also a directory. Then we want to merge them (i.e.
// just apply the metadata from the layer).
if fi, err := os.Lstat(path); err == nil {
if fi.IsDir() && hdr.Name == "." {
continue
}
if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
if err := os.RemoveAll(path); err != nil {
return err
}
}
}
trBuf.Reset(tr)
if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown); err != nil {
return err
}
// Directory mtimes must be handled at the end to avoid further
// file creation in them to modify the directory mtime
if hdr.Typeflag == tar.TypeDir {
dirs = append(dirs, hdr)
}
}
for _, hdr := range dirs {
path := filepath.Join(dest, hdr.Name)
ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)}
if err := syscall.UtimesNano(path, ts); err != nil {
return err
}
}
return nil
}
// ApplyLayer parses a diff in the standard layer format from `layer`, and
// applies it to the directory `dest`.
func ApplyLayer(dest string, layer ArchiveReader) error {
dest = filepath.Clean(dest)
// We need to be able to set any perms
oldmask := syscall.Umask(0)
defer syscall.Umask(oldmask)
layer, err := DecompressStream(layer)
if err != nil {
return err
}
tr := tar.NewReader(layer)
trBuf := pools.BufioReader32KPool.Get(tr)
defer pools.BufioReader32KPool.Put(trBuf)
var dirs []*tar.Header
aufsTempdir := ""
aufsHardlinks := make(map[string]*tar.Header)
// Iterate through the files in the archive.
for {
hdr, err := tr.Next()
if err == io.EOF {
// end of tar archive
break
}
if err != nil {
return err
}
// Normalize name, for safety and for a simple is-root check
hdr.Name = filepath.Clean(hdr.Name)
if !strings.HasSuffix(hdr.Name, "/") {
// Not the root directory, ensure that the parent directory exists.
// This happened in some tests where an image had a tarfile without any
// parent directories.
parent := filepath.Dir(hdr.Name)
parentPath := filepath.Join(dest, parent)
if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {
err = os.MkdirAll(parentPath, 0600)
if err != nil {
return err
}
}
}
// Skip AUFS metadata dirs
if strings.HasPrefix(hdr.Name, ".wh..wh.") {
// Regular files inside /.wh..wh.plnk can be used as hardlink targets
// We don't want this directory, but we need the files in them so that
// such hardlinks can be resolved.
if strings.HasPrefix(hdr.Name, ".wh..wh.plnk") && hdr.Typeflag == tar.TypeReg {
basename := filepath.Base(hdr.Name)
aufsHardlinks[basename] = hdr
if aufsTempdir == "" {
if aufsTempdir, err = ioutil.TempDir("", "dockerplnk"); err != nil {
return err
}
defer os.RemoveAll(aufsTempdir)
}
if err := createTarFile(filepath.Join(aufsTempdir, basename), dest, hdr, tr, true); err != nil {
return err
}
}
continue
}
path := filepath.Join(dest, hdr.Name)
base := filepath.Base(path)
// Prevent symlink breakout
if !strings.HasPrefix(path, dest) {
return breakoutError(fmt.Errorf("%q is outside of %q", path, dest))
}
if strings.HasPrefix(base, ".wh.") {
originalBase := base[len(".wh."):]
originalPath := filepath.Join(filepath.Dir(path), originalBase)
if err := os.RemoveAll(originalPath); err != nil {
return err
}
} else {
// If path exits we almost always just want to remove and replace it.
// The only exception is when it is a directory *and* the file from
// the layer is also a directory. Then we want to merge them (i.e.
// just apply the metadata from the layer).
if fi, err := os.Lstat(path); err == nil {
if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
if err := os.RemoveAll(path); err != nil {
return err
}
}
}
trBuf.Reset(tr)
srcData := io.Reader(trBuf)
srcHdr := hdr
// Hard links into /.wh..wh.plnk don't work, as we don't extract that directory, so
// we manually retarget these into the temporary files we extracted them into
if hdr.Typeflag == tar.TypeLink && strings.HasPrefix(filepath.Clean(hdr.Linkname), ".wh..wh.plnk") {
linkBasename := filepath.Base(hdr.Linkname)
srcHdr = aufsHardlinks[linkBasename]
if srcHdr == nil {
return fmt.Errorf("Invalid aufs hardlink")
}
tmpFile, err := os.Open(filepath.Join(aufsTempdir, linkBasename))
if err != nil {
return err
}
defer tmpFile.Close()
srcData = tmpFile
}
if err := createTarFile(path, dest, srcHdr, srcData, true); err != nil {
return err
}
// Directory mtimes must be handled at the end to avoid further
// file creation in them to modify the directory mtime
if hdr.Typeflag == tar.TypeDir {
dirs = append(dirs, hdr)
}
}
}
for _, hdr := range dirs {
path := filepath.Join(dest, hdr.Name)
ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)}
if err := syscall.UtimesNano(path, ts); err != nil {
return err
}
}
return nil
}
func (ts *tarSum) Read(buf []byte) (int, error) {
if ts.gz == nil {
ts.bufTar = bytes.NewBuffer([]byte{})
ts.bufGz = bytes.NewBuffer([]byte{})
ts.tarR = tar.NewReader(ts.Reader)
ts.tarW = tar.NewWriter(ts.bufTar)
if !ts.DisableCompression {
ts.gz = gzip.NewWriter(ts.bufGz)
} else {
ts.gz = &nopCloseFlusher{Writer: ts.bufGz}
}
ts.h = sha256.New()
ts.h.Reset()
ts.first = true
ts.sums = make(map[string]string)
}
if ts.finished {
return ts.bufGz.Read(buf)
}
if ts.bufData == nil {
switch {
case len(buf) <= buf8K:
ts.bufData = make([]byte, buf8K)
case len(buf) <= buf16K:
ts.bufData = make([]byte, buf16K)
case len(buf) <= buf32K:
ts.bufData = make([]byte, buf32K)
default:
ts.bufData = make([]byte, len(buf))
}
}
buf2 := ts.bufData[:len(buf)-1]
n, err := ts.tarR.Read(buf2)
if err != nil {
if err == io.EOF {
if _, err := ts.h.Write(buf2[:n]); err != nil {
return 0, err
}
if !ts.first {
ts.sums[ts.currentFile] = hex.EncodeToString(ts.h.Sum(nil))
ts.h.Reset()
} else {
ts.first = false
}
currentHeader, err := ts.tarR.Next()
if err != nil {
if err == io.EOF {
if err := ts.gz.Close(); err != nil {
return 0, err
}
ts.finished = true
return n, nil
}
return n, err
}
ts.currentFile = strings.TrimSuffix(strings.TrimPrefix(currentHeader.Name, "./"), "/")
if err := ts.encodeHeader(currentHeader); err != nil {
return 0, err
}
if err := ts.tarW.WriteHeader(currentHeader); err != nil {
return 0, err
}
if _, err := ts.tarW.Write(buf2[:n]); err != nil {
return 0, err
}
ts.tarW.Flush()
if _, err := io.Copy(ts.gz, ts.bufTar); err != nil {
return 0, err
}
ts.gz.Flush()
return ts.bufGz.Read(buf)
}
return n, err
}
// Filling the hash buffer
if _, err = ts.h.Write(buf2[:n]); err != nil {
return 0, err
}
// Filling the tar writter
if _, err = ts.tarW.Write(buf2[:n]); err != nil {
return 0, err
}
ts.tarW.Flush()
// Filling the gz writter
if _, err = io.Copy(ts.gz, ts.bufTar); err != nil {
return 0, err
}
ts.gz.Flush()
return ts.bufGz.Read(buf)
}
// Untar reads a stream of bytes from `archive`, parses it as a tar archive,
// and unpacks it into the directory at `path`.
// The archive may be compressed with one of the following algorithms:
// identity (uncompressed), gzip, bzip2, xz.
// FIXME: specify behavior when target path exists vs. doesn't exist.
func Untar(archive io.Reader, dest string, options *TarOptions) error {
if archive == nil {
return fmt.Errorf("Empty archive")
}
decompressedArchive, err := DecompressStream(archive)
if err != nil {
return err
}
defer decompressedArchive.Close()
tr := tar.NewReader(decompressedArchive)
trBuf := bufio.NewReaderSize(nil, trBufSize)
var dirs []*tar.Header
// Iterate through the files in the archive.
for {
hdr, err := tr.Next()
if err == io.EOF {
// end of tar archive
break
}
if err != nil {
return err
}
// Normalize name, for safety and for a simple is-root check
hdr.Name = filepath.Clean(hdr.Name)
if !strings.HasSuffix(hdr.Name, "/") {
// Not the root directory, ensure that the parent directory exists
parent := filepath.Dir(hdr.Name)
parentPath := filepath.Join(dest, parent)
if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {
err = os.MkdirAll(parentPath, 0777)
if err != nil {
return err
}
}
}
path := filepath.Join(dest, hdr.Name)
// If path exits we almost always just want to remove and replace it
// The only exception is when it is a directory *and* the file from
// the layer is also a directory. Then we want to merge them (i.e.
// just apply the metadata from the layer).
if fi, err := os.Lstat(path); err == nil {
if fi.IsDir() && hdr.Name == "." {
continue
}
if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
if err := os.RemoveAll(path); err != nil {
return err
}
}
}
trBuf.Reset(tr)
if err := createTarFile(path, dest, hdr, trBuf, options == nil || !options.NoLchown); err != nil {
return err
}
// Directory mtimes must be handled at the end to avoid further
// file creation in them to modify the directory mtime
if hdr.Typeflag == tar.TypeDir {
dirs = append(dirs, hdr)
}
}
for _, hdr := range dirs {
path := filepath.Join(dest, hdr.Name)
ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)}
if err := syscall.UtimesNano(path, ts); err != nil {
return err
}
}
return nil
}
