func emptyTarSum(gzip bool) (TarSum, error) { reader, writer := io.Pipe() tarWriter := tar.NewWriter(writer) // Immediately close tarWriter and write-end of the // Pipe in a separate goroutine so we don't block. go func() { tarWriter.Close() writer.Close() }() return NewTarSum(reader, !gzip, Version0) }
func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) { log.Debugf("CopyFileWithTar(%s, %s)", src, dst) srcSt, err := os.Stat(src) if err != nil { return err } if srcSt.IsDir() { return fmt.Errorf("Can't copy a directory") } // Clean up the trailing / if dst[len(dst)-1] == '/' { dst = path.Join(dst, filepath.Base(src)) } // Create the holding directory if necessary if err := os.MkdirAll(filepath.Dir(dst), 0700); err != nil && !os.IsExist(err) { return err } r, w := io.Pipe() errC := promise.Go(func() error { defer w.Close() srcF, err := os.Open(src) if err != nil { return err } defer srcF.Close() hdr, err := tar.FileInfoHeader(srcSt, "") if err != nil { return err } hdr.Name = filepath.Base(dst) tw := tar.NewWriter(w) defer tw.Close() if err := tw.WriteHeader(hdr); err != nil { return err } if _, err := io.Copy(tw, srcF); err != nil { return err } return nil }) defer func() { if er := <-errC; err != nil { err = er } }() return archiver.Untar(r, filepath.Dir(dst), nil) }
// ExportChanges produces an Archive from the provided changes, relative to dir. func ExportChanges(dir string, changes []Change) (Archive, error) { reader, writer := io.Pipe() go func() { ta := &tarAppender{ TarWriter: tar.NewWriter(writer), Buffer: pools.BufioWriter32KPool.Get(nil), SeenFiles: make(map[uint64]string), } // this buffer is needed for the duration of this piped stream defer pools.BufioWriter32KPool.Put(ta.Buffer) // In general we log errors here but ignore them because // during e.g. a diff operation the container can continue // mutating the filesystem and we can see transient errors // from this for _, change := range changes { if change.Kind == ChangeDelete { whiteOutDir := filepath.Dir(change.Path) whiteOutBase := filepath.Base(change.Path) whiteOut := filepath.Join(whiteOutDir, ".wh."+whiteOutBase) timestamp := time.Now() hdr := &tar.Header{ Name: whiteOut[1:], Size: 0, ModTime: timestamp, AccessTime: timestamp, ChangeTime: timestamp, } if err := ta.TarWriter.WriteHeader(hdr); err != nil { log.Debugf("Can't write whiteout header: %s", err) } } else { path := filepath.Join(dir, change.Path) if err := ta.addTarFile(path, change.Path[1:]); err != nil { log.Debugf("Can't add file %s to tar: %s", path, err) } } } // Make sure to check the error on Close. if err := ta.TarWriter.Close(); err != nil { log.Debugf("Can't close layer: %s", err) } if err := writer.Close(); err != nil { log.Debugf("failed close Changes writer: %s", err) } }() return reader, nil }
// make a tar: // * num is the number of files the tar should have // * size is the bytes per file // * isRand is whether the contents of the files should be a random chunk (otherwise it's all zeros) // * realFile will write to a TempFile, instead of an in memory buffer func sizedTar(opts sizedOptions) io.Reader { var ( fh io.ReadWriter err error ) if opts.realFile { fh, err = ioutil.TempFile("", "tarsum") if err != nil { return nil } } else { fh = bytes.NewBuffer([]byte{}) } tarW := tar.NewWriter(fh) for i := int64(0); i < opts.num; i++ { err := tarW.WriteHeader(&tar.Header{ Name: fmt.Sprintf("/testdata%d", i), Mode: 0755, Uid: 0, Gid: 0, Size: opts.size, }) if err != nil { return nil } var rBuf []byte if opts.isRand { rBuf = make([]byte, 8) _, err = rand.Read(rBuf) if err != nil { return nil } } else { rBuf = []byte{0, 0, 0, 0, 0, 0, 0, 0} } for i := int64(0); i < opts.size/int64(8); i++ { tarW.Write(rBuf) } } return fh }
// Generate generates a new archive from the content provided // as input. // // `files` is a sequence of path/content pairs. A new file is // added to the archive for each pair. // If the last pair is incomplete, the file is created with an // empty content. For example: // // Generate("foo.txt", "hello world", "emptyfile") // // The above call will return an archive with 2 files: // * ./foo.txt with content "hello world" // * ./empty with empty content // // FIXME: stream content instead of buffering // FIXME: specify permissions and other archive metadata func Generate(input ...string) (Archive, error) { files := parseStringPairs(input...) buf := new(bytes.Buffer) tw := tar.NewWriter(buf) for _, file := range files { name, content := file[0], file[1] hdr := &tar.Header{ Name: name, Size: int64(len(content)), } if err := tw.WriteHeader(hdr); err != nil { return nil, err } if _, err := tw.Write([]byte(content)); err != nil { return nil, err } } if err := tw.Close(); err != nil { return nil, err } return ioutil.NopCloser(buf), nil }
// TarWithOptions creates an archive from the directory at `path`, only including files whose relative // paths are included in `options.Includes` (if non-nil) or not in `options.Excludes`. func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error) { pipeReader, pipeWriter := io.Pipe() compressWriter, err := CompressStream(pipeWriter, options.Compression) if err != nil { return nil, err } go func() { ta := &tarAppender{ TarWriter: tar.NewWriter(compressWriter), Buffer: pools.BufioWriter32KPool.Get(nil), SeenFiles: make(map[uint64]string), } // this buffer is needed for the duration of this piped stream defer pools.BufioWriter32KPool.Put(ta.Buffer) // In general we log errors here but ignore them because // during e.g. a diff operation the container can continue // mutating the filesystem and we can see transient errors // from this if options.Includes == nil { options.Includes = []string{"."} } var renamedRelFilePath string // For when tar.Options.Name is set for _, include := range options.Includes { filepath.Walk(filepath.Join(srcPath, include), func(filePath string, f os.FileInfo, err error) error { if err != nil { log.Debugf("Tar: Can't stat file %s to tar: %s", srcPath, err) return nil } relFilePath, err := filepath.Rel(srcPath, filePath) if err != nil || (relFilePath == "." && f.IsDir()) { // Error getting relative path OR we are looking // at the root path. Skip in both situations. return nil } skip, err := fileutils.Matches(relFilePath, options.Excludes) if err != nil { log.Debugf("Error matching %s", relFilePath, err) return err } if skip { if f.IsDir() { return filepath.SkipDir } return nil } // Rename the base resource if options.Name != "" && filePath == srcPath+"/"+filepath.Base(relFilePath) { renamedRelFilePath = relFilePath } // Set this to make sure the items underneath also get renamed if options.Name != "" { relFilePath = strings.Replace(relFilePath, renamedRelFilePath, options.Name, 1) } if err := ta.addTarFile(filePath, relFilePath); err != nil { log.Debugf("Can't add file %s to tar: %s", srcPath, err) } return nil }) } // Make sure to check the error on Close. if err := ta.TarWriter.Close(); err != nil { log.Debugf("Can't close tar writer: %s", err) } if err := compressWriter.Close(); err != nil { log.Debugf("Can't close compress writer: %s", err) } if err := pipeWriter.Close(); err != nil { log.Debugf("Can't close pipe writer: %s", err) } }() return pipeReader, nil }