// Writes the given file into tarfn func writeItem(tarfn string, fn string) (pos uint64, err error) { pos = 0 sfh, err := os.Open(fn) defer sfh.Close() if sfi, err := sfh.Stat(); err == nil { hdr := new(tar.Header) hdr.Name = sfi.Name() hdr.Size = sfi.Size() hdr.Mode = int64(sfi.Mode().Perm()) hdr.ModTime = sfi.ModTime() var ( tw *tar.Writer f ReadWriteSeekCloser ) if tw, f, pos, err = OpenForAppend(tarfn); err == nil { defer f.Close() defer tw.Close() //LIFO if err := tw.WriteHeader(hdr); err == nil { if zw, err := gzip.NewWriterLevel(tw, flate.BestCompression); err == nil { defer zw.Close() _, err = io.Copy(zw, sfh) } } } } return }
//WriteGopathSrc tars up files under gopath src func WriteGopathSrc(tw *tar.Writer, excludeDir string) error { gopath := os.Getenv("GOPATH") // Only take the first element of GOPATH gopath = filepath.SplitList(gopath)[0] rootDirectory := filepath.Join(gopath, "src") vmLogger.Infof("rootDirectory = %s", rootDirectory) if err := WriteFolderToTarPackage(tw, rootDirectory, excludeDir, includeFileTypes, nil); err != nil { vmLogger.Errorf("Error writing folder to tar package %s", err) return err } // Add the certificates to tar if viper.GetBool("peer.tls.enabled") { err := WriteFileToPackage(viper.GetString("peer.tls.cert.file"), "src/certs/cert.pem", tw) if err != nil { return fmt.Errorf("Error writing cert file to package: %s", err) } } // Write the tar file out if err := tw.Close(); err != nil { return err } //ioutil.WriteFile("/tmp/chaincode_deployment.tar", inputbuf.Bytes(), 0644) return nil }
//WriteGopathSrc tars up files under gopath src func WriteGopathSrc(tw *tar.Writer, excludeDir string) error { gopath := os.Getenv("GOPATH") if strings.LastIndex(gopath, "/") == len(gopath)-1 { gopath = gopath[:len(gopath)] } rootDirectory := fmt.Sprintf("%s%s%s", os.Getenv("GOPATH"), string(os.PathSeparator), "src") vmLogger.Info("rootDirectory = %s", rootDirectory) //append "/" if necessary if excludeDir != "" && strings.LastIndex(excludeDir, "/") < len(excludeDir)-1 { excludeDir = excludeDir + "/" } rootDirLen := len(rootDirectory) walkFn := func(path string, info os.FileInfo, err error) error { // If path includes .git, ignore if strings.Contains(path, ".git") { return nil } if info.Mode().IsDir() { return nil } //exclude any files with excludeDir prefix. They should already be in the tar if excludeDir != "" && strings.Index(path, excludeDir) == rootDirLen+1 { //1 for "/" return nil } // Because of scoping we can reference the external rootDirectory variable if len(path[rootDirLen:]) == 0 { return nil } newPath := fmt.Sprintf("src%s", path[rootDirLen:]) //newPath := path[len(rootDirectory):] vmLogger.Debug("writing file %s to %s", path, newPath) err = WriteFileToPackage(path, newPath, tw) if err != nil { return fmt.Errorf("Error writing file to package: %s", err) } return nil } if err := filepath.Walk(rootDirectory, walkFn); err != nil { vmLogger.Info("Error walking rootDirectory: %s", err) return err } // Write the tar file out if err := tw.Close(); err != nil { return err } //ioutil.WriteFile("/tmp/chaincode_deployment.tar", inputbuf.Bytes(), 0644) return nil }
func MakeTarball(tarname string, fnames []string) error { log.Printf("tarring %d entrires to %s ...", len(fnames), tarname) tarfile, err := os.Create(tarname) if err != nil { return err } defer tarfile.Close() var tarwriter *tar.Writer if strings.HasSuffix(tarname, ".gz") { zipper := gzip.NewWriter(tarfile) defer zipper.Close() tarwriter = tar.NewWriter(zipper) /* } else if strings.HasSuffix(tarname, ".xz") { p := xz.WriterDefaults p.DictCap = 1 << 24 zipper, err := xz.NewWriterParams(tarfile, &p) //xz.NewWriter(tarfile) if err != nil { return err } defer zipper.Close() tarwriter = tar.NewWriter(zipper) */ } else { tarwriter = tar.NewWriter(tarfile) } defer tarwriter.Close() for _, fname := range fnames { realm, ts, good := util.Parse_FName(fname) if !good { log.Printf("warning: skip ill-named file '%s'", fname) continue // skip } data, err := util.Load(fname) if err != nil { return err } hdr := new(tar.Header) hdr.Name = util.Make_FName(realm, ts, false) hdr.Size = int64(len(data)) hdr.ModTime = ts hdr.Mode = 0644 err = tarwriter.WriteHeader(hdr) if err != nil { return err } log.Printf("tar %d bytes for file %s", hdr.Size, hdr.Name) _, err = tarwriter.Write(data) if err != nil { return err } } log.Printf("%s tarred without errors", tarname) return nil }
func tarball(path string, t *tar.Writer) { path = filepath.Clean(path) info, err := os.Stat(path) if err != nil { return } if t == nil { r, w := io.Pipe() t = tar.NewWriter(w) f, err := os.Create(fmt.Sprintf("%s.tgz", path)) if err != nil { panic(err) } z := gzip.NewWriter(f) defer f.Close() defer z.Close() defer r.Close() defer w.Close() defer t.Close() go io.Copy(z, r) } h, err := tar.FileInfoHeader(info, path) // FIXME what is link arg supposed to be? if err != nil { return } h.Name = path t.WriteHeader(h) if info.IsDir() { infos, err := ioutil.ReadDir(path) if err != nil { return } for _, info := range infos { tgz(fmt.Sprintf("%v/%v", path, info.Name()), t) } } else { f, err := os.Open(path) if err != nil { panic(err) } defer f.Close() if _, err := io.Copy(t, f); err != nil { panic(err) } } }
//Package Java project to tar file from the source path func WriteJavaProjectToPackage(tw *tar.Writer, srcPath string) error { vmLogger.Debugf("Packaging Java project from path %s", srcPath) if err := WriteFolderToTarPackage(tw, srcPath, "", nil, javaExcludeFileTypes); err != nil { vmLogger.Errorf("Error writing folder to tar package %s", err) return err } // Write the tar file out if err := tw.Close(); err != nil { return err } return nil }
func writeTarball(tarball *tar.Writer, archive io.Reader, fileSize int64, name string) error { header := tar.Header{ Name: name, Mode: 0666, Size: fileSize, } tarball.WriteHeader(&header) n, err := io.Copy(tarball, archive) if err != nil { return err } if n != fileSize { return errors.New("upload-deploy: short-write copying to tarball") } return tarball.Close() }
// Read from source tar stream and write training and test to given tar writers func (d DatasetSplitter) transform(source *tar.Reader, train, test *tar.Writer) error { splitter := d.splitter(train, test) for { hdr, err := source.Next() if err == io.EOF { // end of tar archive break } if err != nil { return err } tw := splitter(hdr.Name) if err := tw.WriteHeader(hdr); err != nil { return err } _, err = io.Copy(tw, source) if err != nil { return err } } // close writers if err := train.Close(); err != nil { errMsg := fmt.Errorf("Error closing tar writer: %v", err) logg.LogError(errMsg) return err } if err := test.Close(); err != nil { errMsg := fmt.Errorf("Error closing tar reader: %v", err) logg.LogError(errMsg) return err } return nil }
func makeLayer(t *testing.T, compression archive.Compression) (ddigest.Digest, int64, int64, []byte) { var cwriter io.WriteCloser var uncompressed *ioutils.WriteCounter var twriter *tar.Writer preader, pwriter := io.Pipe() tbuffer := bytes.Buffer{} if compression != archive.Uncompressed { compressor, err := archive.CompressStream(pwriter, compression) if err != nil { t.Fatalf("Error compressing layer: %v", err) } cwriter = compressor uncompressed = ioutils.NewWriteCounter(cwriter) } else { uncompressed = ioutils.NewWriteCounter(pwriter) } twriter = tar.NewWriter(uncompressed) buf := make([]byte, layerSize) n, err := rand.Read(buf) if err != nil { t.Fatalf("Error reading tar data: %v", err) } if n != len(buf) { t.Fatalf("Short read reading tar data: %d < %d", n, len(buf)) } for i := 1024; i < 2048; i++ { buf[i] = 0 } go func() { defer pwriter.Close() if cwriter != nil { defer cwriter.Close() } defer twriter.Close() err := twriter.WriteHeader(&tar.Header{ Name: "/random-single-file", Mode: 0600, Size: int64(len(buf)), ModTime: time.Now(), AccessTime: time.Now(), ChangeTime: time.Now(), Typeflag: tar.TypeReg, }) if err != nil { t.Fatalf("Error writing tar header: %v", err) } n, err := twriter.Write(buf) if err != nil { t.Fatalf("Error writing tar header: %v", err) } if n != len(buf) { t.Fatalf("Short write writing tar header: %d < %d", n, len(buf)) } }() _, err = io.Copy(&tbuffer, preader) if err != nil { t.Fatalf("Error reading layer tar: %v", err) } sum := ddigest.SHA256.FromBytes(tbuffer.Bytes()) return sum, uncompressed.Count, int64(tbuffer.Len()), tbuffer.Bytes() }
// http://blog.ralch.com/tutorial/golang-working-with-tar-and-gzip/ // This function tars the source file/dir. If a tar.Writer is detected // it'll use that as the tar.Writer instead of generating it's own (this is useful // when you want to have selective tarring, such as modules). // Also if tarName is provided it'll use that // returns the name of the target file (if created) and any errors func tarit(source, tarName string, tb *tar.Writer) error { var tarball *tar.Writer target := tarName if tb != nil { tarball = tb } else { filename := filepath.Base(source) if target == "" { target = fmt.Sprintf("%s.tar", filename) } tarfile, err := os.Create(target) if err != nil { return HenchErr(err, nil, "While creating target") } defer tarfile.Close() tarball = tar.NewWriter(tarfile) defer tarball.Close() } info, err := os.Stat(source) if err != nil { return nil } var baseDir string if info.IsDir() { baseDir = filepath.Base(source) } return filepath.Walk(source, func(path string, info os.FileInfo, err error) error { if err != nil { return HenchErr(err, map[string]interface{}{ "path": path, }, "While walking") } header, err := tar.FileInfoHeader(info, info.Name()) if err != nil { return HenchErr(err, map[string]interface{}{ "file": path, "solution": "Golang specific tar package. Submit an issue starting with TAR HEADER", }, "Adding info to tar header") } if baseDir != "" { header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source)) } if err := tarball.WriteHeader(header); err != nil { return HenchErr(err, map[string]interface{}{ "file": path, "solution": "Golang specific tar package. Submit an issue starting with TARBALL", }, "Writing header to tar") } if info.IsDir() { return nil } file, err := os.Open(path) if err != nil { return HenchErr(err, nil, "") } defer file.Close() if _, err := io.Copy(tarball, file); err != nil { return HenchErr(err, map[string]interface{}{ "file": path, "solution": "make sure file exists, correct permissions, or is not corrupted", }, "") } return nil }) }
func writeGopathSrc(tw *tar.Writer, excludeDir string) error { gopath := os.Getenv("GOPATH") if strings.LastIndex(gopath, "/") == len(gopath)-1 { gopath = gopath[:len(gopath)] } rootDirectory := fmt.Sprintf("%s%s%s", os.Getenv("GOPATH"), string(os.PathSeparator), "src") vmLogger.Info("rootDirectory = %s", rootDirectory) //append "/" if necessary if excludeDir != "" && strings.LastIndex(excludeDir, "/") < len(excludeDir)-1 { excludeDir = excludeDir + "/" } rootDirLen := len(rootDirectory) walkFn := func(path string, info os.FileInfo, err error) error { // If path includes .git, ignore if strings.Contains(path, ".git") { return nil } if info.Mode().IsDir() { return nil } //exclude any files with excludeDir prefix. They should already be in the tar if excludeDir != "" && strings.Index(path, excludeDir) == rootDirLen+1 { //1 for "/" return nil } // Because of scoping we can reference the external rootDirectory variable newPath := fmt.Sprintf("src%s", path[rootDirLen:]) //newPath := path[len(rootDirectory):] if len(newPath) == 0 { return nil } fr, err := os.Open(path) if err != nil { return err } defer fr.Close() h, err := tar.FileInfoHeader(info, newPath) if err != nil { vmLogger.Error(fmt.Sprintf("Error getting FileInfoHeader: %s", err)) return err } //Let's take the variance out of the tar, make headers identical everywhere by using zero time var zeroTime time.Time h.AccessTime = zeroTime h.ModTime = zeroTime h.ChangeTime = zeroTime h.Name = newPath if err = tw.WriteHeader(h); err != nil { vmLogger.Error(fmt.Sprintf("Error writing header: %s", err)) return err } if _, err := io.Copy(tw, fr); err != nil { return err } return nil } if err := filepath.Walk(rootDirectory, walkFn); err != nil { vmLogger.Info("Error walking rootDirectory: %s", err) return err } // Write the tar file out if err := tw.Close(); err != nil { return err } //ioutil.WriteFile("/tmp/chaincode_deployment.tar", inputbuf.Bytes(), 0644) return nil }
//Given a writer to a tar stream, import from the filesystem. func ImportFromFilesystem(w *tar.Writer, basePath string) error { defer w.Close() //Get a sane path. tempDir, err := filepath.Abs(filepath.Clean(basePath)) if err != nil { return err } base, err := filepath.EvalSymlinks(tempDir) if err != nil { return err } //Open the metadata file metaFilename := filepath.Join(base, ".guitar") metafile, err := os.Open(metaFilename) defer metafile.Close() if err != nil { return err } buffer := bufio.NewReader(metafile) //Cannot hard link to files that don't exist yet. //We buffer them here, and apply them last. hardLinks := make([]*tar.Header, 0) //Read each line Println("Importing files...") for { //Get bytes until next newline line, err := buffer.ReadBytes('\n') if err == io.EOF { break //Last line has been read; done importing } else if err != nil { return err } //Decode that line's JSON var hdr *format.Header err = json.Unmarshal(line, &hdr) if err != nil { return err } //Convert to tar header format, write to the stream header, err := format.Import(hdr) if err != nil { return err } //Take action on various file types switch header.Typeflag { case tar.TypeLink: //hard link hardLinks = append(hardLinks, header) //save for later case tar.TypeReg, tar.TypeRegA: //regular file filename := filepath.Join(base, header.Name) //Open file, get size file, err := os.Open(filename) if err != nil { return err } info, err := file.Stat() if err != nil { return err } //Set header size header.Size = info.Size() //Write the header err = w.WriteHeader(header) if err != nil { return err } //Write the file _, err = io.Copy(w, file) if err != nil { return Errorf("Could not write file " + filename + ": " + err.Error()) } file.Close() default: //everything //Write the header err = w.WriteHeader(header) if err != nil { return err } } } //Write all the hardlinks for _, hdr := range hardLinks { err = w.WriteHeader(hdr) if err != nil { return err } } return nil }
//WriteGopathSrc tars up files under gopath src func WriteGopathSrc(tw *tar.Writer, excludeDir string) error { gopath := os.Getenv("GOPATH") // Only take the first element of GOPATH gopath = filepath.SplitList(gopath)[0] rootDirectory := filepath.Join(gopath, "src") vmLogger.Infof("rootDirectory = %s", rootDirectory) //append "/" if necessary if excludeDir != "" && strings.LastIndex(excludeDir, "/") < len(excludeDir)-1 { excludeDir = excludeDir + "/" } rootDirLen := len(rootDirectory) walkFn := func(path string, info os.FileInfo, err error) error { // If path includes .git, ignore if strings.Contains(path, ".git") { return nil } if info.Mode().IsDir() { return nil } //exclude any files with excludeDir prefix. They should already be in the tar if excludeDir != "" && strings.Index(path, excludeDir) == rootDirLen+1 { //1 for "/" return nil } // Because of scoping we can reference the external rootDirectory variable if len(path[rootDirLen:]) == 0 { return nil } // we only want 'fileTypes' source files at this point ext := filepath.Ext(path) if _, ok := fileTypes[ext]; ok != true { return nil } newPath := fmt.Sprintf("src%s", path[rootDirLen:]) //newPath := path[len(rootDirectory):] err = WriteFileToPackage(path, newPath, tw) if err != nil { return fmt.Errorf("Error writing file to package: %s", err) } return nil } if err := filepath.Walk(rootDirectory, walkFn); err != nil { vmLogger.Infof("Error walking rootDirectory: %s", err) return err } // Write the tar file out if err := tw.Close(); err != nil { return err } //ioutil.WriteFile("/tmp/chaincode_deployment.tar", inputbuf.Bytes(), 0644) return nil }