// Writes the given file into tarfn
func writeItem(tarfn string, fn string) (pos uint64, err error) {
pos = 0
sfh, err := os.Open(fn)
defer sfh.Close()
if sfi, err := sfh.Stat(); err == nil {
hdr := new(tar.Header)
hdr.Name = sfi.Name()
hdr.Size = sfi.Size()
hdr.Mode = int64(sfi.Mode().Perm())
hdr.ModTime = sfi.ModTime()
var (
tw *tar.Writer
f ReadWriteSeekCloser
)
if tw, f, pos, err = OpenForAppend(tarfn); err == nil {
defer f.Close()
defer tw.Close() //LIFO
if err := tw.WriteHeader(hdr); err == nil {
if zw, err := gzip.NewWriterLevel(tw, flate.BestCompression); err == nil {
defer zw.Close()
_, err = io.Copy(zw, sfh)
}
}
}
}
return
}
//WriteGopathSrc tars up files under gopath src
func WriteGopathSrc(tw *tar.Writer, excludeDir string) error {
gopath := os.Getenv("GOPATH")
// Only take the first element of GOPATH
gopath = filepath.SplitList(gopath)[0]
rootDirectory := filepath.Join(gopath, "src")
vmLogger.Infof("rootDirectory = %s", rootDirectory)
if err := WriteFolderToTarPackage(tw, rootDirectory, excludeDir, includeFileTypes, nil); err != nil {
vmLogger.Errorf("Error writing folder to tar package %s", err)
return err
}
// Add the certificates to tar
if viper.GetBool("peer.tls.enabled") {
err := WriteFileToPackage(viper.GetString("peer.tls.cert.file"), "src/certs/cert.pem", tw)
if err != nil {
return fmt.Errorf("Error writing cert file to package: %s", err)
}
}
// Write the tar file out
if err := tw.Close(); err != nil {
return err
}
//ioutil.WriteFile("/tmp/chaincode_deployment.tar", inputbuf.Bytes(), 0644)
return nil
}
//WriteGopathSrc tars up files under gopath src
func WriteGopathSrc(tw *tar.Writer, excludeDir string) error {
gopath := os.Getenv("GOPATH")
if strings.LastIndex(gopath, "/") == len(gopath)-1 {
gopath = gopath[:len(gopath)]
}
rootDirectory := fmt.Sprintf("%s%s%s", os.Getenv("GOPATH"), string(os.PathSeparator), "src")
vmLogger.Info("rootDirectory = %s", rootDirectory)
//append "/" if necessary
if excludeDir != "" && strings.LastIndex(excludeDir, "/") < len(excludeDir)-1 {
excludeDir = excludeDir + "/"
}
rootDirLen := len(rootDirectory)
walkFn := func(path string, info os.FileInfo, err error) error {
// If path includes .git, ignore
if strings.Contains(path, ".git") {
return nil
}
if info.Mode().IsDir() {
return nil
}
//exclude any files with excludeDir prefix. They should already be in the tar
if excludeDir != "" && strings.Index(path, excludeDir) == rootDirLen+1 {
//1 for "/"
return nil
}
// Because of scoping we can reference the external rootDirectory variable
if len(path[rootDirLen:]) == 0 {
return nil
}
newPath := fmt.Sprintf("src%s", path[rootDirLen:])
//newPath := path[len(rootDirectory):]
vmLogger.Debug("writing file %s to %s", path, newPath)
err = WriteFileToPackage(path, newPath, tw)
if err != nil {
return fmt.Errorf("Error writing file to package: %s", err)
}
return nil
}
if err := filepath.Walk(rootDirectory, walkFn); err != nil {
vmLogger.Info("Error walking rootDirectory: %s", err)
return err
}
// Write the tar file out
if err := tw.Close(); err != nil {
return err
}
//ioutil.WriteFile("/tmp/chaincode_deployment.tar", inputbuf.Bytes(), 0644)
return nil
}
func MakeTarball(tarname string, fnames []string) error {
log.Printf("tarring %d entrires to %s ...", len(fnames), tarname)
tarfile, err := os.Create(tarname)
if err != nil {
return err
}
defer tarfile.Close()
var tarwriter *tar.Writer
if strings.HasSuffix(tarname, ".gz") {
zipper := gzip.NewWriter(tarfile)
defer zipper.Close()
tarwriter = tar.NewWriter(zipper)
/*
} else if strings.HasSuffix(tarname, ".xz") {
p := xz.WriterDefaults
p.DictCap = 1 << 24
zipper, err := xz.NewWriterParams(tarfile, &p) //xz.NewWriter(tarfile)
if err != nil {
return err
}
defer zipper.Close()
tarwriter = tar.NewWriter(zipper)
*/
} else {
tarwriter = tar.NewWriter(tarfile)
}
defer tarwriter.Close()
for _, fname := range fnames {
realm, ts, good := util.Parse_FName(fname)
if !good {
log.Printf("warning: skip ill-named file '%s'", fname)
continue // skip
}
data, err := util.Load(fname)
if err != nil {
return err
}
hdr := new(tar.Header)
hdr.Name = util.Make_FName(realm, ts, false)
hdr.Size = int64(len(data))
hdr.ModTime = ts
hdr.Mode = 0644
err = tarwriter.WriteHeader(hdr)
if err != nil {
return err
}
log.Printf("tar %d bytes for file %s", hdr.Size, hdr.Name)
_, err = tarwriter.Write(data)
if err != nil {
return err
}
}
log.Printf("%s tarred without errors", tarname)
return nil
}
func tarball(path string, t *tar.Writer) {
path = filepath.Clean(path)
info, err := os.Stat(path)
if err != nil {
return
}
if t == nil {
r, w := io.Pipe()
t = tar.NewWriter(w)
f, err := os.Create(fmt.Sprintf("%s.tgz", path))
if err != nil {
panic(err)
}
z := gzip.NewWriter(f)
defer f.Close()
defer z.Close()
defer r.Close()
defer w.Close()
defer t.Close()
go io.Copy(z, r)
}
h, err := tar.FileInfoHeader(info, path) // FIXME what is link arg supposed to be?
if err != nil {
return
}
h.Name = path
t.WriteHeader(h)
if info.IsDir() {
infos, err := ioutil.ReadDir(path)
if err != nil {
return
}
for _, info := range infos {
tgz(fmt.Sprintf("%v/%v", path, info.Name()), t)
}
} else {
f, err := os.Open(path)
if err != nil {
panic(err)
}
defer f.Close()
if _, err := io.Copy(t, f); err != nil {
panic(err)
}
}
}
//Package Java project to tar file from the source path
func WriteJavaProjectToPackage(tw *tar.Writer, srcPath string) error {
vmLogger.Debugf("Packaging Java project from path %s", srcPath)
if err := WriteFolderToTarPackage(tw, srcPath, "", nil, javaExcludeFileTypes); err != nil {
vmLogger.Errorf("Error writing folder to tar package %s", err)
return err
}
// Write the tar file out
if err := tw.Close(); err != nil {
return err
}
return nil
}
func writeTarball(tarball *tar.Writer, archive io.Reader, fileSize int64, name string) error {
header := tar.Header{
Name: name,
Mode: 0666,
Size: fileSize,
}
tarball.WriteHeader(&header)
n, err := io.Copy(tarball, archive)
if err != nil {
return err
}
if n != fileSize {
return errors.New("upload-deploy: short-write copying to tarball")
}
return tarball.Close()
}
// Read from source tar stream and write training and test to given tar writers
func (d DatasetSplitter) transform(source *tar.Reader, train, test *tar.Writer) error {
splitter := d.splitter(train, test)
for {
hdr, err := source.Next()
if err == io.EOF {
// end of tar archive
break
}
if err != nil {
return err
}
tw := splitter(hdr.Name)
if err := tw.WriteHeader(hdr); err != nil {
return err
}
_, err = io.Copy(tw, source)
if err != nil {
return err
}
}
// close writers
if err := train.Close(); err != nil {
errMsg := fmt.Errorf("Error closing tar writer: %v", err)
logg.LogError(errMsg)
return err
}
if err := test.Close(); err != nil {
errMsg := fmt.Errorf("Error closing tar reader: %v", err)
logg.LogError(errMsg)
return err
}
return nil
}
func makeLayer(t *testing.T, compression archive.Compression) (ddigest.Digest, int64, int64, []byte) {
var cwriter io.WriteCloser
var uncompressed *ioutils.WriteCounter
var twriter *tar.Writer
preader, pwriter := io.Pipe()
tbuffer := bytes.Buffer{}
if compression != archive.Uncompressed {
compressor, err := archive.CompressStream(pwriter, compression)
if err != nil {
t.Fatalf("Error compressing layer: %v", err)
}
cwriter = compressor
uncompressed = ioutils.NewWriteCounter(cwriter)
} else {
uncompressed = ioutils.NewWriteCounter(pwriter)
}
twriter = tar.NewWriter(uncompressed)
buf := make([]byte, layerSize)
n, err := rand.Read(buf)
if err != nil {
t.Fatalf("Error reading tar data: %v", err)
}
if n != len(buf) {
t.Fatalf("Short read reading tar data: %d < %d", n, len(buf))
}
for i := 1024; i < 2048; i++ {
buf[i] = 0
}
go func() {
defer pwriter.Close()
if cwriter != nil {
defer cwriter.Close()
}
defer twriter.Close()
err := twriter.WriteHeader(&tar.Header{
Name: "/random-single-file",
Mode: 0600,
Size: int64(len(buf)),
ModTime: time.Now(),
AccessTime: time.Now(),
ChangeTime: time.Now(),
Typeflag: tar.TypeReg,
})
if err != nil {
t.Fatalf("Error writing tar header: %v", err)
}
n, err := twriter.Write(buf)
if err != nil {
t.Fatalf("Error writing tar header: %v", err)
}
if n != len(buf) {
t.Fatalf("Short write writing tar header: %d < %d", n, len(buf))
}
}()
_, err = io.Copy(&tbuffer, preader)
if err != nil {
t.Fatalf("Error reading layer tar: %v", err)
}
sum := ddigest.SHA256.FromBytes(tbuffer.Bytes())
return sum, uncompressed.Count, int64(tbuffer.Len()), tbuffer.Bytes()
}
// http://blog.ralch.com/tutorial/golang-working-with-tar-and-gzip/
// This function tars the source file/dir. If a tar.Writer is detected
// it'll use that as the tar.Writer instead of generating it's own (this is useful
// when you want to have selective tarring, such as modules).
// Also if tarName is provided it'll use that
// returns the name of the target file (if created) and any errors
func tarit(source, tarName string, tb *tar.Writer) error {
var tarball *tar.Writer
target := tarName
if tb != nil {
tarball = tb
} else {
filename := filepath.Base(source)
if target == "" {
target = fmt.Sprintf("%s.tar", filename)
}
tarfile, err := os.Create(target)
if err != nil {
return HenchErr(err, nil, "While creating target")
}
defer tarfile.Close()
tarball = tar.NewWriter(tarfile)
defer tarball.Close()
}
info, err := os.Stat(source)
if err != nil {
return nil
}
var baseDir string
if info.IsDir() {
baseDir = filepath.Base(source)
}
return filepath.Walk(source,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return HenchErr(err, map[string]interface{}{
"path": path,
}, "While walking")
}
header, err := tar.FileInfoHeader(info, info.Name())
if err != nil {
return HenchErr(err, map[string]interface{}{
"file": path,
"solution": "Golang specific tar package. Submit an issue starting with TAR HEADER",
}, "Adding info to tar header")
}
if baseDir != "" {
header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source))
}
if err := tarball.WriteHeader(header); err != nil {
return HenchErr(err, map[string]interface{}{
"file": path,
"solution": "Golang specific tar package. Submit an issue starting with TARBALL",
}, "Writing header to tar")
}
if info.IsDir() {
return nil
}
file, err := os.Open(path)
if err != nil {
return HenchErr(err, nil, "")
}
defer file.Close()
if _, err := io.Copy(tarball, file); err != nil {
return HenchErr(err, map[string]interface{}{
"file": path,
"solution": "make sure file exists, correct permissions, or is not corrupted",
}, "")
}
return nil
})
}
func writeGopathSrc(tw *tar.Writer, excludeDir string) error {
gopath := os.Getenv("GOPATH")
if strings.LastIndex(gopath, "/") == len(gopath)-1 {
gopath = gopath[:len(gopath)]
}
rootDirectory := fmt.Sprintf("%s%s%s", os.Getenv("GOPATH"), string(os.PathSeparator), "src")
vmLogger.Info("rootDirectory = %s", rootDirectory)
//append "/" if necessary
if excludeDir != "" && strings.LastIndex(excludeDir, "/") < len(excludeDir)-1 {
excludeDir = excludeDir + "/"
}
rootDirLen := len(rootDirectory)
walkFn := func(path string, info os.FileInfo, err error) error {
// If path includes .git, ignore
if strings.Contains(path, ".git") {
return nil
}
if info.Mode().IsDir() {
return nil
}
//exclude any files with excludeDir prefix. They should already be in the tar
if excludeDir != "" && strings.Index(path, excludeDir) == rootDirLen+1 { //1 for "/"
return nil
}
// Because of scoping we can reference the external rootDirectory variable
newPath := fmt.Sprintf("src%s", path[rootDirLen:])
//newPath := path[len(rootDirectory):]
if len(newPath) == 0 {
return nil
}
fr, err := os.Open(path)
if err != nil {
return err
}
defer fr.Close()
h, err := tar.FileInfoHeader(info, newPath)
if err != nil {
vmLogger.Error(fmt.Sprintf("Error getting FileInfoHeader: %s", err))
return err
}
//Let's take the variance out of the tar, make headers identical everywhere by using zero time
var zeroTime time.Time
h.AccessTime = zeroTime
h.ModTime = zeroTime
h.ChangeTime = zeroTime
h.Name = newPath
if err = tw.WriteHeader(h); err != nil {
vmLogger.Error(fmt.Sprintf("Error writing header: %s", err))
return err
}
if _, err := io.Copy(tw, fr); err != nil {
return err
}
return nil
}
if err := filepath.Walk(rootDirectory, walkFn); err != nil {
vmLogger.Info("Error walking rootDirectory: %s", err)
return err
}
// Write the tar file out
if err := tw.Close(); err != nil {
return err
}
//ioutil.WriteFile("/tmp/chaincode_deployment.tar", inputbuf.Bytes(), 0644)
return nil
}
//Given a writer to a tar stream, import from the filesystem.
func ImportFromFilesystem(w *tar.Writer, basePath string) error {
defer w.Close()
//Get a sane path.
tempDir, err := filepath.Abs(filepath.Clean(basePath))
if err != nil {
return err
}
base, err := filepath.EvalSymlinks(tempDir)
if err != nil {
return err
}
//Open the metadata file
metaFilename := filepath.Join(base, ".guitar")
metafile, err := os.Open(metaFilename)
defer metafile.Close()
if err != nil {
return err
}
buffer := bufio.NewReader(metafile)
//Cannot hard link to files that don't exist yet.
//We buffer them here, and apply them last.
hardLinks := make([]*tar.Header, 0)
//Read each line
Println("Importing files...")
for {
//Get bytes until next newline
line, err := buffer.ReadBytes('\n')
if err == io.EOF {
break //Last line has been read; done importing
} else if err != nil {
return err
}
//Decode that line's JSON
var hdr *format.Header
err = json.Unmarshal(line, &hdr)
if err != nil {
return err
}
//Convert to tar header format, write to the stream
header, err := format.Import(hdr)
if err != nil {
return err
}
//Take action on various file types
switch header.Typeflag {
case tar.TypeLink: //hard link
hardLinks = append(hardLinks, header) //save for later
case tar.TypeReg, tar.TypeRegA: //regular file
filename := filepath.Join(base, header.Name)
//Open file, get size
file, err := os.Open(filename)
if err != nil {
return err
}
info, err := file.Stat()
if err != nil {
return err
}
//Set header size
header.Size = info.Size()
//Write the header
err = w.WriteHeader(header)
if err != nil {
return err
}
//Write the file
_, err = io.Copy(w, file)
if err != nil {
return Errorf("Could not write file " + filename + ": " + err.Error())
}
file.Close()
default: //everything
//Write the header
err = w.WriteHeader(header)
if err != nil {
return err
}
}
}
//Write all the hardlinks
for _, hdr := range hardLinks {
err = w.WriteHeader(hdr)
if err != nil {
return err
}
}
return nil
}
//WriteGopathSrc tars up files under gopath src
func WriteGopathSrc(tw *tar.Writer, excludeDir string) error {
gopath := os.Getenv("GOPATH")
// Only take the first element of GOPATH
gopath = filepath.SplitList(gopath)[0]
rootDirectory := filepath.Join(gopath, "src")
vmLogger.Infof("rootDirectory = %s", rootDirectory)
//append "/" if necessary
if excludeDir != "" && strings.LastIndex(excludeDir, "/") < len(excludeDir)-1 {
excludeDir = excludeDir + "/"
}
rootDirLen := len(rootDirectory)
walkFn := func(path string, info os.FileInfo, err error) error {
// If path includes .git, ignore
if strings.Contains(path, ".git") {
return nil
}
if info.Mode().IsDir() {
return nil
}
//exclude any files with excludeDir prefix. They should already be in the tar
if excludeDir != "" && strings.Index(path, excludeDir) == rootDirLen+1 {
//1 for "/"
return nil
}
// Because of scoping we can reference the external rootDirectory variable
if len(path[rootDirLen:]) == 0 {
return nil
}
// we only want 'fileTypes' source files at this point
ext := filepath.Ext(path)
if _, ok := fileTypes[ext]; ok != true {
return nil
}
newPath := fmt.Sprintf("src%s", path[rootDirLen:])
//newPath := path[len(rootDirectory):]
err = WriteFileToPackage(path, newPath, tw)
if err != nil {
return fmt.Errorf("Error writing file to package: %s", err)
}
return nil
}
if err := filepath.Walk(rootDirectory, walkFn); err != nil {
vmLogger.Infof("Error walking rootDirectory: %s", err)
return err
}
// Write the tar file out
if err := tw.Close(); err != nil {
return err
}
//ioutil.WriteFile("/tmp/chaincode_deployment.tar", inputbuf.Bytes(), 0644)
return nil
}