Example #1
0
// Writes the given file into tarfn
func writeItem(tarfn string, fn string) (pos uint64, err error) {
	pos = 0
	sfh, err := os.Open(fn)
	defer sfh.Close()
	if sfi, err := sfh.Stat(); err == nil {
		hdr := new(tar.Header)
		hdr.Name = sfi.Name()
		hdr.Size = sfi.Size()
		hdr.Mode = int64(sfi.Mode().Perm())
		hdr.ModTime = sfi.ModTime()
		var (
			tw *tar.Writer
			f  ReadWriteSeekCloser
		)
		if tw, f, pos, err = OpenForAppend(tarfn); err == nil {
			defer f.Close()
			defer tw.Close() //LIFO
			if err := tw.WriteHeader(hdr); err == nil {
				if zw, err := gzip.NewWriterLevel(tw, flate.BestCompression); err == nil {
					defer zw.Close()
					_, err = io.Copy(zw, sfh)
				}
			}
		}
	}
	return
}
Example #2
0
//WriteGopathSrc tars up files under gopath src
func WriteGopathSrc(tw *tar.Writer, excludeDir string) error {
	gopath := os.Getenv("GOPATH")
	// Only take the first element of GOPATH
	gopath = filepath.SplitList(gopath)[0]

	rootDirectory := filepath.Join(gopath, "src")
	vmLogger.Infof("rootDirectory = %s", rootDirectory)

	if err := WriteFolderToTarPackage(tw, rootDirectory, excludeDir, includeFileTypes, nil); err != nil {
		vmLogger.Errorf("Error writing folder to tar package %s", err)
		return err
	}

	// Add the certificates to tar
	if viper.GetBool("peer.tls.enabled") {
		err := WriteFileToPackage(viper.GetString("peer.tls.cert.file"), "src/certs/cert.pem", tw)
		if err != nil {
			return fmt.Errorf("Error writing cert file to package: %s", err)
		}
	}

	// Write the tar file out
	if err := tw.Close(); err != nil {
		return err
	}
	//ioutil.WriteFile("/tmp/chaincode_deployment.tar", inputbuf.Bytes(), 0644)
	return nil
}
Example #3
0
//WriteGopathSrc tars up files under gopath src
func WriteGopathSrc(tw *tar.Writer, excludeDir string) error {
	gopath := os.Getenv("GOPATH")
	if strings.LastIndex(gopath, "/") == len(gopath)-1 {
		gopath = gopath[:len(gopath)]
	}
	rootDirectory := fmt.Sprintf("%s%s%s", os.Getenv("GOPATH"), string(os.PathSeparator), "src")
	vmLogger.Info("rootDirectory = %s", rootDirectory)

	//append "/" if necessary
	if excludeDir != "" && strings.LastIndex(excludeDir, "/") < len(excludeDir)-1 {
		excludeDir = excludeDir + "/"
	}

	rootDirLen := len(rootDirectory)
	walkFn := func(path string, info os.FileInfo, err error) error {

		// If path includes .git, ignore
		if strings.Contains(path, ".git") {
			return nil
		}

		if info.Mode().IsDir() {
			return nil
		}

		//exclude any files with excludeDir prefix. They should already be in the tar
		if excludeDir != "" && strings.Index(path, excludeDir) == rootDirLen+1 {
			//1 for "/"
			return nil
		}
		// Because of scoping we can reference the external rootDirectory variable
		if len(path[rootDirLen:]) == 0 {
			return nil
		}

		newPath := fmt.Sprintf("src%s", path[rootDirLen:])
		//newPath := path[len(rootDirectory):]

		vmLogger.Debug("writing file %s to %s", path, newPath)
		err = WriteFileToPackage(path, newPath, tw)
		if err != nil {
			return fmt.Errorf("Error writing file to package: %s", err)
		}

		return nil
	}

	if err := filepath.Walk(rootDirectory, walkFn); err != nil {
		vmLogger.Info("Error walking rootDirectory: %s", err)
		return err
	}
	// Write the tar file out
	if err := tw.Close(); err != nil {
		return err
	}
	//ioutil.WriteFile("/tmp/chaincode_deployment.tar", inputbuf.Bytes(), 0644)
	return nil
}
Example #4
0
func MakeTarball(tarname string, fnames []string) error {
	log.Printf("tarring %d entrires to %s ...", len(fnames), tarname)
	tarfile, err := os.Create(tarname)
	if err != nil {
		return err
	}
	defer tarfile.Close()
	var tarwriter *tar.Writer
	if strings.HasSuffix(tarname, ".gz") {
		zipper := gzip.NewWriter(tarfile)
		defer zipper.Close()
		tarwriter = tar.NewWriter(zipper)
		/*
			} else if strings.HasSuffix(tarname, ".xz") {
				p := xz.WriterDefaults
				p.DictCap = 1 << 24
				zipper, err := xz.NewWriterParams(tarfile, &p) //xz.NewWriter(tarfile)
				if err != nil {
					return err
				}
				defer zipper.Close()
				tarwriter = tar.NewWriter(zipper)
		*/
	} else {
		tarwriter = tar.NewWriter(tarfile)
	}
	defer tarwriter.Close()

	for _, fname := range fnames {
		realm, ts, good := util.Parse_FName(fname)
		if !good {
			log.Printf("warning: skip ill-named file '%s'", fname)
			continue // skip
		}
		data, err := util.Load(fname)
		if err != nil {
			return err
		}

		hdr := new(tar.Header)
		hdr.Name = util.Make_FName(realm, ts, false)
		hdr.Size = int64(len(data))
		hdr.ModTime = ts
		hdr.Mode = 0644
		err = tarwriter.WriteHeader(hdr)
		if err != nil {
			return err
		}
		log.Printf("tar %d bytes for file %s", hdr.Size, hdr.Name)
		_, err = tarwriter.Write(data)
		if err != nil {
			return err
		}
	}
	log.Printf("%s tarred without errors", tarname)
	return nil
}
Example #5
0
File: main.go Project: S010/test
func tarball(path string, t *tar.Writer) {
	path = filepath.Clean(path)

	info, err := os.Stat(path)
	if err != nil {
		return
	}

	if t == nil {
		r, w := io.Pipe()

		t = tar.NewWriter(w)

		f, err := os.Create(fmt.Sprintf("%s.tgz", path))
		if err != nil {
			panic(err)
		}
		z := gzip.NewWriter(f)

		defer f.Close()
		defer z.Close()
		defer r.Close()
		defer w.Close()
		defer t.Close()

		go io.Copy(z, r)
	}

	h, err := tar.FileInfoHeader(info, path) // FIXME what is link arg supposed to be?
	if err != nil {
		return
	}
	h.Name = path
	t.WriteHeader(h)
	if info.IsDir() {
		infos, err := ioutil.ReadDir(path)
		if err != nil {
			return
		}
		for _, info := range infos {
			tgz(fmt.Sprintf("%v/%v", path, info.Name()), t)
		}
	} else {
		f, err := os.Open(path)
		if err != nil {
			panic(err)
		}
		defer f.Close()

		if _, err := io.Copy(t, f); err != nil {
			panic(err)
		}
	}
}
Example #6
0
//Package Java project to tar file from the source path
func WriteJavaProjectToPackage(tw *tar.Writer, srcPath string) error {

	vmLogger.Debugf("Packaging Java project from path %s", srcPath)

	if err := WriteFolderToTarPackage(tw, srcPath, "", nil, javaExcludeFileTypes); err != nil {

		vmLogger.Errorf("Error writing folder to tar package %s", err)
		return err
	}
	// Write the tar file out
	if err := tw.Close(); err != nil {
		return err
	}
	return nil

}
Example #7
0
func writeTarball(tarball *tar.Writer, archive io.Reader, fileSize int64, name string) error {
	header := tar.Header{
		Name: name,
		Mode: 0666,
		Size: fileSize,
	}
	tarball.WriteHeader(&header)
	n, err := io.Copy(tarball, archive)
	if err != nil {
		return err
	}
	if n != fileSize {
		return errors.New("upload-deploy: short-write copying to tarball")
	}
	return tarball.Close()
}
// Read from source tar stream and write training and test to given tar writers
func (d DatasetSplitter) transform(source *tar.Reader, train, test *tar.Writer) error {

	splitter := d.splitter(train, test)

	for {
		hdr, err := source.Next()

		if err == io.EOF {
			// end of tar archive
			break
		}

		if err != nil {
			return err
		}

		tw := splitter(hdr.Name)

		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}
		_, err = io.Copy(tw, source)
		if err != nil {
			return err
		}

	}

	// close writers
	if err := train.Close(); err != nil {
		errMsg := fmt.Errorf("Error closing tar writer: %v", err)
		logg.LogError(errMsg)
		return err
	}
	if err := test.Close(); err != nil {
		errMsg := fmt.Errorf("Error closing tar reader: %v", err)
		logg.LogError(errMsg)
		return err
	}

	return nil
}
Example #9
0
func makeLayer(t *testing.T, compression archive.Compression) (ddigest.Digest, int64, int64, []byte) {
	var cwriter io.WriteCloser
	var uncompressed *ioutils.WriteCounter
	var twriter *tar.Writer
	preader, pwriter := io.Pipe()
	tbuffer := bytes.Buffer{}
	if compression != archive.Uncompressed {
		compressor, err := archive.CompressStream(pwriter, compression)
		if err != nil {
			t.Fatalf("Error compressing layer: %v", err)
		}
		cwriter = compressor
		uncompressed = ioutils.NewWriteCounter(cwriter)
	} else {
		uncompressed = ioutils.NewWriteCounter(pwriter)
	}
	twriter = tar.NewWriter(uncompressed)
	buf := make([]byte, layerSize)
	n, err := rand.Read(buf)
	if err != nil {
		t.Fatalf("Error reading tar data: %v", err)
	}
	if n != len(buf) {
		t.Fatalf("Short read reading tar data: %d < %d", n, len(buf))
	}
	for i := 1024; i < 2048; i++ {
		buf[i] = 0
	}
	go func() {
		defer pwriter.Close()
		if cwriter != nil {
			defer cwriter.Close()
		}
		defer twriter.Close()
		err := twriter.WriteHeader(&tar.Header{
			Name:       "/random-single-file",
			Mode:       0600,
			Size:       int64(len(buf)),
			ModTime:    time.Now(),
			AccessTime: time.Now(),
			ChangeTime: time.Now(),
			Typeflag:   tar.TypeReg,
		})
		if err != nil {
			t.Fatalf("Error writing tar header: %v", err)
		}
		n, err := twriter.Write(buf)
		if err != nil {
			t.Fatalf("Error writing tar header: %v", err)
		}
		if n != len(buf) {
			t.Fatalf("Short write writing tar header: %d < %d", n, len(buf))
		}
	}()
	_, err = io.Copy(&tbuffer, preader)
	if err != nil {
		t.Fatalf("Error reading layer tar: %v", err)
	}
	sum := ddigest.SHA256.FromBytes(tbuffer.Bytes())
	return sum, uncompressed.Count, int64(tbuffer.Len()), tbuffer.Bytes()
}
Example #10
0
// http://blog.ralch.com/tutorial/golang-working-with-tar-and-gzip/
// This function tars the source file/dir.  If a tar.Writer is detected
// it'll use that as the tar.Writer instead of generating it's own (this is useful
// when you want to have selective tarring, such as modules).
// Also if tarName is provided it'll use that
// returns the name of the target file (if created) and any errors
func tarit(source, tarName string, tb *tar.Writer) error {
	var tarball *tar.Writer
	target := tarName

	if tb != nil {
		tarball = tb
	} else {
		filename := filepath.Base(source)

		if target == "" {
			target = fmt.Sprintf("%s.tar", filename)
		}

		tarfile, err := os.Create(target)
		if err != nil {
			return HenchErr(err, nil, "While creating target")
		}
		defer tarfile.Close()

		tarball = tar.NewWriter(tarfile)
		defer tarball.Close()
	}

	info, err := os.Stat(source)
	if err != nil {
		return nil
	}

	var baseDir string
	if info.IsDir() {
		baseDir = filepath.Base(source)
	}

	return filepath.Walk(source,
		func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return HenchErr(err, map[string]interface{}{
					"path": path,
				}, "While walking")
			}

			header, err := tar.FileInfoHeader(info, info.Name())
			if err != nil {
				return HenchErr(err, map[string]interface{}{
					"file":     path,
					"solution": "Golang specific tar package.  Submit an issue starting with TAR HEADER",
				}, "Adding info to tar header")
			}

			if baseDir != "" {
				header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source))
			}

			if err := tarball.WriteHeader(header); err != nil {
				return HenchErr(err, map[string]interface{}{
					"file":     path,
					"solution": "Golang specific tar package.  Submit an issue starting with TARBALL",
				}, "Writing header to tar")
			}

			if info.IsDir() {
				return nil
			}

			file, err := os.Open(path)
			if err != nil {
				return HenchErr(err, nil, "")
			}
			defer file.Close()

			if _, err := io.Copy(tarball, file); err != nil {
				return HenchErr(err, map[string]interface{}{
					"file":     path,
					"solution": "make sure file exists, correct permissions, or is not corrupted",
				}, "")
			}

			return nil
		})
}
Example #11
0
func writeGopathSrc(tw *tar.Writer, excludeDir string) error {
	gopath := os.Getenv("GOPATH")
	if strings.LastIndex(gopath, "/") == len(gopath)-1 {
		gopath = gopath[:len(gopath)]
	}
	rootDirectory := fmt.Sprintf("%s%s%s", os.Getenv("GOPATH"), string(os.PathSeparator), "src")
	vmLogger.Info("rootDirectory = %s", rootDirectory)

	//append "/" if necessary
	if excludeDir != "" && strings.LastIndex(excludeDir, "/") < len(excludeDir)-1 {
		excludeDir = excludeDir + "/"
	}

	rootDirLen := len(rootDirectory)
	walkFn := func(path string, info os.FileInfo, err error) error {

		// If path includes .git, ignore
		if strings.Contains(path, ".git") {
			return nil
		}

		if info.Mode().IsDir() {
			return nil
		}

		//exclude any files with excludeDir prefix. They should already be in the tar
		if excludeDir != "" && strings.Index(path, excludeDir) == rootDirLen+1 { //1 for "/"
			return nil
		}
		// Because of scoping we can reference the external rootDirectory variable
		newPath := fmt.Sprintf("src%s", path[rootDirLen:])
		//newPath := path[len(rootDirectory):]
		if len(newPath) == 0 {
			return nil
		}

		fr, err := os.Open(path)
		if err != nil {
			return err
		}
		defer fr.Close()

		h, err := tar.FileInfoHeader(info, newPath)
		if err != nil {
			vmLogger.Error(fmt.Sprintf("Error getting FileInfoHeader: %s", err))
			return err
		}
		//Let's take the variance out of the tar, make headers identical everywhere by using zero time
		var zeroTime time.Time
		h.AccessTime = zeroTime
		h.ModTime = zeroTime
		h.ChangeTime = zeroTime
		h.Name = newPath
		if err = tw.WriteHeader(h); err != nil {
			vmLogger.Error(fmt.Sprintf("Error writing header: %s", err))
			return err
		}
		if _, err := io.Copy(tw, fr); err != nil {
			return err
		}
		return nil
	}

	if err := filepath.Walk(rootDirectory, walkFn); err != nil {
		vmLogger.Info("Error walking rootDirectory: %s", err)
		return err
	}
	// Write the tar file out
	if err := tw.Close(); err != nil {
		return err
	}
	//ioutil.WriteFile("/tmp/chaincode_deployment.tar", inputbuf.Bytes(), 0644)
	return nil
}
Example #12
0
//Given a writer to a tar stream, import from the filesystem.
func ImportFromFilesystem(w *tar.Writer, basePath string) error {
	defer w.Close()

	//Get a sane path.
	tempDir, err := filepath.Abs(filepath.Clean(basePath))
	if err != nil {
		return err
	}
	base, err := filepath.EvalSymlinks(tempDir)
	if err != nil {
		return err
	}

	//Open the metadata file
	metaFilename := filepath.Join(base, ".guitar")
	metafile, err := os.Open(metaFilename)
	defer metafile.Close()
	if err != nil {
		return err
	}
	buffer := bufio.NewReader(metafile)

	//Cannot hard link to files that don't exist yet.
	//We buffer them here, and apply them last.
	hardLinks := make([]*tar.Header, 0)

	//Read each line
	Println("Importing files...")
	for {
		//Get bytes until next newline
		line, err := buffer.ReadBytes('\n')
		if err == io.EOF {
			break //Last line has been read; done importing
		} else if err != nil {
			return err
		}

		//Decode that line's JSON
		var hdr *format.Header
		err = json.Unmarshal(line, &hdr)
		if err != nil {
			return err
		}

		//Convert to tar header format, write to the stream
		header, err := format.Import(hdr)
		if err != nil {
			return err
		}

		//Take action on various file types
		switch header.Typeflag {
		case tar.TypeLink: //hard link
			hardLinks = append(hardLinks, header) //save for later

		case tar.TypeReg, tar.TypeRegA: //regular file
			filename := filepath.Join(base, header.Name)

			//Open file, get size
			file, err := os.Open(filename)
			if err != nil {
				return err
			}
			info, err := file.Stat()
			if err != nil {
				return err
			}

			//Set header size
			header.Size = info.Size()

			//Write the header
			err = w.WriteHeader(header)
			if err != nil {
				return err
			}

			//Write the file
			_, err = io.Copy(w, file)
			if err != nil {
				return Errorf("Could not write file " + filename + ": " + err.Error())
			}
			file.Close()
		default: //everything
			//Write the header
			err = w.WriteHeader(header)
			if err != nil {
				return err
			}
		}
	}

	//Write all the hardlinks
	for _, hdr := range hardLinks {
		err = w.WriteHeader(hdr)
		if err != nil {
			return err
		}
	}

	return nil
}
Example #13
0
//WriteGopathSrc tars up files under gopath src
func WriteGopathSrc(tw *tar.Writer, excludeDir string) error {
	gopath := os.Getenv("GOPATH")
	// Only take the first element of GOPATH
	gopath = filepath.SplitList(gopath)[0]

	rootDirectory := filepath.Join(gopath, "src")
	vmLogger.Infof("rootDirectory = %s", rootDirectory)

	//append "/" if necessary
	if excludeDir != "" && strings.LastIndex(excludeDir, "/") < len(excludeDir)-1 {
		excludeDir = excludeDir + "/"
	}

	rootDirLen := len(rootDirectory)
	walkFn := func(path string, info os.FileInfo, err error) error {

		// If path includes .git, ignore
		if strings.Contains(path, ".git") {
			return nil
		}

		if info.Mode().IsDir() {
			return nil
		}

		//exclude any files with excludeDir prefix. They should already be in the tar
		if excludeDir != "" && strings.Index(path, excludeDir) == rootDirLen+1 {
			//1 for "/"
			return nil
		}
		// Because of scoping we can reference the external rootDirectory variable
		if len(path[rootDirLen:]) == 0 {
			return nil
		}

		// we only want 'fileTypes' source files at this point
		ext := filepath.Ext(path)
		if _, ok := fileTypes[ext]; ok != true {
			return nil
		}

		newPath := fmt.Sprintf("src%s", path[rootDirLen:])
		//newPath := path[len(rootDirectory):]

		err = WriteFileToPackage(path, newPath, tw)
		if err != nil {
			return fmt.Errorf("Error writing file to package: %s", err)
		}

		return nil
	}

	if err := filepath.Walk(rootDirectory, walkFn); err != nil {
		vmLogger.Infof("Error walking rootDirectory: %s", err)
		return err
	}
	// Write the tar file out
	if err := tw.Close(); err != nil {
		return err
	}
	//ioutil.WriteFile("/tmp/chaincode_deployment.tar", inputbuf.Bytes(), 0644)
	return nil
}