func normalizeBuildpackArchive(inputFile *os.File, outputFile *os.File) (err error) {
stats, err := inputFile.Stat()
if err != nil {
return
}
reader, err := zip.NewReader(inputFile, stats.Size())
if err != nil {
return
}
contents := reader.File
parentPath, hasBuildpack := findBuildpackPath(contents)
if !hasBuildpack {
return errors.New("Zip archive does not contain a buildpack")
}
writer := zip.NewWriter(outputFile)
for _, file := range contents {
name := file.Name
if parentPath == "." || strings.HasPrefix(name, parentPath) {
var (
r io.ReadCloser
w io.Writer
header *zip.FileHeader
)
fileInfo := file.FileInfo()
header, err = zip.FileInfoHeader(fileInfo)
header.Name = strings.Replace(name, parentPath+"/", "", 1)
r, err = file.Open()
if err != nil {
return
}
w, err = writer.CreateHeader(header)
if err != nil {
return
}
io.Copy(w, r)
err = r.Close()
if err != nil {
return
}
}
}
writer.Close()
outputFile.Seek(0, 0)
return
}
/*
Helper. Recursively zip the contents of a directory tree using the zip.Writer.
Filters so it only includes .rel files
Note: this will not work if there are symbolic links in the src directory tree.
(because Readdir does not follow links.)
*/
func zipSrcDirTree2(w *zip.Writer, directoryPath string, relativeDirName string) (err error) {
var dir *os.File
var filesInDir []os.FileInfo
dir, err = gos.Open(directoryPath)
filesInDir, err = dir.Readdir(0)
if err != nil {
return
}
err = dir.Close()
for _, fileInfo := range filesInDir {
if fileInfo.IsDir() {
subItemPath := directoryPath + "/" + fileInfo.Name()
subItemRelativePath := relativeDirName + "/" + fileInfo.Name()
err = zipSrcDirTree2(w, subItemPath, subItemRelativePath)
if err != nil {
return
}
} else { // plain old file to be added.
if strings.HasSuffix(fileInfo.Name(), ".rel") || strings.HasSuffix(fileInfo.Name(), ".txt") || strings.HasSuffix(fileInfo.Name(), ".html") || strings.HasSuffix(fileInfo.Name(), ".htm") || strings.HasSuffix(fileInfo.Name(), ".css") {
subItemPath := directoryPath + "/" + fileInfo.Name()
subItemRelativePath := relativeDirName + "/" + fileInfo.Name()
var fh *zip.FileHeader
fh, err = zip.FileInfoHeader(fileInfo)
if err != nil {
return
}
fh.Name = subItemRelativePath
var zw io.Writer
zw, err = w.CreateHeader(fh)
if err != nil {
return
}
var f *os.File
f, err = gos.Open(subItemPath)
if err != nil {
return
}
_, err = io.Copy(zw, f)
err = f.Close()
if err != nil {
return
}
}
}
}
return
}
func packFile(srcFile string, recPath string, zw *zip.Writer, fi os.FileInfo) (err error) {
if fi.IsDir() {
// Create zip header
fh := new(zip.FileHeader)
fh.Name = recPath + "/"
fh.UncompressedSize = 0
_, err = zw.CreateHeader(fh)
} else {
// Create zip header
fh := new(zip.FileHeader)
fh.Name = recPath
fh.UncompressedSize = uint32(fi.Size())
fh.SetMode(fi.Mode())
var fw io.Writer
fw, err = zw.CreateHeader(fh)
if err != nil {
return err
}
var f *os.File
f, err = os.Open(srcFile)
if err != nil {
return err
}
_, err = io.Copy(fw, f)
}
return err
}
// CreateNW creates a node-webkit .nw file
func (p Package) CreateNW(zw *zip.Writer, templates Templates, myapp io.Reader, includes string) error {
// Add in a couple of package defaults
p.Main = "index.html"
p.EnvVar = nw.EnvVar
if w, err := zw.Create("package.json"); err != nil {
return err
} else {
if _, err := p.writeJsonTo(w); err != nil {
return err
}
}
filenameTemplates := map[string]string{
"index.html": templates.IndexHtml,
"client.js": templates.ClientJs,
"script.js": templates.ScriptJs}
for filename, str := range filenameTemplates {
if w, err := zw.Create(filename); err != nil {
return err
} else {
if t, err := template.New(filename).Parse(str); err != nil {
return err
} else {
if err := t.Execute(w, p); err != nil {
return err
}
}
}
}
if includes != "" {
if err := copyIncludes(zw, includes); err != nil {
return err
}
}
binHeader := zip.FileHeader{Name: p.Bin}
binHeader.SetMode(0755) // Make it executable
if w, err := zw.CreateHeader(&binHeader); err != nil {
return err
} else {
if _, err := io.Copy(w, myapp); err != nil {
return err
}
}
return nil
}
func makezipfile(w io.Writer, contents zdata) {
const dirname = "test/"
z := zip.NewWriter(w)
for k, v := range contents {
header := zip.FileHeader{
Name: dirname + k,
Method: zip.Store,
}
header.SetModTime(time.Now())
out, _ := z.CreateHeader(&header)
// this should check the number of bytes written, and loop
out.Write([]byte(v))
}
z.Close()
}
func doZipDocs(w http.ResponseWriter, req *http.Request,
path string) {
quit := make(chan bool)
defer close(quit)
ch := make(chan *namedFile)
cherr := make(chan error)
go pathGenerator(path, ch, cherr, quit)
go logErrors("zip", cherr)
w.Header().Set("Content-Disposition",
fmt.Sprintf("attachment; filename=%q", archiveFilename(path, "zip")))
w.Header().Set("Content-Type", "application/zip")
w.WriteHeader(200)
zw := zip.NewWriter(w)
for nf := range ch {
if nf.err != nil {
log.Printf("Error on %v: %v", nf.name, nf.err)
continue
}
fh := zip.FileHeader{
Name: nf.name,
Method: zip.Deflate,
UncompressedSize: uint32(nf.meta.Length),
Comment: nf.meta.OID,
}
fh.SetModTime(nf.meta.Modified)
zf, err := zw.CreateHeader(&fh)
if err != nil {
log.Printf("Error making zip file: %v", err)
// Client will get a broken zip file
return
}
err = copyBlob(zf, nf.meta.OID)
if err != nil {
log.Printf("Error copying blob for %v: %v",
nf.name, err)
// Client will get a broken zip file
return
}
}
zw.Close()
}
// zipInputFiles deterministically builds a zip archive out of input files and
// writes it to the writer. Files are written in the order given.
func zipInputFiles(files []File, w io.Writer, log logging.Logger) error {
writer := zip.NewWriter(w)
defer writer.Close()
// Reports zipping progress to the log each second.
lastReport := time.Time{}
progress := func(count int) {
if time.Since(lastReport) > time.Second {
lastReport = time.Now()
log.Infof("Zipping files: %d files left", len(files)-count)
}
}
for i, in := range files {
progress(i)
// Intentionally do not add timestamp or file mode to make zip archive
// deterministic. See also zip.FileInfoHeader() implementation.
fh := zip.FileHeader{
Name: in.Name(),
Method: zip.Deflate,
}
mode := os.FileMode(0600)
if in.Executable() {
mode |= 0100
}
if in.Symlink() {
mode |= os.ModeSymlink
}
fh.SetMode(mode)
dst, err := writer.CreateHeader(&fh)
if err != nil {
return err
}
if in.Symlink() {
err = zipSymlinkFile(dst, in)
} else {
err = zipRegularFile(dst, in)
}
if err != nil {
return err
}
}
return nil
}
// Zipper zips file contents at a path. Helper to prepare zipped data for upload
// to S3.
func Zipper(zipPath string) (*bytes.Buffer, error) {
f, err := os.Open(zipPath)
if err != nil {
return nil, err
}
defer f.Close()
buffer := bytes.NewBuffer(nil)
w := zip.NewWriter(buffer)
fh := zip.FileHeader{}
fh.Name = filepath.Base(zipPath)
fh.SetMode(0755)
// Add some files to the archive.
var files = []struct {
Name string
FileHandle *os.File
Header *zip.FileHeader
}{
{fh.Name, f, &fh},
}
// Loop through all files listed above
// Can be created using file.Name or a custom file.Header for flexibility.
// To set permissions you need to use the file.Header construct.
for _, file := range files {
var zf io.Writer
if file.Header == nil {
zf, err = w.Create(file.Name)
} else {
zf, err = w.CreateHeader(file.Header)
}
if err != nil {
return nil, err
}
if _, err := io.Copy(zf, f); err != nil {
return nil, err
}
}
if err := w.Close(); err != nil {
return nil, err
}
return buffer, nil
}
func (zwp *ZipWriterPool) GetWriter(fileIndex int64) (io.WriteCloser, error) {
file := zwp.container.Files[fileIndex]
fh := zip.FileHeader{
Name: file.Path,
UncompressedSize64: uint64(file.Size),
Method: zip.Deflate,
}
fh.SetMode(os.FileMode(file.Mode))
fh.SetModTime(time.Now())
w, err := zwp.zw.CreateHeader(&fh)
if err != nil {
return nil, errors.Wrap(err, 1)
}
return &nopWriteCloser{w}, nil
}
func processZipTimestampField(data []byte, file *zip.FileHeader) error {
if !file.Mode().IsDir() && !file.Mode().IsRegular() {
return nil
}
var tsField ZipTimestampField
err := binary.Read(bytes.NewReader(data), binary.LittleEndian, &tsField)
if err != nil {
return err
}
if (tsField.Flags & 1) == 1 {
modTime := time.Unix(int64(tsField.ModTime), 0)
acTime := time.Now()
return os.Chtimes(file.Name, acTime, modTime)
}
return nil
}
// create is for internal use. It allows non-payload files to be written.
func (w *Writer) create(name string) (io.Writer, error) {
// save checksums in case there is an active writer
_ = w.Checksum()
ck := new(Checksum)
w.t.manifest[name] = ck
w.checksum = ck
header := zip.FileHeader{
Name: w.t.dirname + name,
Method: zip.Store,
}
header.SetModTime(time.Now())
out, err := w.z.CreateHeader(&header)
w.hw = util.NewHashWriter(out)
return w.hw, err
}
func dozip(archiveFile string, dir string) {
// file
file := create(archiveFile)
defer file.Close()
// zip
zipWriter := zip.NewWriter(file)
defer zipWriter.Close()
//
walk := func(path string, fileInfo os.FileInfo, err error) error {
if err != nil {
fail("Failed to traverse directory structure %v", err)
}
if fileInfo.Mode().IsDir() {
return nil
}
print(path)
fileToWrite := open(path)
//
var fileHeader *zip.FileHeader
fileHeader, err = zip.FileInfoHeader(fileInfo)
if err != nil {
fail("Failed to create file info header %v", err)
}
fileHeader.Name = path
//
var w io.Writer
w, err = zipWriter.CreateHeader(fileHeader)
if err != nil {
fail("Failed to create zip writer %v", err)
}
_, err = io.Copy(w, fileToWrite)
if err != nil {
fail("Failed to copy (zip writer) %v", err)
}
return nil
}
//
err := filepath.Walk(dir, walk)
if err != nil {
fail("Failed to traverse directory %v %v", dir, err)
}
}
// Deal with file
func zipFile(srcFile string, recPath string, zw *zip.Writer, fi os.FileInfo) {
if fi.IsDir() {
// Create zip header
fh := new(zip.FileHeader)
fh.Name = recPath + "/"
fh.UncompressedSize = 0
_, err := zw.CreateHeader(fh)
handleError(err)
} else {
// Create zip header
fh := new(zip.FileHeader)
fh.Name = recPath
fh.UncompressedSize = uint32(fi.Size())
fw, err := zw.CreateHeader(fh)
handleError(err)
// Read file data
buf := make([]byte, fi.Size())
f, err := os.Open(srcFile)
handleError(err)
_, err = f.Read(buf)
handleError(err)
// Write file data to zip
_, err = fw.Write(buf)
handleError(err)
}
}
func createZipFileEntry(archive *zip.Writer, fh *zip.FileHeader) error {
fh.Method = zip.Deflate
fw, err := archive.CreateHeader(fh)
if err != nil {
return err
}
file, err := os.Open(fh.Name)
if err != nil {
return err
}
_, err = io.Copy(fw, file)
file.Close()
if err != nil {
return err
}
return nil
}
tmpDir, err = ioutil.TempDir(os.TempDir(), "unzip_contents")
Expect(err).NotTo(HaveOccurred())
prevDir, err = os.Getwd()
Expect(err).NotTo(HaveOccurred())
Expect(os.Chdir(tmpDir)).To(Succeed())
tmpFile, err = ioutil.TempFile("", "zipfile")
Expect(err).NotTo(HaveOccurred())
defer tmpFile.Close()
zipWriter := zip.NewWriter(tmpFile)
defer zipWriter.Close()
var (
header *zip.FileHeader
)
header = &zip.FileHeader{Name: "aaa"}
header.SetMode(os.FileMode(0644))
_, err = zipWriter.CreateHeader(header)
Expect(err).NotTo(HaveOccurred())
header = &zip.FileHeader{Name: "bbb/1.txt"}
header.SetMode(os.FileMode(0640))
_, err = zipWriter.CreateHeader(header)
Expect(err).NotTo(HaveOccurred())
header = &zip.FileHeader{Name: "bbb/2.txt"}
header.SetMode(os.FileMode(0600))
_, err = zipWriter.CreateHeader(header)
// zipFiles adds files transferred through the channel to zip (writer)
func zipFiles(dest io.Writer, skipOnError, unsafeArchFn bool, files <-chan ArchFileItem) (err error) {
if dfh, ok := dest.(syncer); ok {
defer func() {
if e := dfh.Sync(); e != nil && err == nil && strings.Index(e.Error(), "invalid argument") < 0 {
err = e
}
}()
}
zfh := zip.NewWriter(dest)
defer func() {
if e := zfh.Close(); e != nil && err == nil {
err = e
}
}()
var (
fi os.FileInfo
zi *zip.FileHeader
w io.Writer
errs []error
openedHere bool
)
appendErr := func(err error) {
if err == nil || err.Error() == "" {
return
}
if errs == nil {
errs = []error{err}
}
}
for item := range files {
openedHere = false
if item.File == nil {
if item.Filename == "" {
continue
}
openedHere = true
if item.File, err = os.Open(item.Filename); err != nil {
err = fmt.Errorf("Zip cannot open %q: %v", item.Filename, err)
if !skipOnError {
return err
}
appendErr(err)
continue
}
}
if fi, err = item.File.Stat(); err != nil {
if openedHere {
_ = item.File.Close()
}
err = fmt.Errorf("error stating %s: %s", item.File, err)
} else if fi == nil {
err = fmt.Errorf("nil stat of %#v", item.File)
}
if err != nil {
if !skipOnError {
return err
}
appendErr(err)
continue
}
logger.Debug("msg", "ZipFiles", "fi", fi)
if zi, err = zip.FileInfoHeader(fi); err != nil {
if openedHere {
_ = item.File.Close()
}
err = fmt.Errorf("cannot convert stat %s to header: %s", item.File, err)
if !skipOnError {
return err
}
appendErr(err)
continue
}
if item.Archive != "" {
zi.Name = item.Archive
} else if unsafeArchFn {
zi.Name = unsafeFn(zi.Name, true)
}
if w, err = zfh.CreateHeader(zi); err != nil {
if openedHere {
_ = item.File.Close()
}
err = fmt.Errorf("error creating header for %q: %v", zi.Name, err)
if !skipOnError {
return err
}
appendErr(err)
continue
}
logger.Debug("msg", "zipping", "arch", item.Archive, "name", zi.Name)
_, err = io.Copy(w, item.File)
if openedHere {
_ = item.File.Close()
}
if err != nil {
err = fmt.Errorf("error writing %s to zipfile: %s", item.File, err)
logger.Warn("msg", "write to zip", "error", err)
if !skipOnError {
return err
}
appendErr(err)
continue
}
}
if errs == nil || len(errs) == 0 {
return nil
}
sarr := make([]string, 0, len(errs))
for _, err = range errs {
sarr = append(sarr, err.Error())
}
return errors.New(strings.Join(sarr, "\n"))
}
func CompressZip(archiveWriter io.Writer, container *tlc.Container, pool wsync.Pool, consumer *state.Consumer) (*archiver.CompressResult, error) {
var err error
var uncompressedSize int64
var compressedSize int64
archiveCounter := counter.NewWriter(archiveWriter)
zipWriter := zip.NewWriter(archiveCounter)
defer zipWriter.Close()
defer func() {
if zipWriter != nil {
if zErr := zipWriter.Close(); err == nil && zErr != nil {
err = errors.Wrap(zErr, 1)
}
}
}()
for _, dir := range container.Dirs {
fh := zip.FileHeader{
Name: dir.Path + "/",
}
fh.SetMode(os.FileMode(dir.Mode))
fh.SetModTime(time.Now())
_, hErr := zipWriter.CreateHeader(&fh)
if hErr != nil {
return nil, errors.Wrap(hErr, 1)
}
}
for fileIndex, file := range container.Files {
fh := zip.FileHeader{
Name: file.Path,
UncompressedSize64: uint64(file.Size),
Method: zip.Deflate,
}
fh.SetMode(os.FileMode(file.Mode))
fh.SetModTime(time.Now())
entryWriter, eErr := zipWriter.CreateHeader(&fh)
if eErr != nil {
return nil, errors.Wrap(eErr, 1)
}
entryReader, eErr := pool.GetReader(int64(fileIndex))
if eErr != nil {
return nil, errors.Wrap(eErr, 1)
}
copiedBytes, eErr := io.Copy(entryWriter, entryReader)
if eErr != nil {
return nil, errors.Wrap(eErr, 1)
}
uncompressedSize += copiedBytes
}
for _, symlink := range container.Symlinks {
fh := zip.FileHeader{
Name: symlink.Path,
}
fh.SetMode(os.FileMode(symlink.Mode))
entryWriter, eErr := zipWriter.CreateHeader(&fh)
if eErr != nil {
return nil, errors.Wrap(eErr, 1)
}
entryWriter.Write([]byte(symlink.Dest))
}
err = zipWriter.Close()
if err != nil {
return nil, errors.Wrap(err, 1)
}
zipWriter = nil
compressedSize = archiveCounter.Count()
return &archiver.CompressResult{
UncompressedSize: uncompressedSize,
CompressedSize: compressedSize,
}, nil
}
func createZipDirectoryEntry(archive *zip.Writer, fh *zip.FileHeader) error {
fh.Name += "/"
_, err := archive.CreateHeader(fh)
return err
}