func (w *archiveWorker) archiveZip(inpath string, size int64, addZipItself bool) (int64, error) { root, err := w.depot.reserveRoot(size) if err != nil { return 0, err } zr, err := czip.OpenReader(inpath) if err != nil { return 0, err } defer zr.Close() var compressedSize int64 for _, zf := range zr.File { cs, err := w.archive(func() (io.ReadCloser, error) { return zf.Open() }, root, zf.FileInfo().Name(), filepath.Join(inpath, zf.FileInfo().Name()), zf.FileInfo().Size()) if err != nil { return 0, err } compressedSize += cs } if addZipItself { cs, err := w.archive(func() (io.ReadCloser, error) { return os.Open(inpath) }, root, filepath.Base(inpath), inpath, size) if err != nil { return 0, err } compressedSize += cs } return compressedSize, nil }
func unzip(path string) error { r, err := czip.OpenReader(path) if err != nil { return err } defer r.Close() for _, fh := range r.File { w, err := os.Create(fh.Name) if err != nil { fmt.Printf("error creating writer for %s: %v\n", fh.Name, err) return err } cr, err := fh.Open() if err != nil { fmt.Printf("error opening reader for %s: %v\n", fh.Name, err) return err } _, err = io.Copy(w, cr) if err != nil { fmt.Printf("error copying from reader to writer for %s: %v\n", fh.Name, err) return err } err = cr.Close() if err != nil { fmt.Printf("error closing reader for %s: %v\n", fh.Name, err) return err } err = w.Close() if err != nil { fmt.Printf("error closing writer for %s: %v\n", fh.Name, err) return err } } return nil }
func main() { flag.Parse() if len(flag.Args()) == 0 { os.Exit(1) } name := flag.Args()[0] zr, err := czip.OpenReader(name) if err != nil { fmt.Fprintf(os.Stderr, "opening zip file %s failed: %v\n", name, err) os.Exit(1) } for _, f := range zr.File { fmt.Printf("Size of %s: %d \n", f.Name, f.UncompressedSize64) dwf, err := os.Create("z-" + f.Name) if err != nil { log.Fatal(err) } rc, err := f.Open() if err != nil { log.Fatal(err) } _, err = io.Copy(dwf, rc) if err != nil { log.Fatal(err) } rc.Close() dwf.Close() } zr.Close() }
func (w *archiveWorker) archiveZip(inpath string, size int64, addZipItself int) (int64, error) { glog.V(4).Infof("archiving zip %s ", inpath) var compressedSize int64 if addZipItself <= 1 { var zfs []zipF if w.pm.useGoZip { zr, err := zip.OpenReader(inpath) if err != nil { return 0, err } defer zr.Close() zfs = make([]zipF, len(zr.File)) for i, zf := range zr.File { zfs[i] = zipF(zf) } } else { zr, err := czip.OpenReader(inpath) if err != nil { return 0, err } defer zr.Close() zfs = make([]zipF, len(zr.File)) for i, zf := range zr.File { zfs[i] = zipF(zf) } } glog.V(4).Infof("zip entries %d: %s", len(zfs), inpath) in := make(chan zipF) out := make(chan zipWorkResult) numWorkers := w.pm.NumWorkers() for i := 0; i < numWorkers; i++ { zw := &zipWorker{ index: i, inpath: inpath, w: w, in: in, out: out, hh: newHashes(), md5crcBuffer: make([]byte, md5.Size+crc32.Size+8), } go zw.Work() } var perr error var nrProcessed int var nrScheduled int expectedResults := numWorkers for _, zf := range zfs { select { case in <- zf: glog.V(4).Infof("scheduled %s from zip %s", zf.FileInfo().Name(), inpath) nrScheduled++ case zwr := <-out: expectedResults-- glog.Warningf("breaking out of the zip loop before all files are scheduled: %s", inpath) if zwr.err != nil { perr = zwr.err break } compressedSize += zwr.compressedSize nrProcessed += zwr.nrProcessed } } close(in) if perr != nil { glog.Errorf("zip error %s: %v", inpath, perr) return 0, perr } glog.V(4).Infof("reading results from zip %s", inpath) for i := 0; i < expectedResults; i++ { zwr := <-out if zwr.err != nil { perr = zwr.err break } compressedSize += zwr.compressedSize nrProcessed += zwr.nrProcessed } if nrProcessed != len(zfs) || nrScheduled != len(zfs) { glog.Warningf("scheduled/processed fewer zip entries: scheduled %d, processed %d, expected %d: %s", nrScheduled, nrProcessed, len(zfs), inpath) } glog.V(4).Infof("scheduled %d, processed %d, expected %d: %s", nrScheduled, nrProcessed, len(zfs), inpath) glog.V(4).Infof("finished archiving contents of zip %s", inpath) if perr != nil { glog.Errorf("zip error %s: %v", inpath, perr) return 0, perr } } if addZipItself >= 1 { cs, err := w.archive(func() (io.ReadCloser, error) { return os.Open(inpath) }, filepath.Base(inpath), inpath, size, w.hh, w.md5crcBuffer) if err != nil { return 0, err } compressedSize += cs } return compressedSize, nil }
func (w *Writer) Close() error { err := w.uw.Close() if err != nil { return err } err = w.bf.Flush() if err != nil { return err } err = w.tf.Close() if err != nil { return err } r, err := czip.OpenReader(w.tf.Name()) if err != nil { return err } fis := make(fileIndices, len(r.File)) for k, zf := range r.File { fis[k] = &fileIndex{ name: torrentCanonicalName(zf.Name), index: k, } } sort.Sort(fis) cw := &countWriter{ w: w.sink, } for _, fi := range fis { fh := r.File[fi.index] fh.Extra = nil fi.offset = cw.count err = writeHeader(cw, fh, fi.name) if err != nil { return err } fr, err := fh.OpenRaw() if err != nil { return err } _, err = io.Copy(cw, fr) if err != nil { return err } } dircrc := crc32.NewIEEE() mw := io.MultiWriter(cw, dircrc) start := cw.count for _, fi := range fis { fh := r.File[fi.index] fh.Extra = nil err = writeCentralHeader(mw, fh, fi.name, fi.offset) if err != nil { return err } } end := cw.count records := uint64(len(fis)) size := uint64(end - start) offset := uint64(start) if records > uint16max || size > uint32max || offset > uint32max { var buf [directory64EndLen + directory64LocLen]byte b := writeBuf(buf[:]) // zip64 end of central directory record b.uint32(directory64EndSignature) b.uint64(directory64EndLen - 12) b.uint16(zipVersion45) // version made by b.uint16(zipVersion45) // version needed to extract b.uint32(0) // number of this disk b.uint32(0) // number of the disk with the start of the central directory b.uint64(records) // total number of entries in the central directory on this disk b.uint64(records) // total number of entries in the central directory b.uint64(size) // size of the central directory b.uint64(offset) // offset of start of central directory with respect to the starting disk number // zip64 end of central directory locator b.uint32(directory64LocSignature) b.uint32(0) // number of the disk with the start of the zip64 end of central directory b.uint64(uint64(end)) // relative offset of the zip64 end of central directory record b.uint32(1) // total number of disks if _, err := cw.Write(buf[:]); err != nil { return err } // store max values in the regular end record to signal that // that the zip64 values should be used instead if records > uint16max { records = uint16max } if size > uint32max { size = uint32max } if offset > uint32max { offset = uint32max } } var buf [directoryEndLen]byte b := writeBuf(buf[:]) b.uint32(uint32(directoryEndSignature)) b.uint16(0) b.uint16(0) b.uint16(uint16(records)) b.uint16(uint16(records)) b.uint32(uint32(size)) b.uint32(uint32(offset)) b.uint16(22) zipcomment := "TORRENTZIPPED-" + strings.ToUpper(hex.EncodeToString(dircrc.Sum(nil))) if _, err := cw.Write(buf[:]); err != nil { return err } if _, err := io.WriteString(cw, zipcomment); err != nil { return err } if err := r.Close(); err != nil { return err } if err := os.Remove(w.tf.Name()); err != nil { return err } return nil }
func (tv *testdataVisitor) visit(path string, f os.FileInfo, err error) error { if filepath.Ext(path) == zipext { tv.t.Logf("testing file %s\n", path) r, err := czip.OpenReader(path) if err != nil { return err } defer r.Close() w, err := ioutil.TempFile(os.TempDir(), "torrentzip_test") if err != nil { return err } hh := sha1.New() zw, err := NewWriter(io.MultiWriter(w, hh)) if err != nil { return err } for _, fh := range r.File { cw, err := zw.Create(fh.Name) if err != nil { return err } cr, err := fh.Open() if err != nil { return err } io.Copy(cw, cr) cr.Close() } err = zw.Close() if err != nil { return err } err = w.Close() if err != nil { return err } testsha1 := hex.EncodeToString(hh.Sum(nil)) goldensha1 := strings.TrimSuffix(filepath.Base(path), zipext) tv.t.Logf("testsha1=%s\n", testsha1) tv.t.Logf("goldensha1=%s\n", goldensha1) if testsha1 != strings.ToLower(goldensha1) { return fmt.Errorf("produced torrentzip for %s differs from golden", path) } err = os.Remove(w.Name()) if err != nil { return err } } return nil }
func testZip(path string, goldensha1 []byte) error { r, err := czip.OpenReader(path) if err != nil { return err } defer r.Close() w, err := ioutil.TempFile(os.TempDir(), "testtorrentzip") if err != nil { return err } hh := sha1.New() zw, err := torrentzip.NewWriter(io.MultiWriter(w, hh)) if err != nil { return err } for _, fh := range r.File { cw, err := zw.Create(fh.Name) if err != nil { return err } cr, err := fh.Open() if err != nil { return err } _, err = io.Copy(cw, cr) if err != nil { return err } err = cr.Close() if err != nil { return err } } err = zw.Close() if err != nil { return err } err = w.Close() if err != nil { return err } err = os.Remove(w.Name()) if err != nil { return err } testsha1 := hh.Sum(nil) if !bytes.Equal(testsha1, goldensha1) { return fmt.Errorf("produced torrentzip for %s differs from golden", path) } return nil }