func (w *archiveWorker) archiveZip(inpath string, size int64, addZipItself bool) (int64, error) {
root, err := w.depot.reserveRoot(size)
if err != nil {
return 0, err
}
zr, err := czip.OpenReader(inpath)
if err != nil {
return 0, err
}
defer zr.Close()
var compressedSize int64
for _, zf := range zr.File {
cs, err := w.archive(func() (io.ReadCloser, error) { return zf.Open() }, root,
zf.FileInfo().Name(), filepath.Join(inpath, zf.FileInfo().Name()), zf.FileInfo().Size())
if err != nil {
return 0, err
}
compressedSize += cs
}
if addZipItself {
cs, err := w.archive(func() (io.ReadCloser, error) { return os.Open(inpath) }, root, filepath.Base(inpath), inpath, size)
if err != nil {
return 0, err
}
compressedSize += cs
}
return compressedSize, nil
}
func unzip(path string) error {
r, err := czip.OpenReader(path)
if err != nil {
return err
}
defer r.Close()
for _, fh := range r.File {
w, err := os.Create(fh.Name)
if err != nil {
fmt.Printf("error creating writer for %s: %v\n", fh.Name, err)
return err
}
cr, err := fh.Open()
if err != nil {
fmt.Printf("error opening reader for %s: %v\n", fh.Name, err)
return err
}
_, err = io.Copy(w, cr)
if err != nil {
fmt.Printf("error copying from reader to writer for %s: %v\n", fh.Name, err)
return err
}
err = cr.Close()
if err != nil {
fmt.Printf("error closing reader for %s: %v\n", fh.Name, err)
return err
}
err = w.Close()
if err != nil {
fmt.Printf("error closing writer for %s: %v\n", fh.Name, err)
return err
}
}
return nil
}
func main() {
flag.Parse()
if len(flag.Args()) == 0 {
os.Exit(1)
}
name := flag.Args()[0]
zr, err := czip.OpenReader(name)
if err != nil {
fmt.Fprintf(os.Stderr, "opening zip file %s failed: %v\n", name, err)
os.Exit(1)
}
for _, f := range zr.File {
fmt.Printf("Size of %s: %d \n", f.Name, f.UncompressedSize64)
dwf, err := os.Create("z-" + f.Name)
if err != nil {
log.Fatal(err)
}
rc, err := f.Open()
if err != nil {
log.Fatal(err)
}
_, err = io.Copy(dwf, rc)
if err != nil {
log.Fatal(err)
}
rc.Close()
dwf.Close()
}
zr.Close()
}
func (w *archiveWorker) archiveZip(inpath string, size int64, addZipItself int) (int64, error) {
glog.V(4).Infof("archiving zip %s ", inpath)
var compressedSize int64
if addZipItself <= 1 {
var zfs []zipF
if w.pm.useGoZip {
zr, err := zip.OpenReader(inpath)
if err != nil {
return 0, err
}
defer zr.Close()
zfs = make([]zipF, len(zr.File))
for i, zf := range zr.File {
zfs[i] = zipF(zf)
}
} else {
zr, err := czip.OpenReader(inpath)
if err != nil {
return 0, err
}
defer zr.Close()
zfs = make([]zipF, len(zr.File))
for i, zf := range zr.File {
zfs[i] = zipF(zf)
}
}
glog.V(4).Infof("zip entries %d: %s", len(zfs), inpath)
in := make(chan zipF)
out := make(chan zipWorkResult)
numWorkers := w.pm.NumWorkers()
for i := 0; i < numWorkers; i++ {
zw := &zipWorker{
index: i,
inpath: inpath,
w: w,
in: in,
out: out,
hh: newHashes(),
md5crcBuffer: make([]byte, md5.Size+crc32.Size+8),
}
go zw.Work()
}
var perr error
var nrProcessed int
var nrScheduled int
expectedResults := numWorkers
for _, zf := range zfs {
select {
case in <- zf:
glog.V(4).Infof("scheduled %s from zip %s", zf.FileInfo().Name(), inpath)
nrScheduled++
case zwr := <-out:
expectedResults--
glog.Warningf("breaking out of the zip loop before all files are scheduled: %s", inpath)
if zwr.err != nil {
perr = zwr.err
break
}
compressedSize += zwr.compressedSize
nrProcessed += zwr.nrProcessed
}
}
close(in)
if perr != nil {
glog.Errorf("zip error %s: %v", inpath, perr)
return 0, perr
}
glog.V(4).Infof("reading results from zip %s", inpath)
for i := 0; i < expectedResults; i++ {
zwr := <-out
if zwr.err != nil {
perr = zwr.err
break
}
compressedSize += zwr.compressedSize
nrProcessed += zwr.nrProcessed
}
if nrProcessed != len(zfs) || nrScheduled != len(zfs) {
glog.Warningf("scheduled/processed fewer zip entries: scheduled %d, processed %d, expected %d: %s",
nrScheduled, nrProcessed, len(zfs), inpath)
}
glog.V(4).Infof("scheduled %d, processed %d, expected %d: %s", nrScheduled, nrProcessed, len(zfs), inpath)
glog.V(4).Infof("finished archiving contents of zip %s", inpath)
if perr != nil {
glog.Errorf("zip error %s: %v", inpath, perr)
return 0, perr
}
}
if addZipItself >= 1 {
cs, err := w.archive(func() (io.ReadCloser, error) { return os.Open(inpath) },
filepath.Base(inpath), inpath, size, w.hh, w.md5crcBuffer)
if err != nil {
return 0, err
}
compressedSize += cs
}
return compressedSize, nil
}
func (w *Writer) Close() error {
err := w.uw.Close()
if err != nil {
return err
}
err = w.bf.Flush()
if err != nil {
return err
}
err = w.tf.Close()
if err != nil {
return err
}
r, err := czip.OpenReader(w.tf.Name())
if err != nil {
return err
}
fis := make(fileIndices, len(r.File))
for k, zf := range r.File {
fis[k] = &fileIndex{
name: torrentCanonicalName(zf.Name),
index: k,
}
}
sort.Sort(fis)
cw := &countWriter{
w: w.sink,
}
for _, fi := range fis {
fh := r.File[fi.index]
fh.Extra = nil
fi.offset = cw.count
err = writeHeader(cw, fh, fi.name)
if err != nil {
return err
}
fr, err := fh.OpenRaw()
if err != nil {
return err
}
_, err = io.Copy(cw, fr)
if err != nil {
return err
}
}
dircrc := crc32.NewIEEE()
mw := io.MultiWriter(cw, dircrc)
start := cw.count
for _, fi := range fis {
fh := r.File[fi.index]
fh.Extra = nil
err = writeCentralHeader(mw, fh, fi.name, fi.offset)
if err != nil {
return err
}
}
end := cw.count
records := uint64(len(fis))
size := uint64(end - start)
offset := uint64(start)
if records > uint16max || size > uint32max || offset > uint32max {
var buf [directory64EndLen + directory64LocLen]byte
b := writeBuf(buf[:])
// zip64 end of central directory record
b.uint32(directory64EndSignature)
b.uint64(directory64EndLen - 12)
b.uint16(zipVersion45) // version made by
b.uint16(zipVersion45) // version needed to extract
b.uint32(0) // number of this disk
b.uint32(0) // number of the disk with the start of the central directory
b.uint64(records) // total number of entries in the central directory on this disk
b.uint64(records) // total number of entries in the central directory
b.uint64(size) // size of the central directory
b.uint64(offset) // offset of start of central directory with respect to the starting disk number
// zip64 end of central directory locator
b.uint32(directory64LocSignature)
b.uint32(0) // number of the disk with the start of the zip64 end of central directory
b.uint64(uint64(end)) // relative offset of the zip64 end of central directory record
b.uint32(1) // total number of disks
if _, err := cw.Write(buf[:]); err != nil {
return err
}
// store max values in the regular end record to signal that
// that the zip64 values should be used instead
if records > uint16max {
records = uint16max
}
if size > uint32max {
size = uint32max
}
if offset > uint32max {
offset = uint32max
}
}
var buf [directoryEndLen]byte
b := writeBuf(buf[:])
b.uint32(uint32(directoryEndSignature))
b.uint16(0)
b.uint16(0)
b.uint16(uint16(records))
b.uint16(uint16(records))
b.uint32(uint32(size))
b.uint32(uint32(offset))
b.uint16(22)
zipcomment := "TORRENTZIPPED-" + strings.ToUpper(hex.EncodeToString(dircrc.Sum(nil)))
if _, err := cw.Write(buf[:]); err != nil {
return err
}
if _, err := io.WriteString(cw, zipcomment); err != nil {
return err
}
if err := r.Close(); err != nil {
return err
}
if err := os.Remove(w.tf.Name()); err != nil {
return err
}
return nil
}
func (tv *testdataVisitor) visit(path string, f os.FileInfo, err error) error {
if filepath.Ext(path) == zipext {
tv.t.Logf("testing file %s\n", path)
r, err := czip.OpenReader(path)
if err != nil {
return err
}
defer r.Close()
w, err := ioutil.TempFile(os.TempDir(), "torrentzip_test")
if err != nil {
return err
}
hh := sha1.New()
zw, err := NewWriter(io.MultiWriter(w, hh))
if err != nil {
return err
}
for _, fh := range r.File {
cw, err := zw.Create(fh.Name)
if err != nil {
return err
}
cr, err := fh.Open()
if err != nil {
return err
}
io.Copy(cw, cr)
cr.Close()
}
err = zw.Close()
if err != nil {
return err
}
err = w.Close()
if err != nil {
return err
}
testsha1 := hex.EncodeToString(hh.Sum(nil))
goldensha1 := strings.TrimSuffix(filepath.Base(path), zipext)
tv.t.Logf("testsha1=%s\n", testsha1)
tv.t.Logf("goldensha1=%s\n", goldensha1)
if testsha1 != strings.ToLower(goldensha1) {
return fmt.Errorf("produced torrentzip for %s differs from golden", path)
}
err = os.Remove(w.Name())
if err != nil {
return err
}
}
return nil
}
func testZip(path string, goldensha1 []byte) error {
r, err := czip.OpenReader(path)
if err != nil {
return err
}
defer r.Close()
w, err := ioutil.TempFile(os.TempDir(), "testtorrentzip")
if err != nil {
return err
}
hh := sha1.New()
zw, err := torrentzip.NewWriter(io.MultiWriter(w, hh))
if err != nil {
return err
}
for _, fh := range r.File {
cw, err := zw.Create(fh.Name)
if err != nil {
return err
}
cr, err := fh.Open()
if err != nil {
return err
}
_, err = io.Copy(cw, cr)
if err != nil {
return err
}
err = cr.Close()
if err != nil {
return err
}
}
err = zw.Close()
if err != nil {
return err
}
err = w.Close()
if err != nil {
return err
}
err = os.Remove(w.Name())
if err != nil {
return err
}
testsha1 := hh.Sum(nil)
if !bytes.Equal(testsha1, goldensha1) {
return fmt.Errorf("produced torrentzip for %s differs from golden", path)
}
return nil
}