Beispiel #1
0
func (w *archiveWorker) archiveZip(inpath string, size int64, addZipItself bool) (int64, error) {
	root, err := w.depot.reserveRoot(size)
	if err != nil {
		return 0, err
	}

	zr, err := czip.OpenReader(inpath)
	if err != nil {
		return 0, err
	}
	defer zr.Close()

	var compressedSize int64

	for _, zf := range zr.File {
		cs, err := w.archive(func() (io.ReadCloser, error) { return zf.Open() }, root,
			zf.FileInfo().Name(), filepath.Join(inpath, zf.FileInfo().Name()), zf.FileInfo().Size())
		if err != nil {
			return 0, err
		}
		compressedSize += cs
	}

	if addZipItself {
		cs, err := w.archive(func() (io.ReadCloser, error) { return os.Open(inpath) }, root, filepath.Base(inpath), inpath, size)
		if err != nil {
			return 0, err
		}
		compressedSize += cs
	}
	return compressedSize, nil
}
Beispiel #2
0
func unzip(path string) error {
	r, err := czip.OpenReader(path)
	if err != nil {
		return err
	}
	defer r.Close()

	for _, fh := range r.File {
		w, err := os.Create(fh.Name)
		if err != nil {
			fmt.Printf("error creating writer for %s: %v\n", fh.Name, err)
			return err
		}
		cr, err := fh.Open()
		if err != nil {
			fmt.Printf("error opening reader for %s: %v\n", fh.Name, err)
			return err
		}

		_, err = io.Copy(w, cr)
		if err != nil {
			fmt.Printf("error copying  from reader to writer for %s: %v\n", fh.Name, err)
			return err
		}

		err = cr.Close()
		if err != nil {
			fmt.Printf("error closing reader for %s: %v\n", fh.Name, err)
			return err
		}

		err = w.Close()
		if err != nil {
			fmt.Printf("error closing writer for %s: %v\n", fh.Name, err)
			return err
		}
	}
	return nil
}
Beispiel #3
0
func main() {
	flag.Parse()

	if len(flag.Args()) == 0 {
		os.Exit(1)
	}

	name := flag.Args()[0]

	zr, err := czip.OpenReader(name)
	if err != nil {
		fmt.Fprintf(os.Stderr, "opening zip file %s failed: %v\n", name, err)
		os.Exit(1)
	}

	for _, f := range zr.File {
		fmt.Printf("Size of %s: %d \n", f.Name, f.UncompressedSize64)

		dwf, err := os.Create("z-" + f.Name)
		if err != nil {
			log.Fatal(err)
		}

		rc, err := f.Open()
		if err != nil {
			log.Fatal(err)
		}
		_, err = io.Copy(dwf, rc)
		if err != nil {
			log.Fatal(err)
		}
		rc.Close()
		dwf.Close()
	}

	zr.Close()
}
Beispiel #4
0
func (w *archiveWorker) archiveZip(inpath string, size int64, addZipItself int) (int64, error) {
	glog.V(4).Infof("archiving zip %s ", inpath)

	var compressedSize int64

	if addZipItself <= 1 {
		var zfs []zipF

		if w.pm.useGoZip {
			zr, err := zip.OpenReader(inpath)
			if err != nil {
				return 0, err
			}
			defer zr.Close()

			zfs = make([]zipF, len(zr.File))
			for i, zf := range zr.File {
				zfs[i] = zipF(zf)
			}
		} else {
			zr, err := czip.OpenReader(inpath)
			if err != nil {
				return 0, err
			}
			defer zr.Close()

			zfs = make([]zipF, len(zr.File))
			for i, zf := range zr.File {
				zfs[i] = zipF(zf)
			}
		}

		glog.V(4).Infof("zip entries %d: %s", len(zfs), inpath)

		in := make(chan zipF)
		out := make(chan zipWorkResult)

		numWorkers := w.pm.NumWorkers()

		for i := 0; i < numWorkers; i++ {
			zw := &zipWorker{
				index:        i,
				inpath:       inpath,
				w:            w,
				in:           in,
				out:          out,
				hh:           newHashes(),
				md5crcBuffer: make([]byte, md5.Size+crc32.Size+8),
			}
			go zw.Work()
		}

		var perr error
		var nrProcessed int
		var nrScheduled int

		expectedResults := numWorkers

		for _, zf := range zfs {
			select {
			case in <- zf:
				glog.V(4).Infof("scheduled %s from zip %s", zf.FileInfo().Name(), inpath)
				nrScheduled++
			case zwr := <-out:
				expectedResults--
				glog.Warningf("breaking out of the zip loop before all files are scheduled: %s", inpath)
				if zwr.err != nil {
					perr = zwr.err
					break
				}
				compressedSize += zwr.compressedSize
				nrProcessed += zwr.nrProcessed
			}
		}

		close(in)

		if perr != nil {
			glog.Errorf("zip error %s: %v", inpath, perr)
			return 0, perr
		}

		glog.V(4).Infof("reading results from zip %s", inpath)

		for i := 0; i < expectedResults; i++ {
			zwr := <-out
			if zwr.err != nil {
				perr = zwr.err
				break
			}
			compressedSize += zwr.compressedSize
			nrProcessed += zwr.nrProcessed
		}

		if nrProcessed != len(zfs) || nrScheduled != len(zfs) {
			glog.Warningf("scheduled/processed fewer zip entries: scheduled %d, processed %d, expected %d: %s",
				nrScheduled, nrProcessed, len(zfs), inpath)
		}

		glog.V(4).Infof("scheduled %d, processed %d, expected %d: %s", nrScheduled, nrProcessed, len(zfs), inpath)
		glog.V(4).Infof("finished archiving contents of zip %s", inpath)

		if perr != nil {
			glog.Errorf("zip error %s: %v", inpath, perr)
			return 0, perr
		}
	}

	if addZipItself >= 1 {
		cs, err := w.archive(func() (io.ReadCloser, error) { return os.Open(inpath) },
			filepath.Base(inpath), inpath, size, w.hh, w.md5crcBuffer)
		if err != nil {
			return 0, err
		}
		compressedSize += cs
	}
	return compressedSize, nil
}
Beispiel #5
0
func (w *Writer) Close() error {
	err := w.uw.Close()
	if err != nil {
		return err
	}

	err = w.bf.Flush()
	if err != nil {
		return err
	}

	err = w.tf.Close()
	if err != nil {
		return err
	}

	r, err := czip.OpenReader(w.tf.Name())
	if err != nil {
		return err
	}

	fis := make(fileIndices, len(r.File))

	for k, zf := range r.File {
		fis[k] = &fileIndex{
			name:  torrentCanonicalName(zf.Name),
			index: k,
		}
	}

	sort.Sort(fis)

	cw := &countWriter{
		w: w.sink,
	}

	for _, fi := range fis {
		fh := r.File[fi.index]
		fh.Extra = nil
		fi.offset = cw.count
		err = writeHeader(cw, fh, fi.name)
		if err != nil {
			return err
		}
		fr, err := fh.OpenRaw()
		if err != nil {
			return err
		}

		_, err = io.Copy(cw, fr)
		if err != nil {
			return err
		}
	}

	dircrc := crc32.NewIEEE()
	mw := io.MultiWriter(cw, dircrc)
	start := cw.count

	for _, fi := range fis {
		fh := r.File[fi.index]
		fh.Extra = nil
		err = writeCentralHeader(mw, fh, fi.name, fi.offset)
		if err != nil {
			return err
		}
	}
	end := cw.count

	records := uint64(len(fis))
	size := uint64(end - start)
	offset := uint64(start)

	if records > uint16max || size > uint32max || offset > uint32max {
		var buf [directory64EndLen + directory64LocLen]byte
		b := writeBuf(buf[:])

		// zip64 end of central directory record
		b.uint32(directory64EndSignature)
		b.uint64(directory64EndLen - 12)
		b.uint16(zipVersion45) // version made by
		b.uint16(zipVersion45) // version needed to extract
		b.uint32(0)            // number of this disk
		b.uint32(0)            // number of the disk with the start of the central directory
		b.uint64(records)      // total number of entries in the central directory on this disk
		b.uint64(records)      // total number of entries in the central directory
		b.uint64(size)         // size of the central directory
		b.uint64(offset)       // offset of start of central directory with respect to the starting disk number

		// zip64 end of central directory locator
		b.uint32(directory64LocSignature)
		b.uint32(0)           // number of the disk with the start of the zip64 end of central directory
		b.uint64(uint64(end)) // relative offset of the zip64 end of central directory record
		b.uint32(1)           // total number of disks

		if _, err := cw.Write(buf[:]); err != nil {
			return err
		}

		// store max values in the regular end record to signal that
		// that the zip64 values should be used instead
		if records > uint16max {
			records = uint16max
		}
		if size > uint32max {
			size = uint32max
		}
		if offset > uint32max {
			offset = uint32max
		}
	}

	var buf [directoryEndLen]byte
	b := writeBuf(buf[:])
	b.uint32(uint32(directoryEndSignature))
	b.uint16(0)
	b.uint16(0)
	b.uint16(uint16(records))
	b.uint16(uint16(records))
	b.uint32(uint32(size))
	b.uint32(uint32(offset))
	b.uint16(22)

	zipcomment := "TORRENTZIPPED-" + strings.ToUpper(hex.EncodeToString(dircrc.Sum(nil)))

	if _, err := cw.Write(buf[:]); err != nil {
		return err
	}
	if _, err := io.WriteString(cw, zipcomment); err != nil {
		return err
	}

	if err := r.Close(); err != nil {
		return err
	}

	if err := os.Remove(w.tf.Name()); err != nil {
		return err
	}

	return nil
}
func (tv *testdataVisitor) visit(path string, f os.FileInfo, err error) error {
	if filepath.Ext(path) == zipext {
		tv.t.Logf("testing file %s\n", path)
		r, err := czip.OpenReader(path)
		if err != nil {
			return err
		}
		defer r.Close()

		w, err := ioutil.TempFile(os.TempDir(), "torrentzip_test")
		if err != nil {
			return err
		}

		hh := sha1.New()

		zw, err := NewWriter(io.MultiWriter(w, hh))
		if err != nil {
			return err
		}

		for _, fh := range r.File {
			cw, err := zw.Create(fh.Name)
			if err != nil {
				return err
			}
			cr, err := fh.Open()
			if err != nil {
				return err
			}

			io.Copy(cw, cr)

			cr.Close()
		}

		err = zw.Close()
		if err != nil {
			return err
		}

		err = w.Close()
		if err != nil {
			return err
		}

		testsha1 := hex.EncodeToString(hh.Sum(nil))
		goldensha1 := strings.TrimSuffix(filepath.Base(path), zipext)

		tv.t.Logf("testsha1=%s\n", testsha1)
		tv.t.Logf("goldensha1=%s\n", goldensha1)

		if testsha1 != strings.ToLower(goldensha1) {
			return fmt.Errorf("produced torrentzip for %s differs from golden", path)
		}

		err = os.Remove(w.Name())
		if err != nil {
			return err
		}
	}
	return nil
}
Beispiel #7
0
func testZip(path string, goldensha1 []byte) error {
	r, err := czip.OpenReader(path)
	if err != nil {
		return err
	}
	defer r.Close()

	w, err := ioutil.TempFile(os.TempDir(), "testtorrentzip")
	if err != nil {
		return err
	}

	hh := sha1.New()

	zw, err := torrentzip.NewWriter(io.MultiWriter(w, hh))
	if err != nil {
		return err
	}

	for _, fh := range r.File {
		cw, err := zw.Create(fh.Name)
		if err != nil {
			return err
		}
		cr, err := fh.Open()
		if err != nil {
			return err
		}

		_, err = io.Copy(cw, cr)
		if err != nil {
			return err
		}

		err = cr.Close()
		if err != nil {
			return err
		}
	}

	err = zw.Close()
	if err != nil {
		return err
	}

	err = w.Close()
	if err != nil {
		return err
	}

	err = os.Remove(w.Name())
	if err != nil {
		return err
	}

	testsha1 := hh.Sum(nil)

	if !bytes.Equal(testsha1, goldensha1) {
		return fmt.Errorf("produced torrentzip for %s differs from golden", path)
	}

	return nil
}