Esempio n. 1
0
func (w *archiveWorker) archive(ro readerOpener, name, path string, size int64, hh *Hashes, md5crcBuffer []byte) (int64, error) {
	r, err := ro()
	if err != nil {
		return 0, err
	}

	br := bufio.NewReader(r)

	err = hh.forReader(br)
	if err != nil {
		r.Close()
		return 0, err
	}
	err = r.Close()
	if err != nil {
		return 0, err
	}

	// if filestat size is different than size read then size read wins
	if size != hh.Size {
		size = hh.Size
	}

	copy(md5crcBuffer[0:md5.Size], hh.Md5)
	copy(md5crcBuffer[md5.Size:md5.Size+crc32.Size], hh.Crc)
	util.Int64ToBytes(size, md5crcBuffer[md5.Size+crc32.Size:])

	rom := new(types.Rom)
	rom.Crc = make([]byte, crc32.Size)
	rom.Md5 = make([]byte, md5.Size)
	rom.Sha1 = make([]byte, sha1.Size)
	copy(rom.Crc, hh.Crc)
	copy(rom.Md5, hh.Md5)
	copy(rom.Sha1, hh.Sha1)
	rom.Name = name
	rom.Size = size
	rom.Path = path

	if !w.pm.noDB {
		if w.pm.onlyneeded {
			dats, err := w.depot.romDB.DatsForRom(rom)
			if err != nil {
				return 0, err
			}

			if len(dats) == 0 {
				return 0, nil
			}
		}

		err = w.depot.romDB.IndexRom(rom)
		if err != nil {
			return 0, err
		}
	}

	sha1Hex := hex.EncodeToString(hh.Sha1)
	exists, _, err := w.depot.RomInDepot(sha1Hex)
	if err != nil {
		return 0, err
	}

	if exists {
		glog.V(4).Infof("%s already in depot, skipping %s/%s", sha1Hex, path, name)
		return 0, nil
	}

	estimatedCompressedSize := size / 5

	root, err := w.depot.reserveRoot(estimatedCompressedSize)
	if err != nil {
		return 0, err
	}

	outpath := pathFromSha1HexEncoding(w.depot.roots[root], sha1Hex, gzipSuffix)

	r, err = ro()
	if err != nil {
		return 0, err
	}
	defer r.Close()

	compressedSize, err := archive(outpath, r, md5crcBuffer)
	if err != nil {
		return 0, err
	}

	w.depot.adjustSize(root, compressedSize-estimatedCompressedSize)
	return compressedSize, nil
}
Esempio n. 2
0
func (w *archiveWorker) archive(ro readerOpener, root int, name, path string, size int64) (int64, error) {
	r, err := ro()
	if err != nil {
		return 0, err
	}

	br := bufio.NewReader(r)

	err = w.hh.forReader(br)
	if err != nil {
		r.Close()
		return 0, err
	}
	err = r.Close()
	if err != nil {
		return 0, err
	}

	copy(w.md5crcBuffer[0:md5.Size], w.hh.Md5)
	copy(w.md5crcBuffer[md5.Size:], w.hh.Crc)

	rom := new(types.Rom)
	rom.Crc = make([]byte, crc32.Size)
	rom.Md5 = make([]byte, md5.Size)
	rom.Sha1 = make([]byte, sha1.Size)
	copy(rom.Crc, w.hh.Crc)
	copy(rom.Md5, w.hh.Md5)
	copy(rom.Sha1, w.hh.Sha1)
	rom.Name = name
	rom.Size = size
	rom.Path = path

	if w.pm.onlyneeded {
		dats, err := w.depot.romDB.DatsForRom(rom)
		if err != nil {
			return 0, err
		}

		needed := false

		for _, dat := range dats {
			if !dat.Artificial {
				needed = true
				break
			}
		}
		if !needed {
			return 0, nil
		}
	}

	err = w.depot.romDB.IndexRom(rom)
	if err != nil {
		return 0, err
	}

	sha1Hex := hex.EncodeToString(w.hh.Sha1)

	outpath := pathFromSha1HexEncoding(w.depot.roots[root], sha1Hex, gzipSuffix)

	exists, err := PathExists(outpath)
	if err != nil {
		return 0, err
	}

	if exists {
		return 0, nil
	}

	r, err = ro()
	if err != nil {
		return 0, err
	}
	defer r.Close()

	compressedSize, err := archive(outpath, r, w.md5crcBuffer)
	if err != nil {
		return 0, err
	}

	w.depot.adjustSize(root, compressedSize)
	return compressedSize, nil
}