func (w *archiveWorker) archive(ro readerOpener, name, path string, size int64, hh *Hashes, md5crcBuffer []byte) (int64, error) {
r, err := ro()
if err != nil {
return 0, err
}
br := bufio.NewReader(r)
err = hh.forReader(br)
if err != nil {
r.Close()
return 0, err
}
err = r.Close()
if err != nil {
return 0, err
}
// if filestat size is different than size read then size read wins
if size != hh.Size {
size = hh.Size
}
copy(md5crcBuffer[0:md5.Size], hh.Md5)
copy(md5crcBuffer[md5.Size:md5.Size+crc32.Size], hh.Crc)
util.Int64ToBytes(size, md5crcBuffer[md5.Size+crc32.Size:])
rom := new(types.Rom)
rom.Crc = make([]byte, crc32.Size)
rom.Md5 = make([]byte, md5.Size)
rom.Sha1 = make([]byte, sha1.Size)
copy(rom.Crc, hh.Crc)
copy(rom.Md5, hh.Md5)
copy(rom.Sha1, hh.Sha1)
rom.Name = name
rom.Size = size
rom.Path = path
if !w.pm.noDB {
if w.pm.onlyneeded {
dats, err := w.depot.romDB.DatsForRom(rom)
if err != nil {
return 0, err
}
if len(dats) == 0 {
return 0, nil
}
}
err = w.depot.romDB.IndexRom(rom)
if err != nil {
return 0, err
}
}
sha1Hex := hex.EncodeToString(hh.Sha1)
exists, _, err := w.depot.RomInDepot(sha1Hex)
if err != nil {
return 0, err
}
if exists {
glog.V(4).Infof("%s already in depot, skipping %s/%s", sha1Hex, path, name)
return 0, nil
}
estimatedCompressedSize := size / 5
root, err := w.depot.reserveRoot(estimatedCompressedSize)
if err != nil {
return 0, err
}
outpath := pathFromSha1HexEncoding(w.depot.roots[root], sha1Hex, gzipSuffix)
r, err = ro()
if err != nil {
return 0, err
}
defer r.Close()
compressedSize, err := archive(outpath, r, md5crcBuffer)
if err != nil {
return 0, err
}
w.depot.adjustSize(root, compressedSize-estimatedCompressedSize)
return compressedSize, nil
}
func (w *archiveWorker) archive(ro readerOpener, root int, name, path string, size int64) (int64, error) {
r, err := ro()
if err != nil {
return 0, err
}
br := bufio.NewReader(r)
err = w.hh.forReader(br)
if err != nil {
r.Close()
return 0, err
}
err = r.Close()
if err != nil {
return 0, err
}
copy(w.md5crcBuffer[0:md5.Size], w.hh.Md5)
copy(w.md5crcBuffer[md5.Size:], w.hh.Crc)
rom := new(types.Rom)
rom.Crc = make([]byte, crc32.Size)
rom.Md5 = make([]byte, md5.Size)
rom.Sha1 = make([]byte, sha1.Size)
copy(rom.Crc, w.hh.Crc)
copy(rom.Md5, w.hh.Md5)
copy(rom.Sha1, w.hh.Sha1)
rom.Name = name
rom.Size = size
rom.Path = path
if w.pm.onlyneeded {
dats, err := w.depot.romDB.DatsForRom(rom)
if err != nil {
return 0, err
}
needed := false
for _, dat := range dats {
if !dat.Artificial {
needed = true
break
}
}
if !needed {
return 0, nil
}
}
err = w.depot.romDB.IndexRom(rom)
if err != nil {
return 0, err
}
sha1Hex := hex.EncodeToString(w.hh.Sha1)
outpath := pathFromSha1HexEncoding(w.depot.roots[root], sha1Hex, gzipSuffix)
exists, err := PathExists(outpath)
if err != nil {
return 0, err
}
if exists {
return 0, nil
}
r, err = ro()
if err != nil {
return 0, err
}
defer r.Close()
compressedSize, err := archive(outpath, r, w.md5crcBuffer)
if err != nil {
return 0, err
}
w.depot.adjustSize(root, compressedSize)
return compressedSize, nil
}