func (kvdb *kvStore) DebugGet(key []byte, size int64) string { var buf bytes.Buffer switch len(key) { case md5.Size: sizeBytes := make([]byte, 8) util.Int64ToBytes(size, sizeBytes) key = append(key, sizeBytes...) sha1s, err := kvdb.md5DB.GetKeySuffixesFor(key) if err != nil { glog.Errorf("error getting from md5DB: %v", err) } else { buf.WriteString(fmt.Sprintf("md5DB -> %s\n", printSha1s(sha1s))) } sha1s, err = kvdb.md5sha1DB.GetKeySuffixesFor(key) if err != nil { glog.Errorf("error getting from md5sha1DB: %v", err) } else { buf.WriteString(fmt.Sprintf("md5sha1DB -> %s\n", printSha1s(sha1s))) } case crc32.Size: sizeBytes := make([]byte, 8) util.Int64ToBytes(size, sizeBytes) key = append(key, sizeBytes...) sha1s, err := kvdb.crcDB.GetKeySuffixesFor(key) if err != nil { glog.Errorf("error getting from crcDB: %v", err) } else { buf.WriteString(fmt.Sprintf("crcDB -> %s\n", printSha1s(sha1s))) } sha1s, err = kvdb.crcsha1DB.GetKeySuffixesFor(key) if err != nil { glog.Errorf("error getting from crcsha1DB: %v", err) } else { buf.WriteString(fmt.Sprintf("crcsha1DB -> %s\n", printSha1s(sha1s))) } case sha1.Size: sha1s, err := kvdb.sha1DB.GetKeySuffixesFor(key) if err != nil { glog.Errorf("error getting from sha1DB: %v", err) } else { buf.WriteString(fmt.Sprintf("sha1DB -> %s\n", printSha1s(sha1s))) } default: glog.Errorf("found unknown hash size: %d", len(key)) return "" } return buf.String() }
func (ar *Rom) Md5WithSizeKey() []byte { if ar.Md5 == nil { return nil } n := keySizeMd5 key := make([]byte, n+8) copy(key[:n], ar.Md5) util.Int64ToBytes(ar.Size, key[n:]) return key }
func (ar *Rom) CrcWithSizeKey() []byte { if ar.Crc == nil { return nil } n := keySizeCrc key := make([]byte, n+8) copy(key[:n], ar.Crc) util.Int64ToBytes(ar.Size, key[n:]) return key }
func (ar *Rom) Md5WithSizeAndSha1Key(sha1Bytes []byte) []byte { if sha1Bytes == nil { sha1Bytes = ar.Sha1 } if ar.Md5 == nil || sha1Bytes == nil { return nil } key := make([]byte, keySizeMd5+8+keySizeSha1) copy(key[:keySizeMd5], ar.Md5) util.Int64ToBytes(ar.Size, key[keySizeMd5:keySizeMd5+8]) copy(key[keySizeMd5+8:], sha1Bytes) return key }
func (ar *Rom) CrcWithSizeAndSha1Key(sha1Bytes []byte) []byte { if sha1Bytes == nil { sha1Bytes = ar.Sha1 } if ar.Crc == nil || sha1Bytes == nil { return nil } key := make([]byte, keySizeCrc+8+keySizeSha1) copy(key[:keySizeCrc], ar.Crc) util.Int64ToBytes(ar.Size, key[keySizeCrc:keySizeCrc+8]) copy(key[keySizeCrc+8:], sha1Bytes) return key }
func (w *archiveWorker) archive(ro readerOpener, name, path string, size int64, hh *Hashes, md5crcBuffer []byte) (int64, error) { r, err := ro() if err != nil { return 0, err } br := bufio.NewReader(r) err = hh.forReader(br) if err != nil { r.Close() return 0, err } err = r.Close() if err != nil { return 0, err } // if filestat size is different than size read then size read wins if size != hh.Size { size = hh.Size } copy(md5crcBuffer[0:md5.Size], hh.Md5) copy(md5crcBuffer[md5.Size:md5.Size+crc32.Size], hh.Crc) util.Int64ToBytes(size, md5crcBuffer[md5.Size+crc32.Size:]) rom := new(types.Rom) rom.Crc = make([]byte, crc32.Size) rom.Md5 = make([]byte, md5.Size) rom.Sha1 = make([]byte, sha1.Size) copy(rom.Crc, hh.Crc) copy(rom.Md5, hh.Md5) copy(rom.Sha1, hh.Sha1) rom.Name = name rom.Size = size rom.Path = path if !w.pm.noDB { if w.pm.onlyneeded { dats, err := w.depot.romDB.DatsForRom(rom) if err != nil { return 0, err } if len(dats) == 0 { return 0, nil } } err = w.depot.romDB.IndexRom(rom) if err != nil { return 0, err } } sha1Hex := hex.EncodeToString(hh.Sha1) exists, _, err := w.depot.RomInDepot(sha1Hex) if err != nil { return 0, err } if exists { glog.V(4).Infof("%s already in depot, skipping %s/%s", sha1Hex, path, name) return 0, nil } estimatedCompressedSize := size / 5 root, err := w.depot.reserveRoot(estimatedCompressedSize) if err != nil { return 0, err } outpath := pathFromSha1HexEncoding(w.depot.roots[root], sha1Hex, gzipSuffix) r, err = ro() if err != nil { return 0, err } defer r.Close() compressedSize, err := archive(outpath, r, md5crcBuffer) if err != nil { return 0, err } w.depot.adjustSize(root, compressedSize-estimatedCompressedSize) return compressedSize, nil }