func (dbd *dbDeduper) Seen(r *types.Rom) (bool, error) {
if len(r.Sha1) > 0 {
val, err := dbd.sha1DB.Get(ro, r.Sha1)
if err != nil {
return false, err
}
return len(val) == 1 && val[0] == 1, nil
}
if len(r.Md5) > 0 {
val, err := dbd.md5DB.Get(ro, r.Md5WithSizeKey())
if err != nil {
return false, err
}
return len(val) == 1 && val[0] == 1, nil
}
if len(r.Crc) > 0 {
val, err := dbd.crcDB.Get(ro, r.CrcWithSizeKey())
if err != nil {
return false, err
}
return len(val) == 1 && val[0] == 1, nil
}
return false, nil
}
func (kvdb *kvStore) CompleteRom(rom *types.Rom) error {
if rom.Sha1 != nil {
return nil
}
if rom.Md5 != nil {
dBytes, err := kvdb.md5sha1DB.Get(rom.Md5)
if err != nil {
return err
}
if len(dBytes) >= sha1.Size {
rom.Sha1 = dBytes
} else {
glog.Warningf("no mapping from MD5 %s to SHA1", hex.EncodeToString(rom.Md5))
}
return nil
}
if rom.Crc != nil {
dBytes, err := kvdb.crcsha1DB.Get(rom.Crc)
if err != nil {
return err
}
if len(dBytes) >= sha1.Size {
rom.Sha1 = dBytes
} else {
glog.Warningf("no mapping from CRC %s to SHA1", hex.EncodeToString(rom.Crc))
}
}
return nil
}
func (kvb *kvBatch) IndexRom(rom *types.Rom) error {
glog.V(4).Infof("indexing rom %s", rom.Name)
if rom.Sha1 != nil {
if rom.Crc != nil {
glog.V(4).Infof("declaring crc %s -> sha1 %s mapping", hex.EncodeToString(rom.Crc), hex.EncodeToString(rom.Sha1))
err := kvb.crcsha1Batch.Set(rom.CrcWithSizeAndSha1Key(nil), oneValue)
if err != nil {
return err
}
kvb.size += int64(sha1.Size)
}
if rom.Md5 != nil {
glog.V(4).Infof("declaring md5 %s -> sha1 %s mapping", hex.EncodeToString(rom.Md5), hex.EncodeToString(rom.Sha1))
err := kvb.md5sha1Batch.Set(rom.Md5WithSizeAndSha1Key(nil), oneValue)
if err != nil {
return err
}
kvb.size += int64(sha1.Size)
}
} else {
glog.V(4).Infof("indexing rom %s with missing SHA1", rom.Name)
}
return nil
}
func RomFromGZDepotFile(inpath string) (*types.Rom, error) {
rom := new(types.Rom)
fileName := filepath.Base(inpath)
sha1Hex := strings.TrimSuffix(fileName, filepath.Ext(fileName))
sha1, err := hex.DecodeString(sha1Hex)
if err != nil {
return nil, err
}
rom.Sha1 = sha1
return rom, nil
}
func (rs *RombaService) lookup(cmd *commander.Command, args []string) error {
for _, arg := range args {
hash, err := hex.DecodeString(arg)
if err != nil {
return err
}
if len(hash) == sha1.Size {
dat, err := rs.romDB.GetDat(hash)
if err != nil {
return err
}
if dat != nil {
fmt.Fprintf(cmd.Stdout, "dat with sha1 %s = %s\n", arg, types.PrintShortDat(dat))
}
}
r := new(types.Rom)
switch len(hash) {
case md5.Size:
r.Md5 = hash
case crc32.Size:
r.Crc = hash
case sha1.Size:
r.Sha1 = hash
default:
return fmt.Errorf("found unknown hash size: %d", len(hash))
}
dats, err := rs.romDB.DatsForRom(r)
if err != nil {
return err
}
err = rs.romDB.CompleteRom(r)
if err != nil {
return err
}
if len(dats) > 0 {
fmt.Fprintf(cmd.Stdout, "rom in %s\n", types.PrintRomInDats(dats))
}
}
return nil
}
func (md *memoryDeduper) Declare(r *types.Rom) error {
md.mutex.Lock()
defer md.mutex.Unlock()
if len(r.Crc) > 0 {
md.crcs[string(r.CrcWithSizeKey())] = true
}
if len(r.Md5) > 0 {
md.md5s[string(r.Md5WithSizeKey())] = true
}
if len(r.Sha1) > 0 {
md.sha1s[string(r.Sha1)] = true
}
return nil
}
func (rs *RombaService) lookupRom(cmd *commander.Command, r *types.Rom, outpath string) error {
err := rs.romDB.CompleteRom(r)
if err != nil {
return err
}
if r.Sha1 != nil {
sha1Str := hex.EncodeToString(r.Sha1)
inDepot, hh, rompath, size, err := rs.depot.SHA1InDepot(sha1Str)
if err != nil {
return err
}
if inDepot {
fmt.Fprintf(cmd.Stdout, "-----------------\n")
fmt.Fprintf(cmd.Stdout, "rom file %s in depot\n", rompath)
fmt.Fprintf(cmd.Stdout, "crc = %s\n", hex.EncodeToString(hh.Crc))
fmt.Fprintf(cmd.Stdout, "md5 = %s\n", hex.EncodeToString(hh.Md5))
fmt.Fprintf(cmd.Stdout, "size = %d\n", size)
r.Crc = hh.Crc
r.Md5 = hh.Md5
if outpath != "" {
worker.Cp(rompath, filepath.Join(outpath, filepath.Base(rompath)))
}
}
}
dats, err := rs.romDB.DatsForRom(r)
if err != nil {
return err
}
if len(dats) > 0 {
fmt.Fprintf(cmd.Stdout, "-----------------\n")
fmt.Fprintf(cmd.Stdout, "rom found in:\n")
for _, dat := range dats {
dn := dat.NarrowToRom(r)
if dn != nil {
fmt.Fprintf(cmd.Stdout, "%s\n", types.PrintDat(dn))
}
}
}
return nil
}
func (md *memoryDeduper) Seen(r *types.Rom) (bool, error) {
md.mutex.Lock()
defer md.mutex.Unlock()
if len(r.Sha1) > 0 && md.sha1s[string(r.Sha1)] {
return true, nil
}
if len(r.Md5) > 0 && md.md5s[string(r.Md5WithSizeKey())] {
return true, nil
}
if len(r.Crc) > 0 && md.crcs[string(r.CrcWithSizeKey())] {
return true, nil
}
return false, nil
}
func (gw *gameWalker) visit(path string, f os.FileInfo, err error) error {
if f == nil || f.Name() == ".DS_Store" {
return nil
}
if f.IsDir() {
return nil
}
hh, err := HashesForFile(path)
if err != nil {
return err
}
romName, err := filepath.Rel(gw.gamepath, path)
if err != nil {
return err
}
rom := new(types.Rom)
rom.Name = romName
rom.Size = f.Size()
rom.Crc = hh.Crc
rom.Md5 = hh.Md5
rom.Sha1 = hh.Sha1
gw.game.Roms = append(gw.game.Roms, rom)
return nil
}
func lookupByHash(hash []byte) (bool, error) {
found := false
if len(hash) == sha1.Size {
dat, err := romDB.GetDat(hash)
if err != nil {
return false, err
}
if dat != nil {
fmt.Printf("dat = %s\n", types.PrintDat(dat))
found = true
}
}
r := new(types.Rom)
switch len(hash) {
case md5.Size:
r.Md5 = hash
case crc32.Size:
r.Crc = hash
case sha1.Size:
r.Sha1 = hash
default:
return false, fmt.Errorf("found unknown hash size: %d", len(hash))
}
dats, err := romDB.DatsForRom(r)
if err != nil {
return false, err
}
for _, dat := range dats {
fmt.Printf("dat = %s\n", types.PrintDat(dat))
}
found = found || len(dats) > 0
return found, nil
}
func (dbd *dbDeduper) Declare(r *types.Rom) error {
if len(r.Crc) > 0 {
err := dbd.crcDB.Put(wo, r.CrcWithSizeKey(), trueVal)
if err != nil {
return err
}
}
if len(r.Md5) > 0 {
err := dbd.md5DB.Put(wo, r.Md5WithSizeKey(), trueVal)
if err != nil {
return err
}
}
if len(r.Sha1) > 0 {
err := dbd.sha1DB.Put(wo, r.Sha1, trueVal)
if err != nil {
return err
}
}
return nil
}
func fixHashes(rom *types.Rom) {
if rom.Crc != nil {
strV := string(rom.Crc)
if strV != "" {
v, err := hex.DecodeString(string(rom.Crc))
if err != nil {
rom.Crc = nil
}
rom.Crc = v
} else {
rom.Crc = nil
}
}
if rom.Md5 != nil {
strV := string(rom.Md5)
if strV != "" {
v, err := hex.DecodeString(string(rom.Md5))
if err != nil {
rom.Md5 = nil
}
rom.Md5 = v
} else {
rom.Md5 = nil
}
}
if rom.Sha1 != nil {
strV := string(rom.Sha1)
if strV != "" {
v, err := hex.DecodeString(string(rom.Sha1))
if err != nil {
rom.Sha1 = nil
}
rom.Sha1 = v
} else {
rom.Sha1 = nil
}
}
}
func (kvdb *kvStore) CompleteRom(rom *types.Rom) error {
if rom.Sha1 != nil {
return nil
}
if rom.Md5 != nil {
dBytes, err := kvdb.md5sha1DB.GetKeySuffixesFor(rom.Md5WithSizeKey())
if err != nil {
return err
}
rom.Sha1 = dBytes
return nil
}
if rom.Crc != nil {
dBytes, err := kvdb.crcsha1DB.GetKeySuffixesFor(rom.CrcWithSizeKey())
if err != nil {
return err
}
rom.Sha1 = dBytes
}
return nil
}
func TestDB(t *testing.T) {
dbDir, err := ioutil.TempDir("", "rombadb")
if err != nil {
t.Fatalf("cannot create temp dir for test db: %v", err)
}
t.Logf("creating test db in %s\n", dbDir)
krdb, err := db.New(dbDir)
if err != nil {
t.Fatalf("failed to open db: %v", err)
}
dat, sha1Bytes, err := parser.ParseDat(strings.NewReader(datText), "testing/dat")
if err != nil {
t.Fatalf("failed to parse test dat: %v", err)
}
err = krdb.IndexDat(dat, sha1Bytes)
if err != nil {
t.Fatalf("failed to index test dat: %v", err)
}
datFromDb, err := krdb.GetDat(sha1Bytes)
if err != nil {
t.Fatalf("failed to retrieve test dat: %v", err)
}
if !datFromDb.Equals(dat) {
fmt.Printf("datFromDb=%s\n", string(types.PrintDat(datFromDb)))
fmt.Printf("dat=%s\n", string(types.PrintDat(dat)))
t.Fatalf("dat differs from db dat")
}
romSha1Bytes, err := hex.DecodeString("80353cb168dc5d7cc1dce57971f4ea2640a50ac4")
if err != nil {
t.Fatalf("failed to hex decode: %v", err)
}
rom := new(types.Rom)
rom.Sha1 = romSha1Bytes
dats, err := krdb.DatsForRom(rom)
if err != nil {
t.Fatalf("failed to retrieve dats for rom: %v", err)
}
if len(dats) != 1 {
t.Fatalf("couldn't find dats for rom")
}
datFromDb = dats[0]
if !datFromDb.Equals(dat) {
fmt.Printf("datFromDb=%s\n", string(types.PrintDat(datFromDb)))
fmt.Printf("dat=%s\n", string(types.PrintDat(dat)))
t.Fatalf("dat differs from db dat")
}
err = krdb.Close()
if err != nil {
t.Fatalf("failed to close db: %v", err)
}
err = os.RemoveAll(dbDir)
if err != nil {
t.Fatalf("failed to remove test db dir %s: %v", dbDir, err)
}
}
func (kvdb *kvStore) FilteredDatsForRom(rom *types.Rom, filter func(*types.Dat) bool) ([]*types.Dat, []*types.Dat, error) {
var dBytes []byte
if rom.Sha1 != nil {
bs, err := kvdb.sha1DB.GetKeySuffixesFor(rom.Sha1)
if err != nil {
return nil, nil, err
}
if bs != nil {
dBytes = append(dBytes, bs...)
}
}
if rom.Md5 != nil {
bs, err := kvdb.md5DB.GetKeySuffixesFor(rom.Md5WithSizeKey())
if err != nil {
return nil, nil, err
}
if bs != nil {
dBytes = append(dBytes, bs...)
}
}
if rom.Crc != nil {
bs, err := kvdb.crcDB.GetKeySuffixesFor(rom.CrcWithSizeKey())
if err != nil {
return nil, nil, err
}
if bs != nil {
dBytes = append(dBytes, bs...)
}
}
if dBytes == nil {
return nil, nil, nil
}
var dats []*types.Dat
var rejectedDats []*types.Dat
seen := make(map[string]bool)
for i := 0; i < len(dBytes); i += sha1.Size {
sha1Bytes := dBytes[i : i+sha1.Size]
if seen[string(sha1Bytes)] {
continue
}
seen[string(sha1Bytes)] = true
dat, err := kvdb.GetDat(sha1Bytes)
if err != nil {
return nil, nil, err
}
if dat != nil {
if filter(dat) {
dats = append(dats, dat)
} else {
rejectedDats = append(rejectedDats, dat)
}
}
}
return dats, rejectedDats, nil
}
func (rs *RombaService) lookup(cmd *commander.Command, args []string) error {
size := cmd.Flag.Lookup("size").Value.Get().(int64)
outpath := cmd.Flag.Lookup("out").Value.Get().(string)
for _, arg := range args {
fmt.Fprintf(cmd.Stdout, "----------------------------------------\n")
fmt.Fprintf(cmd.Stdout, "key: %s\n", arg)
if strings.HasPrefix(arg, "0x") {
arg = arg[2:]
}
hash, err := hex.DecodeString(arg)
if err != nil {
return err
}
if len(hash) == sha1.Size {
dat, err := rs.romDB.GetDat(hash)
if err != nil {
return err
}
if dat != nil {
fmt.Fprintf(cmd.Stdout, "-----------------\n")
fmt.Fprintf(cmd.Stdout, "dat with sha1 %s = %s\n", arg, types.PrintShortDat(dat))
}
}
if size != -1 || len(hash) == sha1.Size {
r := new(types.Rom)
r.Size = size
switch len(hash) {
case md5.Size:
r.Md5 = hash
case crc32.Size:
r.Crc = hash
case sha1.Size:
r.Sha1 = hash
default:
return fmt.Errorf("found unknown hash size: %d", len(hash))
}
err = rs.lookupRom(cmd, r, outpath)
if err != nil {
return err
}
fmt.Fprintf(cmd.Stdout, "-----------------\n")
fmt.Fprintf(cmd.Stdout, "DebugGet:\n%s\n", rs.romDB.DebugGet(hash, size))
} else {
suffixes, err := rs.romDB.ResolveHash(hash)
if err != nil {
return err
}
for i := 0; i < len(suffixes); i += sha1.Size + 8 {
r := new(types.Rom)
r.Size = util.BytesToInt64(suffixes[i : i+8])
switch len(hash) {
case md5.Size:
r.Md5 = hash
case crc32.Size:
r.Crc = hash
default:
return fmt.Errorf("found unknown hash size: %d", len(hash))
}
r.Sha1 = suffixes[i+8 : i+8+sha1.Size]
err = rs.lookupRom(cmd, r, outpath)
if err != nil {
return err
}
}
}
}
return nil
}
func (w *archiveWorker) archive(ro readerOpener, name, path string, size int64, hh *Hashes, md5crcBuffer []byte) (int64, error) {
r, err := ro()
if err != nil {
return 0, err
}
br := bufio.NewReader(r)
err = hh.forReader(br)
if err != nil {
r.Close()
return 0, err
}
err = r.Close()
if err != nil {
return 0, err
}
// if filestat size is different than size read then size read wins
if size != hh.Size {
size = hh.Size
}
copy(md5crcBuffer[0:md5.Size], hh.Md5)
copy(md5crcBuffer[md5.Size:md5.Size+crc32.Size], hh.Crc)
util.Int64ToBytes(size, md5crcBuffer[md5.Size+crc32.Size:])
rom := new(types.Rom)
rom.Crc = make([]byte, crc32.Size)
rom.Md5 = make([]byte, md5.Size)
rom.Sha1 = make([]byte, sha1.Size)
copy(rom.Crc, hh.Crc)
copy(rom.Md5, hh.Md5)
copy(rom.Sha1, hh.Sha1)
rom.Name = name
rom.Size = size
rom.Path = path
if !w.pm.noDB {
if w.pm.onlyneeded {
dats, err := w.depot.romDB.DatsForRom(rom)
if err != nil {
return 0, err
}
if len(dats) == 0 {
return 0, nil
}
}
err = w.depot.romDB.IndexRom(rom)
if err != nil {
return 0, err
}
}
sha1Hex := hex.EncodeToString(hh.Sha1)
exists, _, err := w.depot.RomInDepot(sha1Hex)
if err != nil {
return 0, err
}
if exists {
glog.V(4).Infof("%s already in depot, skipping %s/%s", sha1Hex, path, name)
return 0, nil
}
estimatedCompressedSize := size / 5
root, err := w.depot.reserveRoot(estimatedCompressedSize)
if err != nil {
return 0, err
}
outpath := pathFromSha1HexEncoding(w.depot.roots[root], sha1Hex, gzipSuffix)
r, err = ro()
if err != nil {
return 0, err
}
defer r.Close()
compressedSize, err := archive(outpath, r, md5crcBuffer)
if err != nil {
return 0, err
}
w.depot.adjustSize(root, compressedSize-estimatedCompressedSize)
return compressedSize, nil
}
func (w *archiveWorker) archive(ro readerOpener, root int, name, path string, size int64) (int64, error) {
r, err := ro()
if err != nil {
return 0, err
}
br := bufio.NewReader(r)
err = w.hh.forReader(br)
if err != nil {
r.Close()
return 0, err
}
err = r.Close()
if err != nil {
return 0, err
}
copy(w.md5crcBuffer[0:md5.Size], w.hh.Md5)
copy(w.md5crcBuffer[md5.Size:], w.hh.Crc)
rom := new(types.Rom)
rom.Crc = make([]byte, crc32.Size)
rom.Md5 = make([]byte, md5.Size)
rom.Sha1 = make([]byte, sha1.Size)
copy(rom.Crc, w.hh.Crc)
copy(rom.Md5, w.hh.Md5)
copy(rom.Sha1, w.hh.Sha1)
rom.Name = name
rom.Size = size
rom.Path = path
if w.pm.onlyneeded {
dats, err := w.depot.romDB.DatsForRom(rom)
if err != nil {
return 0, err
}
needed := false
for _, dat := range dats {
if !dat.Artificial {
needed = true
break
}
}
if !needed {
return 0, nil
}
}
err = w.depot.romDB.IndexRom(rom)
if err != nil {
return 0, err
}
sha1Hex := hex.EncodeToString(w.hh.Sha1)
outpath := pathFromSha1HexEncoding(w.depot.roots[root], sha1Hex, gzipSuffix)
exists, err := PathExists(outpath)
if err != nil {
return 0, err
}
if exists {
return 0, nil
}
r, err = ro()
if err != nil {
return 0, err
}
defer r.Close()
compressedSize, err := archive(outpath, r, w.md5crcBuffer)
if err != nil {
return 0, err
}
w.depot.adjustSize(root, compressedSize)
return compressedSize, nil
}