func (kvb *kvBatch) IndexRom(rom *types.Rom) error { //glog.Infof("indexing rom %s", rom.Name) dats, err := kvb.db.DatsForRom(rom) if err != nil { return err } if len(dats) > 0 { if rom.Crc != nil && rom.Sha1 != nil { //glog.Infof("declaring crc %s -> sha1 %s ampping", hex.EncodeToString(rom.Crc), hex.EncodeToString(rom.Sha1)) err = kvb.crcsha1Batch.Append(rom.Crc, rom.Sha1) if err != nil { return err } kvb.size += int64(sha1.Size) } if rom.Md5 != nil && rom.Sha1 != nil { //glog.Infof("declaring md5 %s -> sha1 %s ampping", hex.EncodeToString(rom.Md5), hex.EncodeToString(rom.Sha1)) err = kvb.md5sha1Batch.Append(rom.Md5, rom.Sha1) if err != nil { return err } kvb.size += int64(sha1.Size) } return nil } if rom.Sha1 == nil { glog.Warningf("indexing rom %s with missing SHA1", rom.Name) } dat := new(types.Dat) dat.Artificial = true dat.Generation = kvb.db.generation dat.Name = fmt.Sprintf("Artificial Dat for %s", rom.Name) dat.Path = rom.Path game := new(types.Game) game.Roms = []*types.Rom{rom} dat.Games = []*types.Game{game} var buf bytes.Buffer gobEncoder := gob.NewEncoder(&buf) err = gobEncoder.Encode(dat) if err != nil { return err } hh := sha1.New() _, err = io.Copy(hh, &buf) if err != nil { return err } return kvb.IndexDat(dat, hh.Sum(nil)) }
func (kvb *kvBatch) IndexDat(dat *types.Dat, sha1Bytes []byte) error { glog.V(4).Infof("indexing dat %s", dat.Name) if sha1Bytes == nil { return fmt.Errorf("sha1 is nil for %s", dat.Path) } dat.Generation = kvb.db.generation var buf bytes.Buffer gobEncoder := gob.NewEncoder(&buf) err := gobEncoder.Encode(dat) if err != nil { return err } exists, err := kvb.db.datsDB.Exists(sha1Bytes) if err != nil { return fmt.Errorf("failed to lookup sha1 indexing dats: %v", err) } kvb.datsBatch.Set(sha1Bytes, buf.Bytes()) kvb.size += int64(sha1.Size + buf.Len()) if !exists { for _, g := range dat.Games { glog.V(4).Infof("indexing game %s", g.Name) for _, r := range g.Roms { if r.Sha1 != nil { err = kvb.sha1Batch.Set(r.Sha1Sha1Key(sha1Bytes), oneValue) if err != nil { return err } kvb.size += int64(sha1.Size) } if r.Md5 != nil { err = kvb.md5Batch.Set(r.Md5WithSizeAndSha1Key(sha1Bytes), oneValue) if err != nil { return err } kvb.size += int64(sha1.Size) if r.Sha1 != nil { glog.V(4).Infof("declaring md5 %s -> sha1 %s mapping", hex.EncodeToString(r.Md5), hex.EncodeToString(r.Sha1)) err = kvb.md5sha1Batch.Set(r.Md5WithSizeAndSha1Key(nil), oneValue) if err != nil { return err } kvb.size += int64(sha1.Size) } } if r.Crc != nil { err = kvb.crcBatch.Set(r.CrcWithSizeAndSha1Key(sha1Bytes), oneValue) if err != nil { return err } kvb.size += int64(sha1.Size) if r.Sha1 != nil { glog.V(4).Infof("declaring crc %s -> sha1 %s mapping", hex.EncodeToString(r.Crc), hex.EncodeToString(r.Sha1)) err = kvb.crcsha1Batch.Set(r.CrcWithSizeAndSha1Key(nil), oneValue) if err != nil { return err } kvb.size += int64(sha1.Size) } } } } } return nil }
func (kvb *kvBatch) IndexDat(dat *types.Dat, sha1Bytes []byte) error { glog.Infof("indexing dat %s", dat.Name) if sha1Bytes == nil { return fmt.Errorf("sha1 is nil for %s", dat.Path) } dat.Generation = kvb.db.generation var buf bytes.Buffer gobEncoder := gob.NewEncoder(&buf) err := gobEncoder.Encode(dat) if err != nil { return err } var exists bool if dat.Artificial { exists = false } else { existsSha1, err := kvb.db.datsDB.Exists(sha1Bytes) if err != nil { return fmt.Errorf("failed to lookup sha1 indexing dats: %v", err) } exists = existsSha1 } kvb.datsBatch.Set(sha1Bytes, buf.Bytes()) kvb.size += int64(sha1.Size + buf.Len()) if !exists { for _, g := range dat.Games { for _, r := range g.Roms { if r.Sha1 != nil { err = kvb.sha1Batch.Append(r.Sha1, sha1Bytes) if err != nil { return err } kvb.size += int64(sha1.Size) } if r.Md5 != nil { err = kvb.md5Batch.Append(r.Md5, sha1Bytes) if err != nil { return err } kvb.size += int64(sha1.Size) if r.Sha1 != nil { //glog.Infof("declaring md5 %s -> sha1 %s ampping", hex.EncodeToString(r.Md5), hex.EncodeToString(r.Sha1)) err = kvb.md5sha1Batch.Append(r.Md5, r.Sha1) if err != nil { return err } kvb.size += int64(sha1.Size) } } if r.Crc != nil { err = kvb.crcBatch.Append(r.Crc, sha1Bytes) if err != nil { return err } kvb.size += int64(sha1.Size) if r.Sha1 != nil { //glog.Infof("declaring crc %s -> sha1 %s ampping", hex.EncodeToString(r.Crc), hex.EncodeToString(r.Sha1)) err = kvb.crcsha1Batch.Append(r.Crc, r.Sha1) if err != nil { return err } kvb.size += int64(sha1.Size) } } } } } return nil }