func Dedup(d *types.Dat, deduper Deduper) (*types.Dat, error) {
dc := new(types.Dat)
dc.CopyHeader(d)
for _, g := range d.Games {
gc := new(types.Game)
gc.CopyHeader(g)
for _, r := range g.Roms {
if !r.Valid() {
continue
}
seen, err := deduper.Seen(r)
if err != nil {
return nil, err
}
if !seen {
gc.Roms = append(gc.Roms, r)
err = deduper.Declare(r)
if err != nil {
return nil, err
}
}
}
if len(gc.Roms) > 0 {
dc.Games = append(dc.Games, gc)
}
}
if len(dc.Games) > 0 {
return dc, nil
}
return nil, nil
}
func (rw *romWalker) visit(path string, f os.FileInfo, err error) error {
if f == nil || f.Name() == ".DS_Store" {
return nil
}
if f.IsDir() {
return nil
}
hh, err := HashesForFile(path)
if err != nil {
return err
}
romName, err := filepath.Rel(rw.sourcePath, path)
if err != nil {
return err
}
rom := new(types.Rom)
rom.Name = romName
rom.Size = f.Size()
rom.Crc = hh.Crc
rom.Md5 = hh.Md5
rom.Sha1 = hh.Sha1
game := new(types.Game)
game.Name = romName
game.Roms = append(game.Roms, rom)
rw.dat.Games = append(rw.dat.Games, game)
return nil
}
func (kvb *kvBatch) IndexRom(rom *types.Rom) error {
//glog.Infof("indexing rom %s", rom.Name)
dats, err := kvb.db.DatsForRom(rom)
if err != nil {
return err
}
if len(dats) > 0 {
if rom.Crc != nil && rom.Sha1 != nil {
//glog.Infof("declaring crc %s -> sha1 %s ampping", hex.EncodeToString(rom.Crc), hex.EncodeToString(rom.Sha1))
err = kvb.crcsha1Batch.Append(rom.Crc, rom.Sha1)
if err != nil {
return err
}
kvb.size += int64(sha1.Size)
}
if rom.Md5 != nil && rom.Sha1 != nil {
//glog.Infof("declaring md5 %s -> sha1 %s ampping", hex.EncodeToString(rom.Md5), hex.EncodeToString(rom.Sha1))
err = kvb.md5sha1Batch.Append(rom.Md5, rom.Sha1)
if err != nil {
return err
}
kvb.size += int64(sha1.Size)
}
return nil
}
if rom.Sha1 == nil {
glog.Warningf("indexing rom %s with missing SHA1", rom.Name)
}
dat := new(types.Dat)
dat.Artificial = true
dat.Generation = kvb.db.generation
dat.Name = fmt.Sprintf("Artificial Dat for %s", rom.Name)
dat.Path = rom.Path
game := new(types.Game)
game.Roms = []*types.Rom{rom}
dat.Games = []*types.Game{game}
var buf bytes.Buffer
gobEncoder := gob.NewEncoder(&buf)
err = gobEncoder.Encode(dat)
if err != nil {
return err
}
hh := sha1.New()
_, err = io.Copy(hh, &buf)
if err != nil {
return err
}
return kvb.IndexDat(dat, hh.Sum(nil))
}
func (depot *Depot) buildGame(game *types.Game, gamePath string,
unzipGame bool, deduper dedup.Deduper) (*types.Game, bool, error) {
var gameTorrent *torrentzip.Writer
var err error
glog.V(4).Infof("building game %s with path %s", game.Name, gamePath)
if unzipGame {
err := os.Mkdir(gamePath, 0777)
if err != nil {
glog.Errorf("error mkdir %s: %v", gamePath, err)
return nil, false, err
}
} else {
gameDir := filepath.Dir(game.Name)
if gameDir != "." {
// name has dirs in it
err := os.MkdirAll(filepath.Dir(gamePath), 0777)
if err != nil {
glog.Errorf("error mkdir %s: %v", filepath.Dir(gamePath), err)
return nil, false, err
}
}
gameFile, err := os.Create(gamePath + zipSuffix)
if err != nil {
glog.Errorf("error creating zip file %s: %v", gamePath+zipSuffix, err)
return nil, false, err
}
defer gameFile.Close()
gameTorrent, err = torrentzip.NewWriterWithTemp(gameFile, config.GlobalConfig.General.TmpDir)
if err != nil {
glog.Errorf("error writing to torrentzip file %s: %v", gamePath+zipSuffix, err)
return nil, false, err
}
defer gameTorrent.Close()
}
var fixGame *types.Game
foundRom := false
for _, rom := range game.Roms {
err = depot.romDB.CompleteRom(rom)
if err != nil {
glog.Errorf("error completing rom %s: %v", rom.Name, err)
return nil, false, err
}
if rom.Sha1 == nil && rom.Size > 0 {
if fixGame == nil {
fixGame = new(types.Game)
fixGame.Name = game.Name
fixGame.Description = game.Description
}
fixGame.Roms = append(fixGame.Roms, rom)
continue
}
romGZ, err := depot.OpenRomGZ(rom)
if err != nil {
glog.Errorf("error opening rom %s from depot: %v", rom.Name, err)
return nil, false, err
}
if romGZ == nil {
if glog.V(2) {
glog.Warningf("game %s has missing rom %s (sha1 %s)", game.Name, rom.Name, hex.EncodeToString(rom.Sha1))
}
seenRom, err := deduper.Seen(rom)
if err != nil {
return nil, false, err
}
if !seenRom {
err = deduper.Declare(rom)
if err != nil {
glog.Errorf("error deduping rom %s: %v", rom.Name, err)
return nil, false, err
}
if fixGame == nil {
fixGame = new(types.Game)
fixGame.Name = game.Name
fixGame.Description = game.Description
}
fixGame.Roms = append(fixGame.Roms, rom)
}
continue
}
foundRom = true
src, err := gzip.NewReader(romGZ)
if err != nil {
glog.Errorf("error opening rom gz file %s: %v", rom.Name, err)
return nil, false, err
}
var dstWriter io.WriteCloser
if unzipGame {
romPath := filepath.Join(gamePath, rom.Name)
if strings.ContainsRune(rom.Name, filepath.Separator) {
err := os.MkdirAll(filepath.Dir(romPath), 0777)
if err != nil {
glog.Errorf("error mkdir %s: %v", filepath.Dir(romPath), err)
return nil, false, err
}
}
dst, err := os.Create(romPath)
if err != nil {
glog.Errorf("error creating destination rom file %s: %v", dst, err)
return nil, false, err
}
dstWriter = dst
} else {
dst, err := gameTorrent.Create(rom.Name)
if err != nil {
glog.Errorf("error creating torrentzip rom entry %s: %v", rom.Name, err)
return nil, false, err
}
dstWriter = nopWriterCloser{dst}
}
_, err = io.Copy(dstWriter, src)
if err != nil {
glog.Errorf("error copying rom %s: %v", rom.Name, err)
return nil, false, err
}
src.Close()
dstWriter.Close()
romGZ.Close()
}
return fixGame, foundRom, nil
}
func (depot *Depot) fixdatGame(game *types.Game, gamePath string,
unzipGame bool, deduper dedup.Deduper) (*types.Game, error) {
var err error
var fixGame *types.Game
for _, rom := range game.Roms {
err = depot.romDB.CompleteRom(rom)
if err != nil {
glog.Errorf("error completing rom %s: %v", rom.Name, err)
return nil, err
}
if rom.Sha1 == nil {
if rom.Size > 0 {
if fixGame == nil {
fixGame = new(types.Game)
fixGame.Name = game.Name
fixGame.Description = game.Description
}
fixGame.Roms = append(fixGame.Roms, rom)
}
continue
}
sha1Hex := hex.EncodeToString(rom.Sha1)
exists, _, err := depot.RomInDepot(sha1Hex)
if err != nil {
glog.Errorf("error checking rom %s in depot: %v", rom.Name, err)
return nil, err
}
if !exists {
if glog.V(2) {
glog.Warningf("game %s has missing rom %s (sha1 %s)", game.Name, rom.Name, hex.EncodeToString(rom.Sha1))
}
seenRom, err := deduper.Seen(rom)
if err != nil {
return nil, err
}
if !seenRom {
err = deduper.Declare(rom)
if err != nil {
glog.Errorf("error deduping rom %s: %v", rom.Name, err)
return nil, err
}
if fixGame == nil {
fixGame = new(types.Game)
fixGame.Name = game.Name
fixGame.Description = game.Description
}
fixGame.Roms = append(fixGame.Roms, rom)
}
continue
}
}
return fixGame, nil
}
func (depot *Depot) buildGame(game *types.Game, datPath string) (*types.Game, error) {
gamePath := filepath.Join(datPath, game.Name+zipSuffix)
gameFile, err := os.Create(gamePath)
if err != nil {
return nil, err
}
defer gameFile.Close()
gameTorrent, err := torrentzip.NewWriter(gameFile)
if err != nil {
return nil, err
}
defer gameTorrent.Close()
var fixGame *types.Game
for _, rom := range game.Roms {
if rom.Sha1 == nil {
glog.Warningf("game %s has rom with missing SHA1 %s", game.Name, rom.Name)
if fixGame == nil {
fixGame = new(types.Game)
fixGame.Name = game.Name
fixGame.Description = game.Description
}
fixGame.Roms = append(fixGame.Roms, rom)
continue
}
romGZ, err := depot.OpenRomGZ(rom)
if err != nil {
return nil, err
}
if romGZ == nil {
glog.Warningf("game %s has missing rom %s (sha1 %s)", game.Name, rom.Name, hex.EncodeToString(rom.Sha1))
if fixGame == nil {
fixGame = new(types.Game)
fixGame.Name = game.Name
fixGame.Description = game.Description
}
fixGame.Roms = append(fixGame.Roms, rom)
continue
}
src, err := cgzip.NewReader(romGZ)
if err != nil {
return nil, err
}
dst, err := gameTorrent.Create(rom.Name)
if err != nil {
return nil, err
}
_, err = io.Copy(dst, src)
if err != nil {
return nil, err
}
src.Close()
romGZ.Close()
}
return fixGame, nil
}