Exemple #1
0
func Dedup(d *types.Dat, deduper Deduper) (*types.Dat, error) {
	dc := new(types.Dat)
	dc.CopyHeader(d)

	for _, g := range d.Games {
		gc := new(types.Game)
		gc.CopyHeader(g)
		for _, r := range g.Roms {
			if !r.Valid() {
				continue
			}
			seen, err := deduper.Seen(r)
			if err != nil {
				return nil, err
			}
			if !seen {
				gc.Roms = append(gc.Roms, r)
				err = deduper.Declare(r)
				if err != nil {
					return nil, err
				}
			}
		}
		if len(gc.Roms) > 0 {
			dc.Games = append(dc.Games, gc)
		}
	}

	if len(dc.Games) > 0 {
		return dc, nil
	}
	return nil, nil
}
Exemple #2
0
func (rw *romWalker) visit(path string, f os.FileInfo, err error) error {
	if f == nil || f.Name() == ".DS_Store" {
		return nil
	}
	if f.IsDir() {
		return nil
	}

	hh, err := HashesForFile(path)
	if err != nil {
		return err
	}

	romName, err := filepath.Rel(rw.sourcePath, path)
	if err != nil {
		return err
	}

	rom := new(types.Rom)
	rom.Name = romName
	rom.Size = f.Size()
	rom.Crc = hh.Crc
	rom.Md5 = hh.Md5
	rom.Sha1 = hh.Sha1

	game := new(types.Game)
	game.Name = romName

	game.Roms = append(game.Roms, rom)

	rw.dat.Games = append(rw.dat.Games, game)
	return nil
}
Exemple #3
0
func (kvb *kvBatch) IndexRom(rom *types.Rom) error {
	//glog.Infof("indexing rom %s", rom.Name)

	dats, err := kvb.db.DatsForRom(rom)
	if err != nil {
		return err
	}

	if len(dats) > 0 {
		if rom.Crc != nil && rom.Sha1 != nil {
			//glog.Infof("declaring crc %s -> sha1 %s ampping", hex.EncodeToString(rom.Crc), hex.EncodeToString(rom.Sha1))
			err = kvb.crcsha1Batch.Append(rom.Crc, rom.Sha1)
			if err != nil {
				return err
			}
			kvb.size += int64(sha1.Size)
		}
		if rom.Md5 != nil && rom.Sha1 != nil {
			//glog.Infof("declaring md5 %s -> sha1 %s ampping", hex.EncodeToString(rom.Md5), hex.EncodeToString(rom.Sha1))
			err = kvb.md5sha1Batch.Append(rom.Md5, rom.Sha1)
			if err != nil {
				return err
			}
			kvb.size += int64(sha1.Size)
		}
		return nil
	}

	if rom.Sha1 == nil {
		glog.Warningf("indexing rom %s with missing SHA1", rom.Name)
	}

	dat := new(types.Dat)
	dat.Artificial = true
	dat.Generation = kvb.db.generation
	dat.Name = fmt.Sprintf("Artificial Dat for %s", rom.Name)
	dat.Path = rom.Path
	game := new(types.Game)
	game.Roms = []*types.Rom{rom}
	dat.Games = []*types.Game{game}

	var buf bytes.Buffer

	gobEncoder := gob.NewEncoder(&buf)
	err = gobEncoder.Encode(dat)
	if err != nil {
		return err
	}

	hh := sha1.New()
	_, err = io.Copy(hh, &buf)
	if err != nil {
		return err
	}

	return kvb.IndexDat(dat, hh.Sum(nil))
}
Exemple #4
0
func populateGame(srcpath string, gameDirInfo os.FileInfo) (*types.Game, error) {
	game := new(types.Game)
	baseName := gameDirInfo.Name()

	game.Name = baseName
	game.Description = baseName

	gw := &gameWalker{
		game:     game,
		gamepath: filepath.Join(srcpath, baseName),
	}

	err := filepath.Walk(gw.gamepath, gw.visit)
	if err != nil {
		return nil, err
	}

	return game, nil
}
Exemple #5
0
func (depot *Depot) buildGame(game *types.Game, gamePath string,
	unzipGame bool, deduper dedup.Deduper) (*types.Game, bool, error) {

	var gameTorrent *torrentzip.Writer
	var err error

	glog.V(4).Infof("building game %s with path %s", game.Name, gamePath)

	if unzipGame {
		err := os.Mkdir(gamePath, 0777)
		if err != nil {
			glog.Errorf("error mkdir %s: %v", gamePath, err)
			return nil, false, err
		}
	} else {
		gameDir := filepath.Dir(game.Name)
		if gameDir != "." {
			// name has dirs in it
			err := os.MkdirAll(filepath.Dir(gamePath), 0777)
			if err != nil {
				glog.Errorf("error mkdir %s: %v", filepath.Dir(gamePath), err)
				return nil, false, err
			}
		}

		gameFile, err := os.Create(gamePath + zipSuffix)
		if err != nil {
			glog.Errorf("error creating zip file %s: %v", gamePath+zipSuffix, err)
			return nil, false, err
		}
		defer gameFile.Close()

		gameTorrent, err = torrentzip.NewWriterWithTemp(gameFile, config.GlobalConfig.General.TmpDir)
		if err != nil {
			glog.Errorf("error writing to torrentzip file %s: %v", gamePath+zipSuffix, err)
			return nil, false, err
		}
		defer gameTorrent.Close()
	}

	var fixGame *types.Game

	foundRom := false

	for _, rom := range game.Roms {
		err = depot.romDB.CompleteRom(rom)
		if err != nil {
			glog.Errorf("error completing rom %s: %v", rom.Name, err)
			return nil, false, err
		}

		if rom.Sha1 == nil && rom.Size > 0 {
			if fixGame == nil {
				fixGame = new(types.Game)
				fixGame.Name = game.Name
				fixGame.Description = game.Description
			}

			fixGame.Roms = append(fixGame.Roms, rom)
			continue
		}

		romGZ, err := depot.OpenRomGZ(rom)
		if err != nil {
			glog.Errorf("error opening rom %s from depot: %v", rom.Name, err)
			return nil, false, err
		}

		if romGZ == nil {
			if glog.V(2) {
				glog.Warningf("game %s has missing rom %s (sha1 %s)", game.Name, rom.Name, hex.EncodeToString(rom.Sha1))
			}

			seenRom, err := deduper.Seen(rom)
			if err != nil {
				return nil, false, err
			}

			if !seenRom {
				err = deduper.Declare(rom)
				if err != nil {
					glog.Errorf("error deduping rom %s: %v", rom.Name, err)
					return nil, false, err
				}

				if fixGame == nil {
					fixGame = new(types.Game)
					fixGame.Name = game.Name
					fixGame.Description = game.Description
				}

				fixGame.Roms = append(fixGame.Roms, rom)
			}
			continue
		}

		foundRom = true

		src, err := gzip.NewReader(romGZ)
		if err != nil {
			glog.Errorf("error opening rom gz file %s: %v", rom.Name, err)
			return nil, false, err
		}

		var dstWriter io.WriteCloser

		if unzipGame {
			romPath := filepath.Join(gamePath, rom.Name)
			if strings.ContainsRune(rom.Name, filepath.Separator) {
				err := os.MkdirAll(filepath.Dir(romPath), 0777)
				if err != nil {
					glog.Errorf("error mkdir %s: %v", filepath.Dir(romPath), err)
					return nil, false, err
				}
			}
			dst, err := os.Create(romPath)
			if err != nil {
				glog.Errorf("error creating destination rom file %s: %v", dst, err)
				return nil, false, err
			}
			dstWriter = dst
		} else {
			dst, err := gameTorrent.Create(rom.Name)
			if err != nil {
				glog.Errorf("error creating torrentzip rom entry %s: %v", rom.Name, err)
				return nil, false, err
			}
			dstWriter = nopWriterCloser{dst}
		}
		_, err = io.Copy(dstWriter, src)
		if err != nil {
			glog.Errorf("error copying rom %s: %v", rom.Name, err)
			return nil, false, err
		}

		src.Close()
		dstWriter.Close()

		romGZ.Close()
	}
	return fixGame, foundRom, nil
}
Exemple #6
0
func (depot *Depot) fixdatGame(game *types.Game, gamePath string,
	unzipGame bool, deduper dedup.Deduper) (*types.Game, error) {

	var err error

	var fixGame *types.Game

	for _, rom := range game.Roms {
		err = depot.romDB.CompleteRom(rom)
		if err != nil {
			glog.Errorf("error completing rom %s: %v", rom.Name, err)
			return nil, err
		}

		if rom.Sha1 == nil {
			if rom.Size > 0 {
				if fixGame == nil {
					fixGame = new(types.Game)
					fixGame.Name = game.Name
					fixGame.Description = game.Description
				}

				fixGame.Roms = append(fixGame.Roms, rom)
			}
			continue
		}

		sha1Hex := hex.EncodeToString(rom.Sha1)
		exists, _, err := depot.RomInDepot(sha1Hex)
		if err != nil {
			glog.Errorf("error checking rom %s in depot: %v", rom.Name, err)
			return nil, err
		}

		if !exists {
			if glog.V(2) {
				glog.Warningf("game %s has missing rom %s (sha1 %s)", game.Name, rom.Name, hex.EncodeToString(rom.Sha1))
			}

			seenRom, err := deduper.Seen(rom)
			if err != nil {
				return nil, err
			}

			if !seenRom {
				err = deduper.Declare(rom)
				if err != nil {
					glog.Errorf("error deduping rom %s: %v", rom.Name, err)
					return nil, err
				}

				if fixGame == nil {
					fixGame = new(types.Game)
					fixGame.Name = game.Name
					fixGame.Description = game.Description
				}

				fixGame.Roms = append(fixGame.Roms, rom)
			}
			continue
		}
	}
	return fixGame, nil
}
Exemple #7
0
func (depot *Depot) buildGame(game *types.Game, datPath string) (*types.Game, error) {
	gamePath := filepath.Join(datPath, game.Name+zipSuffix)
	gameFile, err := os.Create(gamePath)
	if err != nil {
		return nil, err
	}
	defer gameFile.Close()

	gameTorrent, err := torrentzip.NewWriter(gameFile)
	if err != nil {
		return nil, err
	}
	defer gameTorrent.Close()

	var fixGame *types.Game

	for _, rom := range game.Roms {
		if rom.Sha1 == nil {
			glog.Warningf("game %s has rom with missing SHA1 %s", game.Name, rom.Name)
			if fixGame == nil {
				fixGame = new(types.Game)
				fixGame.Name = game.Name
				fixGame.Description = game.Description
			}

			fixGame.Roms = append(fixGame.Roms, rom)
			continue
		}

		romGZ, err := depot.OpenRomGZ(rom)
		if err != nil {
			return nil, err
		}

		if romGZ == nil {
			glog.Warningf("game %s has missing rom %s (sha1 %s)", game.Name, rom.Name, hex.EncodeToString(rom.Sha1))
			if fixGame == nil {
				fixGame = new(types.Game)
				fixGame.Name = game.Name
				fixGame.Description = game.Description
			}

			fixGame.Roms = append(fixGame.Roms, rom)
			continue
		}

		src, err := cgzip.NewReader(romGZ)
		if err != nil {
			return nil, err
		}

		dst, err := gameTorrent.Create(rom.Name)
		if err != nil {
			return nil, err
		}

		_, err = io.Copy(dst, src)
		if err != nil {
			return nil, err
		}

		src.Close()
		romGZ.Close()
	}
	return fixGame, nil
}