func Dedup(d *types.Dat, deduper Deduper) (*types.Dat, error) {
dc := new(types.Dat)
dc.CopyHeader(d)
for _, g := range d.Games {
gc := new(types.Game)
gc.CopyHeader(g)
for _, r := range g.Roms {
if !r.Valid() {
continue
}
seen, err := deduper.Seen(r)
if err != nil {
return nil, err
}
if !seen {
gc.Roms = append(gc.Roms, r)
err = deduper.Declare(r)
if err != nil {
return nil, err
}
}
}
if len(gc.Roms) > 0 {
dc.Games = append(dc.Games, gc)
}
}
if len(dc.Games) > 0 {
return dc, nil
}
return nil, nil
}
func (rs *RombaService) dir2dat(cmd *commander.Command, args []string) error {
outpath := cmd.Flag.Lookup("out").Value.Get().(string)
if err := os.MkdirAll(outpath, 0777); err != nil {
return err
}
srcpath := cmd.Flag.Lookup("source").Value.Get().(string)
srcInfo, err := os.Stat(srcpath)
if err != nil {
return err
}
if !srcInfo.IsDir() {
return fmt.Errorf("%s is not a directory", srcpath)
}
dat := new(types.Dat)
dat.Name = cmd.Flag.Lookup("name").Value.Get().(string)
dat.Description = cmd.Flag.Lookup("description").Value.Get().(string)
err = archive.Dir2Dat(dat, srcpath, outpath)
if err != nil {
return err
}
fmt.Fprintf(cmd.Stdout, "dir2dat successfully completed a DAT in %s for directory %s", outpath, srcpath)
return nil
}
func ParseXml(r io.Reader, path string) (*types.Dat, []byte, error) {
br := bufio.NewReader(r)
hr := hashingReader{
ir: br,
h: sha1.New(),
}
lr := lineCountingReader{
ir: hr,
}
d := new(types.Dat)
decoder := xml.NewDecoder(lr)
err := decoder.Decode(d)
if err != nil {
return nil, nil, fmt.Errorf("xml parsing error %d: %v", lr.line, err)
}
for _, g := range d.Games {
for _, rom := range g.Roms {
fixHashes(rom)
}
for _, rom := range g.Disks {
fixHashes(rom)
}
for _, rom := range g.Parts {
fixHashes(rom)
}
for _, rom := range g.Regions {
fixHashes(rom)
}
}
for _, g := range d.Software {
for _, rom := range g.Roms {
fixHashes(rom)
}
for _, rom := range g.Disks {
fixHashes(rom)
}
for _, rom := range g.Parts {
fixHashes(rom)
}
for _, rom := range g.Regions {
fixHashes(rom)
}
}
d.Normalize()
d.Path = path
return d, hr.h.Sum(nil), nil
}
func Dir2Dat(dat *types.Dat, srcpath, outpath string) error {
glog.Infof("composing DAT from source %s into output dir %s", srcpath, outpath)
fis, err := ioutil.ReadDir(srcpath)
if err != nil {
return err
}
for _, fi := range fis {
if fi.IsDir() {
game, err := populateGame(srcpath, fi)
if err != nil {
return err
}
dat.Games = append(dat.Games, game)
}
}
outfilename := filepath.Join(outpath, dat.Name+datSuffix)
outf, err := os.Create(outfilename)
if err != nil {
return err
}
defer outf.Close()
outbuf := bufio.NewWriter(outf)
defer outbuf.Flush()
return types.ComposeDat(dat, outbuf)
}
func (depot *Depot) BuildDat(dat *types.Dat, outpath string) (bool, error) {
datPath := filepath.Join(outpath, dat.Name)
err := os.Mkdir(datPath, 0777)
if err != nil {
return false, err
}
var fixDat *types.Dat
for _, game := range dat.Games {
fixGame, err := depot.buildGame(game, datPath)
if err != nil {
return false, err
}
if fixGame != nil {
if fixDat == nil {
fixDat = new(types.Dat)
fixDat.Name = dat.Name
fixDat.Description = dat.Description
fixDat.Path = dat.Path
}
fixDat.Games = append(fixDat.Games, fixGame)
}
}
if fixDat != nil {
fixDatPath := filepath.Join(outpath, fixPrefix+dat.Name+datSuffix)
fixFile, err := os.Create(fixDatPath)
if err != nil {
return false, err
}
defer fixFile.Close()
fixWriter := bufio.NewWriter(fixFile)
defer fixWriter.Flush()
err = types.ComposeDat(fixDat, fixWriter)
if err != nil {
return false, err
}
}
return fixDat == nil, nil
}
func writeDiffDat(diffDat *types.Dat, outPath string) error {
diffDat.Path = outPath
diffFile, err := os.Create(outPath)
if err != nil {
return err
}
defer diffFile.Close()
diffWriter := bufio.NewWriter(diffFile)
defer diffWriter.Flush()
return types.ComposeCompliantDat(diffDat, diffWriter)
}
func (kvb *kvBatch) IndexRom(rom *types.Rom) error {
//glog.Infof("indexing rom %s", rom.Name)
dats, err := kvb.db.DatsForRom(rom)
if err != nil {
return err
}
if len(dats) > 0 {
if rom.Crc != nil && rom.Sha1 != nil {
//glog.Infof("declaring crc %s -> sha1 %s ampping", hex.EncodeToString(rom.Crc), hex.EncodeToString(rom.Sha1))
err = kvb.crcsha1Batch.Append(rom.Crc, rom.Sha1)
if err != nil {
return err
}
kvb.size += int64(sha1.Size)
}
if rom.Md5 != nil && rom.Sha1 != nil {
//glog.Infof("declaring md5 %s -> sha1 %s ampping", hex.EncodeToString(rom.Md5), hex.EncodeToString(rom.Sha1))
err = kvb.md5sha1Batch.Append(rom.Md5, rom.Sha1)
if err != nil {
return err
}
kvb.size += int64(sha1.Size)
}
return nil
}
if rom.Sha1 == nil {
glog.Warningf("indexing rom %s with missing SHA1", rom.Name)
}
dat := new(types.Dat)
dat.Artificial = true
dat.Generation = kvb.db.generation
dat.Name = fmt.Sprintf("Artificial Dat for %s", rom.Name)
dat.Path = rom.Path
game := new(types.Game)
game.Roms = []*types.Rom{rom}
dat.Games = []*types.Game{game}
var buf bytes.Buffer
gobEncoder := gob.NewEncoder(&buf)
err = gobEncoder.Encode(dat)
if err != nil {
return err
}
hh := sha1.New()
_, err = io.Copy(hh, &buf)
if err != nil {
return err
}
return kvb.IndexDat(dat, hh.Sum(nil))
}
func (kvb *kvBatch) IndexDat(dat *types.Dat, sha1Bytes []byte) error {
glog.V(4).Infof("indexing dat %s", dat.Name)
if sha1Bytes == nil {
return fmt.Errorf("sha1 is nil for %s", dat.Path)
}
dat.Generation = kvb.db.generation
var buf bytes.Buffer
gobEncoder := gob.NewEncoder(&buf)
err := gobEncoder.Encode(dat)
if err != nil {
return err
}
exists, err := kvb.db.datsDB.Exists(sha1Bytes)
if err != nil {
return fmt.Errorf("failed to lookup sha1 indexing dats: %v", err)
}
kvb.datsBatch.Set(sha1Bytes, buf.Bytes())
kvb.size += int64(sha1.Size + buf.Len())
if !exists {
for _, g := range dat.Games {
glog.V(4).Infof("indexing game %s", g.Name)
for _, r := range g.Roms {
if r.Sha1 != nil {
err = kvb.sha1Batch.Set(r.Sha1Sha1Key(sha1Bytes), oneValue)
if err != nil {
return err
}
kvb.size += int64(sha1.Size)
}
if r.Md5 != nil {
err = kvb.md5Batch.Set(r.Md5WithSizeAndSha1Key(sha1Bytes), oneValue)
if err != nil {
return err
}
kvb.size += int64(sha1.Size)
if r.Sha1 != nil {
glog.V(4).Infof("declaring md5 %s -> sha1 %s mapping", hex.EncodeToString(r.Md5), hex.EncodeToString(r.Sha1))
err = kvb.md5sha1Batch.Set(r.Md5WithSizeAndSha1Key(nil), oneValue)
if err != nil {
return err
}
kvb.size += int64(sha1.Size)
}
}
if r.Crc != nil {
err = kvb.crcBatch.Set(r.CrcWithSizeAndSha1Key(sha1Bytes), oneValue)
if err != nil {
return err
}
kvb.size += int64(sha1.Size)
if r.Sha1 != nil {
glog.V(4).Infof("declaring crc %s -> sha1 %s mapping", hex.EncodeToString(r.Crc), hex.EncodeToString(r.Sha1))
err = kvb.crcsha1Batch.Set(r.CrcWithSizeAndSha1Key(nil), oneValue)
if err != nil {
return err
}
kvb.size += int64(sha1.Size)
}
}
}
}
}
return nil
}
func ParseXml(r io.Reader, path string) (*types.Dat, []byte, error) {
br := bufio.NewReader(r)
hr := hashingReader{
ir: br,
h: sha1.New(),
}
lr := lineCountingReader{
ir: hr,
}
d := new(types.Dat)
decoder := xml.NewDecoder(lr)
err := decoder.Decode(d)
if err != nil {
derrStr := fmt.Sprintf("error in file %s on line %d: %v", path, lr.line, err)
derr := XMLParseError.NewWith(derrStr, setErrorFilePath(path), setErrorLineNumber(lr.line))
return nil, nil, derr
}
for _, g := range d.Games {
for _, rom := range g.Roms {
fixHashes(rom)
}
for _, rom := range g.Parts {
fixHashes(rom)
}
for _, rom := range g.Regions {
fixHashes(rom)
}
}
for _, g := range d.Software {
for _, rom := range g.Roms {
fixHashes(rom)
}
for _, rom := range g.Parts {
fixHashes(rom)
}
for _, rom := range g.Regions {
fixHashes(rom)
}
}
for _, g := range d.Machines {
for _, rom := range g.Roms {
fixHashes(rom)
}
for _, rom := range g.Parts {
fixHashes(rom)
}
for _, rom := range g.Regions {
fixHashes(rom)
}
}
d.Normalize()
d.Path = path
return d, hr.h.Sum(nil), nil
}
func (depot *Depot) BuildDat(dat *types.Dat, outpath string, numSubworkers int, deduper dedup.Deduper) (bool, error) {
datPath := filepath.Join(outpath, dat.Name)
err := os.Mkdir(datPath, 0777)
if err != nil {
return false, err
}
fixDat := new(types.Dat)
fixDat.FixDat = true
fixDat.Name = "fix_" + dat.Name
fixDat.Description = dat.Description
fixDat.Path = dat.Path
fixDat.UnzipGames = dat.UnzipGames
wc := make(chan *types.Game)
erc := make(chan error)
closeC := make(chan bool)
mutex := new(sync.Mutex)
for i := 0; i < numSubworkers; i++ {
gb := new(gameBuilder)
gb.depot = depot
gb.wc = wc
gb.erc = erc
gb.mutex = mutex
gb.datPath = datPath
gb.fixDat = fixDat
gb.index = i
gb.deduper = deduper
gb.closeC = closeC
go gb.work()
}
var minionErr error
endLoop:
for _, game := range dat.Games {
select {
case wc <- game:
case err := <-erc:
minionErr = err
break endLoop
}
}
close(wc)
finishedSubworkers := 0
endLoop2:
for {
glog.V(4).Infof("builder master: finished so far %d", finishedSubworkers)
select {
case <-closeC:
glog.V(4).Infof("builder master: finished another subworker")
finishedSubworkers++
if finishedSubworkers == numSubworkers {
break endLoop2
}
case err := <-erc:
glog.V(4).Infof("builder master: minion error")
minionErr = err
}
}
if minionErr != nil {
return false, minionErr
}
if len(fixDat.Games) > 0 {
fixDatPath := filepath.Join(outpath, fixPrefix+dat.Name+datSuffix)
fixFile, err := os.Create(fixDatPath)
if err != nil {
return false, err
}
defer fixFile.Close()
fixWriter := bufio.NewWriter(fixFile)
defer fixWriter.Flush()
err = types.ComposeCompliantDat(fixDat, fixWriter)
if err != nil {
return false, err
}
}
return len(fixDat.Games) > 0, nil
}
func (kvb *kvBatch) IndexDat(dat *types.Dat, sha1Bytes []byte) error {
glog.Infof("indexing dat %s", dat.Name)
if sha1Bytes == nil {
return fmt.Errorf("sha1 is nil for %s", dat.Path)
}
dat.Generation = kvb.db.generation
var buf bytes.Buffer
gobEncoder := gob.NewEncoder(&buf)
err := gobEncoder.Encode(dat)
if err != nil {
return err
}
var exists bool
if dat.Artificial {
exists = false
} else {
existsSha1, err := kvb.db.datsDB.Exists(sha1Bytes)
if err != nil {
return fmt.Errorf("failed to lookup sha1 indexing dats: %v", err)
}
exists = existsSha1
}
kvb.datsBatch.Set(sha1Bytes, buf.Bytes())
kvb.size += int64(sha1.Size + buf.Len())
if !exists {
for _, g := range dat.Games {
for _, r := range g.Roms {
if r.Sha1 != nil {
err = kvb.sha1Batch.Append(r.Sha1, sha1Bytes)
if err != nil {
return err
}
kvb.size += int64(sha1.Size)
}
if r.Md5 != nil {
err = kvb.md5Batch.Append(r.Md5, sha1Bytes)
if err != nil {
return err
}
kvb.size += int64(sha1.Size)
if r.Sha1 != nil {
//glog.Infof("declaring md5 %s -> sha1 %s ampping", hex.EncodeToString(r.Md5), hex.EncodeToString(r.Sha1))
err = kvb.md5sha1Batch.Append(r.Md5, r.Sha1)
if err != nil {
return err
}
kvb.size += int64(sha1.Size)
}
}
if r.Crc != nil {
err = kvb.crcBatch.Append(r.Crc, sha1Bytes)
if err != nil {
return err
}
kvb.size += int64(sha1.Size)
if r.Sha1 != nil {
//glog.Infof("declaring crc %s -> sha1 %s ampping", hex.EncodeToString(r.Crc), hex.EncodeToString(r.Sha1))
err = kvb.crcsha1Batch.Append(r.Crc, r.Sha1)
if err != nil {
return err
}
kvb.size += int64(sha1.Size)
}
}
}
}
}
return nil
}