func (depot *Depot) Purge(backupDir string, numWorkers int, workDepot string, fromDats string,
pt worker.ProgressTracker) (string, error) {
pm := new(purgeMaster)
pm.depot = depot
pm.pt = pt
pm.numWorkers = numWorkers
absBackupDir, err := filepath.Abs(backupDir)
if err != nil {
return "", err
}
pm.backupDir = absBackupDir
if backupDir == "" {
return "", errors.New("no backup dir specified")
}
err = os.MkdirAll(backupDir, 0777)
if err != nil {
return "", err
}
if fromDats == "" {
wds := depot.roots
if len(workDepot) > 0 {
wds = []string{workDepot}
}
return worker.Work("purge roms", wds, pm)
} else {
var dats []*types.Dat
err = filepath.Walk(fromDats, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() && (strings.HasSuffix(path, ".dat") || strings.HasSuffix(path, ".xml")) {
dat, _, err := parser.Parse(path)
if err != nil {
return err
}
dats = append(dats, dat)
}
return nil
})
if err != nil {
return "", err
}
rdi := newRomsFromDatIterator(depot, dats)
return worker.WorkPathIterator("purge roms", rdi, pm)
}
}
func (pw *refreshWorker) Process(path string, size int64) error {
if pw.romBatch.Size() >= MaxBatchSize {
glog.Infof("flushing batch of size %d", pw.romBatch.Size())
err := pw.romBatch.Flush()
if err != nil {
return fmt.Errorf("failed to flush: %v", err)
}
}
dat, sha1Bytes, err := parser.Parse(path)
if err != nil {
return err
}
return pw.romBatch.IndexDat(dat, sha1Bytes)
}
func (pw *refreshWorker) Process(path string, size int64) error {
if pw.romBatch.Size() >= MaxBatchSize {
glog.V(3).Infof("flushing batch of size %d", pw.romBatch.Size())
err := pw.romBatch.Flush()
if err != nil {
return fmt.Errorf("failed to flush: %v", err)
}
}
dat, sha1Bytes, err := parser.Parse(path)
if err != nil {
return err
}
if pw.pm.missingSha1sWriter != nil && dat.MissingSha1s {
_, err = fmt.Fprintln(pw.pm.missingSha1sWriter, dat.Path)
if err != nil {
return err
}
}
return pw.romBatch.IndexDat(dat, sha1Bytes)
}
func (pw *parseWorker) Process(path string, size int64) error {
_, _, err := parser.Parse(path)
return err
}
func (rs *RombaService) diffdat(cmd *commander.Command, args []string) error {
oldDatPath := cmd.Flag.Lookup("old").Value.Get().(string)
newDatPath := cmd.Flag.Lookup("new").Value.Get().(string)
outPath := cmd.Flag.Lookup("out").Value.Get().(string)
givenName := cmd.Flag.Lookup("name").Value.Get().(string)
givenDescription := cmd.Flag.Lookup("description").Value.Get().(string)
if oldDatPath == "" {
fmt.Fprintf(cmd.Stdout, "-old argument required")
return errors.New("missing old argument")
}
if newDatPath == "" {
fmt.Fprintf(cmd.Stdout, "-new argument required")
return errors.New("missing new argument")
}
if outPath == "" {
fmt.Fprintf(cmd.Stdout, "-out argument required")
return errors.New("missing out argument")
}
glog.Infof("diffdat new dat %s and old dat %s into %s", newDatPath, oldDatPath, outPath)
oldDat, _, err := parser.Parse(oldDatPath)
if err != nil {
return err
}
newDat, _, err := parser.Parse(newDatPath)
if err != nil {
return err
}
if givenName == "" {
givenName = strings.TrimSuffix(filepath.Base(outPath), filepath.Ext(outPath))
}
if givenDescription == "" {
givenDescription = givenName
}
dd, err := dedup.NewLevelDBDeduper()
if err != nil {
return err
}
defer dd.Close()
err = dedup.Declare(oldDat, dd)
if err != nil {
return err
}
diffDat, err := dedup.Dedup(newDat, dd)
if err != nil {
return err
}
diffDat = diffDat.FilterRoms(func(r *types.Rom) bool {
return r.Size > 0
})
var endMsg string
if diffDat != nil {
diffDat.Name = givenName
diffDat.Description = givenDescription
diffDat.Path = outPath
diffFile, err := os.Create(outPath)
if err != nil {
return err
}
defer diffFile.Close()
diffWriter := bufio.NewWriter(diffFile)
defer diffWriter.Flush()
err = types.ComposeCompliantDat(diffDat, diffWriter)
if err != nil {
return err
}
endMsg = fmt.Sprintf("diffdat finished, %d games with diffs found, written diffdat file %s",
len(diffDat.Games), outPath)
} else {
endMsg = "diffdat finished, no diffs found, no diffdat file written"
}
glog.Infof(endMsg)
fmt.Fprintf(cmd.Stdout, endMsg)
rs.broadCastProgress(time.Now(), false, true, endMsg)
return nil
}
func (rs *RombaService) ediffdatWork(cmd *commander.Command, args []string) error {
oldDatPath := cmd.Flag.Lookup("old").Value.Get().(string)
newDatPath := cmd.Flag.Lookup("new").Value.Get().(string)
outPath := cmd.Flag.Lookup("out").Value.Get().(string)
if oldDatPath == "" {
fmt.Fprintf(cmd.Stdout, "-old argument required")
return errors.New("missing old argument")
}
if newDatPath == "" {
fmt.Fprintf(cmd.Stdout, "-new argument required")
return errors.New("missing new argument")
}
if outPath == "" {
fmt.Fprintf(cmd.Stdout, "-out argument required")
return errors.New("missing out argument")
}
err := os.MkdirAll(outPath, 0777)
if err != nil {
return err
}
glog.Infof("ediffdat new dat %s and old dat %s into %s", newDatPath, oldDatPath, outPath)
dd, err := dedup.NewLevelDBDeduper()
if err != nil {
return err
}
defer dd.Close()
err = filepath.Walk(oldDatPath, func(path string, info os.FileInfo, err error) error {
if info.IsDir() {
return nil
}
ext := filepath.Ext(path)
if ext == ".dat" || ext == ".xml" {
rs.pt.DeclareFile(path)
oldDat, _, err := parser.Parse(path)
if err != nil {
return err
}
err = dedup.Declare(oldDat, dd)
if err != nil {
return err
}
rs.pt.AddBytesFromFile(info.Size(), false)
}
return nil
})
if err != nil {
return err
}
err = filepath.Walk(newDatPath, func(path string, info os.FileInfo, err error) error {
if info.IsDir() {
return nil
}
ext := filepath.Ext(path)
if ext == ".dat" || ext == ".xml" {
rs.pt.DeclareFile(path)
newDat, _, err := parser.Parse(path)
if err != nil {
return err
}
oneDiffDat, err := dedup.Dedup(newDat, dd)
if err != nil {
return err
}
if oneDiffDat != nil {
oneDiffDat = oneDiffDat.FilterRoms(func(r *types.Rom) bool {
return r.Size > 0
})
}
if oneDiffDat != nil {
err = writeDiffDat(oneDiffDat, filepath.Join(outPath, oneDiffDat.Name+".dat"))
}
rs.pt.AddBytesFromFile(info.Size(), err != nil)
return err
}
return nil
})
if err != nil {
return err
}
return nil
}
func (pw *buildWorker) Process(path string, size int64) error {
hashes, err := archive.HashesForFile(path)
if err != nil {
return err
}
dat, err := pw.pm.rs.romDB.GetDat(hashes.Sha1)
if err != nil {
return err
}
if dat == nil {
glog.Warningf("did not find a DAT for %s, parsing it", path)
dat, _, err = parser.Parse(path)
if err != nil {
return err
}
glog.V(4).Infof("parsed dat=%s", types.PrintShortDat(dat))
}
reldatdir, err := filepath.Rel(pw.pm.commonRootPath, filepath.Dir(path))
if err != nil {
return err
}
datdir := filepath.Join(pw.pm.outpath, reldatdir)
glog.Infof("buildWorker processing %s, reldatdir=%s, datdir=%s", path, reldatdir, datdir)
err = os.MkdirAll(datdir, 0777)
if err != nil {
return err
}
for _, game := range dat.Games {
for _, rom := range game.Roms {
err = pw.pm.rs.romDB.CompleteRom(rom)
if err != nil {
return err
}
}
}
datInComplete := false
if pw.pm.fixdatOnly {
datInComplete, err = pw.pm.rs.depot.FixDat(dat, datdir, pw.pm.numSubWorkers, pw.pm.deduper)
} else {
datInComplete, err = pw.pm.rs.depot.BuildDat(dat, datdir, pw.pm.numSubWorkers, pw.pm.deduper)
}
if err != nil {
return err
}
glog.Infof("finished building dat %s in directory %s", dat.Name, datdir)
if datInComplete {
glog.Info("dat has missing roms")
}
return nil
}