func (rs *RombaService) diffdat(cmd *commander.Command, args []string) error {
oldDatPath := cmd.Flag.Lookup("old").Value.Get().(string)
newDatPath := cmd.Flag.Lookup("new").Value.Get().(string)
outPath := cmd.Flag.Lookup("out").Value.Get().(string)
givenName := cmd.Flag.Lookup("name").Value.Get().(string)
givenDescription := cmd.Flag.Lookup("description").Value.Get().(string)
if oldDatPath == "" {
fmt.Fprintf(cmd.Stdout, "-old argument required")
return errors.New("missing old argument")
}
if newDatPath == "" {
fmt.Fprintf(cmd.Stdout, "-new argument required")
return errors.New("missing new argument")
}
if outPath == "" {
fmt.Fprintf(cmd.Stdout, "-out argument required")
return errors.New("missing out argument")
}
glog.Infof("diffdat new dat %s and old dat %s into %s", newDatPath, oldDatPath, outPath)
oldDat, _, err := parser.Parse(oldDatPath)
if err != nil {
return err
}
newDat, _, err := parser.Parse(newDatPath)
if err != nil {
return err
}
if givenName == "" {
givenName = strings.TrimSuffix(filepath.Base(outPath), filepath.Ext(outPath))
}
if givenDescription == "" {
givenDescription = givenName
}
dd, err := dedup.NewLevelDBDeduper()
if err != nil {
return err
}
defer dd.Close()
err = dedup.Declare(oldDat, dd)
if err != nil {
return err
}
diffDat, err := dedup.Dedup(newDat, dd)
if err != nil {
return err
}
diffDat = diffDat.FilterRoms(func(r *types.Rom) bool {
return r.Size > 0
})
var endMsg string
if diffDat != nil {
diffDat.Name = givenName
diffDat.Description = givenDescription
diffDat.Path = outPath
diffFile, err := os.Create(outPath)
if err != nil {
return err
}
defer diffFile.Close()
diffWriter := bufio.NewWriter(diffFile)
defer diffWriter.Flush()
err = types.ComposeCompliantDat(diffDat, diffWriter)
if err != nil {
return err
}
endMsg = fmt.Sprintf("diffdat finished, %d games with diffs found, written diffdat file %s",
len(diffDat.Games), outPath)
} else {
endMsg = "diffdat finished, no diffs found, no diffdat file written"
}
glog.Infof(endMsg)
fmt.Fprintf(cmd.Stdout, endMsg)
rs.broadCastProgress(time.Now(), false, true, endMsg)
return nil
}
func (rs *RombaService) build(cmd *commander.Command, args []string) error {
rs.jobMutex.Lock()
defer rs.jobMutex.Unlock()
if rs.busy {
p := rs.pt.GetProgress()
fmt.Fprintf(cmd.Stdout, "still busy with %s: (%d of %d files) and (%s of %s) \n", rs.jobName,
p.FilesSoFar, p.TotalFiles, humanize.IBytes(uint64(p.BytesSoFar)), humanize.IBytes(uint64(p.TotalBytes)))
return nil
}
outpath := cmd.Flag.Lookup("out").Value.Get().(string)
if outpath == "" {
fmt.Fprintf(cmd.Stdout, "-out flag is required")
return nil
}
fixdatOnly := cmd.Flag.Lookup("fixdatOnly").Value.Get().(bool)
numWorkers := cmd.Flag.Lookup("workers").Value.Get().(int)
numSubWorkers := cmd.Flag.Lookup("subworkers").Value.Get().(int)
if !filepath.IsAbs(outpath) {
absoutpath, err := filepath.Abs(outpath)
if err != nil {
return err
}
outpath = absoutpath
}
if err := os.MkdirAll(outpath, 0777); err != nil {
return err
}
deduper, err := dedup.NewLevelDBDeduper()
if err != nil {
return err
}
rs.pt.Reset()
rs.busy = true
rs.jobName = "build"
go func() {
glog.Infof("service starting build")
rs.broadCastProgress(time.Now(), true, false, "")
ticker := time.NewTicker(time.Second * 5)
stopTicker := make(chan bool)
go func() {
glog.Infof("starting progress broadcaster")
for {
select {
case t := <-ticker.C:
rs.broadCastProgress(t, false, false, "")
case <-stopTicker:
glog.Info("stopped progress broadcaster")
return
}
}
}()
pm := &buildMaster{
outpath: outpath,
rs: rs,
numWorkers: numWorkers,
numSubWorkers: numSubWorkers,
pt: rs.pt,
fixdatOnly: fixdatOnly,
deduper: deduper,
}
endMsg, err := worker.Work("building dats", args, pm)
if err != nil {
glog.Errorf("error building dats: %v", err)
}
ticker.Stop()
stopTicker <- true
err = archive.DeleteEmptyFolders(outpath)
if err != nil {
glog.Errorf("error building dats: %v", err)
}
rs.jobMutex.Lock()
rs.busy = false
rs.jobName = ""
rs.jobMutex.Unlock()
rs.broadCastProgress(time.Now(), false, true, endMsg)
glog.Infof("service finished build")
}()
fmt.Fprintf(cmd.Stdout, "started build")
return nil
}
func (rs *RombaService) ediffdatWork(cmd *commander.Command, args []string) error {
oldDatPath := cmd.Flag.Lookup("old").Value.Get().(string)
newDatPath := cmd.Flag.Lookup("new").Value.Get().(string)
outPath := cmd.Flag.Lookup("out").Value.Get().(string)
if oldDatPath == "" {
fmt.Fprintf(cmd.Stdout, "-old argument required")
return errors.New("missing old argument")
}
if newDatPath == "" {
fmt.Fprintf(cmd.Stdout, "-new argument required")
return errors.New("missing new argument")
}
if outPath == "" {
fmt.Fprintf(cmd.Stdout, "-out argument required")
return errors.New("missing out argument")
}
err := os.MkdirAll(outPath, 0777)
if err != nil {
return err
}
glog.Infof("ediffdat new dat %s and old dat %s into %s", newDatPath, oldDatPath, outPath)
dd, err := dedup.NewLevelDBDeduper()
if err != nil {
return err
}
defer dd.Close()
err = filepath.Walk(oldDatPath, func(path string, info os.FileInfo, err error) error {
if info.IsDir() {
return nil
}
ext := filepath.Ext(path)
if ext == ".dat" || ext == ".xml" {
rs.pt.DeclareFile(path)
oldDat, _, err := parser.Parse(path)
if err != nil {
return err
}
err = dedup.Declare(oldDat, dd)
if err != nil {
return err
}
rs.pt.AddBytesFromFile(info.Size(), false)
}
return nil
})
if err != nil {
return err
}
err = filepath.Walk(newDatPath, func(path string, info os.FileInfo, err error) error {
if info.IsDir() {
return nil
}
ext := filepath.Ext(path)
if ext == ".dat" || ext == ".xml" {
rs.pt.DeclareFile(path)
newDat, _, err := parser.Parse(path)
if err != nil {
return err
}
oneDiffDat, err := dedup.Dedup(newDat, dd)
if err != nil {
return err
}
if oneDiffDat != nil {
oneDiffDat = oneDiffDat.FilterRoms(func(r *types.Rom) bool {
return r.Size > 0
})
}
if oneDiffDat != nil {
err = writeDiffDat(oneDiffDat, filepath.Join(outPath, oneDiffDat.Name+".dat"))
}
rs.pt.AddBytesFromFile(info.Size(), err != nil)
return err
}
return nil
})
if err != nil {
return err
}
return nil
}
func (rs *RombaService) datstats(cmd *commander.Command, args []string) error {
rs.jobMutex.Lock()
defer rs.jobMutex.Unlock()
deduper, err := dedup.NewLevelDBDeduper()
if err != nil {
return err
}
defer deduper.Close()
dts := &datStats{
h: hdrhistogram.New(0, 1000000000000, 5),
}
err = rs.romDB.ForEachDat(func(dat *types.Dat) error {
if dat.Generation != rs.romDB.Generation() {
return nil
}
dedat, err := dedup.Dedup(dat, deduper)
if err != nil {
return err
}
if dedat == nil {
return nil
}
dts.nDats = dts.nDats + 1
for _, g := range dedat.Games {
dts.nGames = dts.nGames + 1
for _, r := range g.Roms {
dts.h.RecordValue(r.Size)
dts.nRoms = dts.nRoms + 1
dts.totalSize = dts.totalSize + uint64(r.Size)
if r.Size <= 4000 {
dts.nRomsBelow4k = dts.nRomsBelow4k + 1
}
}
}
return nil
})
if err != nil {
return err
}
bs := dts.h.CumulativeDistribution()
fmt.Fprintf(cmd.Stdout, "number of dats = %d\n", dts.nDats)
fmt.Fprintf(cmd.Stdout, "number of games = %d\n", dts.nGames)
fmt.Fprintf(cmd.Stdout, "number of roms = %d\n", dts.nRoms)
fmt.Fprintf(cmd.Stdout, "total rom size = %s\n", humanize.IBytes(dts.totalSize))
fmt.Fprintf(cmd.Stdout, "number of roms below 4k size = %d\n\n", dts.nRomsBelow4k)
fmt.Fprintf(cmd.Stdout, "rom size cumulative distribution = \n")
fmt.Fprintf(cmd.Stdout, "count, percentile, file size\n")
for i := 0; i < len(bs); i++ {
b := bs[i]
vstr := humanize.IBytes(uint64(b.ValueAt))
if (i < len(bs)-1 && vstr != humanize.IBytes(uint64(bs[i+1].ValueAt))) || (i == len(bs)-1) {
fmt.Fprintf(cmd.Stdout, "%d, %.8f, %s\n", b.Count, b.Quantile, humanize.IBytes(uint64(b.ValueAt)))
}
}
fmt.Fprintf(cmd.Stdout, "\nrom size histogram = \n")
fmt.Fprintf(cmd.Stdout, "count, file size\n")
var lastCount int64
for _, b := range bs {
count := b.Count - lastCount
if count > 0 {
fmt.Fprintf(cmd.Stdout, "%d, %s\n", count, humanize.IBytes(uint64(b.ValueAt)))
}
lastCount = b.Count
}
return nil
}