示例#1
0
func (rs *RombaService) diffdat(cmd *commander.Command, args []string) error {
	oldDatPath := cmd.Flag.Lookup("old").Value.Get().(string)
	newDatPath := cmd.Flag.Lookup("new").Value.Get().(string)
	outPath := cmd.Flag.Lookup("out").Value.Get().(string)
	givenName := cmd.Flag.Lookup("name").Value.Get().(string)
	givenDescription := cmd.Flag.Lookup("description").Value.Get().(string)

	if oldDatPath == "" {
		fmt.Fprintf(cmd.Stdout, "-old argument required")
		return errors.New("missing old argument")
	}
	if newDatPath == "" {
		fmt.Fprintf(cmd.Stdout, "-new argument required")
		return errors.New("missing new argument")
	}
	if outPath == "" {
		fmt.Fprintf(cmd.Stdout, "-out argument required")
		return errors.New("missing out argument")
	}

	glog.Infof("diffdat new dat %s and old dat %s into %s", newDatPath, oldDatPath, outPath)

	oldDat, _, err := parser.Parse(oldDatPath)
	if err != nil {
		return err
	}

	newDat, _, err := parser.Parse(newDatPath)
	if err != nil {
		return err
	}

	if givenName == "" {
		givenName = strings.TrimSuffix(filepath.Base(outPath), filepath.Ext(outPath))
	}

	if givenDescription == "" {
		givenDescription = givenName
	}

	dd, err := dedup.NewLevelDBDeduper()
	if err != nil {
		return err
	}
	defer dd.Close()

	err = dedup.Declare(oldDat, dd)
	if err != nil {
		return err
	}

	diffDat, err := dedup.Dedup(newDat, dd)
	if err != nil {
		return err
	}

	diffDat = diffDat.FilterRoms(func(r *types.Rom) bool {
		return r.Size > 0
	})

	var endMsg string

	if diffDat != nil {
		diffDat.Name = givenName
		diffDat.Description = givenDescription
		diffDat.Path = outPath

		diffFile, err := os.Create(outPath)
		if err != nil {
			return err
		}
		defer diffFile.Close()

		diffWriter := bufio.NewWriter(diffFile)
		defer diffWriter.Flush()

		err = types.ComposeCompliantDat(diffDat, diffWriter)
		if err != nil {
			return err
		}

		endMsg = fmt.Sprintf("diffdat finished, %d games with diffs found, written diffdat file %s",
			len(diffDat.Games), outPath)
	} else {
		endMsg = "diffdat finished, no diffs found, no diffdat file written"

	}

	glog.Infof(endMsg)
	fmt.Fprintf(cmd.Stdout, endMsg)
	rs.broadCastProgress(time.Now(), false, true, endMsg)

	return nil
}
示例#2
0
func (rs *RombaService) build(cmd *commander.Command, args []string) error {
	rs.jobMutex.Lock()
	defer rs.jobMutex.Unlock()

	if rs.busy {
		p := rs.pt.GetProgress()

		fmt.Fprintf(cmd.Stdout, "still busy with %s: (%d of %d files) and (%s of %s) \n", rs.jobName,
			p.FilesSoFar, p.TotalFiles, humanize.IBytes(uint64(p.BytesSoFar)), humanize.IBytes(uint64(p.TotalBytes)))
		return nil
	}

	outpath := cmd.Flag.Lookup("out").Value.Get().(string)
	if outpath == "" {
		fmt.Fprintf(cmd.Stdout, "-out flag is required")
		return nil
	}

	fixdatOnly := cmd.Flag.Lookup("fixdatOnly").Value.Get().(bool)

	numWorkers := cmd.Flag.Lookup("workers").Value.Get().(int)
	numSubWorkers := cmd.Flag.Lookup("subworkers").Value.Get().(int)

	if !filepath.IsAbs(outpath) {
		absoutpath, err := filepath.Abs(outpath)
		if err != nil {
			return err
		}
		outpath = absoutpath
	}

	if err := os.MkdirAll(outpath, 0777); err != nil {
		return err
	}

	deduper, err := dedup.NewLevelDBDeduper()
	if err != nil {
		return err
	}

	rs.pt.Reset()
	rs.busy = true
	rs.jobName = "build"

	go func() {
		glog.Infof("service starting build")
		rs.broadCastProgress(time.Now(), true, false, "")
		ticker := time.NewTicker(time.Second * 5)
		stopTicker := make(chan bool)
		go func() {
			glog.Infof("starting progress broadcaster")
			for {
				select {
				case t := <-ticker.C:
					rs.broadCastProgress(t, false, false, "")
				case <-stopTicker:
					glog.Info("stopped progress broadcaster")
					return
				}
			}
		}()

		pm := &buildMaster{
			outpath:       outpath,
			rs:            rs,
			numWorkers:    numWorkers,
			numSubWorkers: numSubWorkers,
			pt:            rs.pt,
			fixdatOnly:    fixdatOnly,
			deduper:       deduper,
		}

		endMsg, err := worker.Work("building dats", args, pm)
		if err != nil {
			glog.Errorf("error building dats: %v", err)
		}

		ticker.Stop()
		stopTicker <- true

		err = archive.DeleteEmptyFolders(outpath)
		if err != nil {
			glog.Errorf("error building dats: %v", err)
		}

		rs.jobMutex.Lock()
		rs.busy = false
		rs.jobName = ""
		rs.jobMutex.Unlock()

		rs.broadCastProgress(time.Now(), false, true, endMsg)
		glog.Infof("service finished build")
	}()

	fmt.Fprintf(cmd.Stdout, "started build")
	return nil
}
示例#3
0
func (rs *RombaService) ediffdatWork(cmd *commander.Command, args []string) error {
	oldDatPath := cmd.Flag.Lookup("old").Value.Get().(string)
	newDatPath := cmd.Flag.Lookup("new").Value.Get().(string)
	outPath := cmd.Flag.Lookup("out").Value.Get().(string)

	if oldDatPath == "" {
		fmt.Fprintf(cmd.Stdout, "-old argument required")
		return errors.New("missing old argument")
	}
	if newDatPath == "" {
		fmt.Fprintf(cmd.Stdout, "-new argument required")
		return errors.New("missing new argument")
	}
	if outPath == "" {
		fmt.Fprintf(cmd.Stdout, "-out argument required")
		return errors.New("missing out argument")
	}

	err := os.MkdirAll(outPath, 0777)
	if err != nil {
		return err
	}

	glog.Infof("ediffdat new dat %s and old dat %s into %s", newDatPath, oldDatPath, outPath)

	dd, err := dedup.NewLevelDBDeduper()
	if err != nil {
		return err
	}
	defer dd.Close()

	err = filepath.Walk(oldDatPath, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}

		ext := filepath.Ext(path)
		if ext == ".dat" || ext == ".xml" {
			rs.pt.DeclareFile(path)

			oldDat, _, err := parser.Parse(path)
			if err != nil {
				return err
			}

			err = dedup.Declare(oldDat, dd)
			if err != nil {
				return err
			}

			rs.pt.AddBytesFromFile(info.Size(), false)
		}
		return nil
	})
	if err != nil {
		return err
	}

	err = filepath.Walk(newDatPath, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}

		ext := filepath.Ext(path)
		if ext == ".dat" || ext == ".xml" {
			rs.pt.DeclareFile(path)

			newDat, _, err := parser.Parse(path)
			if err != nil {
				return err
			}

			oneDiffDat, err := dedup.Dedup(newDat, dd)
			if err != nil {
				return err
			}

			if oneDiffDat != nil {
				oneDiffDat = oneDiffDat.FilterRoms(func(r *types.Rom) bool {
					return r.Size > 0
				})
			}

			if oneDiffDat != nil {
				err = writeDiffDat(oneDiffDat, filepath.Join(outPath, oneDiffDat.Name+".dat"))
			}

			rs.pt.AddBytesFromFile(info.Size(), err != nil)
			return err
		}
		return nil
	})
	if err != nil {
		return err
	}

	return nil
}
示例#4
0
func (rs *RombaService) datstats(cmd *commander.Command, args []string) error {
	rs.jobMutex.Lock()
	defer rs.jobMutex.Unlock()

	deduper, err := dedup.NewLevelDBDeduper()
	if err != nil {
		return err
	}
	defer deduper.Close()

	dts := &datStats{
		h: hdrhistogram.New(0, 1000000000000, 5),
	}

	err = rs.romDB.ForEachDat(func(dat *types.Dat) error {
		if dat.Generation != rs.romDB.Generation() {
			return nil
		}
		dedat, err := dedup.Dedup(dat, deduper)
		if err != nil {
			return err
		}

		if dedat == nil {
			return nil
		}

		dts.nDats = dts.nDats + 1
		for _, g := range dedat.Games {
			dts.nGames = dts.nGames + 1
			for _, r := range g.Roms {
				dts.h.RecordValue(r.Size)
				dts.nRoms = dts.nRoms + 1
				dts.totalSize = dts.totalSize + uint64(r.Size)
				if r.Size <= 4000 {
					dts.nRomsBelow4k = dts.nRomsBelow4k + 1
				}
			}
		}
		return nil
	})
	if err != nil {
		return err
	}

	bs := dts.h.CumulativeDistribution()

	fmt.Fprintf(cmd.Stdout, "number of dats = %d\n", dts.nDats)
	fmt.Fprintf(cmd.Stdout, "number of games = %d\n", dts.nGames)
	fmt.Fprintf(cmd.Stdout, "number of roms = %d\n", dts.nRoms)
	fmt.Fprintf(cmd.Stdout, "total rom size = %s\n", humanize.IBytes(dts.totalSize))
	fmt.Fprintf(cmd.Stdout, "number of roms below 4k size = %d\n\n", dts.nRomsBelow4k)

	fmt.Fprintf(cmd.Stdout, "rom size cumulative distribution = \n")
	fmt.Fprintf(cmd.Stdout, "count, percentile, file size\n")
	for i := 0; i < len(bs); i++ {
		b := bs[i]

		vstr := humanize.IBytes(uint64(b.ValueAt))

		if (i < len(bs)-1 && vstr != humanize.IBytes(uint64(bs[i+1].ValueAt))) || (i == len(bs)-1) {
			fmt.Fprintf(cmd.Stdout, "%d, %.8f, %s\n", b.Count, b.Quantile, humanize.IBytes(uint64(b.ValueAt)))
		}
	}

	fmt.Fprintf(cmd.Stdout, "\nrom size histogram = \n")
	fmt.Fprintf(cmd.Stdout, "count, file size\n")
	var lastCount int64
	for _, b := range bs {
		count := b.Count - lastCount
		if count > 0 {
			fmt.Fprintf(cmd.Stdout, "%d, %s\n", count, humanize.IBytes(uint64(b.ValueAt)))
		}
		lastCount = b.Count
	}

	return nil
}