func (depot *Depot) Merge(paths []string, resumePath string, onlyneeded bool, numWorkers int,
logDir string, pt worker.ProgressTracker, skipInitialScan bool) (string, error) {
resumeLogPath := filepath.Join(logDir, fmt.Sprintf("merge-resume-%s.log", time.Now().Format(ResumeDateFormat)))
resumeLogFile, err := os.Create(resumeLogPath)
if err != nil {
return "", err
}
resumeLogWriter := bufio.NewWriter(resumeLogFile)
resumePoint := ""
if len(resumePath) > 0 {
resumePoint, err = extractResumePoint(resumePath, numWorkers)
if err != nil {
return "", err
}
}
glog.Infof("resuming with path %s", resumePoint)
pm := new(mergeMaster)
pm.depot = depot
pm.resumePath = resumePoint
pm.pt = pt
pm.numWorkers = numWorkers
pm.soFar = make(chan *completed)
pm.resumeLogWriter = resumeLogWriter
pm.resumeLogFile = resumeLogFile
pm.onlyneeded = onlyneeded
pm.skipInitialScan = skipInitialScan
go loopObserver(pm.numWorkers, pm.soFar, pm.depot, pm.resumeLogWriter)
return worker.Work("merge roms", paths, pm)
}
func main() {
flag.Usage = usage
help := flag.Bool("help", false, "show this message")
version := flag.Bool("version", false, "show version")
flag.Parse()
if *help {
flag.Usage()
os.Exit(0)
}
if *version {
fmt.Fprintf(os.Stderr, "%s version %s, Copyright (c) 2013 Uwe Hoffmann. All rights reserved.\n", os.Args[0], versionStr)
os.Exit(0)
}
runtime.GOMAXPROCS(numWorkers)
_, err := worker.Work("parse dats", flag.Args(), new(parseMaster))
if err != nil {
fmt.Fprintf(os.Stderr, " error: %v\n", err)
os.Exit(1)
}
}
func (depot *Depot) Archive(paths []string, resumePath string, includezips bool, onlyneeded bool, numWorkers int,
logDir string, pt worker.ProgressTracker) (string, error) {
resumeLogPath := filepath.Join(logDir, fmt.Sprintf("archive-resume-%s.log", time.Now().Format("2006-01-02-15_04_05")))
resumeLogFile, err := os.Create(resumeLogPath)
if err != nil {
return "", err
}
resumeLogWriter := bufio.NewWriter(resumeLogFile)
pm := new(archiveMaster)
pm.depot = depot
pm.resumePath = resumePath
pm.pt = pt
pm.numWorkers = numWorkers
pm.soFar = make(chan *completed)
pm.resumeLogWriter = resumeLogWriter
pm.resumeLogFile = resumeLogFile
pm.includezips = includezips
pm.onlyneeded = onlyneeded
go pm.loopObserver(resumeLogWriter)
return worker.Work("archive roms", paths, pm)
}
func Refresh(romdb RomDB, datsPath string, numWorkers int, pt worker.ProgressTracker, missingSha1s string) (string, error) {
err := romdb.OrphanDats()
if err != nil {
return "", err
}
var missingSha1sWriter io.Writer
if missingSha1s != "" {
missingSha1sFile, err := os.Create(missingSha1s)
if err != nil {
return "", err
}
defer missingSha1sFile.Close()
missingSha1sBuf := bufio.NewWriter(missingSha1sFile)
defer missingSha1sBuf.Flush()
missingSha1sWriter = missingSha1sBuf
}
pm := &refreshMaster{
romdb: romdb,
numWorkers: numWorkers,
pt: pt,
missingSha1sWriter: missingSha1sWriter,
}
return worker.Work("refresh dats", []string{datsPath}, pm)
}
func (depot *Depot) Purge(backupDir string, numWorkers int, workDepot string, fromDats string,
pt worker.ProgressTracker) (string, error) {
pm := new(purgeMaster)
pm.depot = depot
pm.pt = pt
pm.numWorkers = numWorkers
absBackupDir, err := filepath.Abs(backupDir)
if err != nil {
return "", err
}
pm.backupDir = absBackupDir
if backupDir == "" {
return "", errors.New("no backup dir specified")
}
err = os.MkdirAll(backupDir, 0777)
if err != nil {
return "", err
}
if fromDats == "" {
wds := depot.roots
if len(workDepot) > 0 {
wds = []string{workDepot}
}
return worker.Work("purge roms", wds, pm)
} else {
var dats []*types.Dat
err = filepath.Walk(fromDats, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() && (strings.HasSuffix(path, ".dat") || strings.HasSuffix(path, ".xml")) {
dat, _, err := parser.Parse(path)
if err != nil {
return err
}
dats = append(dats, dat)
}
return nil
})
if err != nil {
return "", err
}
rdi := newRomsFromDatIterator(depot, dats)
return worker.WorkPathIterator("purge roms", rdi, pm)
}
}
func Refresh(romdb RomDB, datsPath string, numWorkers int, pt worker.ProgressTracker) (string, error) {
err := romdb.OrphanDats()
if err != nil {
return "", err
}
pm := &refreshMaster{
romdb: romdb,
numWorkers: numWorkers,
pt: pt,
}
return worker.Work("refresh dats", []string{datsPath}, pm)
}
func (rs *RombaService) build(cmd *commander.Command, args []string) error {
rs.jobMutex.Lock()
defer rs.jobMutex.Unlock()
if rs.busy {
p := rs.pt.GetProgress()
fmt.Fprintf(cmd.Stdout, "still busy with %s: (%d of %d files) and (%s of %s) \n", rs.jobName,
p.FilesSoFar, p.TotalFiles, humanize.Bytes(uint64(p.BytesSoFar)), humanize.Bytes(uint64(p.TotalBytes)))
return nil
}
outpath := cmd.Flag.Lookup("out").Value.Get().(string)
if !filepath.IsAbs(outpath) {
absoutpath, err := filepath.Abs(outpath)
if err != nil {
return err
}
outpath = absoutpath
}
if err := os.MkdirAll(outpath, 0777); err != nil {
return err
}
rs.pt.Reset()
rs.busy = true
rs.jobName = "build"
go func() {
glog.Infof("service starting build")
rs.broadCastProgress(time.Now(), true, false, "")
ticker := time.NewTicker(time.Second * 5)
stopTicker := make(chan bool)
go func() {
glog.Infof("starting progress broadcaster")
for {
select {
case t := <-ticker.C:
rs.broadCastProgress(t, false, false, "")
case <-stopTicker:
glog.Info("stopped progress broadcaster")
return
}
}
}()
pm := &buildMaster{
outpath: outpath,
rs: rs,
numWorkers: rs.numWorkers,
pt: rs.pt,
}
endMsg, err := worker.Work("building dats", args, pm)
if err != nil {
glog.Errorf("error building dats: %v", err)
}
ticker.Stop()
stopTicker <- true
rs.jobMutex.Lock()
rs.busy = false
rs.jobName = ""
rs.jobMutex.Unlock()
rs.broadCastProgress(time.Now(), false, true, endMsg)
glog.Infof("service finished build")
}()
fmt.Fprintf(cmd.Stdout, "started build")
return nil
}