// CrawlROMs crawls the rom directory and processes the files. func CrawlROMs(gl *rom.GameListXML, sources []ds.DS, xmlOpts *rom.XMLOpts, gameOpts *rom.GameOpts) error { var ct http.RoundTripper = NewCancelTransport(http.DefaultTransport.(*http.Transport)) http.DefaultClient.Transport = ct existing := make(map[string]struct{}) if !dirExists(xmlOpts.RomDir) { log.Printf("ERR %s: does not exists", xmlOpts.RomDir) return nil } extraMap := make(map[string]struct{}) if *extraExt != "" { extraSlice := strings.Split(*extraExt, ",") for _, e := range extraSlice { if e[0] != '.' { extraMap["."+e] = struct{}{} } else { extraMap[e] = struct{}{} } } } for _, x := range gl.GameList { switch { case *appendOut: p, err := filepath.Rel(xmlOpts.RomXMLDir, x.Path) if err != nil { log.Printf("Can't find original path: %s", x.Path) } f := filepath.Join(xmlOpts.RomDir, p) existing[f] = struct{}{} case *refreshOut: existing[x.Path] = struct{}{} } } var wg sync.WaitGroup results := make(chan *rom.GameXML, *workers) roms := make(chan *rom.ROM, 2**workers) for i := 0; i < *workers; i++ { wg.Add(1) go worker(sources, xmlOpts, gameOpts, results, roms, &wg) } go func() { defer wg.Done() for r := range results { if _, ok := existing[r.Path]; ok && *refreshOut { for i, g := range gl.GameList { if g.Path != r.Path { continue } copy(gl.GameList[i:], gl.GameList[i+1:]) gl.GameList = gl.GameList[:len(gl.GameList)-1] } } gl.Append(r) } }() var stop bool sig := make(chan os.Signal, 1) signal.Notify(sig, os.Interrupt) defer signal.Stop(sig) go func() { for { <-sig if !stop { stop = true log.Println("Stopping, ctrl-c again to stop now.") ct.(*CancelTransport).Stop() for _ = range roms { } continue } panic("AHHHH!") } }() bins := make(map[string]struct{}) if !*mame { walker := fs.Walk(xmlOpts.RomDir) for walker.Step() { if stop { break } if err := walker.Err(); err != nil { return err } f := walker.Path() if b := filepath.Base(f); b != "." && strings.HasPrefix(b, ".") { walker.SkipDir() continue } r, err := rom.NewROM(f) if err != nil { log.Printf("ERR: Processing: %s, %s", f, err) continue } if !r.Cue { continue } for _, b := range r.Bins { bins[b] = struct{}{} } bins[f] = struct{}{} if _, ok := existing[f]; !*refreshOut && ok { log.Printf("INFO: Skipping %s, already in gamelist.", f) continue } roms <- r } } walker := fs.Walk(xmlOpts.RomDir) for walker.Step() { if stop { break } if err := walker.Err(); err != nil { return err } f := walker.Path() if b := filepath.Base(f); b != "." && strings.HasPrefix(b, ".") { walker.SkipDir() continue } if _, ok := existing[f]; !*refreshOut && ok { log.Printf("INFO: Skipping %s, already in gamelist.", f) continue } r, err := rom.NewROM(f) if err != nil { log.Printf("ERR: Processing: %s, %s", f, err) continue } _, isExtra := extraMap[r.Ext] if *mame { if r.Ext == ".zip" || r.Ext == ".7z" || isExtra { roms <- r } continue } _, ok := bins[f] if !ok && (rh.KnownExt(r.Ext) || r.Ext == ".svm" || isExtra) { roms <- r } } close(roms) wg.Wait() wg.Add(1) close(results) wg.Wait() if stop { return UserCanceled } else { return nil } }
// CrawlROMs crawls the rom directory and processes the files. func CrawlROMs(gl *rom.GameListXML, sources []ds.DS, xmlOpts *rom.XMLOpts, gameOpts *rom.GameOpts) error { var ct http.RoundTripper = NewCancelTransport(http.DefaultTransport.(*http.Transport)) http.DefaultClient.Transport = ct existing := make(map[string]struct{}) for _, x := range gl.GameList { p, err := filepath.Rel(xmlOpts.RomXMLDir, x.Path) if err != nil { log.Printf("Can't find original path: %s", x.Path) } f := filepath.Join(xmlOpts.RomDir, p) existing[f] = struct{}{} } var wg sync.WaitGroup results := make(chan *rom.GameXML, *workers) roms := make(chan *rom.ROM, 2**workers) for i := 0; i < *workers; i++ { wg.Add(1) go worker(sources, xmlOpts, gameOpts, results, roms, &wg) } go func() { defer wg.Done() for r := range results { gl.Append(r) } }() var stop bool sig := make(chan os.Signal, 1) signal.Notify(sig, os.Interrupt) defer signal.Stop(sig) go func() { for { <-sig if !stop { stop = true log.Println("Stopping, ctrl-c again to stop now.") ct.(*CancelTransport).Stop() for _ = range roms { } continue } panic("AHHHH!") } }() bins := make(map[string]struct{}) if !*mame { walker := fs.Walk(xmlOpts.RomDir) for walker.Step() { if stop { break } if err := walker.Err(); err != nil { return err } f := walker.Path() r, err := rom.NewROM(f) if err != nil { log.Printf("ERR: Processing: %s, %s", f, err) continue } if !r.Cue { continue } for _, b := range r.Bins { bins[b] = struct{}{} } bins[f] = struct{}{} if _, ok := existing[f]; ok { log.Printf("INFO: Skipping %s, already in gamelist.", f) continue } roms <- r } } walker := fs.Walk(xmlOpts.RomDir) for walker.Step() { if stop { break } if err := walker.Err(); err != nil { return err } f := walker.Path() if _, ok := existing[f]; ok { log.Printf("INFO: Skipping %s, already in gamelist.", f) continue } r, err := rom.NewROM(f) if err != nil { log.Printf("ERR: Processing: %s, %s", f, err) continue } if *mame { if r.Ext == ".zip" || r.Ext == ".7z" { roms <- r } continue } _, ok := bins[f] if !ok && rh.KnownExt(r.Ext) { roms <- r } } close(roms) wg.Wait() wg.Add(1) close(results) wg.Wait() if stop { return UserCanceled } else { return nil } }
// CrawlROMs crawls the rom directory and processes the files. func CrawlROMs(gl *rom.GameListXML, sources []ds.DS, xmlOpts *rom.XMLOpts, gameOpts *rom.GameOpts) error { var missingCSV *csv.Writer var gdbDS *ds.GDB if *missing != "" { f, err := os.Create(*missing) if err != nil { return err } missingCSV = csv.NewWriter(f) defer func() { missingCSV.Flush() if err := missingCSV.Error(); err != nil { log.Fatal(err) } f.Close() }() if err := missingCSV.Write([]string{"Game", "Error", "Hash", "Extra"}); err != nil { return err } for _, d := range sources { switch d := d.(type) { case *ds.GDB: gdbDS = d } } } var ct http.RoundTripper = NewCancelTransport(http.DefaultTransport.(*http.Transport)) http.DefaultClient.Transport = ct existing := make(map[string]struct{}) if !dirExists(xmlOpts.RomDir) { log.Printf("ERR %s: does not exists", xmlOpts.RomDir) return nil } extraMap := make(map[string]struct{}) if *extraExt != "" { extraSlice := strings.Split(*extraExt, ",") for _, e := range extraSlice { if e[0] != '.' { extraMap["."+e] = struct{}{} } else { extraMap[e] = struct{}{} } } } for _, x := range gl.GameList { switch { case *appendOut: p, err := filepath.Rel(xmlOpts.RomXMLDir, x.Path) if err != nil { log.Printf("Can't find original path: %s", x.Path) } f := filepath.Join(xmlOpts.RomDir, p) existing[f] = struct{}{} case *refreshOut: existing[x.Path] = struct{}{} } } var wg sync.WaitGroup results := make(chan Result, *workers) roms := make(chan *rom.ROM, 2**workers) for i := 0; i < *workers; i++ { wg.Add(1) go worker(sources, xmlOpts, gameOpts, results, roms, &wg) } go func() { defer wg.Done() for r := range results { if r.XML == nil { if *missing == "" { continue } files := []string{r.ROM.Path} if r.ROM.Cue { files = append(files, r.ROM.Bins...) } for _, file := range files { var hash, extra string if gdbDS != nil { var err error hash, err = gdbDS.Hash(file) if err != nil { log.Printf("ERR: Can't hash file %s", file) } name := gdbDS.GetName(file) if name != "" && r.Err == ds.NotFoundErr { extra = "hash found but no GDB ID" } } if err := missingCSV.Write([]string{file, r.Err.Error(), hash, extra}); err != nil { log.Printf("ERR: Can't write to %s", *missing) } } continue } if r.XML.Image == "" && *missing != "" { var hash string if gdbDS != nil { var err error hash, err = gdbDS.Hash(r.ROM.Path) if err != nil { log.Printf("ERR: Can't hash file %s", r.ROM.Path) } } if err := missingCSV.Write([]string{r.ROM.FileName, "", hash, "missing image"}); err != nil { log.Printf("ERR: Can't write to %s", *missing) } } if _, ok := existing[r.XML.Path]; ok && *refreshOut { for i, g := range gl.GameList { if g.Path != r.XML.Path { continue } copy(gl.GameList[i:], gl.GameList[i+1:]) gl.GameList = gl.GameList[:len(gl.GameList)-1] } } gl.Append(r.XML) } }() var stop bool sig := make(chan os.Signal, 1) signal.Notify(sig, os.Interrupt) defer signal.Stop(sig) go func() { for { <-sig if !stop { stop = true log.Println("Stopping, ctrl-c again to stop now.") ct.(*CancelTransport).Stop() for _ = range roms { } continue } panic("AHHHH!") } }() bins := make(map[string]struct{}) if !*mame { walker := fs.Walk(xmlOpts.RomDir) for walker.Step() { if stop { break } if err := walker.Err(); err != nil { return err } f := walker.Path() if b := filepath.Base(f); b != "." && strings.HasPrefix(b, ".") { walker.SkipDir() continue } r, err := rom.NewROM(f) if err != nil { log.Printf("ERR: Processing: %s, %s", f, err) continue } if !r.Cue { continue } for _, b := range r.Bins { bins[b] = struct{}{} } bins[f] = struct{}{} if _, ok := existing[f]; !*refreshOut && ok { log.Printf("INFO: Skipping %s, already in gamelist.", f) continue } roms <- r } } walker := fs.Walk(xmlOpts.RomDir) for walker.Step() { if stop { break } if err := walker.Err(); err != nil { return err } f := walker.Path() if b := filepath.Base(f); b != "." && strings.HasPrefix(b, ".") { walker.SkipDir() continue } if _, ok := existing[f]; !*refreshOut && ok { log.Printf("INFO: Skipping %s, already in gamelist.", f) continue } r, err := rom.NewROM(f) if err != nil { log.Printf("ERR: Processing: %s, %s", f, err) continue } _, isExtra := extraMap[r.Ext] if *mame { if r.Ext == ".zip" || r.Ext == ".7z" || isExtra { roms <- r } continue } _, ok := bins[f] if !ok && (rh.KnownExt(r.Ext) || r.Ext == ".svm" || isExtra) { roms <- r } } close(roms) wg.Wait() wg.Add(1) close(results) wg.Wait() if stop { return UserCanceled } else { return nil } }