// CrawlROMs crawls the rom directory and processes the files.
func CrawlROMs(gl *rom.GameListXML, sources []ds.DS, xmlOpts *rom.XMLOpts, gameOpts *rom.GameOpts) error {
var ct http.RoundTripper = NewCancelTransport(http.DefaultTransport.(*http.Transport))
http.DefaultClient.Transport = ct
existing := make(map[string]struct{})
if !dirExists(xmlOpts.RomDir) {
log.Printf("ERR %s: does not exists", xmlOpts.RomDir)
return nil
}
for _, x := range gl.GameList {
switch {
case *appendOut:
p, err := filepath.Rel(xmlOpts.RomXMLDir, x.Path)
if err != nil {
log.Printf("Can't find original path: %s", x.Path)
}
f := filepath.Join(xmlOpts.RomDir, p)
existing[f] = struct{}{}
case *refreshOut:
existing[x.Path] = struct{}{}
}
}
var wg sync.WaitGroup
results := make(chan *rom.GameXML, *workers)
roms := make(chan *rom.ROM, 2**workers)
for i := 0; i < *workers; i++ {
wg.Add(1)
go worker(sources, xmlOpts, gameOpts, results, roms, &wg)
}
go func() {
defer wg.Done()
for r := range results {
if _, ok := existing[r.Path]; ok && *refreshOut {
for i, g := range gl.GameList {
if g.Path != r.Path {
continue
}
copy(gl.GameList[i:], gl.GameList[i+1:])
gl.GameList = gl.GameList[:len(gl.GameList)-1]
}
}
gl.Append(r)
}
}()
var stop bool
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt)
defer signal.Stop(sig)
go func() {
for {
<-sig
if !stop {
stop = true
log.Println("Stopping, ctrl-c again to stop now.")
ct.(*CancelTransport).Stop()
for _ = range roms {
}
continue
}
panic("AHHHH!")
}
}()
bins := make(map[string]struct{})
if !*mame {
walker := fs.Walk(xmlOpts.RomDir)
for walker.Step() {
if stop {
break
}
if err := walker.Err(); err != nil {
return err
}
f := walker.Path()
r, err := rom.NewROM(f)
if err != nil {
log.Printf("ERR: Processing: %s, %s", f, err)
continue
}
if !r.Cue {
continue
}
for _, b := range r.Bins {
bins[b] = struct{}{}
}
bins[f] = struct{}{}
if _, ok := existing[f]; !*refreshOut && ok {
log.Printf("INFO: Skipping %s, already in gamelist.", f)
continue
}
roms <- r
}
}
walker := fs.Walk(xmlOpts.RomDir)
for walker.Step() {
if stop {
break
}
if err := walker.Err(); err != nil {
return err
}
f := walker.Path()
if _, ok := existing[f]; !*refreshOut && ok {
log.Printf("INFO: Skipping %s, already in gamelist.", f)
continue
}
r, err := rom.NewROM(f)
if err != nil {
log.Printf("ERR: Processing: %s, %s", f, err)
continue
}
if *mame {
if r.Ext == ".zip" || r.Ext == ".7z" {
roms <- r
}
continue
}
_, ok := bins[f]
if !ok && (rh.KnownExt(r.Ext) || r.Ext == ".svm") {
roms <- r
}
}
close(roms)
wg.Wait()
wg.Add(1)
close(results)
wg.Wait()
if stop {
return UserCanceled
} else {
return nil
}
}
// CrawlROMs crawls the rom directory and processes the files.
func CrawlROMs(gl *rom.GameListXML, sources []ds.DS, xmlOpts *rom.XMLOpts, gameOpts *rom.GameOpts) error {
var missingCSV *csv.Writer
var gdbDS *ds.GDB
if *missing != "" {
f, err := os.Create(*missing)
if err != nil {
return err
}
missingCSV = csv.NewWriter(f)
defer func() {
missingCSV.Flush()
if err := missingCSV.Error(); err != nil {
log.Fatal(err)
}
f.Close()
}()
if err := missingCSV.Write([]string{"Game", "Error", "Hash", "Extra"}); err != nil {
return err
}
for _, d := range sources {
switch d := d.(type) {
case *ds.GDB:
gdbDS = d
}
}
}
var ct http.RoundTripper = NewCancelTransport(http.DefaultTransport.(*http.Transport))
http.DefaultClient.Transport = ct
existing := make(map[string]struct{})
if !dirExists(xmlOpts.RomDir) {
log.Printf("ERR %s: does not exists", xmlOpts.RomDir)
return nil
}
extraMap := make(map[string]struct{})
if *extraExt != "" {
extraSlice := strings.Split(*extraExt, ",")
for _, e := range extraSlice {
if e[0] != '.' {
extraMap["."+e] = struct{}{}
} else {
extraMap[e] = struct{}{}
}
}
}
for _, x := range gl.GameList {
switch {
case *appendOut:
p, err := filepath.Rel(xmlOpts.RomXMLDir, x.Path)
if err != nil {
log.Printf("Can't find original path: %s", x.Path)
}
f := filepath.Join(xmlOpts.RomDir, p)
existing[f] = struct{}{}
case *refreshOut:
existing[x.Path] = struct{}{}
}
}
var wg sync.WaitGroup
results := make(chan Result, *workers)
roms := make(chan *rom.ROM, 2**workers)
for i := 0; i < *workers; i++ {
wg.Add(1)
go worker(sources, xmlOpts, gameOpts, results, roms, &wg)
}
go func() {
defer wg.Done()
for r := range results {
if r.XML == nil {
if *missing == "" {
continue
}
files := []string{r.ROM.Path}
if r.ROM.Cue {
files = append(files, r.ROM.Bins...)
}
for _, file := range files {
var hash, extra string
if gdbDS != nil {
var err error
hash, err = gdbDS.Hash(file)
if err != nil {
log.Printf("ERR: Can't hash file %s", file)
}
name := gdbDS.GetName(file)
if name != "" && r.Err == ds.NotFoundErr {
extra = "hash found but no GDB ID"
}
}
if err := missingCSV.Write([]string{file, r.Err.Error(), hash, extra}); err != nil {
log.Printf("ERR: Can't write to %s", *missing)
}
}
continue
}
if r.XML.Image == "" && *missing != "" {
var hash string
if gdbDS != nil {
var err error
hash, err = gdbDS.Hash(r.ROM.Path)
if err != nil {
log.Printf("ERR: Can't hash file %s", r.ROM.Path)
}
}
if err := missingCSV.Write([]string{r.ROM.FileName, "", hash, "missing image"}); err != nil {
log.Printf("ERR: Can't write to %s", *missing)
}
}
if _, ok := existing[r.XML.Path]; ok && *refreshOut {
for i, g := range gl.GameList {
if g.Path != r.XML.Path {
continue
}
copy(gl.GameList[i:], gl.GameList[i+1:])
gl.GameList = gl.GameList[:len(gl.GameList)-1]
}
}
gl.Append(r.XML)
}
}()
var stop bool
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt)
defer signal.Stop(sig)
go func() {
for {
<-sig
if !stop {
stop = true
log.Println("Stopping, ctrl-c again to stop now.")
ct.(*CancelTransport).Stop()
for _ = range roms {
}
continue
}
panic("AHHHH!")
}
}()
bins := make(map[string]struct{})
if !*mame {
walker := fs.Walk(xmlOpts.RomDir)
for walker.Step() {
if stop {
break
}
if err := walker.Err(); err != nil {
return err
}
f := walker.Path()
if b := filepath.Base(f); b != "." && strings.HasPrefix(b, ".") {
walker.SkipDir()
continue
}
r, err := rom.NewROM(f)
if err != nil {
log.Printf("ERR: Processing: %s, %s", f, err)
continue
}
if !r.Cue {
continue
}
for _, b := range r.Bins {
bins[b] = struct{}{}
}
bins[f] = struct{}{}
if _, ok := existing[f]; !*refreshOut && ok {
log.Printf("INFO: Skipping %s, already in gamelist.", f)
continue
}
roms <- r
}
}
walker := fs.Walk(xmlOpts.RomDir)
for walker.Step() {
if stop {
break
}
if err := walker.Err(); err != nil {
return err
}
f := walker.Path()
if b := filepath.Base(f); b != "." && strings.HasPrefix(b, ".") {
walker.SkipDir()
continue
}
if _, ok := existing[f]; !*refreshOut && ok {
log.Printf("INFO: Skipping %s, already in gamelist.", f)
continue
}
r, err := rom.NewROM(f)
if err != nil {
log.Printf("ERR: Processing: %s, %s", f, err)
continue
}
_, isExtra := extraMap[r.Ext]
if *mame {
if r.Ext == ".zip" || r.Ext == ".7z" || isExtra {
roms <- r
}
continue
}
_, ok := bins[f]
if !ok && (rh.KnownExt(r.Ext) || r.Ext == ".svm" || isExtra) {
roms <- r
}
}
close(roms)
wg.Wait()
wg.Add(1)
close(results)
wg.Wait()
if stop {
return UserCanceled
} else {
return nil
}
}