func writeTo(writer *csv.Writer, in interface{}) error {
inValue, inType := getConcreteReflectValueAndType(in) // Get the concrete type (not pointer) (Slice<?> or Array<?>)
if err := ensureInType(inType); err != nil {
return err
}
inInnerWasPointer, inInnerType := getConcreteContainerInnerType(inType) // Get the concrete inner type (not pointer) (Container<"?">)
if err := ensureInInnerType(inInnerType); err != nil {
return err
}
inInnerStructInfo := getStructInfo(inInnerType) // Get the inner struct info to get CSV annotations
csvHeadersLabels := make([]string, len(inInnerStructInfo.Fields))
for i, fieldInfo := range inInnerStructInfo.Fields { // Used to write the header (first line) in CSV
csvHeadersLabels[i] = fieldInfo.getFirstKey()
}
if err := writer.Write(csvHeadersLabels); err != nil {
return err
}
inLen := inValue.Len()
for i := 0; i < inLen; i++ { // Iterate over container rows
for j, fieldInfo := range inInnerStructInfo.Fields {
csvHeadersLabels[j] = ""
inInnerFieldValue, err := getInnerField(inValue.Index(i), inInnerWasPointer, fieldInfo.IndexChain) // Get the correct field header <-> position
if err != nil {
return err
}
csvHeadersLabels[j] = inInnerFieldValue
}
if err := writer.Write(csvHeadersLabels); err != nil {
return err
}
}
writer.Flush()
return writer.Error()
}
func (p *csvProcessor) flush(writer *csv.Writer, err error) error {
if err != nil {
return err
}
writer.Flush()
return writer.Error()
}
// NewWithCsvWriter writes all of the Rows in a Table to a CSV file using the options in the CSV writer.
// It assumes that all Rows have the same headers. Columns are written in alphabetical order.
func NewWithCsvWriter(writer *csv.Writer) optimus.Sink {
return func(source optimus.Table) error {
defer source.Stop()
headers := []string{}
wroteHeader := false
for row := range source.Rows() {
if !wroteHeader {
headers = convertRowToHeader(row)
sort.Strings(headers)
if err := writer.Write(headers); err != nil {
return err
}
wroteHeader = true
}
if err := writer.Write(convertRowToRecord(row, headers)); err != nil {
return err
}
}
if source.Err() != nil {
return source.Err()
}
writer.Flush()
if writer.Error() != nil {
return writer.Error()
}
return nil
}
}
func writeFromChan(writer *csv.Writer, c <-chan interface{}) error {
// Get the first value. It wil determine the header structure.
firstValue := <-c
inValue, inType := getConcreteReflectValueAndType(firstValue) // Get the concrete type
if err := ensureStructOrPtr(inType); err != nil {
return err
}
inInnerWasPointer := inType.Kind() == reflect.Ptr
inInnerStructInfo := getStructInfo(inType) // Get the inner struct info to get CSV annotations
csvHeadersLabels := make([]string, len(inInnerStructInfo.Fields))
for i, fieldInfo := range inInnerStructInfo.Fields { // Used to write the header (first line) in CSV
csvHeadersLabels[i] = fieldInfo.getFirstKey()
}
if err := writer.Write(csvHeadersLabels); err != nil {
return err
}
write := func(val reflect.Value) error {
for j, fieldInfo := range inInnerStructInfo.Fields {
csvHeadersLabels[j] = ""
inInnerFieldValue, err := getInnerField(val, inInnerWasPointer, fieldInfo.IndexChain) // Get the correct field header <-> position
if err != nil {
return err
}
csvHeadersLabels[j] = inInnerFieldValue
}
if err := writer.Write(csvHeadersLabels); err != nil {
return err
}
return nil
}
if err := write(inValue); err != nil {
return err
}
for v := range c {
val, _ := getConcreteReflectValueAndType(v) // Get the concrete type (not pointer) (Slice<?> or Array<?>)
if err := ensureStructOrPtr(inType); err != nil {
return err
}
if err := write(val); err != nil {
return err
}
}
writer.Flush()
return writer.Error()
}
func (w *ReportWriter) writeCsvOne(writer *csv.Writer, report Report) error {
if w.dialect.HasMetadata {
preamble := make([]string, 4)
if len(report.Path) > 0 {
preamble[0] = "# File"
preamble[1] = report.Path
preamble[2] = report.Filename
preamble[3] = report.MD5hex
writer.Write(preamble)
}
preamble[0] = "# Field"
preamble[1] = fmt.Sprint(len(report.Fields))
if report.HasHeader {
preamble[2] = "(has header)"
} else {
preamble[2] = ""
}
preamble[3] = ""
writer.Write(preamble)
preamble[0] = "# Record"
preamble[1] = fmt.Sprint(report.Records)
preamble[2] = ""
writer.Write(preamble)
}
// Put header line.
if w.dialect.HasHeader {
writer.Write(ReportOutputFields)
}
// Put each field report.
for i, f := range report.Fields {
r := f.format(report.Records)
r[0] = fmt.Sprint(i + 1)
writer.Write(r)
}
return writer.Error()
}
// CrawlROMs crawls the rom directory and processes the files.
func CrawlROMs(gl *rom.GameListXML, sources []ds.DS, xmlOpts *rom.XMLOpts, gameOpts *rom.GameOpts) error {
var missingCSV *csv.Writer
var gdbDS *ds.GDB
if *missing != "" {
f, err := os.Create(*missing)
if err != nil {
return err
}
missingCSV = csv.NewWriter(f)
defer func() {
missingCSV.Flush()
if err := missingCSV.Error(); err != nil {
log.Fatal(err)
}
f.Close()
}()
if err := missingCSV.Write([]string{"Game", "Error", "Hash", "Extra"}); err != nil {
return err
}
for _, d := range sources {
switch d := d.(type) {
case *ds.GDB:
gdbDS = d
}
}
}
var ct http.RoundTripper = NewCancelTransport(http.DefaultTransport.(*http.Transport))
http.DefaultClient.Transport = ct
existing := make(map[string]struct{})
if !dirExists(xmlOpts.RomDir) {
log.Printf("ERR %s: does not exists", xmlOpts.RomDir)
return nil
}
extraMap := make(map[string]struct{})
if *extraExt != "" {
extraSlice := strings.Split(*extraExt, ",")
for _, e := range extraSlice {
if e[0] != '.' {
extraMap["."+e] = struct{}{}
} else {
extraMap[e] = struct{}{}
}
}
}
for _, x := range gl.GameList {
switch {
case *appendOut:
p, err := filepath.Rel(xmlOpts.RomXMLDir, x.Path)
if err != nil {
log.Printf("Can't find original path: %s", x.Path)
}
f := filepath.Join(xmlOpts.RomDir, p)
existing[f] = struct{}{}
case *refreshOut:
existing[x.Path] = struct{}{}
}
}
var wg sync.WaitGroup
results := make(chan Result, *workers)
roms := make(chan *rom.ROM, 2**workers)
for i := 0; i < *workers; i++ {
wg.Add(1)
go worker(sources, xmlOpts, gameOpts, results, roms, &wg)
}
go func() {
defer wg.Done()
for r := range results {
if r.XML == nil {
if *missing == "" {
continue
}
files := []string{r.ROM.Path}
if r.ROM.Cue {
files = append(files, r.ROM.Bins...)
}
for _, file := range files {
var hash, extra string
if gdbDS != nil {
var err error
hash, err = gdbDS.Hash(file)
if err != nil {
log.Printf("ERR: Can't hash file %s", file)
}
name := gdbDS.GetName(file)
if name != "" && r.Err == ds.NotFoundErr {
extra = "hash found but no GDB ID"
}
}
if err := missingCSV.Write([]string{file, r.Err.Error(), hash, extra}); err != nil {
log.Printf("ERR: Can't write to %s", *missing)
}
}
continue
}
if r.XML.Image == "" && *missing != "" {
var hash string
if gdbDS != nil {
var err error
hash, err = gdbDS.Hash(r.ROM.Path)
if err != nil {
log.Printf("ERR: Can't hash file %s", r.ROM.Path)
}
}
if err := missingCSV.Write([]string{r.ROM.FileName, "", hash, "missing image"}); err != nil {
log.Printf("ERR: Can't write to %s", *missing)
}
}
if _, ok := existing[r.XML.Path]; ok && *refreshOut {
for i, g := range gl.GameList {
if g.Path != r.XML.Path {
continue
}
copy(gl.GameList[i:], gl.GameList[i+1:])
gl.GameList = gl.GameList[:len(gl.GameList)-1]
}
}
gl.Append(r.XML)
}
}()
var stop bool
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt)
defer signal.Stop(sig)
go func() {
for {
<-sig
if !stop {
stop = true
log.Println("Stopping, ctrl-c again to stop now.")
ct.(*CancelTransport).Stop()
for _ = range roms {
}
continue
}
panic("AHHHH!")
}
}()
bins := make(map[string]struct{})
if !*mame {
walker := fs.Walk(xmlOpts.RomDir)
for walker.Step() {
if stop {
break
}
if err := walker.Err(); err != nil {
return err
}
f := walker.Path()
if b := filepath.Base(f); b != "." && strings.HasPrefix(b, ".") {
walker.SkipDir()
continue
}
r, err := rom.NewROM(f)
if err != nil {
log.Printf("ERR: Processing: %s, %s", f, err)
continue
}
if !r.Cue {
continue
}
for _, b := range r.Bins {
bins[b] = struct{}{}
}
bins[f] = struct{}{}
if _, ok := existing[f]; !*refreshOut && ok {
log.Printf("INFO: Skipping %s, already in gamelist.", f)
continue
}
roms <- r
}
}
walker := fs.Walk(xmlOpts.RomDir)
for walker.Step() {
if stop {
break
}
if err := walker.Err(); err != nil {
return err
}
f := walker.Path()
if b := filepath.Base(f); b != "." && strings.HasPrefix(b, ".") {
walker.SkipDir()
continue
}
if _, ok := existing[f]; !*refreshOut && ok {
log.Printf("INFO: Skipping %s, already in gamelist.", f)
continue
}
r, err := rom.NewROM(f)
if err != nil {
log.Printf("ERR: Processing: %s, %s", f, err)
continue
}
_, isExtra := extraMap[r.Ext]
if *mame {
if r.Ext == ".zip" || r.Ext == ".7z" || isExtra {
roms <- r
}
continue
}
_, ok := bins[f]
if !ok && (rh.KnownExt(r.Ext) || r.Ext == ".svm" || isExtra) {
roms <- r
}
}
close(roms)
wg.Wait()
wg.Add(1)
close(results)
wg.Wait()
if stop {
return UserCanceled
} else {
return nil
}
}