func writeTo(writer *csv.Writer, in interface{}) error { inValue, inType := getConcreteReflectValueAndType(in) // Get the concrete type (not pointer) (Slice<?> or Array<?>) if err := ensureInType(inType); err != nil { return err } inInnerWasPointer, inInnerType := getConcreteContainerInnerType(inType) // Get the concrete inner type (not pointer) (Container<"?">) if err := ensureInInnerType(inInnerType); err != nil { return err } inInnerStructInfo := getStructInfo(inInnerType) // Get the inner struct info to get CSV annotations csvHeadersLabels := make([]string, len(inInnerStructInfo.Fields)) for i, fieldInfo := range inInnerStructInfo.Fields { // Used to write the header (first line) in CSV csvHeadersLabels[i] = fieldInfo.getFirstKey() } if err := writer.Write(csvHeadersLabels); err != nil { return err } inLen := inValue.Len() for i := 0; i < inLen; i++ { // Iterate over container rows for j, fieldInfo := range inInnerStructInfo.Fields { csvHeadersLabels[j] = "" inInnerFieldValue, err := getInnerField(inValue.Index(i), inInnerWasPointer, fieldInfo.IndexChain) // Get the correct field header <-> position if err != nil { return err } csvHeadersLabels[j] = inInnerFieldValue } if err := writer.Write(csvHeadersLabels); err != nil { return err } } writer.Flush() return writer.Error() }
func (p *csvProcessor) flush(writer *csv.Writer, err error) error { if err != nil { return err } writer.Flush() return writer.Error() }
// NewWithCsvWriter writes all of the Rows in a Table to a CSV file using the options in the CSV writer. // It assumes that all Rows have the same headers. Columns are written in alphabetical order. func NewWithCsvWriter(writer *csv.Writer) optimus.Sink { return func(source optimus.Table) error { defer source.Stop() headers := []string{} wroteHeader := false for row := range source.Rows() { if !wroteHeader { headers = convertRowToHeader(row) sort.Strings(headers) if err := writer.Write(headers); err != nil { return err } wroteHeader = true } if err := writer.Write(convertRowToRecord(row, headers)); err != nil { return err } } if source.Err() != nil { return source.Err() } writer.Flush() if writer.Error() != nil { return writer.Error() } return nil } }
func writeFromChan(writer *csv.Writer, c <-chan interface{}) error { // Get the first value. It wil determine the header structure. firstValue := <-c inValue, inType := getConcreteReflectValueAndType(firstValue) // Get the concrete type if err := ensureStructOrPtr(inType); err != nil { return err } inInnerWasPointer := inType.Kind() == reflect.Ptr inInnerStructInfo := getStructInfo(inType) // Get the inner struct info to get CSV annotations csvHeadersLabels := make([]string, len(inInnerStructInfo.Fields)) for i, fieldInfo := range inInnerStructInfo.Fields { // Used to write the header (first line) in CSV csvHeadersLabels[i] = fieldInfo.getFirstKey() } if err := writer.Write(csvHeadersLabels); err != nil { return err } write := func(val reflect.Value) error { for j, fieldInfo := range inInnerStructInfo.Fields { csvHeadersLabels[j] = "" inInnerFieldValue, err := getInnerField(val, inInnerWasPointer, fieldInfo.IndexChain) // Get the correct field header <-> position if err != nil { return err } csvHeadersLabels[j] = inInnerFieldValue } if err := writer.Write(csvHeadersLabels); err != nil { return err } return nil } if err := write(inValue); err != nil { return err } for v := range c { val, _ := getConcreteReflectValueAndType(v) // Get the concrete type (not pointer) (Slice<?> or Array<?>) if err := ensureStructOrPtr(inType); err != nil { return err } if err := write(val); err != nil { return err } } writer.Flush() return writer.Error() }
func (w *ReportWriter) writeCsvOne(writer *csv.Writer, report Report) error { if w.dialect.HasMetadata { preamble := make([]string, 4) if len(report.Path) > 0 { preamble[0] = "# File" preamble[1] = report.Path preamble[2] = report.Filename preamble[3] = report.MD5hex writer.Write(preamble) } preamble[0] = "# Field" preamble[1] = fmt.Sprint(len(report.Fields)) if report.HasHeader { preamble[2] = "(has header)" } else { preamble[2] = "" } preamble[3] = "" writer.Write(preamble) preamble[0] = "# Record" preamble[1] = fmt.Sprint(report.Records) preamble[2] = "" writer.Write(preamble) } // Put header line. if w.dialect.HasHeader { writer.Write(ReportOutputFields) } // Put each field report. for i, f := range report.Fields { r := f.format(report.Records) r[0] = fmt.Sprint(i + 1) writer.Write(r) } return writer.Error() }
// CrawlROMs crawls the rom directory and processes the files. func CrawlROMs(gl *rom.GameListXML, sources []ds.DS, xmlOpts *rom.XMLOpts, gameOpts *rom.GameOpts) error { var missingCSV *csv.Writer var gdbDS *ds.GDB if *missing != "" { f, err := os.Create(*missing) if err != nil { return err } missingCSV = csv.NewWriter(f) defer func() { missingCSV.Flush() if err := missingCSV.Error(); err != nil { log.Fatal(err) } f.Close() }() if err := missingCSV.Write([]string{"Game", "Error", "Hash", "Extra"}); err != nil { return err } for _, d := range sources { switch d := d.(type) { case *ds.GDB: gdbDS = d } } } var ct http.RoundTripper = NewCancelTransport(http.DefaultTransport.(*http.Transport)) http.DefaultClient.Transport = ct existing := make(map[string]struct{}) if !dirExists(xmlOpts.RomDir) { log.Printf("ERR %s: does not exists", xmlOpts.RomDir) return nil } extraMap := make(map[string]struct{}) if *extraExt != "" { extraSlice := strings.Split(*extraExt, ",") for _, e := range extraSlice { if e[0] != '.' { extraMap["."+e] = struct{}{} } else { extraMap[e] = struct{}{} } } } for _, x := range gl.GameList { switch { case *appendOut: p, err := filepath.Rel(xmlOpts.RomXMLDir, x.Path) if err != nil { log.Printf("Can't find original path: %s", x.Path) } f := filepath.Join(xmlOpts.RomDir, p) existing[f] = struct{}{} case *refreshOut: existing[x.Path] = struct{}{} } } var wg sync.WaitGroup results := make(chan Result, *workers) roms := make(chan *rom.ROM, 2**workers) for i := 0; i < *workers; i++ { wg.Add(1) go worker(sources, xmlOpts, gameOpts, results, roms, &wg) } go func() { defer wg.Done() for r := range results { if r.XML == nil { if *missing == "" { continue } files := []string{r.ROM.Path} if r.ROM.Cue { files = append(files, r.ROM.Bins...) } for _, file := range files { var hash, extra string if gdbDS != nil { var err error hash, err = gdbDS.Hash(file) if err != nil { log.Printf("ERR: Can't hash file %s", file) } name := gdbDS.GetName(file) if name != "" && r.Err == ds.NotFoundErr { extra = "hash found but no GDB ID" } } if err := missingCSV.Write([]string{file, r.Err.Error(), hash, extra}); err != nil { log.Printf("ERR: Can't write to %s", *missing) } } continue } if r.XML.Image == "" && *missing != "" { var hash string if gdbDS != nil { var err error hash, err = gdbDS.Hash(r.ROM.Path) if err != nil { log.Printf("ERR: Can't hash file %s", r.ROM.Path) } } if err := missingCSV.Write([]string{r.ROM.FileName, "", hash, "missing image"}); err != nil { log.Printf("ERR: Can't write to %s", *missing) } } if _, ok := existing[r.XML.Path]; ok && *refreshOut { for i, g := range gl.GameList { if g.Path != r.XML.Path { continue } copy(gl.GameList[i:], gl.GameList[i+1:]) gl.GameList = gl.GameList[:len(gl.GameList)-1] } } gl.Append(r.XML) } }() var stop bool sig := make(chan os.Signal, 1) signal.Notify(sig, os.Interrupt) defer signal.Stop(sig) go func() { for { <-sig if !stop { stop = true log.Println("Stopping, ctrl-c again to stop now.") ct.(*CancelTransport).Stop() for _ = range roms { } continue } panic("AHHHH!") } }() bins := make(map[string]struct{}) if !*mame { walker := fs.Walk(xmlOpts.RomDir) for walker.Step() { if stop { break } if err := walker.Err(); err != nil { return err } f := walker.Path() if b := filepath.Base(f); b != "." && strings.HasPrefix(b, ".") { walker.SkipDir() continue } r, err := rom.NewROM(f) if err != nil { log.Printf("ERR: Processing: %s, %s", f, err) continue } if !r.Cue { continue } for _, b := range r.Bins { bins[b] = struct{}{} } bins[f] = struct{}{} if _, ok := existing[f]; !*refreshOut && ok { log.Printf("INFO: Skipping %s, already in gamelist.", f) continue } roms <- r } } walker := fs.Walk(xmlOpts.RomDir) for walker.Step() { if stop { break } if err := walker.Err(); err != nil { return err } f := walker.Path() if b := filepath.Base(f); b != "." && strings.HasPrefix(b, ".") { walker.SkipDir() continue } if _, ok := existing[f]; !*refreshOut && ok { log.Printf("INFO: Skipping %s, already in gamelist.", f) continue } r, err := rom.NewROM(f) if err != nil { log.Printf("ERR: Processing: %s, %s", f, err) continue } _, isExtra := extraMap[r.Ext] if *mame { if r.Ext == ".zip" || r.Ext == ".7z" || isExtra { roms <- r } continue } _, ok := bins[f] if !ok && (rh.KnownExt(r.Ext) || r.Ext == ".svm" || isExtra) { roms <- r } } close(roms) wg.Wait() wg.Add(1) close(results) wg.Wait() if stop { return UserCanceled } else { return nil } }