func writeCsv(ch chan []string) {
var writer *csv.Writer
if outputFn == "" {
writer = csv.NewWriter(os.Stdout)
} else {
file, err := os.Create(outputFn)
if err != nil {
fmt.Println("Error:", err)
os.Exit(1)
}
defer file.Close()
writer = csv.NewWriter(file)
}
r, _ := utf8.DecodeRuneInString(outputSep)
writer.Comma = r
for row := range ch {
err := writer.Write(row)
if err != nil {
fmt.Println("Error:", err)
close(ch)
return
}
}
writer.Flush()
}
// csvWriter specific method
func (cdre *CdrExporter) writeCsv(csvWriter *csv.Writer) error {
csvWriter.Comma = cdre.fieldSeparator
if len(cdre.header) != 0 {
if err := csvWriter.Write(cdre.header); err != nil {
return err
}
}
for _, cdrContent := range cdre.content {
if err := csvWriter.Write(cdrContent); err != nil {
return err
}
}
if len(cdre.trailer) != 0 {
if err := csvWriter.Write(cdre.trailer); err != nil {
return err
}
}
csvWriter.Flush()
return nil
}
func main() {
flag.Parse()
if *showVersion {
fmt.Printf("json2csv v1.1\n")
return
}
var reader *bufio.Reader
var writer *csv.Writer
if *inputFile != "" {
file, err := os.OpenFile(*inputFile, os.O_RDONLY, 0600)
if err != nil {
log.Printf("Error %s opening input file %v", err, *inputFile)
os.Exit(1)
}
reader = bufio.NewReader(file)
} else {
reader = bufio.NewReader(os.Stdin)
}
if *outputFile != "" {
file, err := os.OpenFile(*outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
log.Printf("Error %s opening output file %v", err, *outputFile)
os.Exit(1)
}
writer = csv.NewWriter(file)
} else {
writer = csv.NewWriter(os.Stdout)
}
delim, _ := utf8.DecodeRuneInString(*outputDelim)
writer.Comma = delim
json2csv(reader, writer, keys, *printHeader)
}
func executeDiff(cmd *cobra.Command, args []string) {
var (
err error
match *regexp.Regexp
ignore *regexp.Regexp
csvMap = make(map[string][]string)
fisList = make([]FileInfos, 0)
q = make(chan info)
wg = new(sync.WaitGroup)
)
if len(args) == 0 {
cmd.Help()
return
}
// Get glob file args.
args, err = core.GetGlobArgs(args)
if err != nil {
log.Fatalln(err)
}
// Recheck args.
if len(args) <= 1 {
cmd.Help()
return
}
// Load csv and store.
for _, csvPath := range args {
fmt.Println("Open:", csvPath)
c, err := os.Open(csvPath)
if err != nil {
log.Fatalln(err)
}
defer c.Close()
var reader *csv.Reader
if sjisIn {
reader = csv.NewReader(transform.NewReader(c, japanese.ShiftJIS.NewDecoder()))
} else {
reader = csv.NewReader(c)
}
reader.Comma = '\t'
// Skip header.
_, err = reader.Read()
if err != nil {
log.Fatalln(err)
}
left, err := reader.ReadAll()
if err != nil {
log.Fatalln(err)
}
// Change data to FileInfos struct.
fis := make(FileInfos, 0)
for _, r := range left {
fis = append(fis, *csvToFileInfo(r))
}
fisList = append(fisList, fis)
}
// Compile if given matches and ignores.
if len(matches) != 0 {
match, err = core.CompileStrs(matches)
if err != nil {
log.Fatalln(err)
}
}
if len(ignores) != 0 {
ignore, err = core.CompileStrs(ignores)
if err != nil {
log.Fatalln(err)
}
}
for i, one := range fisList {
wg.Add(1)
go func(i int, one FileInfos) {
defer wg.Done()
// Diff fileinfo.
for _, oneFi := range one {
if fileOnly && oneFi.Type == DIR {
continue
}
if dirOnly && oneFi.Type == FILE {
continue
}
// Ignore check.
if ignore != nil && ignore.MatchString(oneFi.Full) {
continue
}
// Match check.
if match != nil && !match.MatchString(oneFi.Full) {
continue
}
for j, other := range fisList {
if i == j {
continue
}
// Get other's same full path info.
otherFi, err := findFileInfo(other, oneFi)
if err == nil {
// Diff Time.
if oneFi.Time != otherFi.Time {
q <- info{
path: args[i],
index: i,
full: oneFi.Full,
diff: FileTime,
value: oneFi.Time,
ford: oneFi.Type,
}
}
// Diff Size.
if oneFi.Size != otherFi.Size {
q <- info{
path: args[i],
index: i,
full: oneFi.Full,
diff: FileSize,
value: oneFi.Size,
ford: oneFi.Type,
}
}
// Diff Mode.
if oneFi.Mode != otherFi.Mode {
q <- info{
path: args[i],
index: i,
full: oneFi.Full,
diff: FileMode,
value: oneFi.Mode,
ford: oneFi.Type,
}
}
} else {
q <- info{
path: args[i],
index: i,
full: oneFi.Full,
diff: FileFull,
value: oneFi.Full,
ford: oneFi.Type,
}
}
}
}
}(i, one)
}
// Async wait.
go func() {
wg.Wait()
close(q)
}()
// Receive diff and store to array.
for info := range q {
cnt++
if !silent {
fmt.Fprintf(os.Stderr, "Count: %d\r", cnt)
}
key := info.full + fmt.Sprint(info.diff)
if _, ok := csvMap[key]; ok {
csvMap[key][info.index+3] = info.value
} else {
s := make([]string, len(args)+3)
s[0] = info.full
s[1] = info.ford
s[2] = fmt.Sprint(info.diff)
s[info.index+3] = info.value
csvMap[key] = s
}
}
if len(csvMap) == 0 {
fmt.Println("There is no difference !")
return
}
// Output to csv.
os.MkdirAll(filepath.Dir(out), os.ModePerm)
c, err := os.Create(out)
if err != nil {
log.Fatalln(err)
}
defer c.Close()
var writer *csv.Writer
if sjisOut {
writer = csv.NewWriter(transform.NewWriter(c, japanese.ShiftJIS.NewEncoder()))
} else {
writer = csv.NewWriter(c)
}
writer.Comma = '\t'
writer.UseCRLF = true
// Write header.
err = writer.Write(append(strings.Split(DiffHeader, "\t"), args...))
if err != nil {
log.Fatalln(err)
}
// map to array.
var csvArray records
for _, v := range csvMap {
csvArray = append(csvArray, v)
}
// sort
if sorts == "" {
sorts = "0,2"
}
sort.Sort(csvArray)
for _, v := range csvArray {
err = writer.Write(v)
if err != nil {
log.Fatalln(err)
}
}
writer.Flush()
fmt.Printf("Write to [%s]. ([%d] row)\n", out, cnt)
}
func (s *SpreadsheetGenerator) Generate(writer io.Writer) error {
var csvWriter *csv.Writer
if s.Format != FORMAT_FIXED_WIDTH {
// Make the writer based on the format
csvWriter = csv.NewWriter(writer)
switch s.Format {
case FORMAT_CSV:
csvWriter.Comma = ','
case FORMAT_TSV:
csvWriter.Comma = '\t'
case FORMAT_PSV:
csvWriter.Comma = '|'
}
}
s.csvWriter = csvWriter
s.writer = writer
if s.ShowColumnHeaders {
headers := make([]string, len(s.Columns))
for i, c := range s.Columns {
headers[i] = c.Header
}
err := s.writeRow(headers)
if err != nil {
return err
}
}
// Figure out which columns we need to keep track of to get aggregations on the footer. The value spit out by the getter for that column must be parseable as a float for it to be added to the aggregations.
totalRows := 0
columnsToTrack := make(map[int][]float64)
for i, c := range s.Columns {
// If it has $sum, $mean, $median, $mode, $totalUnempty, or $totalEmpty, we need to track it so we can show it in the footer
if strings.Contains(c.Footer, ".mean") || strings.Contains(c.Footer, ".median") || strings.Contains(c.Footer, ".sum") || strings.Contains(c.Footer, ".mode") || strings.Contains(c.Footer, ".totalUnempty") || strings.Contains(c.Footer, ".totalEmpty") {
columnsToTrack[i] = []float64{}
}
}
for s.DataSource.HasNext() {
next, err := s.DataSource.Next()
if err != nil {
return err
}
totalRows++
row := make([]string, len(s.Columns))
for i, c := range s.Columns {
val := next.Get(c.Value, c.Default)
row[i] = val
// Do we need to keep track of it?
if colval, ok := columnsToTrack[i]; ok {
if len(val) > 0 {
// If it's numerical, add the number. Otherwise add 1.
f, err := strconv.ParseFloat(val, 64)
if err != nil {
columnsToTrack[i] = append(colval, 1.0)
} else {
columnsToTrack[i] = append(colval, f)
}
} else {
columnsToTrack[i] = append(colval, 0.0)
}
}
}
err = s.writeRow(row)
}
if s.ShowColumnFooters {
footer := make([]string, len(s.Columns))
for i, c := range s.Columns {
if aggregate, ok := columnsToTrack[i]; ok {
// Make the different aggregate values
list := floatlist.Floatlist(aggregate)
agg := &footerAggregation{
Sum: list.Sum(),
Mean: list.Mean(),
Median: list.Median(),
Mode: list.Mode(),
TotalUnempty: list.GetCountByValue(1.0),
TotalEmpty: list.GetCountByValue(0.0),
}
datum := &data.Datum{}
datum.SetSource(agg, "mapTo")
footer[i] = datum.Get(c.Footer, "")
} else {
footer[i] = ""
}
}
s.writeRow(footer)
}
return nil
}
