Пример #1
0
func writeCsv(ch chan []string) {
	var writer *csv.Writer
	if outputFn == "" {
		writer = csv.NewWriter(os.Stdout)
	} else {
		file, err := os.Create(outputFn)
		if err != nil {
			fmt.Println("Error:", err)
			os.Exit(1)
		}
		defer file.Close()
		writer = csv.NewWriter(file)
	}

	r, _ := utf8.DecodeRuneInString(outputSep)
	writer.Comma = r

	for row := range ch {
		err := writer.Write(row)
		if err != nil {
			fmt.Println("Error:", err)
			close(ch)
			return
		}
	}
	writer.Flush()
}
Пример #2
0
// csvWriter specific method
func (cdre *CdrExporter) writeCsv(csvWriter *csv.Writer) error {
	csvWriter.Comma = cdre.fieldSeparator
	if len(cdre.header) != 0 {
		if err := csvWriter.Write(cdre.header); err != nil {
			return err
		}
	}
	for _, cdrContent := range cdre.content {
		if err := csvWriter.Write(cdrContent); err != nil {
			return err
		}
	}
	if len(cdre.trailer) != 0 {
		if err := csvWriter.Write(cdre.trailer); err != nil {
			return err
		}
	}
	csvWriter.Flush()
	return nil
}
Пример #3
0
func main() {
	flag.Parse()

	if *showVersion {
		fmt.Printf("json2csv v1.1\n")
		return
	}

	var reader *bufio.Reader
	var writer *csv.Writer
	if *inputFile != "" {
		file, err := os.OpenFile(*inputFile, os.O_RDONLY, 0600)
		if err != nil {
			log.Printf("Error %s opening input file %v", err, *inputFile)
			os.Exit(1)
		}
		reader = bufio.NewReader(file)
	} else {
		reader = bufio.NewReader(os.Stdin)
	}

	if *outputFile != "" {
		file, err := os.OpenFile(*outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
		if err != nil {
			log.Printf("Error %s opening output file %v", err, *outputFile)
			os.Exit(1)
		}
		writer = csv.NewWriter(file)
	} else {
		writer = csv.NewWriter(os.Stdout)
	}

	delim, _ := utf8.DecodeRuneInString(*outputDelim)
	writer.Comma = delim

	json2csv(reader, writer, keys, *printHeader)
}
Пример #4
0
func executeDiff(cmd *cobra.Command, args []string) {

	var (
		err error

		match  *regexp.Regexp
		ignore *regexp.Regexp

		csvMap  = make(map[string][]string)
		fisList = make([]FileInfos, 0)
		q       = make(chan info)
		wg      = new(sync.WaitGroup)
	)

	if len(args) == 0 {
		cmd.Help()
		return
	}

	// Get glob file args.
	args, err = core.GetGlobArgs(args)
	if err != nil {
		log.Fatalln(err)
	}

	// Recheck args.
	if len(args) <= 1 {
		cmd.Help()
		return
	}

	// Load csv and store.
	for _, csvPath := range args {
		fmt.Println("Open:", csvPath)
		c, err := os.Open(csvPath)
		if err != nil {
			log.Fatalln(err)
		}
		defer c.Close()
		var reader *csv.Reader
		if sjisIn {
			reader = csv.NewReader(transform.NewReader(c, japanese.ShiftJIS.NewDecoder()))
		} else {
			reader = csv.NewReader(c)
		}
		reader.Comma = '\t'
		// Skip header.
		_, err = reader.Read()
		if err != nil {
			log.Fatalln(err)
		}
		left, err := reader.ReadAll()
		if err != nil {
			log.Fatalln(err)
		}

		// Change data to FileInfos struct.
		fis := make(FileInfos, 0)
		for _, r := range left {
			fis = append(fis, *csvToFileInfo(r))
		}
		fisList = append(fisList, fis)
	}

	// Compile if given matches and ignores.
	if len(matches) != 0 {
		match, err = core.CompileStrs(matches)
		if err != nil {
			log.Fatalln(err)
		}
	}
	if len(ignores) != 0 {
		ignore, err = core.CompileStrs(ignores)
		if err != nil {
			log.Fatalln(err)
		}
	}

	for i, one := range fisList {
		wg.Add(1)
		go func(i int, one FileInfos) {
			defer wg.Done()

			// Diff fileinfo.
			for _, oneFi := range one {
				if fileOnly && oneFi.Type == DIR {
					continue
				}
				if dirOnly && oneFi.Type == FILE {
					continue
				}

				// Ignore check.
				if ignore != nil && ignore.MatchString(oneFi.Full) {
					continue
				}

				// Match check.
				if match != nil && !match.MatchString(oneFi.Full) {
					continue
				}

				for j, other := range fisList {
					if i == j {
						continue
					}

					// Get other's same full path info.
					otherFi, err := findFileInfo(other, oneFi)
					if err == nil {
						// Diff Time.
						if oneFi.Time != otherFi.Time {
							q <- info{
								path:  args[i],
								index: i,
								full:  oneFi.Full,
								diff:  FileTime,
								value: oneFi.Time,
								ford:  oneFi.Type,
							}
						}
						// Diff Size.
						if oneFi.Size != otherFi.Size {
							q <- info{
								path:  args[i],
								index: i,
								full:  oneFi.Full,
								diff:  FileSize,
								value: oneFi.Size,
								ford:  oneFi.Type,
							}
						}
						// Diff Mode.
						if oneFi.Mode != otherFi.Mode {
							q <- info{
								path:  args[i],
								index: i,
								full:  oneFi.Full,
								diff:  FileMode,
								value: oneFi.Mode,
								ford:  oneFi.Type,
							}
						}
					} else {
						q <- info{
							path:  args[i],
							index: i,
							full:  oneFi.Full,
							diff:  FileFull,
							value: oneFi.Full,
							ford:  oneFi.Type,
						}
					}
				}
			}
		}(i, one)
	}

	// Async wait.
	go func() {
		wg.Wait()
		close(q)
	}()

	// Receive diff and store to array.
	for info := range q {
		cnt++
		if !silent {
			fmt.Fprintf(os.Stderr, "Count: %d\r", cnt)
		}
		key := info.full + fmt.Sprint(info.diff)
		if _, ok := csvMap[key]; ok {
			csvMap[key][info.index+3] = info.value
		} else {
			s := make([]string, len(args)+3)
			s[0] = info.full
			s[1] = info.ford
			s[2] = fmt.Sprint(info.diff)
			s[info.index+3] = info.value
			csvMap[key] = s
		}
	}

	if len(csvMap) == 0 {
		fmt.Println("There is no difference !")
		return
	}

	// Output to csv.
	os.MkdirAll(filepath.Dir(out), os.ModePerm)
	c, err := os.Create(out)
	if err != nil {
		log.Fatalln(err)
	}
	defer c.Close()
	var writer *csv.Writer
	if sjisOut {
		writer = csv.NewWriter(transform.NewWriter(c, japanese.ShiftJIS.NewEncoder()))
	} else {
		writer = csv.NewWriter(c)
	}
	writer.Comma = '\t'
	writer.UseCRLF = true

	// Write header.
	err = writer.Write(append(strings.Split(DiffHeader, "\t"), args...))
	if err != nil {
		log.Fatalln(err)
	}

	// map to array.
	var csvArray records
	for _, v := range csvMap {
		csvArray = append(csvArray, v)
	}

	// sort
	if sorts == "" {
		sorts = "0,2"
	}
	sort.Sort(csvArray)

	for _, v := range csvArray {
		err = writer.Write(v)
		if err != nil {
			log.Fatalln(err)
		}
	}
	writer.Flush()
	fmt.Printf("Write to [%s]. ([%d] row)\n", out, cnt)
}
Пример #5
0
func (s *SpreadsheetGenerator) Generate(writer io.Writer) error {
	var csvWriter *csv.Writer
	if s.Format != FORMAT_FIXED_WIDTH {
		// Make the writer based on the format
		csvWriter = csv.NewWriter(writer)
		switch s.Format {
		case FORMAT_CSV:
			csvWriter.Comma = ','
		case FORMAT_TSV:
			csvWriter.Comma = '\t'
		case FORMAT_PSV:
			csvWriter.Comma = '|'
		}
	}

	s.csvWriter = csvWriter
	s.writer = writer

	if s.ShowColumnHeaders {
		headers := make([]string, len(s.Columns))
		for i, c := range s.Columns {
			headers[i] = c.Header
		}

		err := s.writeRow(headers)
		if err != nil {
			return err
		}
	}

	// Figure out which columns we need to keep track of to get aggregations on the footer. The value spit out by the getter for that column must be parseable as a float for it to be added to the aggregations.
	totalRows := 0

	columnsToTrack := make(map[int][]float64)

	for i, c := range s.Columns {
		// If it has $sum, $mean, $median, $mode, $totalUnempty, or $totalEmpty, we need to track it so we can show it in the footer
		if strings.Contains(c.Footer, ".mean") || strings.Contains(c.Footer, ".median") || strings.Contains(c.Footer, ".sum") || strings.Contains(c.Footer, ".mode") || strings.Contains(c.Footer, ".totalUnempty") || strings.Contains(c.Footer, ".totalEmpty") {
			columnsToTrack[i] = []float64{}
		}
	}

	for s.DataSource.HasNext() {
		next, err := s.DataSource.Next()

		if err != nil {
			return err
		}
		totalRows++

		row := make([]string, len(s.Columns))

		for i, c := range s.Columns {
			val := next.Get(c.Value, c.Default)
			row[i] = val

			// Do we need to keep track of it?
			if colval, ok := columnsToTrack[i]; ok {
				if len(val) > 0 {
					// If it's numerical, add the number. Otherwise add 1.
					f, err := strconv.ParseFloat(val, 64)
					if err != nil {
						columnsToTrack[i] = append(colval, 1.0)
					} else {
						columnsToTrack[i] = append(colval, f)
					}
				} else {
					columnsToTrack[i] = append(colval, 0.0)
				}

			}
		}

		err = s.writeRow(row)
	}

	if s.ShowColumnFooters {
		footer := make([]string, len(s.Columns))
		for i, c := range s.Columns {
			if aggregate, ok := columnsToTrack[i]; ok {
				// Make the different aggregate values
				list := floatlist.Floatlist(aggregate)
				agg := &footerAggregation{
					Sum:          list.Sum(),
					Mean:         list.Mean(),
					Median:       list.Median(),
					Mode:         list.Mode(),
					TotalUnempty: list.GetCountByValue(1.0),
					TotalEmpty:   list.GetCountByValue(0.0),
				}

				datum := &data.Datum{}
				datum.SetSource(agg, "mapTo")
				footer[i] = datum.Get(c.Footer, "")

			} else {
				footer[i] = ""
			}
		}

		s.writeRow(footer)
	}
	return nil
}