Exemple #1
0
func main() {
	if len(os.Args) <= 1 {
		showUse()
		os.Exit(1)
	}

	inputFiles := os.Args[1:]
	n := len(inputFiles)
	if n == 0 {
		fmt.Fprintf(os.Stderr, "no input files given\n")
		showUse()
		os.Exit(1)
	}

	const MB = 1024 * 1024
	outputStream := tf.NewFrameWriter(os.Stdout, MB)

	strms := make([]*tf.BufferedFrameReader, n)
	for i := 0; i < n; i++ {
		if !FileExists(inputFiles[i]) {
			fmt.Fprintf(os.Stderr, "path '%s' not found\n", inputFiles[i])
			os.Exit(1)
		}
		f, err := os.Open(inputFiles[i])
		if err != nil {
			fmt.Fprintf(os.Stderr, "could not open path '%s': '%s'\n",
				inputFiles[i], err)
			os.Exit(1)
		}
		strms[i] = tf.NewBufferedFrameReader(f, MB, "")
	}

	// okay, now create and merge streams
	err := outputStream.Merge(strms...)
	outputStream.Sync()
	panicOn(err)
}
Exemple #2
0
func main() {
	myflags := flag.NewFlagSet("tfsort", flag.ExitOnError)
	cfg := &tf.TfsortConfig{}
	cfg.DefineFlags(myflags)

	err := myflags.Parse(os.Args[1:])
	err = cfg.ValidateConfig()
	if err != nil {
		usage(err, myflags)
	}

	leftover := myflags.Args()
	//Q("leftover = %v", leftover)
	if len(leftover) == 0 {
		fmt.Fprintf(os.Stderr, "no input files given\n")
		showUse(myflags)
		os.Exit(1)
	}

	wrote := []*os.File{}
	wroteTmp := []string{}
	for _, inputFile := range leftover {
		//P("starting on inputFile '%s'", inputFile)
		if !FileExists(inputFile) {
			fmt.Fprintf(os.Stderr, "input file '%s' does not exist.\n", inputFile)
			os.Exit(1)
		}

		frames, err := tf.ReadAllFrames(inputFile)
		panicOn(err)

		sort.Stable(tf.TimeSorter(frames))

		writeFile := inputFile + ".sorted"
		of, err := os.Create(writeFile)
		panicOn(err)
		wrote = append(wrote, of)
		wroteTmp = append(wroteTmp, writeFile)

		fw := tf.NewFrameWriter(of, 1024*1024)
		fw.Frames = frames
		_, err = fw.WriteTo(of)
		panicOn(err)
		fw.Sync()
		of.Seek(0, 0)
	}

	// INVAR: individual files are sorted, now merge to stdout

	const MB = 1024 * 1024
	outputStream := tf.NewFrameWriter(os.Stdout, MB)

	strms := make([]*tf.BufferedFrameReader, len(wrote))
	for i := range wrote {
		strms[i] = tf.NewBufferedFrameReader(wrote[i], MB, "")
	}

	// okay, now create and merge streams
	err = outputStream.Merge(strms...)
	outputStream.Sync()
	panicOn(err)

	if !cfg.KeepTmpFiles {
		for _, w := range wroteTmp {
			os.Remove(w)
		}
	}

}
Exemple #3
0
func main() {
	myflags := flag.NewFlagSet("tfindex", flag.ExitOnError)
	cfg := &tf.TfindexConfig{}
	cfg.DefineFlags(myflags)

	err := myflags.Parse(os.Args[1:])
	err = cfg.ValidateConfig()
	if err != nil {
		usage(err, myflags)
	}

	leftover := myflags.Args()
	//Q("leftover = %v", leftover)
	if len(leftover) == 0 {
		fmt.Fprintf(os.Stderr, "no input files given\n")
		showUse(myflags)
		os.Exit(1)
	}

	i := int64(1)
nextfile:
	for _, inputFile := range leftover {
		//P("starting on inputFile '%s'", inputFile)
		if !FileExists(inputFile) {
			fmt.Fprintf(os.Stderr, "input file '%s' does not exist.\n", inputFile)
			os.Exit(1)
		}

		f, err := os.Open(inputFile)
		panicOn(err)
		fr := tf.NewFrameReader(f, 1024*1024)

		writeFile := inputFile + ".idx"
		of, err := os.Create(writeFile)
		panicOn(err)

		fw := tf.NewFrameWriter(of, 1024*1024)

		var offset int64
		var frame tf.Frame
		var nextTm time.Time
		var nbytes int64

		for ; err == nil; i++ {
			_, nbytes, err, _ = fr.NextFrame(&frame)
			if err != nil {
				if err == io.EOF {
					fw.Flush()
					fw.Sync()
					of.Close()
					continue nextfile
				}
				fmt.Fprintf(os.Stderr, "tfindex error from fr.NextFrame() at i=%v: '%v'\n", i, err)
				os.Exit(1)
			}
			unix := frame.Tm()
			tm := time.Unix(0, unix)
			trunc := tm.Truncate(time.Minute)

			if i == 0 {
				first, err := tf.NewFrame(tm, tf.EvOneInt64, 0, offset, nil)
				panicOn(err)
				fw.Append(first)
				nextTm = trunc.Add(time.Minute)
			} else if tm.After(nextTm) {
				next, err := tf.NewFrame(tm, tf.EvOneInt64, 0, offset, nil)
				panicOn(err)
				fw.Append(next)
				nextTm = trunc.Add(time.Minute)
			}
			offset += nbytes
		}
	}
}