Exemple #1
0
func main() {
	myflags := flag.NewFlagSet("tfgroup", flag.ExitOnError)
	cfg := &tf.TfgroupConfig{}
	cfg.DefineFlags(myflags)

	err := myflags.Parse(os.Args[1:])
	err = cfg.ValidateConfig()
	if err != nil || cfg.Help {
		usage(err, myflags)
	}

	leftover := myflags.Args()
	//p("leftover = %v", leftover)
	if len(leftover) != 0 {
		fmt.Fprintf(os.Stderr, "tfgroup reads stdin and writes stdout, no args allowed.\n")
		showUse(myflags)
		os.Exit(1)
	}

	i := int64(1)

	f := os.Stdin
	panicOn(err)
	//buf := make([]byte, 1024*1024)
	fr := tf.NewFrameReader(f, 1024*1024)

	var frame tf.Frame

	var nextMin time.Time
	var countLastMin int64
	incr := time.Minute
	for ; err == nil; i++ {
		_, _, err = fr.NextFrame(&frame)
		if err != nil {
			if err == io.EOF {
				break
			}
			fmt.Fprintf(os.Stderr, "tfgroup error from fr.NextFrame() at i=%v: '%v'\n", i, err)
			os.Exit(1)
		}
		cur := time.Unix(0, frame.Tm())
		if nextMin.IsZero() {
			nextMin = cur.Add(incr)
		}
		for cur.After(nextMin) {
			fmt.Printf("%v %v\n", countLastMin, nextMin)
			nextMin = nextMin.Add(incr)
			countLastMin = 0
		}
		countLastMin++
	}
	fmt.Printf("%v %v\n", countLastMin, nextMin)
}
Exemple #2
0
func main() {
	myflags := flag.NewFlagSet("tfsum", flag.ExitOnError)
	cfg := &tf.TfsumConfig{}
	cfg.DefineFlags(myflags)

	err := myflags.Parse(os.Args[1:])
	err = cfg.ValidateConfig()
	if err != nil || cfg.Help {
		usage(err, myflags)
	}

	leftover := myflags.Args()
	//p("leftover = %v", leftover)
	if len(leftover) != 0 {
		fmt.Fprintf(os.Stderr, "tfsum reads stdin and writes stdout, no args allowed.\n")
		showUse(myflags)
		os.Exit(1)
	}

	i := int64(1)

	f := os.Stdin
	panicOn(err)
	buf := make([]byte, 1024*1024)
	fr := tf.NewFrameReader(f, 1024*1024)

	var frame tf.Frame

	for ; err == nil; i++ {
		_, _, err, _ = fr.NextFrame(&frame)
		if err != nil {
			if err == io.EOF {
				return
			}
			fmt.Fprintf(os.Stderr, "tfcat error from fr.NextFrame() at i=%v: '%v'\n", i, err)
			os.Exit(1)
		}
		hash := frame.Blake2b()
		chk := int64(binary.LittleEndian.Uint64(hash[:8]))
		newf, err := tf.NewMarshalledFrame(buf, time.Unix(0, frame.Tm()), tf.EvOneInt64, 0, chk, nil)
		panicOn(err)
		_, err = os.Stdout.Write(newf)
		panicOn(err)
	}
}
Exemple #3
0
func main() {
	myflags := flag.NewFlagSet("tfindex", flag.ExitOnError)
	cfg := &tf.TfindexConfig{}
	cfg.DefineFlags(myflags)

	err := myflags.Parse(os.Args[1:])
	err = cfg.ValidateConfig()
	if err != nil {
		usage(err, myflags)
	}

	leftover := myflags.Args()
	//Q("leftover = %v", leftover)
	if len(leftover) == 0 {
		fmt.Fprintf(os.Stderr, "no input files given\n")
		showUse(myflags)
		os.Exit(1)
	}

	i := int64(1)
nextfile:
	for _, inputFile := range leftover {
		//P("starting on inputFile '%s'", inputFile)
		if !FileExists(inputFile) {
			fmt.Fprintf(os.Stderr, "input file '%s' does not exist.\n", inputFile)
			os.Exit(1)
		}

		f, err := os.Open(inputFile)
		panicOn(err)
		fr := tf.NewFrameReader(f, 1024*1024)

		writeFile := inputFile + ".idx"
		of, err := os.Create(writeFile)
		panicOn(err)

		fw := tf.NewFrameWriter(of, 1024*1024)

		var offset int64
		var frame tf.Frame
		var nextTm time.Time
		var nbytes int64

		for ; err == nil; i++ {
			_, nbytes, err, _ = fr.NextFrame(&frame)
			if err != nil {
				if err == io.EOF {
					fw.Flush()
					fw.Sync()
					of.Close()
					continue nextfile
				}
				fmt.Fprintf(os.Stderr, "tfindex error from fr.NextFrame() at i=%v: '%v'\n", i, err)
				os.Exit(1)
			}
			unix := frame.Tm()
			tm := time.Unix(0, unix)
			trunc := tm.Truncate(time.Minute)

			if i == 0 {
				first, err := tf.NewFrame(tm, tf.EvOneInt64, 0, offset, nil)
				panicOn(err)
				fw.Append(first)
				nextTm = trunc.Add(time.Minute)
			} else if tm.After(nextTm) {
				next, err := tf.NewFrame(tm, tf.EvOneInt64, 0, offset, nil)
				panicOn(err)
				fw.Append(next)
				nextTm = trunc.Add(time.Minute)
			}
			offset += nbytes
		}
	}
}