func main() {
myflags := flag.NewFlagSet("tfgroup", flag.ExitOnError)
cfg := &tf.TfgroupConfig{}
cfg.DefineFlags(myflags)
err := myflags.Parse(os.Args[1:])
err = cfg.ValidateConfig()
if err != nil || cfg.Help {
usage(err, myflags)
}
leftover := myflags.Args()
//p("leftover = %v", leftover)
if len(leftover) != 0 {
fmt.Fprintf(os.Stderr, "tfgroup reads stdin and writes stdout, no args allowed.\n")
showUse(myflags)
os.Exit(1)
}
i := int64(1)
f := os.Stdin
panicOn(err)
//buf := make([]byte, 1024*1024)
fr := tf.NewFrameReader(f, 1024*1024)
var frame tf.Frame
var nextMin time.Time
var countLastMin int64
incr := time.Minute
for ; err == nil; i++ {
_, _, err = fr.NextFrame(&frame)
if err != nil {
if err == io.EOF {
break
}
fmt.Fprintf(os.Stderr, "tfgroup error from fr.NextFrame() at i=%v: '%v'\n", i, err)
os.Exit(1)
}
cur := time.Unix(0, frame.Tm())
if nextMin.IsZero() {
nextMin = cur.Add(incr)
}
for cur.After(nextMin) {
fmt.Printf("%v %v\n", countLastMin, nextMin)
nextMin = nextMin.Add(incr)
countLastMin = 0
}
countLastMin++
}
fmt.Printf("%v %v\n", countLastMin, nextMin)
}
// copy the raw TMFRAME bytes of messageCount messages read from
// inputPath to w
func SendRawBytes(inputPath string, writeFrameCount int, w io.Writer, skipFrameCount int) {
f := prepInput(inputPath)
defer f.Close()
var err error
fr := tf.NewFrameReader(f, 1024*1024)
var frame tf.Frame
skipByteCount := int64(0)
writeByteCount := int64(0)
for i := 0; i < skipFrameCount; i++ {
_, nbytes, err, _ := fr.NextFrame(&frame)
if err != nil {
panic(err)
//os.Exit(0)
}
skipByteCount += nbytes
}
for i := 0; i < writeFrameCount; i++ {
_, nbytes, err, _ := fr.NextFrame(&frame)
if err != nil {
panic(err)
//os.Exit(0)
}
writeByteCount += nbytes
}
//fmt.Fprintf(os.Stderr, "skipByteCount: %v, writeByteCount: %v, skipFramCount: %v, writeFrameCount: %v\n", skipByteCount, writeByteCount, skipFrameCount, writeFrameCount)
// seek back to beginning to copy just those bytes
_, err = f.Seek(skipByteCount, 0)
panicOn(err)
if writeFrameCount == 0 {
// write them all
_, err = io.Copy(w, f)
panicOn(err)
} else {
var wrote int64
wrote, err = io.CopyN(w, f, writeByteCount)
if wrote != writeByteCount {
panic(fmt.Sprintf("short write: %v vs %v expected", wrote, writeByteCount))
}
panicOn(err)
}
}
func main() {
myflags := flag.NewFlagSet("tfsum", flag.ExitOnError)
cfg := &tf.TfsumConfig{}
cfg.DefineFlags(myflags)
err := myflags.Parse(os.Args[1:])
err = cfg.ValidateConfig()
if err != nil || cfg.Help {
usage(err, myflags)
}
leftover := myflags.Args()
//p("leftover = %v", leftover)
if len(leftover) != 0 {
fmt.Fprintf(os.Stderr, "tfsum reads stdin and writes stdout, no args allowed.\n")
showUse(myflags)
os.Exit(1)
}
i := int64(1)
f := os.Stdin
panicOn(err)
buf := make([]byte, 1024*1024)
fr := tf.NewFrameReader(f, 1024*1024)
var frame tf.Frame
for ; err == nil; i++ {
_, _, err, _ = fr.NextFrame(&frame)
if err != nil {
if err == io.EOF {
return
}
fmt.Fprintf(os.Stderr, "tfcat error from fr.NextFrame() at i=%v: '%v'\n", i, err)
os.Exit(1)
}
hash := frame.Blake2b()
chk := int64(binary.LittleEndian.Uint64(hash[:8]))
newf, err := tf.NewMarshalledFrame(buf, time.Unix(0, frame.Tm()), tf.EvOneInt64, 0, chk, nil)
panicOn(err)
_, err = os.Stdout.Write(newf)
panicOn(err)
}
}
//export ReadTmFrame
//
// ReadTmFrame reads a TMFRAME file and returns the frames as
// an R dataframe.
//
func ReadTmFrame(path_ C.SEXP) C.SEXP {
// s must be a RAWSXP
if C.TYPEOF(path_) != C.STRSXP {
C.ReportErrorToR_NoReturn(C.CString("ReadTmFrame() error: path is not a string path to TMFRAME file."))
}
cpath := C.R_CHAR(C.STRING_ELT(path_, 0))
path := C.GoString(cpath)
if !FileExists(path) {
C.ReportErrorToR_NoReturn(C.CString(fmt.Sprintf("ReadTmFrame() error: bad path '%s'; does not exist", path)))
}
/// begin TMFRAME read
f, err := os.Open(path)
if err != nil {
C.ReportErrorToR_NoReturn(C.CString(fmt.Sprintf("ReadTmFrame() error, could not open path '%s': '%s'", path, err)))
}
i := int64(1)
fr := tf.NewFrameReader(f, 1024*1024)
var frame *tf.Frame
//var raw []byte
res := []*tf.Frame{}
toploop:
for ; err == nil; i++ {
frame, _, err, _ = fr.NextFrame(nil)
if err != nil {
if err == io.EOF {
break toploop
}
C.ReportErrorToR_NoReturn(C.CString(fmt.Sprintf("ReadTmFrame() error reading '%s', fr.NextFrame() at i=%v gave error: '%v'",
path, i, err)))
}
res = append(res, frame)
} // end for toploop
if len(res) > 0 {
return tmFramesToR(res)
}
return C.R_NilValue
}
func FollowFile(path string, cfg *tf.TfcatConfig) {
if !FileExists(path) {
fmt.Fprintf(os.Stderr, "input file '%s' does not exist.\n", path)
os.Exit(1)
}
watcher, err := fsnotify.NewWatcher()
panicOn(err)
defer watcher.Close()
f, err := os.Open(path)
panicOn(err)
// move to end for tailing
_, err = f.Seek(0, 2)
panicOn(err)
err = watcher.Add(path)
panicOn(err)
fr := tf.NewFrameReader(f, 1024*1024)
var frame tf.Frame
i := int64(1)
nextFrame:
for {
_, _, err, _ = fr.NextFrame(&frame)
if err != nil {
if err == io.EOF {
select {
case event := <-watcher.Events:
if event.Op&fsnotify.Write == fsnotify.Write {
continue nextFrame
}
}
}
fmt.Fprintf(os.Stderr, "tfcat error from fr.NextFrame(): '%v'\n", err)
os.Exit(1)
}
frame.DisplayFrame(os.Stdout, i, cfg.PrettyPrint, cfg.SkipPayload, cfg.Rreadable)
i++
}
}
func main() {
myflags := flag.NewFlagSet("tffilter", flag.ExitOnError)
cfg := &tf.TffilterConfig{}
cfg.DefineFlags(myflags)
err := myflags.Parse(os.Args[1:])
err = cfg.ValidateConfig()
if err != nil {
usage(err, myflags)
}
if cfg.Help {
fmt.Fprintf(os.Stderr, "help requested:\n")
usage(nil, myflags)
}
leftover := myflags.Args()
regs := leftover
//p("regs = %v", regs)
if cfg.RegexFile != "" {
regs, err = tf.ReadNewlineDelimFile(cfg.RegexFile)
if err != nil {
fmt.Fprintf(os.Stderr, "error reading -regexfile '%s': '%s'\n", cfg.RegexFile, err)
usage(err, myflags)
}
if len(regs) == 0 {
fmt.Fprintf(os.Stderr, "no regex given from file '%s': specify at least one regex to filter with.\n", cfg.RegexFile)
showUse(myflags)
os.Exit(1)
}
} else {
if len(regs) == 0 || (len(regs) == 1 && strings.HasPrefix(regs[0], "-h")) {
fmt.Fprintf(os.Stderr, "no regex given: specify at least one regex to filter with.\n")
showUse(myflags)
os.Exit(1)
}
}
// INVAR: regs specified, and len(regs) > 0
arrRegex := make([]*regexp.Regexp, 0)
for i := range regs {
field := regs[i]
//fmt.Fprintf(os.Stderr, "compiling regex %d: '%s'\n", i, field)
re := regexp.MustCompile(field)
arrRegex = append(arrRegex, re)
}
i := int64(1)
fr := tf.NewFrameReader(os.Stdin, 1024*1024)
var frame tf.Frame
var raw []byte
n := len(regs)
var allMatched, anyMatched bool
toploop:
for ; err == nil; i++ {
_, _, err, raw = fr.NextFrame(&frame)
if err != nil {
if err == io.EOF {
break toploop
}
fmt.Fprintf(os.Stderr, "tffilter error from fr.NextFrame() at i=%v: '%v'\n", i, err)
os.Exit(1)
}
str := frame.Stringify(-1, false, false, false)
// match regex
matchN := 0
var o string
var sub []string
for _, r := range arrRegex {
if cfg.Sub {
sub = r.FindStringSubmatch(str)
//p("sub back from r.FindStringSubmatch(str='%s') is '%#v' from regex '%s'", str, sub, regs[j])
if sub == nil {
o = ""
} else {
o = "hit"
}
} else {
o = r.FindString(str)
}
//fmt.Fprintf(os.Stderr, "tffilter at i=%v, matching frame '%s' against regex '%s': output is: '%s'\n", j, str, regs[j], o)
if o != "" {
matchN++
switch {
// we have a match
case cfg.Any:
// found at least one match, we can stop under Any
switch {
case cfg.ExcludeMatches:
continue toploop
case !cfg.ExcludeMatches:
goto writeout
}
}
} else {
// not a match
switch {
case !cfg.Any && cfg.ExcludeMatches:
// we've got to match all n in order to exclude, so we know now that we will include.
goto writeout
case !cfg.Any && !cfg.ExcludeMatches:
// not a match, and all must match to survive the filter, so we can stop
continue toploop
}
}
}
allMatched = (matchN == n)
anyMatched = (matchN > 0)
switch {
case cfg.Any && cfg.ExcludeMatches:
if anyMatched {
continue toploop
}
goto writeout
case cfg.Any && !cfg.ExcludeMatches:
if anyMatched {
goto writeout
}
continue toploop
case !cfg.Any && cfg.ExcludeMatches:
if allMatched {
continue toploop
}
goto writeout
case !cfg.Any && !cfg.ExcludeMatches:
if allMatched {
goto writeout
}
continue toploop
}
writeout:
if cfg.Sub {
// sub-expression matching and reporting only the sub matches
//p("sub = %#v", sub)
sub = sub[1:]
//p("sub[1:] = %#v", sub)
nsub := len(sub)
if nsub > 0 {
for k := range sub {
fmt.Printf("%s", sub[k])
if k < nsub-1 {
fmt.Printf(" ", sub[k])
}
}
fmt.Printf("\n")
}
} else {
// full record matching
_, err = os.Stdout.Write(raw)
if err != nil {
fmt.Fprintf(os.Stderr, "tffilter stopping at: '%s'", err)
}
}
} // end for toploop
//fmt.Fprintf(os.Stderr, "field='%s': found %v matches.\n", field, matchCount)
}
func main() {
myflags := flag.NewFlagSet("tfindex", flag.ExitOnError)
cfg := &tf.TfindexConfig{}
cfg.DefineFlags(myflags)
err := myflags.Parse(os.Args[1:])
err = cfg.ValidateConfig()
if err != nil {
usage(err, myflags)
}
leftover := myflags.Args()
//Q("leftover = %v", leftover)
if len(leftover) == 0 {
fmt.Fprintf(os.Stderr, "no input files given\n")
showUse(myflags)
os.Exit(1)
}
i := int64(1)
nextfile:
for _, inputFile := range leftover {
//P("starting on inputFile '%s'", inputFile)
if !FileExists(inputFile) {
fmt.Fprintf(os.Stderr, "input file '%s' does not exist.\n", inputFile)
os.Exit(1)
}
f, err := os.Open(inputFile)
panicOn(err)
fr := tf.NewFrameReader(f, 1024*1024)
writeFile := inputFile + ".idx"
of, err := os.Create(writeFile)
panicOn(err)
fw := tf.NewFrameWriter(of, 1024*1024)
var offset int64
var frame tf.Frame
var nextTm time.Time
var nbytes int64
for ; err == nil; i++ {
_, nbytes, err, _ = fr.NextFrame(&frame)
if err != nil {
if err == io.EOF {
fw.Flush()
fw.Sync()
of.Close()
continue nextfile
}
fmt.Fprintf(os.Stderr, "tfindex error from fr.NextFrame() at i=%v: '%v'\n", i, err)
os.Exit(1)
}
unix := frame.Tm()
tm := time.Unix(0, unix)
trunc := tm.Truncate(time.Minute)
if i == 0 {
first, err := tf.NewFrame(tm, tf.EvOneInt64, 0, offset, nil)
panicOn(err)
fw.Append(first)
nextTm = trunc.Add(time.Minute)
} else if tm.After(nextTm) {
next, err := tf.NewFrame(tm, tf.EvOneInt64, 0, offset, nil)
panicOn(err)
fw.Append(next)
nextTm = trunc.Add(time.Minute)
}
offset += nbytes
}
}
}
func main() {
myflags := flag.NewFlagSet("tfcat", flag.ExitOnError)
cfg := &tf.TfcatConfig{}
cfg.DefineFlags(myflags)
err := myflags.Parse(os.Args[1:])
err = cfg.ValidateConfig()
if err != nil {
usage(err, myflags)
}
leftover := myflags.Args()
//Q("leftover = %v", leftover)
if len(leftover) == 0 && cfg.ReadStdin == false {
fmt.Fprintf(os.Stderr, "no input files given\n")
showUse(myflags)
os.Exit(1)
}
GlobalPrettyPrint = cfg.PrettyPrint
if cfg.Follow {
if len(leftover) != 1 {
if cfg.ReadStdin {
fmt.Fprintf(os.Stderr, "cannot follow stdin\n")
showUse(myflags)
os.Exit(1)
}
fmt.Fprintf(os.Stderr, "can only follow a single file\n")
showUse(myflags)
os.Exit(1)
}
FollowFile(leftover[0], cfg)
return
}
if cfg.ReadStdin {
if len(leftover) > 0 {
fmt.Fprintf(os.Stderr, "if reading from stdin, cannot also read from files\n")
showUse(myflags)
os.Exit(1)
}
leftover = []string{"stdin"}
}
if cfg.RawCount > 0 || cfg.RawSkip > 0 {
if len(leftover) != 1 {
fmt.Fprintf(os.Stderr, "can only copy raw messages from one file\n")
showUse(myflags)
os.Exit(1)
}
SendRawBytes(leftover[0], cfg.RawCount, os.Stdout, cfg.RawSkip)
return
}
i := int64(1)
nextfile:
for _, inputFile := range leftover {
f := prepInput(inputFile)
defer f.Close()
//P("starting on inputFile '%s'", inputFile)
fr := tf.NewFrameReader(f, 1024*1024)
var frame tf.Frame
for ; err == nil; i++ {
_, _, err, _ = fr.NextFrame(&frame)
if err != nil {
if err == io.EOF {
continue nextfile
}
fmt.Fprintf(os.Stderr, "tfcat error from fr.NextFrame() at i=%v: '%v'\n", i, err)
os.Exit(1)
}
frame.DisplayFrame(os.Stdout, i, cfg.PrettyPrint, cfg.SkipPayload, cfg.Rreadable)
}
}
}
func main() {
myflags := flag.NewFlagSet("tfcat", flag.ExitOnError)
cfg := &TfcatConfig{}
cfg.DefineFlags(myflags)
err := myflags.Parse(os.Args[1:])
err = cfg.ValidateConfig()
if err != nil {
usage(err, myflags)
}
leftover := myflags.Args()
//Q("leftover = %v", leftover)
if len(leftover) == 0 {
fmt.Fprintf(os.Stderr, "no input files given\n")
showUse(myflags)
os.Exit(1)
}
GlobalPrettyPrint = cfg.PrettyPrint
i := int64(0)
nextfile:
for _, inputFile := range leftover {
//P("starting on inputFile '%s'", inputFile)
if !FileExists(inputFile) {
fmt.Fprintf(os.Stderr, "input file '%s' does not exist.\n", inputFile)
os.Exit(1)
}
f, err := os.Open(inputFile)
panicOn(err)
fr := tf.NewFrameReader(f, 1024*1024)
var frame *tf.Frame
for ; err == nil; i++ {
frame, _, err = fr.NextFrame()
if err != nil {
if err == io.EOF {
continue nextfile
}
fmt.Fprintf(os.Stderr, "tfcat error from fr.NextFrame() at i=%v: '%v'\n", i, err)
os.Exit(1)
}
fmt.Printf("%v", frame)
evtnum := frame.GetEvtnum()
if evtnum == tf.EvJson {
pp := prettyPrintJson(cfg.PrettyPrint, frame.Data)
fmt.Printf(" %s", string(pp))
}
if evtnum == tf.EvMsgpKafka || evtnum == tf.EvMsgpack {
// decode msgpack to json with ugorji/go/codec
var iface interface{}
dec := codec.NewDecoderBytes(frame.Data, &msgpHelper.mh)
err := dec.Decode(&iface)
panicOn(err)
//Q("iface = '%#v'", iface)
var w bytes.Buffer
enc := codec.NewEncoder(&w, &msgpHelper.jh)
err = enc.Encode(&iface)
panicOn(err)
pp := prettyPrintJson(cfg.PrettyPrint, w.Bytes())
fmt.Printf(" %s", string(pp))
}
fmt.Printf("\n")
}
}
}