func benchScan(cmd *cobra.Command, args []string) {
readConfig()
iscan, ifile := openInputFile(infile)
defer ifile.Close()
var lines []string
var totalSize int
n := 0
for iscan.Scan() {
line := iscan.Text()
if len(line) == 0 || line[0] == '#' {
continue
}
n++
lines = append(lines, line)
totalSize += len(line)
}
profile()
now := time.Now()
if workers == 1 {
scanner := sequence.NewScanner()
for _, line := range lines {
scanMessage(scanner, line)
}
} else {
var wg sync.WaitGroup
msgpipe := make(chan string, 10000)
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
scanner := sequence.NewScanner()
for line := range msgpipe {
scanMessage(scanner, line)
}
}()
}
for _, line := range lines {
msgpipe <- line
}
close(msgpipe)
wg.Wait()
}
since := time.Since(now)
log.Printf("Scanned %d messages in %.2f secs, ~ %.2f msgs/sec, ~ %.2f MB/sec", n, float64(since)/float64(time.Second), float64(n)/(float64(since)/float64(time.Second)), float64(totalSize)/float64(mbyte)/(float64(since)/float64(time.Second)))
close(quit)
<-done
}
func scan(cmd *cobra.Command, args []string) {
readConfig()
scanner := sequence.NewScanner()
if infile != "" {
// Open input file
iscan, ifile := openInputFile(infile)
defer ifile.Close()
ofile := openOutputFile(outfile)
defer ofile.Close()
for iscan.Scan() {
line := iscan.Text()
if len(line) == 0 || line[0] == '#' {
continue
}
seq := scanMessage(scanner, line)
fmt.Fprintf(ofile, "%s\n\n", seq.PrintTokens())
}
} else if len(args) == 1 && args[0] != "" {
seq := scanMessage(scanner, args[0])
fmt.Println(seq.PrintTokens())
} else {
log.Fatal("Invalid input file or string specified")
}
}
func buildParser() *sequence.Parser {
parser := sequence.NewParser()
if patfile == "" {
return parser
}
var files []string
if fi, err := os.Stat(patfile); err != nil {
log.Fatal(err)
} else if fi.Mode().IsDir() {
files = getDirOfFiles(patfile)
} else {
files = append(files, patfile)
}
scanner := sequence.NewScanner()
for _, file := range files {
// Open pattern file
pscan, pfile := openInputFile(file)
for pscan.Scan() {
line := pscan.Text()
if len(line) == 0 || line[0] == '#' {
continue
}
seq, err := scanner.Scan(line)
if err != nil {
log.Fatal(err)
}
if err := parser.Add(seq); err != nil {
log.Fatal(err)
}
}
pfile.Close()
}
return parser
}
func parse(cmd *cobra.Command, args []string) {
readConfig()
if infile == "" {
log.Fatal("Invalid input file specified")
}
profile()
parser := buildParser()
scanner := sequence.NewScanner()
iscan, ifile := openInputFile(infile)
defer ifile.Close()
ofile := openOutputFile(outfile)
defer ofile.Close()
n := 0
now := time.Now()
for iscan.Scan() {
line := iscan.Text()
if len(line) == 0 || line[0] == '#' {
continue
}
n++
seq := scanMessage(scanner, line)
seq, err := parser.Parse(seq)
if err != nil {
log.Printf("Error (%s) parsing: %s", err, line)
} else {
fmt.Fprintf(ofile, "%s\n%s\n\n", line, seq.PrintTokens())
}
}
since := time.Since(now)
log.Printf("Parsed %d messages in %.2f secs, ~ %.2f msgs/sec", n, float64(since)/float64(time.Second), float64(n)/(float64(since)/float64(time.Second)))
close(quit)
<-done
}
func analyze(cmd *cobra.Command, args []string) {
readConfig()
if infile == "" {
log.Fatal("Invalid input file specified")
}
profile()
parser := buildParser()
analyzer := sequence.NewAnalyzer()
scanner := sequence.NewScanner()
// Open input file
iscan, ifile := openInputFile(infile)
defer ifile.Close()
// For all the log messages, if we can't parse it, then let's add it to the
// analyzer for pattern analysis
for iscan.Scan() {
line := iscan.Text()
if len(line) == 0 || line[0] == '#' {
continue
}
seq := scanMessage(scanner, line)
if _, err := parser.Parse(seq); err != nil {
analyzer.Add(seq)
}
}
ifile.Close()
analyzer.Finalize()
iscan, ifile = openInputFile(infile)
defer ifile.Close()
pmap := make(map[string]struct {
ex string
cnt int
})
amap := make(map[string]struct {
ex string
cnt int
})
n := 0
// Now that we have built the analyzer, let's go through each log message again
// to determine the unique patterns
for iscan.Scan() {
line := iscan.Text()
if len(line) == 0 || line[0] == '#' {
continue
}
n++
seq := scanMessage(scanner, line)
pseq, err := parser.Parse(seq)
if err == nil {
pat := pseq.String()
stat, ok := pmap[pat]
if !ok {
stat = struct {
ex string
cnt int
}{}
}
stat.ex = line
stat.cnt++
pmap[pat] = stat
} else {
aseq, err := analyzer.Analyze(seq)
if err != nil {
log.Printf("Error analyzing: %s", line)
} else {
pat := aseq.String()
stat, ok := amap[pat]
if !ok {
stat = struct {
ex string
cnt int
}{}
}
stat.ex = line
stat.cnt++
amap[pat] = stat
}
}
}
ofile := openOutputFile(outfile)
defer ofile.Close()
for pat, stat := range pmap {
fmt.Fprintf(ofile, "%s\n# %d log messages matched\n# %s\n\n", pat, stat.cnt, stat.ex)
}
for pat, stat := range amap {
fmt.Fprintf(ofile, "%s\n# %d log messages matched\n# %s\n\n", pat, stat.cnt, stat.ex)
}
log.Printf("Analyzed %d messages, found %d unique patterns, %d are new.", n, len(pmap)+len(amap), len(amap))
}