Exemplo n.º 1
0
func benchScan(cmd *cobra.Command, args []string) {
	readConfig()

	iscan, ifile := openInputFile(infile)
	defer ifile.Close()

	var lines []string
	var totalSize int
	n := 0

	for iscan.Scan() {
		line := iscan.Text()
		if len(line) == 0 || line[0] == '#' {
			continue
		}

		n++
		lines = append(lines, line)
		totalSize += len(line)
	}

	profile()

	now := time.Now()

	if workers == 1 {
		scanner := sequence.NewScanner()
		for _, line := range lines {
			scanMessage(scanner, line)
		}
	} else {
		var wg sync.WaitGroup
		msgpipe := make(chan string, 10000)

		for i := 0; i < workers; i++ {
			wg.Add(1)
			go func() {
				defer wg.Done()
				scanner := sequence.NewScanner()

				for line := range msgpipe {
					scanMessage(scanner, line)
				}
			}()
		}

		for _, line := range lines {
			msgpipe <- line
		}
		close(msgpipe)

		wg.Wait()
	}

	since := time.Since(now)
	log.Printf("Scanned %d messages in %.2f secs, ~ %.2f msgs/sec, ~ %.2f MB/sec", n, float64(since)/float64(time.Second), float64(n)/(float64(since)/float64(time.Second)), float64(totalSize)/float64(mbyte)/(float64(since)/float64(time.Second)))
	close(quit)
	<-done
}
Exemplo n.º 2
0
func scan(cmd *cobra.Command, args []string) {
	readConfig()

	scanner := sequence.NewScanner()

	if infile != "" {
		// Open input file
		iscan, ifile := openInputFile(infile)
		defer ifile.Close()

		ofile := openOutputFile(outfile)
		defer ofile.Close()

		for iscan.Scan() {
			line := iscan.Text()
			if len(line) == 0 || line[0] == '#' {
				continue
			}

			seq := scanMessage(scanner, line)
			fmt.Fprintf(ofile, "%s\n\n", seq.PrintTokens())
		}
	} else if len(args) == 1 && args[0] != "" {
		seq := scanMessage(scanner, args[0])
		fmt.Println(seq.PrintTokens())
	} else {
		log.Fatal("Invalid input file or string specified")
	}
}
Exemplo n.º 3
0
func buildParser() *sequence.Parser {
	parser := sequence.NewParser()

	if patfile == "" {
		return parser
	}

	var files []string

	if fi, err := os.Stat(patfile); err != nil {
		log.Fatal(err)
	} else if fi.Mode().IsDir() {
		files = getDirOfFiles(patfile)
	} else {
		files = append(files, patfile)
	}

	scanner := sequence.NewScanner()

	for _, file := range files {
		// Open pattern file
		pscan, pfile := openInputFile(file)

		for pscan.Scan() {
			line := pscan.Text()
			if len(line) == 0 || line[0] == '#' {
				continue
			}

			seq, err := scanner.Scan(line)
			if err != nil {
				log.Fatal(err)
			}

			if err := parser.Add(seq); err != nil {
				log.Fatal(err)
			}
		}

		pfile.Close()
	}

	return parser
}
Exemplo n.º 4
0
func parse(cmd *cobra.Command, args []string) {
	readConfig()

	if infile == "" {
		log.Fatal("Invalid input file specified")
	}

	profile()

	parser := buildParser()
	scanner := sequence.NewScanner()

	iscan, ifile := openInputFile(infile)
	defer ifile.Close()

	ofile := openOutputFile(outfile)
	defer ofile.Close()

	n := 0
	now := time.Now()

	for iscan.Scan() {
		line := iscan.Text()
		if len(line) == 0 || line[0] == '#' {
			continue
		}
		n++

		seq := scanMessage(scanner, line)

		seq, err := parser.Parse(seq)
		if err != nil {
			log.Printf("Error (%s) parsing: %s", err, line)
		} else {
			fmt.Fprintf(ofile, "%s\n%s\n\n", line, seq.PrintTokens())
		}
	}

	since := time.Since(now)
	log.Printf("Parsed %d messages in %.2f secs, ~ %.2f msgs/sec", n, float64(since)/float64(time.Second), float64(n)/(float64(since)/float64(time.Second)))
	close(quit)
	<-done
}
Exemplo n.º 5
0
func analyze(cmd *cobra.Command, args []string) {
	readConfig()

	if infile == "" {
		log.Fatal("Invalid input file specified")
	}

	profile()

	parser := buildParser()
	analyzer := sequence.NewAnalyzer()
	scanner := sequence.NewScanner()

	// Open input file
	iscan, ifile := openInputFile(infile)
	defer ifile.Close()

	// For all the log messages, if we can't parse it, then let's add it to the
	// analyzer for pattern analysis
	for iscan.Scan() {
		line := iscan.Text()
		if len(line) == 0 || line[0] == '#' {
			continue
		}

		seq := scanMessage(scanner, line)

		if _, err := parser.Parse(seq); err != nil {
			analyzer.Add(seq)
		}
	}

	ifile.Close()
	analyzer.Finalize()

	iscan, ifile = openInputFile(infile)
	defer ifile.Close()

	pmap := make(map[string]struct {
		ex  string
		cnt int
	})
	amap := make(map[string]struct {
		ex  string
		cnt int
	})
	n := 0

	// Now that we have built the analyzer, let's go through each log message again
	// to determine the unique patterns
	for iscan.Scan() {
		line := iscan.Text()
		if len(line) == 0 || line[0] == '#' {
			continue
		}
		n++

		seq := scanMessage(scanner, line)

		pseq, err := parser.Parse(seq)
		if err == nil {
			pat := pseq.String()
			stat, ok := pmap[pat]
			if !ok {
				stat = struct {
					ex  string
					cnt int
				}{}
			}
			stat.ex = line
			stat.cnt++
			pmap[pat] = stat
		} else {
			aseq, err := analyzer.Analyze(seq)
			if err != nil {
				log.Printf("Error analyzing: %s", line)
			} else {
				pat := aseq.String()
				stat, ok := amap[pat]
				if !ok {
					stat = struct {
						ex  string
						cnt int
					}{}
				}
				stat.ex = line
				stat.cnt++
				amap[pat] = stat
			}
		}
	}

	ofile := openOutputFile(outfile)
	defer ofile.Close()

	for pat, stat := range pmap {
		fmt.Fprintf(ofile, "%s\n# %d log messages matched\n# %s\n\n", pat, stat.cnt, stat.ex)
	}

	for pat, stat := range amap {
		fmt.Fprintf(ofile, "%s\n# %d log messages matched\n# %s\n\n", pat, stat.cnt, stat.ex)
	}

	log.Printf("Analyzed %d messages, found %d unique patterns, %d are new.", n, len(pmap)+len(amap), len(amap))
}