func Main(args []string) {
var end bool
var oneshot bool
var useBookmark bool
var bookmarkPath string
var verbose bool
flagset := flag.NewFlagSet("evereader", flag.ExitOnError)
flagset.BoolVar(&end, "end", false, "Start at end of file")
flagset.BoolVar(&oneshot, "oneshot", false, "One shot mode (exit on EOF)")
flagset.BoolVar(&useBookmark, "bookmark", false, "Bookmark location")
flagset.StringVar(&bookmarkPath, "bookmark-path", "", "Path to bookmark file")
flagset.BoolVarP(&verbose, "verbose", "v", false, "Verbose output")
flagset.Parse(args[1:])
if verbose {
log.SetLevel(log.DEBUG)
}
if len(flagset.Args()) == 0 {
log.Fatal("No input files provided.")
} else if len(flagset.Args()) > 1 {
log.Fatal("Only one input file allowed.")
}
inputFilename := flagset.Args()[0]
// If useBookmark but no path, set a default.
if useBookmark && bookmarkPath == "" {
bookmarkPath = fmt.Sprintf("%s.bookmark", inputFilename)
}
encoder := json.NewEncoder(os.Stdout)
reader, err := evereader.New(flagset.Args()[0])
if err != nil {
log.Fatal(err)
}
// Initialize bookmark.
var bookmarker *evereader.Bookmarker = nil
if useBookmark {
bookmarker = &evereader.Bookmarker{
Filename: bookmarkPath,
Reader: reader,
}
err := bookmarker.Init(end)
if err != nil {
log.Fatal(err)
}
} else if end {
log.Info("Jumping to end of file.")
err := reader.SkipToEnd()
if err != nil {
log.Fatal(err)
}
}
for {
eof := false
event, err := reader.Next()
if err != nil {
if err == io.EOF {
if oneshot {
break
}
eof = true
} else {
log.Fatal(err)
}
}
if eof {
time.Sleep(1 * time.Second)
} else {
encoder.Encode(event)
if useBookmark {
bookmark := bookmarker.GetBookmark()
bookmarker.WriteBookmark(bookmark)
}
}
}
}
func Main(args []string) {
var end bool
var oneshot bool
var useBookmark bool
var bookmarkPath string
var verbose bool
var dbfile string
flagset := flag.NewFlagSet("sqliteimport", flag.ExitOnError)
flagset.StringVarP(&dbfile, "database", "D", "", "Database filename")
flagset.BoolVar(&end, "end", false, "Start at end of file")
flagset.BoolVar(&oneshot, "oneshot", false, "One shot mode (exit on EOF)")
flagset.BoolVar(&useBookmark, "bookmark", false, "Bookmark location")
flagset.StringVar(&bookmarkPath, "bookmark-path", "", "Path to bookmark file")
flagset.BoolVarP(&verbose, "verbose", "v", false, "Verbose output")
flagset.Parse(args)
if verbose {
log.SetLevel(log.DEBUG)
}
if dbfile == "" {
log.Fatal("Database filename must be provided.")
}
if len(flagset.Args()) == 0 {
log.Fatal("No input files provided.")
} else if len(flagset.Args()) > 1 {
log.Fatal("Only one input file allowed.")
}
inputFilename := flagset.Args()[0]
// If useBookmark but no path, set a default.
if useBookmark && bookmarkPath == "" {
bookmarkPath = fmt.Sprintf("%s.bookmark", inputFilename)
}
reader, err := evereader.New(flagset.Args()[0])
if err != nil {
log.Fatal(err)
}
// Initialize bookmark.
var bookmarker *evereader.Bookmarker = nil
if useBookmark {
bookmarker = &evereader.Bookmarker{
Filename: bookmarkPath,
Reader: reader,
}
err := bookmarker.Init(end)
if err != nil {
log.Fatal(err)
}
} else if end {
log.Info("Jumping to end of file.")
err := reader.SkipToEnd()
if err != nil {
log.Fatal(err)
}
}
db, err := sqlite.NewSqliteService(dbfile)
if err != nil {
log.Fatal(err)
}
if err := db.Migrate(); err != nil {
log.Fatal(err)
}
indexer, err := sqlite.NewSqliteIndexer(db)
if err != nil {
log.Fatal(err)
}
count := uint64(0)
lastStatTs := time.Now()
lastStatCount := uint64(0)
// Number of EOFs in last stat interval.
eofs := uint64(0)
tagsFilter := eve.TagsFilter{}
for {
eof := false
event, err := reader.Next()
if err != nil {
if err == io.EOF {
eof = true
eofs++
} else {
log.Fatal(err)
}
}
if event != nil {
tagsFilter.Filter(event)
indexer.IndexRawEve(event)
count++
if useBookmark {
bookmark := bookmarker.GetBookmark()
bookmarker.WriteBookmark(bookmark)
}
}
now := time.Now()
if now.Sub(lastStatTs).Seconds() > 1 {
log.Info("Total: %d; Last interval: %d; Avg: %.2f/s, EOFs: %d",
count,
count-lastStatCount,
float64(count-lastStatCount)/(now.Sub(lastStatTs).Seconds()),
eofs)
lastStatTs = now
lastStatCount = count
eofs = 0
indexer.Flush()
}
if eof {
if oneshot {
break
} else {
indexer.Flush()
time.Sleep(100 * time.Millisecond)
}
}
}
now := time.Now()
log.Info("Total: %d; Last interval: %d; Avg: %.2f/s, EOFs: %d",
count,
count-lastStatCount,
float64(count-lastStatCount)/(now.Sub(lastStatTs).Seconds()),
eofs)
indexer.Flush()
}
func Main(args []string) {
conf := configure(args)
if conf.BatchSize < 1 {
log.Fatal("Batch size must be greater than 0")
}
if conf.Url == "" {
log.Error("error: --elasticsearch is a required parameter")
usage()
os.Exit(1)
}
if conf.InputFilename == "" {
log.Fatal("error: no input file provided")
}
if conf.Bookmark && conf.BookmarkPath == "" {
conf.BookmarkPath = fmt.Sprintf("%s.bookmark", conf.InputFilename)
log.Info("Using bookmark file %s", conf.BookmarkPath)
}
es := elasticsearch.New(conf.Url)
es.DisableCertCheck(conf.DisableCertificateCheck)
if conf.Username != "" || conf.Password != "" {
if err := es.SetUsernamePassword(conf.Username,
conf.Password); err != nil {
log.Fatal("Failed to set username and password: %v", err)
}
}
response, err := es.Ping()
if err != nil {
log.Fatal("error: failed to ping Elastic Search:", err)
}
log.Info("Connected to Elastic Search v%s (cluster:%s; name: %s)",
response.Version.Number, response.ClusterName, response.Name)
majorVersion := response.MajorVersion()
// Check if the template exists.
templateExists, err := es.CheckTemplate(conf.Index)
if !templateExists {
log.Info("Template %s does not exist, creating...", conf.Index)
err = es.LoadTemplate(conf.Index, majorVersion)
if err != nil {
log.Fatal("Failed to create template:", err)
}
} else {
log.Info("Template %s exists, will not create.", conf.Index)
}
var geoipFilter *eve.GeoipFilter
tagsFilter := &eve.TagsFilter{}
if !conf.DisableGeoIp {
geoipdb, err := geoip.NewGeoIpDb(conf.GeoIpDatabase)
if err != nil {
log.Notice("Failed to load GeoIP database: %v", err)
} else {
log.Info("Using GeoIP database %s, %s", geoipdb.Type(), geoipdb.BuildDate())
geoipFilter = eve.NewGeoipFilter(geoipdb)
}
}
indexer := elasticsearch.NewIndexer(es, conf.DisableCertificateCheck)
indexer.IndexPrefix = conf.Index
reader, err := evereader.New(conf.InputFilename)
if err != nil {
log.Fatal(err)
}
// Initialize bookmarking...
var bookmarker *evereader.Bookmarker = nil
if conf.Bookmark {
bookmarker = &evereader.Bookmarker{
Filename: conf.BookmarkPath,
Reader: reader,
}
err := bookmarker.Init(conf.End)
if err != nil {
log.Fatal(err)
}
} else if conf.End {
log.Info("Jumping to end of file.")
err := reader.SkipToEnd()
if err != nil {
log.Fatal(err)
}
}
count := uint64(0)
lastStatTs := time.Now()
lastStatCount := uint64(0)
startTime := time.Now()
// Number of EOFs in last stat interval.
eofs := uint64(0)
go func() {
err := indexer.Run()
if err != nil {
log.Fatal("Elastic Search indexer connection unexpectedly closed:", err)
} else {
log.Debug("Indexer exited without issue.")
}
}()
for {
eof := false
event, err := reader.Next()
if err != nil {
if err == io.EOF {
eof = true
eofs++
} else if _, ok := err.(evereader.MalformedEventError); ok {
log.Error("Failed to read event but will continue: %v", err)
} else {
log.Fatalf("Unrecoverable error reading event: %v", err)
}
}
if event != nil {
if geoipFilter != nil {
geoipFilter.AddGeoIP(event)
}
tagsFilter.Filter(event)
if conf.stdout {
asJson, err := json.Marshal(event)
if err != nil {
log.Error("Failed to print event as json: %v", err)
} else {
fmt.Println(string(asJson))
}
}
indexer.IndexRawEvent(event)
count++
}
if eof || (count > 0 && count%conf.BatchSize == 0) {
var bookmark *evereader.Bookmark = nil
if conf.Bookmark {
bookmark = bookmarker.GetBookmark()
}
response, err := indexer.FlushConnection()
if err != nil {
log.Fatal(err)
}
if response != nil {
log.Debug("Indexed %d events {errors=%v}", len(response.Items),
response.Errors)
}
if conf.Bookmark {
bookmarker.WriteBookmark(bookmark)
}
}
now := time.Now()
if now.Sub(lastStatTs).Seconds() > 1 && now.Second() == 0 {
// Calculate the lag in bytes, that is the number of bytes behind
// the end of file we are.
lag, err := GetLag(reader)
if err != nil {
log.Error("Failed to calculate lag: %v", err)
}
log.Info("Total: %d; Last minute: %d; Avg: %.2f/s, EOFs: %d; Lag (bytes): %d",
count,
count-lastStatCount,
float64(count-lastStatCount)/(now.Sub(lastStatTs).Seconds()),
eofs,
lag)
lastStatTs = now
lastStatCount = count
eofs = 0
}
if eof {
if conf.oneshot {
break
} else {
time.Sleep(1 * time.Second)
}
}
}
totalTime := time.Since(startTime)
if conf.oneshot {
log.Info("Indexed %d events: time=%.2fs; avg=%d/s", count, totalTime.Seconds(),
uint64(float64(count)/totalTime.Seconds()))
}
}