Beispiel #1
0
func Main(args []string) {

	var end bool
	var oneshot bool
	var useBookmark bool
	var bookmarkPath string
	var verbose bool

	flagset := flag.NewFlagSet("evereader", flag.ExitOnError)
	flagset.BoolVar(&end, "end", false, "Start at end of file")
	flagset.BoolVar(&oneshot, "oneshot", false, "One shot mode (exit on EOF)")
	flagset.BoolVar(&useBookmark, "bookmark", false, "Bookmark location")
	flagset.StringVar(&bookmarkPath, "bookmark-path", "", "Path to bookmark file")
	flagset.BoolVarP(&verbose, "verbose", "v", false, "Verbose output")
	flagset.Parse(args[1:])

	if verbose {
		log.SetLevel(log.DEBUG)
	}

	if len(flagset.Args()) == 0 {
		log.Fatal("No input files provided.")
	} else if len(flagset.Args()) > 1 {
		log.Fatal("Only one input file allowed.")
	}
	inputFilename := flagset.Args()[0]

	// If useBookmark but no path, set a default.
	if useBookmark && bookmarkPath == "" {
		bookmarkPath = fmt.Sprintf("%s.bookmark", inputFilename)
	}

	encoder := json.NewEncoder(os.Stdout)

	reader, err := evereader.New(flagset.Args()[0])
	if err != nil {
		log.Fatal(err)
	}

	// Initialize bookmark.
	var bookmarker *evereader.Bookmarker = nil
	if useBookmark {
		bookmarker = &evereader.Bookmarker{
			Filename: bookmarkPath,
			Reader:   reader,
		}
		err := bookmarker.Init(end)
		if err != nil {
			log.Fatal(err)
		}
	} else if end {
		log.Info("Jumping to end of file.")
		err := reader.SkipToEnd()
		if err != nil {
			log.Fatal(err)
		}
	}

	for {
		eof := false
		event, err := reader.Next()
		if err != nil {
			if err == io.EOF {
				if oneshot {
					break
				}
				eof = true
			} else {
				log.Fatal(err)
			}
		}

		if eof {
			time.Sleep(1 * time.Second)
		} else {
			encoder.Encode(event)

			if useBookmark {
				bookmark := bookmarker.GetBookmark()
				bookmarker.WriteBookmark(bookmark)
			}
		}
	}
}
Beispiel #2
0
func Main(args []string) {

	var end bool
	var oneshot bool
	var useBookmark bool
	var bookmarkPath string
	var verbose bool
	var dbfile string

	flagset := flag.NewFlagSet("sqliteimport", flag.ExitOnError)
	flagset.StringVarP(&dbfile, "database", "D", "", "Database filename")
	flagset.BoolVar(&end, "end", false, "Start at end of file")
	flagset.BoolVar(&oneshot, "oneshot", false, "One shot mode (exit on EOF)")
	flagset.BoolVar(&useBookmark, "bookmark", false, "Bookmark location")
	flagset.StringVar(&bookmarkPath, "bookmark-path", "", "Path to bookmark file")
	flagset.BoolVarP(&verbose, "verbose", "v", false, "Verbose output")
	flagset.Parse(args)

	if verbose {
		log.SetLevel(log.DEBUG)
	}

	if dbfile == "" {
		log.Fatal("Database filename must be provided.")
	}

	if len(flagset.Args()) == 0 {
		log.Fatal("No input files provided.")
	} else if len(flagset.Args()) > 1 {
		log.Fatal("Only one input file allowed.")
	}
	inputFilename := flagset.Args()[0]

	// If useBookmark but no path, set a default.
	if useBookmark && bookmarkPath == "" {
		bookmarkPath = fmt.Sprintf("%s.bookmark", inputFilename)
	}

	reader, err := evereader.New(flagset.Args()[0])
	if err != nil {
		log.Fatal(err)
	}

	// Initialize bookmark.
	var bookmarker *evereader.Bookmarker = nil
	if useBookmark {
		bookmarker = &evereader.Bookmarker{
			Filename: bookmarkPath,
			Reader:   reader,
		}
		err := bookmarker.Init(end)
		if err != nil {
			log.Fatal(err)
		}
	} else if end {
		log.Info("Jumping to end of file.")
		err := reader.SkipToEnd()
		if err != nil {
			log.Fatal(err)
		}
	}

	db, err := sqlite.NewSqliteService(dbfile)
	if err != nil {
		log.Fatal(err)
	}
	if err := db.Migrate(); err != nil {
		log.Fatal(err)
	}

	indexer, err := sqlite.NewSqliteIndexer(db)
	if err != nil {
		log.Fatal(err)
	}

	count := uint64(0)
	lastStatTs := time.Now()
	lastStatCount := uint64(0)

	// Number of EOFs in last stat interval.
	eofs := uint64(0)

	tagsFilter := eve.TagsFilter{}

	for {
		eof := false
		event, err := reader.Next()
		if err != nil {
			if err == io.EOF {
				eof = true
				eofs++
			} else {
				log.Fatal(err)
			}
		}

		if event != nil {

			tagsFilter.Filter(event)

			indexer.IndexRawEve(event)
			count++

			if useBookmark {
				bookmark := bookmarker.GetBookmark()
				bookmarker.WriteBookmark(bookmark)
			}
		}

		now := time.Now()

		if now.Sub(lastStatTs).Seconds() > 1 {
			log.Info("Total: %d; Last interval: %d; Avg: %.2f/s, EOFs: %d",
				count,
				count-lastStatCount,
				float64(count-lastStatCount)/(now.Sub(lastStatTs).Seconds()),
				eofs)
			lastStatTs = now
			lastStatCount = count
			eofs = 0
			indexer.Flush()
		}

		if eof {
			if oneshot {
				break
			} else {
				indexer.Flush()
				time.Sleep(100 * time.Millisecond)
			}
		}

	}

	now := time.Now()

	log.Info("Total: %d; Last interval: %d; Avg: %.2f/s, EOFs: %d",
		count,
		count-lastStatCount,
		float64(count-lastStatCount)/(now.Sub(lastStatTs).Seconds()),
		eofs)

	indexer.Flush()
}
Beispiel #3
0
func Main(args []string) {

	conf := configure(args)

	if conf.BatchSize < 1 {
		log.Fatal("Batch size must be greater than 0")
	}

	if conf.Url == "" {
		log.Error("error: --elasticsearch is a required parameter")
		usage()
		os.Exit(1)
	}

	if conf.InputFilename == "" {
		log.Fatal("error: no input file provided")
	}

	if conf.Bookmark && conf.BookmarkPath == "" {
		conf.BookmarkPath = fmt.Sprintf("%s.bookmark", conf.InputFilename)
		log.Info("Using bookmark file %s", conf.BookmarkPath)
	}

	es := elasticsearch.New(conf.Url)
	es.DisableCertCheck(conf.DisableCertificateCheck)
	if conf.Username != "" || conf.Password != "" {
		if err := es.SetUsernamePassword(conf.Username,
			conf.Password); err != nil {
			log.Fatal("Failed to set username and password: %v", err)
		}
	}
	response, err := es.Ping()
	if err != nil {
		log.Fatal("error: failed to ping Elastic Search:", err)
	}
	log.Info("Connected to Elastic Search v%s (cluster:%s; name: %s)",
		response.Version.Number, response.ClusterName, response.Name)
	majorVersion := response.MajorVersion()

	// Check if the template exists.
	templateExists, err := es.CheckTemplate(conf.Index)
	if !templateExists {
		log.Info("Template %s does not exist, creating...", conf.Index)
		err = es.LoadTemplate(conf.Index, majorVersion)
		if err != nil {
			log.Fatal("Failed to create template:", err)
		}
	} else {
		log.Info("Template %s exists, will not create.", conf.Index)
	}

	var geoipFilter *eve.GeoipFilter
	tagsFilter := &eve.TagsFilter{}

	if !conf.DisableGeoIp {
		geoipdb, err := geoip.NewGeoIpDb(conf.GeoIpDatabase)
		if err != nil {
			log.Notice("Failed to load GeoIP database: %v", err)
		} else {
			log.Info("Using GeoIP database %s, %s", geoipdb.Type(), geoipdb.BuildDate())
			geoipFilter = eve.NewGeoipFilter(geoipdb)
		}
	}

	indexer := elasticsearch.NewIndexer(es, conf.DisableCertificateCheck)
	indexer.IndexPrefix = conf.Index

	reader, err := evereader.New(conf.InputFilename)
	if err != nil {
		log.Fatal(err)
	}

	// Initialize bookmarking...
	var bookmarker *evereader.Bookmarker = nil
	if conf.Bookmark {
		bookmarker = &evereader.Bookmarker{
			Filename: conf.BookmarkPath,
			Reader:   reader,
		}
		err := bookmarker.Init(conf.End)
		if err != nil {
			log.Fatal(err)
		}
	} else if conf.End {
		log.Info("Jumping to end of file.")
		err := reader.SkipToEnd()
		if err != nil {
			log.Fatal(err)
		}
	}

	count := uint64(0)
	lastStatTs := time.Now()
	lastStatCount := uint64(0)
	startTime := time.Now()

	// Number of EOFs in last stat interval.
	eofs := uint64(0)

	go func() {
		err := indexer.Run()
		if err != nil {
			log.Fatal("Elastic Search indexer connection unexpectedly closed:", err)
		} else {
			log.Debug("Indexer exited without issue.")
		}
	}()

	for {
		eof := false
		event, err := reader.Next()
		if err != nil {
			if err == io.EOF {
				eof = true
				eofs++
			} else if _, ok := err.(evereader.MalformedEventError); ok {
				log.Error("Failed to read event but will continue: %v", err)
			} else {
				log.Fatalf("Unrecoverable error reading event: %v", err)
			}
		}

		if event != nil {

			if geoipFilter != nil {
				geoipFilter.AddGeoIP(event)
			}

			tagsFilter.Filter(event)

			if conf.stdout {
				asJson, err := json.Marshal(event)
				if err != nil {
					log.Error("Failed to print event as json: %v", err)
				} else {
					fmt.Println(string(asJson))
				}
			}

			indexer.IndexRawEvent(event)
			count++
		}

		if eof || (count > 0 && count%conf.BatchSize == 0) {
			var bookmark *evereader.Bookmark = nil

			if conf.Bookmark {
				bookmark = bookmarker.GetBookmark()
			}

			response, err := indexer.FlushConnection()
			if err != nil {
				log.Fatal(err)
			}
			if response != nil {
				log.Debug("Indexed %d events {errors=%v}", len(response.Items),
					response.Errors)
			}

			if conf.Bookmark {
				bookmarker.WriteBookmark(bookmark)
			}
		}

		now := time.Now()
		if now.Sub(lastStatTs).Seconds() > 1 && now.Second() == 0 {

			// Calculate the lag in bytes, that is the number of bytes behind
			// the end of file we are.
			lag, err := GetLag(reader)
			if err != nil {
				log.Error("Failed to calculate lag: %v", err)
			}

			log.Info("Total: %d; Last minute: %d; Avg: %.2f/s, EOFs: %d; Lag (bytes): %d",
				count,
				count-lastStatCount,
				float64(count-lastStatCount)/(now.Sub(lastStatTs).Seconds()),
				eofs,
				lag)
			lastStatTs = now
			lastStatCount = count
			eofs = 0
		}

		if eof {
			if conf.oneshot {
				break
			} else {
				time.Sleep(1 * time.Second)
			}
		}
	}

	totalTime := time.Since(startTime)

	if conf.oneshot {
		log.Info("Indexed %d events: time=%.2fs; avg=%d/s", count, totalTime.Seconds(),
			uint64(float64(count)/totalTime.Seconds()))
	}
}