func TestAggregateSum(t *testing.T) {
log.SetLevel(log.ERROR)
es := New("http://10.16.1.10:9200")
size := int64(10)
eventType := "netflow"
query := NewEventQuery()
query.AddFilter(TermQuery("event_type", eventType))
aggType := "keyword"
agg := "src_ip"
sum := m{
"sum": m{
"sum": m{
"field": "netflow.pkts",
},
},
}
if aggType == "keyword" {
query.Aggs[agg] = map[string]interface{}{
"terms": map[string]interface{}{
"field": fmt.Sprintf("%s.%s", agg, es.keyword),
"size": size,
},
}
} else {
query.Aggs[agg] = map[string]interface{}{
"terms": map[string]interface{}{
"field": agg,
"size": size,
},
}
}
query.Aggs[agg].(map[string]interface{})["aggs"] = sum
fmt.Println(ToJsonPretty(query))
response, err := es.Search(query)
if err != nil {
t.Fatal(err)
}
fmt.Println(ToJsonPretty(response))
}
func Main(args []string) {
var end bool
var oneshot bool
var useBookmark bool
var bookmarkPath string
var verbose bool
var dbfile string
flagset := flag.NewFlagSet("sqliteimport", flag.ExitOnError)
flagset.StringVarP(&dbfile, "database", "D", "", "Database filename")
flagset.BoolVar(&end, "end", false, "Start at end of file")
flagset.BoolVar(&oneshot, "oneshot", false, "One shot mode (exit on EOF)")
flagset.BoolVar(&useBookmark, "bookmark", false, "Bookmark location")
flagset.StringVar(&bookmarkPath, "bookmark-path", "", "Path to bookmark file")
flagset.BoolVarP(&verbose, "verbose", "v", false, "Verbose output")
flagset.Parse(args)
if verbose {
log.SetLevel(log.DEBUG)
}
if dbfile == "" {
log.Fatal("Database filename must be provided.")
}
if len(flagset.Args()) == 0 {
log.Fatal("No input files provided.")
} else if len(flagset.Args()) > 1 {
log.Fatal("Only one input file allowed.")
}
inputFilename := flagset.Args()[0]
// If useBookmark but no path, set a default.
if useBookmark && bookmarkPath == "" {
bookmarkPath = fmt.Sprintf("%s.bookmark", inputFilename)
}
reader, err := evereader.New(flagset.Args()[0])
if err != nil {
log.Fatal(err)
}
// Initialize bookmark.
var bookmarker *evereader.Bookmarker = nil
if useBookmark {
bookmarker = &evereader.Bookmarker{
Filename: bookmarkPath,
Reader: reader,
}
err := bookmarker.Init(end)
if err != nil {
log.Fatal(err)
}
} else if end {
log.Info("Jumping to end of file.")
err := reader.SkipToEnd()
if err != nil {
log.Fatal(err)
}
}
db, err := sqlite.NewSqliteService(dbfile)
if err != nil {
log.Fatal(err)
}
if err := db.Migrate(); err != nil {
log.Fatal(err)
}
indexer, err := sqlite.NewSqliteIndexer(db)
if err != nil {
log.Fatal(err)
}
count := uint64(0)
lastStatTs := time.Now()
lastStatCount := uint64(0)
// Number of EOFs in last stat interval.
eofs := uint64(0)
tagsFilter := eve.TagsFilter{}
for {
eof := false
event, err := reader.Next()
if err != nil {
if err == io.EOF {
eof = true
eofs++
} else {
log.Fatal(err)
}
}
if event != nil {
tagsFilter.Filter(event)
indexer.IndexRawEve(event)
count++
if useBookmark {
bookmark := bookmarker.GetBookmark()
bookmarker.WriteBookmark(bookmark)
}
}
now := time.Now()
if now.Sub(lastStatTs).Seconds() > 1 {
log.Info("Total: %d; Last interval: %d; Avg: %.2f/s, EOFs: %d",
count,
count-lastStatCount,
float64(count-lastStatCount)/(now.Sub(lastStatTs).Seconds()),
eofs)
lastStatTs = now
lastStatCount = count
eofs = 0
indexer.Flush()
}
if eof {
if oneshot {
break
} else {
indexer.Flush()
time.Sleep(100 * time.Millisecond)
}
}
}
now := time.Now()
log.Info("Total: %d; Last interval: %d; Avg: %.2f/s, EOFs: %d",
count,
count-lastStatCount,
float64(count-lastStatCount)/(now.Sub(lastStatTs).Seconds()),
eofs)
indexer.Flush()
}
func Main(args []string) {
var end bool
var oneshot bool
var useBookmark bool
var bookmarkPath string
var verbose bool
flagset := flag.NewFlagSet("evereader", flag.ExitOnError)
flagset.BoolVar(&end, "end", false, "Start at end of file")
flagset.BoolVar(&oneshot, "oneshot", false, "One shot mode (exit on EOF)")
flagset.BoolVar(&useBookmark, "bookmark", false, "Bookmark location")
flagset.StringVar(&bookmarkPath, "bookmark-path", "", "Path to bookmark file")
flagset.BoolVarP(&verbose, "verbose", "v", false, "Verbose output")
flagset.Parse(args[1:])
if verbose {
log.SetLevel(log.DEBUG)
}
if len(flagset.Args()) == 0 {
log.Fatal("No input files provided.")
} else if len(flagset.Args()) > 1 {
log.Fatal("Only one input file allowed.")
}
inputFilename := flagset.Args()[0]
// If useBookmark but no path, set a default.
if useBookmark && bookmarkPath == "" {
bookmarkPath = fmt.Sprintf("%s.bookmark", inputFilename)
}
encoder := json.NewEncoder(os.Stdout)
reader, err := evereader.New(flagset.Args()[0])
if err != nil {
log.Fatal(err)
}
// Initialize bookmark.
var bookmarker *evereader.Bookmarker = nil
if useBookmark {
bookmarker = &evereader.Bookmarker{
Filename: bookmarkPath,
Reader: reader,
}
err := bookmarker.Init(end)
if err != nil {
log.Fatal(err)
}
} else if end {
log.Info("Jumping to end of file.")
err := reader.SkipToEnd()
if err != nil {
log.Fatal(err)
}
}
for {
eof := false
event, err := reader.Next()
if err != nil {
if err == io.EOF {
if oneshot {
break
}
eof = true
} else {
log.Fatal(err)
}
}
if eof {
time.Sleep(1 * time.Second)
} else {
encoder.Encode(event)
if useBookmark {
bookmark := bookmarker.GetBookmark()
bookmarker.WriteBookmark(bookmark)
}
}
}
}
func Main(args []string) {
var err error
log.Info("This is EveBox Server version %v (rev: %v)", core.BuildVersion, core.BuildRev)
flagset := flag.NewFlagSet("server", flag.ExitOnError)
flagset.StringVarP(&opts.ElasticSearchUri, "elasticsearch", "e", "", "Elastic Search URI (default: http://localhost:9200")
flagset.StringVarP(&opts.ElasticSearchIndex, "index", "i", "", "Elastic Search Index (default: logstash)")
flagset.StringVarP(&opts.Port, "port", "p", "5636", "Port to bind to")
flagset.StringVarP(&opts.Host, "host", "", "0.0.0.0", "Host to bind to")
flagset.StringVarP(&opts.DevServerUri, "dev", "", "", "Frontend development server URI")
flagset.BoolVarP(&opts.Version, "version", "", false, "Show version")
flagset.StringVarP(&opts.Config, "config", "c", "", "Configuration filename")
flagset.BoolVarP(&opts.NoCheckCertificate, "no-check-certificate", "k", false, "Disable certificate check for Elastic Search")
flagset.BoolVarP(&opts.Sqlite, "sqlite", "", false, "Use SQLite for the event store")
flagset.Parse(args[0:])
if opts.Version {
VersionMain()
return
}
log.SetLevel(log.DEBUG)
// If no configuration was provided, see if evebox.yaml exists
// in the current directory.
if opts.Config == "" {
_, err = os.Stat("./evebox.yaml")
if err == nil {
opts.Config = "./evebox.yaml"
}
}
if opts.Config != "" {
log.Printf("Loading configuration file %s.\n", opts.Config)
conf, err = config.LoadConfig(opts.Config)
if err != nil {
log.Fatal(err)
}
}
conf.ElasticSearchIndex = getElasticSearchIndex()
log.Info("Using ElasticSearch Index %s.", conf.ElasticSearchIndex)
appContext := server.AppContext{
Config: conf,
}
elasticSearch := elasticsearch.New(getElasticSearchUrl())
elasticSearch.SetEventIndex(conf.ElasticSearchIndex)
elasticSearch.InitKeyword()
pingResponse, err := elasticSearch.Ping()
if err != nil {
log.Error("Failed to ping Elastic Search: %v", err)
} else {
log.Info("Connected to Elastic Search (version: %s)",
pingResponse.Version.Number)
}
appContext.ElasticSearch = elasticSearch
appContext.EventService = elasticsearch.NewEventService(elasticSearch)
appContext.AlertQueryService = elasticsearch.NewAlertQueryService(elasticSearch)
appContext.EventQueryService = elasticsearch.NewEventQueryService(elasticSearch)
appContext.ReportService = elasticsearch.NewReportService(elasticSearch)
appContext.Vars.DevWebAppServerUrl = opts.DevServerUri
var datastoreType string = "elasticsearch"
if opts.Sqlite {
datastoreType = "sqlite"
}
if datastoreType == "elasticsearch" {
appContext.DataStore, err = elasticsearch.NewDataStore(elasticSearch)
if err != nil {
log.Fatal(err)
}
} else if datastoreType == "sqlite" {
appContext.DataStore, err = sqlite.NewDataStore()
if err != nil {
log.Fatal(err)
}
}
httpServer := server.NewServer(appContext)
err = httpServer.Start(opts.Host + ":" + opts.Port)
if err != nil {
log.Fatal(err)
}
}
func configure(args []string) Config {
flagset = flag.NewFlagSet("import", flag.ExitOnError)
flagset.Usage = usage
configFilename := flagset.StringP("config", "c", "", "Configuration file")
verbose := flagset.BoolP("verbose", "v", false, "Verbose output")
elasticSearchUri := flagset.StringP("elasticsearch", "e", "", "Elastic Search URL")
username := flagset.StringP("username", "u", "", "Username")
password := flagset.StringP("password", "p", "", "Password")
noCheckCertificate := flagset.BoolP("no-check-certificate", "k", false, "Disable certificate check")
index := flagset.String("index", DEFAULT_INDEX, "Elastic Search index prefix")
oneshot := flagset.Bool("oneshot", false, "One shot mode (exit on EOF)")
stdout := flagset.Bool("stdout", false, "Print events to stdout")
end := flagset.Bool("end", false, "Start at end of file")
batchSize := flagset.Uint64("batch-size", 1000, "Batch import size")
useBookmark := flagset.Bool("bookmark", false, "Bookmark location")
bookmarkPath := flagset.String("bookmark-path", "", "Path to bookmark file")
noGeoIp := flagset.Bool("no-geoip", false, "Disable GeoIP lookups")
geoIpDatabase := flagset.String("geoip-database", "", "Path to GeoIP (v2) database file")
flagset.Parse(args[1:])
if *verbose {
log.Info("Setting log level to debug")
log.SetLevel(log.DEBUG)
}
configWrapper := ConfigWrapper{
Config: Config{
Index: DEFAULT_INDEX,
BatchSize: 1000,
},
}
if *configFilename != "" {
log.Debug("Loading configuration file %s", *configFilename)
err := config.LoadConfigTo(*configFilename, &configWrapper)
if err != nil {
log.Fatal(err)
}
}
conf := configWrapper.Config
flagset.Visit(func(flag *flag.Flag) {
log.Debug("Found command line argument %s -> %s", flag.Name,
flag.Value.String())
switch flag.Name {
case "elasticsearch":
conf.Url = *elasticSearchUri
case "username":
conf.Username = *username
case "password":
conf.Password = *password
case "no-check-certificate":
conf.DisableCertificateCheck = *noCheckCertificate
case "index":
conf.Index = *index
case "oneshot":
conf.oneshot = *oneshot
case "stdout":
conf.stdout = *stdout
case "end":
conf.End = *end
case "batch-size":
conf.BatchSize = *batchSize
case "bookmark":
conf.Bookmark = *useBookmark
case "bookmark-path":
conf.BookmarkPath = *bookmarkPath
case "no-geoip":
conf.DisableGeoIp = *noGeoIp
case "geoip-database":
conf.GeoIpDatabase = *geoIpDatabase
case "verbose":
conf.Verbose = *verbose
case "config":
default:
log.Notice("Unhandle configuration flag %s", flag.Name)
}
})
if len(flagset.Args()) == 1 {
conf.InputFilename = flagset.Args()[0]
} else if len(flagset.Args()) > 1 {
log.Fatal("Multiple input filenames not allowed")
}
return conf
}
