func (f *GeoipFilter) AddGeoIP(event RawEveEvent) {
if f.db == nil {
return
}
srcip, ok := event["src_ip"].(string)
if ok && !IsRFC1918(srcip) {
gip, err := f.db.LookupString(srcip)
if err != nil {
log.Debug("Failed to lookup geoip for %s", srcip)
}
// Need at least a continent code.
if gip.ContinentCode != "" {
event["geoip"] = gip
}
}
if event["geoip"] == nil {
destip, ok := event["dest_ip"].(string)
if ok && !IsRFC1918(destip) {
gip, err := f.db.LookupString(destip)
if err != nil {
log.Debug("Failed to lookup geoip for %s", destip)
}
// Need at least a continent code.
if gip.ContinentCode != "" {
event["geoip"] = gip
}
}
}
}
func (m *Migrator) Migrate() error {
var currentVersion int
nextVersion := 0
rows, err := m.db.Query("select max(version) from schema")
if err == nil {
if rows.Next() {
if err := rows.Scan(¤tVersion); err != nil {
return err
}
nextVersion = currentVersion + 1
}
log.Debug("Current database schema version: %d", currentVersion)
} else {
log.Debug("Initializing database.")
}
for {
script, err := resources.AssetString(fmt.Sprintf("sqlite/V%d.sql", nextVersion))
if err != nil {
break
}
log.Info("Updating database to version %d.", nextVersion)
tx, err := m.db.Begin()
if err != nil {
return err
}
_, err = tx.Exec(script)
if err != nil {
return err
}
err = m.setVersion(tx, nextVersion)
if err != nil {
return err
}
err = tx.Commit()
if err != nil {
return err
}
nextVersion++
}
return nil
}
func (s *DataStore) ArchiveAlertGroup(p core.AlertGroupQueryParams) error {
sql := `UPDATE events SET archived = 1 WHERE`
builder := SqlBuilder{}
builder.WhereEquals("archived", 0)
builder.WhereEquals(
"json_extract(events.source, '$.alert.signature_id')",
p.SignatureID)
builder.WhereEquals(
"json_extract(events.source, '$.src_ip')",
p.SrcIP)
builder.WhereEquals(
"json_extract(events.source, '$.dest_ip')",
p.DstIP)
if p.MaxTimestamp != "" {
ts, err := eveTs2SqliteTs(p.MaxTimestamp)
if err != nil {
return err
}
builder.WhereLte("timestamp", ts)
}
sql = strings.Replace(sql, "WHERE", builder.BuildWhere(), 1)
start := time.Now()
r, err := s.db.DB.Exec(sql, builder.args...)
if err != nil {
return err
}
rows, _ := r.RowsAffected()
log.Debug("Archived %d alerts.", rows)
duration := time.Now().Sub(start).Seconds()
log.Debug("Archive query time: %v", duration)
return nil
}
func (es *ElasticSearch) GetTemplate(name string) (JsonMap, error) {
log.Debug("Fetching template [%s]", name)
response, err := es.HttpClient.Get(fmt.Sprintf("_template/%s", name))
if err != nil {
return nil, err
}
template := JsonMap{}
if err := es.Decode(response, &template); err != nil {
return nil, err
}
return template, nil
}
// GetKeywordType is a crude way of determining if the template is using
// Logstash 5 keyword type, or Logstash 2 "raw" type.
func (es *ElasticSearch) GetKeywordType(index string) (string, error) {
if index == "" {
index = es.EventBaseIndex
}
template, err := es.GetTemplate(index)
if err != nil {
log.Warning("Failed to get template from Elastic Search, keyword resolution delayed.")
return "", nil
}
version := template.GetMap(index).Get("version")
log.Debug("Found template version %v", version)
dynamicTemplates := template.GetMap(index).
GetMap("mappings").
GetMap("_default_").
GetMapList("dynamic_templates")
if dynamicTemplates == nil {
log.Warning("Failed to parse template, keyword resolution delayed.")
log.Warning("Template: %s", util.ToJson(template))
return "", nil
}
for _, entry := range dynamicTemplates {
if entry["string_fields"] != nil {
mappingType := entry.GetMap("string_fields").
GetMap("mapping").
GetMap("fields").
GetMap("keyword").
Get("type")
if mappingType == "keyword" {
return "keyword", nil
}
if entry.GetMap("string_fields").GetMap("mapping").GetMap("fields").GetMap("raw") != nil {
return "raw", nil
}
}
}
log.Warning("Failed to parse template, keyword resolution delayed.")
log.Warning("Template: %s", util.ToJson(template))
return "", nil
}
// RemoveTagsFromAlertGroup removes the given tags from all alerts matching
// the provided parameters.
func (s *EventService) RemoveTagsFromAlertGroup(p core.AlertGroupQueryParams, tags []string) error {
filter := []interface{}{
ExistsQuery("event_type"),
KeywordTermQuery("event_type", "alert", s.es.keyword),
RangeQuery{
Field: "timestamp",
Gte: p.MinTimestamp,
Lte: p.MaxTimestamp,
},
KeywordTermQuery("src_ip", p.SrcIP, s.es.keyword),
KeywordTermQuery("dest_ip", p.DstIP, s.es.keyword),
TermQuery("alert.signature_id", p.SignatureID),
}
for _, tag := range tags {
filter = append(filter, TermQuery("tags", tag))
}
query := m{
"query": m{
"bool": m{
"filter": filter,
},
},
"_source": "tags",
"sort": l{
"_doc",
},
"size": 10000,
}
log.Println(util.ToJson(query))
searchResponse, err := s.es.SearchScroll(query, "1m")
if err != nil {
log.Error("Failed to initialize scroll: %v", err)
return err
}
scrollID := searchResponse.ScrollId
for {
log.Debug("Search response total: %d; hits: %d",
searchResponse.Hits.Total, len(searchResponse.Hits.Hits))
if len(searchResponse.Hits.Hits) == 0 {
break
}
// We do this in a retry loop as some documents may fail to be
// updated. Most likely rejected due to max thread count or
// something.
maxRetries := 5
retries := 0
for {
retry, err := bulkUpdateTags(s.es, searchResponse.Hits.Hits,
nil, tags)
if err != nil {
log.Error("BulkAddTags failed: %v", err)
return err
}
if !retry {
break
}
retries++
if retries > maxRetries {
log.Warning("Errors occurred archive events, not all events may have been archived.")
break
}
}
// Get next set of events to archive.
searchResponse, err = s.es.Scroll(scrollID, "1m")
if err != nil {
log.Error("Failed to fetch from scroll: %v", err)
return err
}
}
response, err := s.es.DeleteScroll(scrollID)
if err != nil {
log.Error("Failed to delete scroll id: %v", err)
}
io.Copy(ioutil.Discard, response.Body)
s.es.Refresh()
return nil
}
func (r *Router) GET(path string, handler http.Handler) {
log.Debug("Adding GET route: %s", path)
r.router.Handle(path, handler).Methods("GET")
}
func (s *DataStore) EventQuery(options core.EventQueryOptions) (interface{}, error) {
size := int64(500)
if options.Size > 0 {
size = options.Size
}
sql := `select events.id, events.timestamp, events.source`
sqlBuilder := SqlBuilder{}
sqlBuilder.From("events")
if options.EventType != "" {
sqlBuilder.WhereEquals("json_extract(events.source, '$.event_type')", options.EventType)
}
fts := []string{}
if options.QueryString != "" {
words, _ := shellwords.Parse(options.QueryString)
for _, word := range words {
log.Debug("Word: %s", word)
parts := strings.SplitN(word, "=", 2)
if len(parts) == 2 {
field := parts[0]
valuestr := parts[1]
var arg interface{}
valueint, err := strconv.ParseInt(valuestr, 0, 64)
if err == nil {
arg = valueint
} else {
arg = valuestr
}
sqlBuilder.WhereEquals(
fmt.Sprintf(" json_extract(events.source, '$.%s')", field),
arg)
} else {
fts = append(fts, fmt.Sprintf("\"%s\"", parts[0]))
}
}
}
if options.MaxTs != "" {
maxTs, err := time.Parse("2006-01-02T15:04:05.999999", options.MaxTs)
if err != nil {
return nil, fmt.Errorf("Bad timestamp: %s", options.MaxTs)
}
sqlBuilder.WhereLte("datetime(events.timestamp)", maxTs)
}
if options.MinTs != "" {
minTs, err := time.Parse("2006-01-02T15:04:05.999999", options.MinTs)
if err != nil {
return nil, fmt.Errorf("Bad timestamp: %s", options.MinTs)
}
sqlBuilder.WhereGte("datetime(events.timestamp)", minTs)
}
if len(fts) > 0 {
sqlBuilder.From("events_fts")
sqlBuilder.Where("events.id == events_fts.id")
sqlBuilder.Where(fmt.Sprintf("events_fts MATCH '%s'", strings.Join(fts, " AND ")))
}
sql += sqlBuilder.BuildFrom()
if sqlBuilder.HasWhere() {
sql += sqlBuilder.BuildWhere()
}
sql += fmt.Sprintf(" ORDER BY timestamp DESC")
sql += fmt.Sprintf(" LIMIT %d", size)
log.Println(sql)
rows, err := s.db.Query(sql, sqlBuilder.args...)
if err != nil {
return nil, err
}
events := []interface{}{}
for rows.Next() {
var rawSource []byte
var id uuid.UUID
var timestamp string
err = rows.Scan(&id, ×tamp, &rawSource)
if err != nil {
return nil, err
}
source := map[string]interface{}{}
decoder := json.NewDecoder(bytes.NewReader(rawSource))
decoder.UseNumber()
err = decoder.Decode(&source)
if err != nil {
return nil, err
}
source["@timestamp"] = timestamp
events = append(events, map[string]interface{}{
"_id": id.String(),
"_source": source,
})
}
return map[string]interface{}{
"data": events,
}, nil
}
func (er *EveReader) Reopen() error {
log.Debug("Reopening %s", er.path)
er.file.Close()
return er.OpenFile()
}
func configure(args []string) Config {
flagset = flag.NewFlagSet("import", flag.ExitOnError)
flagset.Usage = usage
configFilename := flagset.StringP("config", "c", "", "Configuration file")
verbose := flagset.BoolP("verbose", "v", false, "Verbose output")
elasticSearchUri := flagset.StringP("elasticsearch", "e", "", "Elastic Search URL")
username := flagset.StringP("username", "u", "", "Username")
password := flagset.StringP("password", "p", "", "Password")
noCheckCertificate := flagset.BoolP("no-check-certificate", "k", false, "Disable certificate check")
index := flagset.String("index", DEFAULT_INDEX, "Elastic Search index prefix")
oneshot := flagset.Bool("oneshot", false, "One shot mode (exit on EOF)")
stdout := flagset.Bool("stdout", false, "Print events to stdout")
end := flagset.Bool("end", false, "Start at end of file")
batchSize := flagset.Uint64("batch-size", 1000, "Batch import size")
useBookmark := flagset.Bool("bookmark", false, "Bookmark location")
bookmarkPath := flagset.String("bookmark-path", "", "Path to bookmark file")
noGeoIp := flagset.Bool("no-geoip", false, "Disable GeoIP lookups")
geoIpDatabase := flagset.String("geoip-database", "", "Path to GeoIP (v2) database file")
flagset.Parse(args[1:])
if *verbose {
log.Info("Setting log level to debug")
log.SetLevel(log.DEBUG)
}
configWrapper := ConfigWrapper{
Config: Config{
Index: DEFAULT_INDEX,
BatchSize: 1000,
},
}
if *configFilename != "" {
log.Debug("Loading configuration file %s", *configFilename)
err := config.LoadConfigTo(*configFilename, &configWrapper)
if err != nil {
log.Fatal(err)
}
}
conf := configWrapper.Config
flagset.Visit(func(flag *flag.Flag) {
log.Debug("Found command line argument %s -> %s", flag.Name,
flag.Value.String())
switch flag.Name {
case "elasticsearch":
conf.Url = *elasticSearchUri
case "username":
conf.Username = *username
case "password":
conf.Password = *password
case "no-check-certificate":
conf.DisableCertificateCheck = *noCheckCertificate
case "index":
conf.Index = *index
case "oneshot":
conf.oneshot = *oneshot
case "stdout":
conf.stdout = *stdout
case "end":
conf.End = *end
case "batch-size":
conf.BatchSize = *batchSize
case "bookmark":
conf.Bookmark = *useBookmark
case "bookmark-path":
conf.BookmarkPath = *bookmarkPath
case "no-geoip":
conf.DisableGeoIp = *noGeoIp
case "geoip-database":
conf.GeoIpDatabase = *geoIpDatabase
case "verbose":
conf.Verbose = *verbose
case "config":
default:
log.Notice("Unhandle configuration flag %s", flag.Name)
}
})
if len(flagset.Args()) == 1 {
conf.InputFilename = flagset.Args()[0]
} else if len(flagset.Args()) > 1 {
log.Fatal("Multiple input filenames not allowed")
}
return conf
}
func Main(args []string) {
conf := configure(args)
if conf.BatchSize < 1 {
log.Fatal("Batch size must be greater than 0")
}
if conf.Url == "" {
log.Error("error: --elasticsearch is a required parameter")
usage()
os.Exit(1)
}
if conf.InputFilename == "" {
log.Fatal("error: no input file provided")
}
if conf.Bookmark && conf.BookmarkPath == "" {
conf.BookmarkPath = fmt.Sprintf("%s.bookmark", conf.InputFilename)
log.Info("Using bookmark file %s", conf.BookmarkPath)
}
es := elasticsearch.New(conf.Url)
es.DisableCertCheck(conf.DisableCertificateCheck)
if conf.Username != "" || conf.Password != "" {
if err := es.SetUsernamePassword(conf.Username,
conf.Password); err != nil {
log.Fatal("Failed to set username and password: %v", err)
}
}
response, err := es.Ping()
if err != nil {
log.Fatal("error: failed to ping Elastic Search:", err)
}
log.Info("Connected to Elastic Search v%s (cluster:%s; name: %s)",
response.Version.Number, response.ClusterName, response.Name)
majorVersion := response.MajorVersion()
// Check if the template exists.
templateExists, err := es.CheckTemplate(conf.Index)
if !templateExists {
log.Info("Template %s does not exist, creating...", conf.Index)
err = es.LoadTemplate(conf.Index, majorVersion)
if err != nil {
log.Fatal("Failed to create template:", err)
}
} else {
log.Info("Template %s exists, will not create.", conf.Index)
}
var geoipFilter *eve.GeoipFilter
tagsFilter := &eve.TagsFilter{}
if !conf.DisableGeoIp {
geoipdb, err := geoip.NewGeoIpDb(conf.GeoIpDatabase)
if err != nil {
log.Notice("Failed to load GeoIP database: %v", err)
} else {
log.Info("Using GeoIP database %s, %s", geoipdb.Type(), geoipdb.BuildDate())
geoipFilter = eve.NewGeoipFilter(geoipdb)
}
}
indexer := elasticsearch.NewIndexer(es, conf.DisableCertificateCheck)
indexer.IndexPrefix = conf.Index
reader, err := evereader.New(conf.InputFilename)
if err != nil {
log.Fatal(err)
}
// Initialize bookmarking...
var bookmarker *evereader.Bookmarker = nil
if conf.Bookmark {
bookmarker = &evereader.Bookmarker{
Filename: conf.BookmarkPath,
Reader: reader,
}
err := bookmarker.Init(conf.End)
if err != nil {
log.Fatal(err)
}
} else if conf.End {
log.Info("Jumping to end of file.")
err := reader.SkipToEnd()
if err != nil {
log.Fatal(err)
}
}
count := uint64(0)
lastStatTs := time.Now()
lastStatCount := uint64(0)
startTime := time.Now()
// Number of EOFs in last stat interval.
eofs := uint64(0)
go func() {
err := indexer.Run()
if err != nil {
log.Fatal("Elastic Search indexer connection unexpectedly closed:", err)
} else {
log.Debug("Indexer exited without issue.")
}
}()
for {
eof := false
event, err := reader.Next()
if err != nil {
if err == io.EOF {
eof = true
eofs++
} else if _, ok := err.(evereader.MalformedEventError); ok {
log.Error("Failed to read event but will continue: %v", err)
} else {
log.Fatalf("Unrecoverable error reading event: %v", err)
}
}
if event != nil {
if geoipFilter != nil {
geoipFilter.AddGeoIP(event)
}
tagsFilter.Filter(event)
if conf.stdout {
asJson, err := json.Marshal(event)
if err != nil {
log.Error("Failed to print event as json: %v", err)
} else {
fmt.Println(string(asJson))
}
}
indexer.IndexRawEvent(event)
count++
}
if eof || (count > 0 && count%conf.BatchSize == 0) {
var bookmark *evereader.Bookmark = nil
if conf.Bookmark {
bookmark = bookmarker.GetBookmark()
}
response, err := indexer.FlushConnection()
if err != nil {
log.Fatal(err)
}
if response != nil {
log.Debug("Indexed %d events {errors=%v}", len(response.Items),
response.Errors)
}
if conf.Bookmark {
bookmarker.WriteBookmark(bookmark)
}
}
now := time.Now()
if now.Sub(lastStatTs).Seconds() > 1 && now.Second() == 0 {
// Calculate the lag in bytes, that is the number of bytes behind
// the end of file we are.
lag, err := GetLag(reader)
if err != nil {
log.Error("Failed to calculate lag: %v", err)
}
log.Info("Total: %d; Last minute: %d; Avg: %.2f/s, EOFs: %d; Lag (bytes): %d",
count,
count-lastStatCount,
float64(count-lastStatCount)/(now.Sub(lastStatTs).Seconds()),
eofs,
lag)
lastStatTs = now
lastStatCount = count
eofs = 0
}
if eof {
if conf.oneshot {
break
} else {
time.Sleep(1 * time.Second)
}
}
}
totalTime := time.Since(startTime)
if conf.oneshot {
log.Info("Indexed %d events: time=%.2fs; avg=%d/s", count, totalTime.Seconds(),
uint64(float64(count)/totalTime.Seconds()))
}
}
