// GetKeywordType is a crude way of determining if the template is using
// Logstash 5 keyword type, or Logstash 2 "raw" type.
func (es *ElasticSearch) GetKeywordType(index string) (string, error) {
if index == "" {
index = es.EventBaseIndex
}
template, err := es.GetTemplate(index)
if err != nil {
log.Warning("Failed to get template from Elastic Search, keyword resolution delayed.")
return "", nil
}
version := template.GetMap(index).Get("version")
log.Debug("Found template version %v", version)
dynamicTemplates := template.GetMap(index).
GetMap("mappings").
GetMap("_default_").
GetMapList("dynamic_templates")
if dynamicTemplates == nil {
log.Warning("Failed to parse template, keyword resolution delayed.")
log.Warning("Template: %s", util.ToJson(template))
return "", nil
}
for _, entry := range dynamicTemplates {
if entry["string_fields"] != nil {
mappingType := entry.GetMap("string_fields").
GetMap("mapping").
GetMap("fields").
GetMap("keyword").
Get("type")
if mappingType == "keyword" {
return "keyword", nil
}
if entry.GetMap("string_fields").GetMap("mapping").GetMap("fields").GetMap("raw") != nil {
return "raw", nil
}
}
}
log.Warning("Failed to parse template, keyword resolution delayed.")
log.Warning("Template: %s", util.ToJson(template))
return "", nil
}
func (es *ElasticSearch) Search(query interface{}) (*SearchResponse, error) {
if es.keyword == "" {
log.Warning("Search keyword not known, trying again.")
es.InitKeyword()
}
path := fmt.Sprintf("%s/_search", es.EventSearchIndex)
response, err := es.HttpClient.PostJson(path, query)
if err != nil {
return nil, errors.WithStack(&DatastoreError{
Message: "Failed to connect to Elastic Search",
Cause: err,
})
}
result := SearchResponse{}
if err := es.Decode(response, &result); err != nil {
log.Println("Failed to decode response...")
return nil, err
}
return &result, nil
}
// RemoveTagsFromAlertGroup removes the given tags from all alerts matching
// the provided parameters.
func (s *EventService) RemoveTagsFromAlertGroup(p core.AlertGroupQueryParams, tags []string) error {
filter := []interface{}{
ExistsQuery("event_type"),
KeywordTermQuery("event_type", "alert", s.es.keyword),
RangeQuery{
Field: "timestamp",
Gte: p.MinTimestamp,
Lte: p.MaxTimestamp,
},
KeywordTermQuery("src_ip", p.SrcIP, s.es.keyword),
KeywordTermQuery("dest_ip", p.DstIP, s.es.keyword),
TermQuery("alert.signature_id", p.SignatureID),
}
for _, tag := range tags {
filter = append(filter, TermQuery("tags", tag))
}
query := m{
"query": m{
"bool": m{
"filter": filter,
},
},
"_source": "tags",
"sort": l{
"_doc",
},
"size": 10000,
}
log.Println(util.ToJson(query))
searchResponse, err := s.es.SearchScroll(query, "1m")
if err != nil {
log.Error("Failed to initialize scroll: %v", err)
return err
}
scrollID := searchResponse.ScrollId
for {
log.Debug("Search response total: %d; hits: %d",
searchResponse.Hits.Total, len(searchResponse.Hits.Hits))
if len(searchResponse.Hits.Hits) == 0 {
break
}
// We do this in a retry loop as some documents may fail to be
// updated. Most likely rejected due to max thread count or
// something.
maxRetries := 5
retries := 0
for {
retry, err := bulkUpdateTags(s.es, searchResponse.Hits.Hits,
nil, tags)
if err != nil {
log.Error("BulkAddTags failed: %v", err)
return err
}
if !retry {
break
}
retries++
if retries > maxRetries {
log.Warning("Errors occurred archive events, not all events may have been archived.")
break
}
}
// Get next set of events to archive.
searchResponse, err = s.es.Scroll(scrollID, "1m")
if err != nil {
log.Error("Failed to fetch from scroll: %v", err)
return err
}
}
response, err := s.es.DeleteScroll(scrollID)
if err != nil {
log.Error("Failed to delete scroll id: %v", err)
}
io.Copy(ioutil.Discard, response.Body)
s.es.Refresh()
return nil
}
func (s *ReportService) ReportAggs(agg string, options core.ReportOptions) (interface{}, error) {
size := int64(10)
query := NewEventQuery()
// Event type...
if options.EventType != "" {
query.EventType(options.EventType)
}
// Narrow the type even further...
if options.DnsType != "" {
query.AddFilter(TermQuery("dns.type", options.DnsType))
}
if options.QueryString != "" {
query.AddFilter(QueryString(options.QueryString))
}
if options.AddressFilter != "" {
query.ShouldHaveIp(options.AddressFilter, s.es.keyword)
}
if options.TimeRange != "" {
err := query.AddTimeRangeFilter(options.TimeRange)
if err != nil {
return nil, err
}
}
if options.Size > 0 {
size = options.Size
}
aggregations := map[string]string{
// Generic.
"src_ip": "keyword",
"dest_ip": "keyword",
"src_port": "term",
"dest_port": "term",
// Alert.
"alert.category": "keyword",
"alert.signature": "keyword",
// DNS.
"dns.rrname": "keyword",
"dns.rrtype": "keyword",
"dns.rcode": "keyword",
"dns.rdata": "keyword",
// SSH.
"ssh.client.software_version": "keyword",
"ssh.server.software_version": "keyword",
}
aggType := aggregations[agg]
if aggType == "" {
log.Warning("Unknown aggregation type for %s, will use term.", agg)
aggType = "term"
}
if aggType == "keyword" {
query.Aggs[agg] = map[string]interface{}{
"terms": map[string]interface{}{
"field": fmt.Sprintf("%s.%s", agg, s.es.keyword),
"size": size,
},
}
} else {
query.Aggs[agg] = map[string]interface{}{
"terms": map[string]interface{}{
"field": agg,
"size": size,
},
}
}
response, err := s.es.Search(query)
if err != nil {
return nil, err
}
// Unwrap response.
buckets := JsonMap(response.Aggregations[agg].(map[string]interface{})).GetMapList("buckets")
data := []map[string]interface{}{}
for _, bucket := range buckets {
data = append(data, map[string]interface{}{
"key": bucket["key"],
"count": bucket["doc_count"],
})
}
return map[string]interface{}{
"data": data,
}, nil
}