// GetKeywordType is a crude way of determining if the template is using
// Logstash 5 keyword type, or Logstash 2 "raw" type.
func (es *ElasticSearch) GetKeywordType(index string) (string, error) {
if index == "" {
index = es.EventBaseIndex
}
template, err := es.GetTemplate(index)
if err != nil {
log.Warning("Failed to get template from Elastic Search, keyword resolution delayed.")
return "", nil
}
version := template.GetMap(index).Get("version")
log.Debug("Found template version %v", version)
dynamicTemplates := template.GetMap(index).
GetMap("mappings").
GetMap("_default_").
GetMapList("dynamic_templates")
if dynamicTemplates == nil {
log.Warning("Failed to parse template, keyword resolution delayed.")
log.Warning("Template: %s", util.ToJson(template))
return "", nil
}
for _, entry := range dynamicTemplates {
if entry["string_fields"] != nil {
mappingType := entry.GetMap("string_fields").
GetMap("mapping").
GetMap("fields").
GetMap("keyword").
Get("type")
if mappingType == "keyword" {
return "keyword", nil
}
if entry.GetMap("string_fields").GetMap("mapping").GetMap("fields").GetMap("raw") != nil {
return "raw", nil
}
}
}
log.Warning("Failed to parse template, keyword resolution delayed.")
log.Warning("Template: %s", util.ToJson(template))
return "", nil
}
func logBulkUpdateError(item map[string]interface{}) {
update, ok := item["update"].(map[string]interface{})
if !ok || update == nil {
return
}
error := update["error"]
if error == nil {
return
}
log.Notice("Elastic Search bulk update error: %s", util.ToJson(error))
}
// RemoveTagsFromAlertGroup removes the given tags from all alerts matching
// the provided parameters.
func (s *EventService) RemoveTagsFromAlertGroup(p core.AlertGroupQueryParams, tags []string) error {
filter := []interface{}{
ExistsQuery("event_type"),
KeywordTermQuery("event_type", "alert", s.es.keyword),
RangeQuery{
Field: "timestamp",
Gte: p.MinTimestamp,
Lte: p.MaxTimestamp,
},
KeywordTermQuery("src_ip", p.SrcIP, s.es.keyword),
KeywordTermQuery("dest_ip", p.DstIP, s.es.keyword),
TermQuery("alert.signature_id", p.SignatureID),
}
for _, tag := range tags {
filter = append(filter, TermQuery("tags", tag))
}
query := m{
"query": m{
"bool": m{
"filter": filter,
},
},
"_source": "tags",
"sort": l{
"_doc",
},
"size": 10000,
}
log.Println(util.ToJson(query))
searchResponse, err := s.es.SearchScroll(query, "1m")
if err != nil {
log.Error("Failed to initialize scroll: %v", err)
return err
}
scrollID := searchResponse.ScrollId
for {
log.Debug("Search response total: %d; hits: %d",
searchResponse.Hits.Total, len(searchResponse.Hits.Hits))
if len(searchResponse.Hits.Hits) == 0 {
break
}
// We do this in a retry loop as some documents may fail to be
// updated. Most likely rejected due to max thread count or
// something.
maxRetries := 5
retries := 0
for {
retry, err := bulkUpdateTags(s.es, searchResponse.Hits.Hits,
nil, tags)
if err != nil {
log.Error("BulkAddTags failed: %v", err)
return err
}
if !retry {
break
}
retries++
if retries > maxRetries {
log.Warning("Errors occurred archive events, not all events may have been archived.")
break
}
}
// Get next set of events to archive.
searchResponse, err = s.es.Scroll(scrollID, "1m")
if err != nil {
log.Error("Failed to fetch from scroll: %v", err)
return err
}
}
response, err := s.es.DeleteScroll(scrollID)
if err != nil {
log.Error("Failed to delete scroll id: %v", err)
}
io.Copy(ioutil.Discard, response.Body)
s.es.Refresh()
return nil
}
// bulkUpdateTags will add and/or remvoe tags from a set of documents using
// the Elastic Search bulk API.
func bulkUpdateTags(es *ElasticSearch, documents []map[string]interface{},
addTags []string, rmTags []string) (bool, error) {
bulk := make([]string, 0)
for _, item := range documents {
doc := JsonMap(item)
currentTags := doc.GetMap("_source").GetAsStrings("tags")
tags := make([]string, 0)
for _, tag := range currentTags {
if rmTags == nil || !StringSliceContains(rmTags, tag) {
tags = append(tags, tag)
}
}
for _, tag := range addTags {
if !StringSliceContains(tags, tag) {
tags = append(tags, tag)
}
}
id := doc.Get("_id").(string)
docType := doc.Get("_type").(string)
index := doc.Get("_index").(string)
command := m{
"update": m{
"_id": id,
"_type": docType,
"_index": index,
},
}
bulk = append(bulk, util.ToJson(command))
partial := m{
"doc": m{
"tags": tags,
},
}
bulk = append(bulk, util.ToJson(partial))
}
// Needs to finish with a new line.
bulk = append(bulk, "")
bulkString := strings.Join(bulk, "\n")
response, err := es.HttpClient.PostString("_bulk", "application/json", bulkString)
if err != nil {
log.Error("Failed to update event tags: %v", err)
return false, err
}
retry := false
if response.StatusCode != http.StatusOK {
return retry, NewElasticSearchError(response)
} else {
bulkResponse := BulkResponse{}
if err := es.Decode(response, &bulkResponse); err != nil {
log.Error("Failed to decode bulk response: %v", err)
} else {
log.Info("Tags updated on %d events; errors=%v",
len(bulkResponse.Items), bulkResponse.Errors)
if bulkResponse.Errors {
retry = true
for _, item := range bulkResponse.Items {
logBulkUpdateError(item)
}
}
}
}
return retry, nil
}