Exemple #1
0
// GetKeywordType is a crude way of determining if the template is using
// Logstash 5 keyword type, or Logstash 2 "raw" type.
func (es *ElasticSearch) GetKeywordType(index string) (string, error) {
	if index == "" {
		index = es.EventBaseIndex
	}
	template, err := es.GetTemplate(index)
	if err != nil {
		log.Warning("Failed to get template from Elastic Search, keyword resolution delayed.")
		return "", nil
	}

	version := template.GetMap(index).Get("version")
	log.Debug("Found template version %v", version)

	dynamicTemplates := template.GetMap(index).
		GetMap("mappings").
		GetMap("_default_").
		GetMapList("dynamic_templates")
	if dynamicTemplates == nil {
		log.Warning("Failed to parse template, keyword resolution delayed.")
		log.Warning("Template: %s", util.ToJson(template))
		return "", nil
	}
	for _, entry := range dynamicTemplates {
		if entry["string_fields"] != nil {
			mappingType := entry.GetMap("string_fields").
				GetMap("mapping").
				GetMap("fields").
				GetMap("keyword").
				Get("type")
			if mappingType == "keyword" {
				return "keyword", nil
			}

			if entry.GetMap("string_fields").GetMap("mapping").GetMap("fields").GetMap("raw") != nil {
				return "raw", nil
			}
		}
	}
	log.Warning("Failed to parse template, keyword resolution delayed.")
	log.Warning("Template: %s", util.ToJson(template))
	return "", nil
}
Exemple #2
0
func logBulkUpdateError(item map[string]interface{}) {
	update, ok := item["update"].(map[string]interface{})
	if !ok || update == nil {
		return
	}
	error := update["error"]
	if error == nil {
		return
	}
	log.Notice("Elastic Search bulk update error: %s", util.ToJson(error))
}
Exemple #3
0
// RemoveTagsFromAlertGroup removes the given tags from all alerts matching
// the provided parameters.
func (s *EventService) RemoveTagsFromAlertGroup(p core.AlertGroupQueryParams, tags []string) error {

	filter := []interface{}{
		ExistsQuery("event_type"),
		KeywordTermQuery("event_type", "alert", s.es.keyword),
		RangeQuery{
			Field: "timestamp",
			Gte:   p.MinTimestamp,
			Lte:   p.MaxTimestamp,
		},
		KeywordTermQuery("src_ip", p.SrcIP, s.es.keyword),
		KeywordTermQuery("dest_ip", p.DstIP, s.es.keyword),
		TermQuery("alert.signature_id", p.SignatureID),
	}

	for _, tag := range tags {
		filter = append(filter, TermQuery("tags", tag))
	}

	query := m{
		"query": m{
			"bool": m{
				"filter": filter,
			},
		},
		"_source": "tags",
		"sort": l{
			"_doc",
		},
		"size": 10000,
	}

	log.Println(util.ToJson(query))

	searchResponse, err := s.es.SearchScroll(query, "1m")
	if err != nil {
		log.Error("Failed to initialize scroll: %v", err)
		return err
	}

	scrollID := searchResponse.ScrollId

	for {

		log.Debug("Search response total: %d; hits: %d",
			searchResponse.Hits.Total, len(searchResponse.Hits.Hits))

		if len(searchResponse.Hits.Hits) == 0 {
			break
		}

		// We do this in a retry loop as some documents may fail to be
		// updated. Most likely rejected due to max thread count or
		// something.
		maxRetries := 5
		retries := 0
		for {
			retry, err := bulkUpdateTags(s.es, searchResponse.Hits.Hits,
				nil, tags)
			if err != nil {
				log.Error("BulkAddTags failed: %v", err)
				return err
			}
			if !retry {
				break
			}
			retries++
			if retries > maxRetries {
				log.Warning("Errors occurred archive events, not all events may have been archived.")
				break
			}
		}

		// Get next set of events to archive.
		searchResponse, err = s.es.Scroll(scrollID, "1m")
		if err != nil {
			log.Error("Failed to fetch from scroll: %v", err)
			return err
		}

	}

	response, err := s.es.DeleteScroll(scrollID)
	if err != nil {
		log.Error("Failed to delete scroll id: %v", err)
	}
	io.Copy(ioutil.Discard, response.Body)

	s.es.Refresh()

	return nil
}
Exemple #4
0
// bulkUpdateTags will add and/or remvoe tags from a set of documents using
// the Elastic Search bulk API.
func bulkUpdateTags(es *ElasticSearch, documents []map[string]interface{},
	addTags []string, rmTags []string) (bool, error) {

	bulk := make([]string, 0)

	for _, item := range documents {
		doc := JsonMap(item)

		currentTags := doc.GetMap("_source").GetAsStrings("tags")
		tags := make([]string, 0)

		for _, tag := range currentTags {
			if rmTags == nil || !StringSliceContains(rmTags, tag) {
				tags = append(tags, tag)
			}
		}

		for _, tag := range addTags {
			if !StringSliceContains(tags, tag) {
				tags = append(tags, tag)
			}
		}

		id := doc.Get("_id").(string)
		docType := doc.Get("_type").(string)
		index := doc.Get("_index").(string)

		command := m{
			"update": m{
				"_id":    id,
				"_type":  docType,
				"_index": index,
			},
		}
		bulk = append(bulk, util.ToJson(command))

		partial := m{
			"doc": m{
				"tags": tags,
			},
		}
		bulk = append(bulk, util.ToJson(partial))
	}

	// Needs to finish with a new line.
	bulk = append(bulk, "")
	bulkString := strings.Join(bulk, "\n")
	response, err := es.HttpClient.PostString("_bulk", "application/json", bulkString)
	if err != nil {
		log.Error("Failed to update event tags: %v", err)
		return false, err
	}

	retry := false

	if response.StatusCode != http.StatusOK {
		return retry, NewElasticSearchError(response)
	} else {
		bulkResponse := BulkResponse{}

		if err := es.Decode(response, &bulkResponse); err != nil {
			log.Error("Failed to decode bulk response: %v", err)
		} else {
			log.Info("Tags updated on %d events; errors=%v",
				len(bulkResponse.Items), bulkResponse.Errors)
			if bulkResponse.Errors {
				retry = true
				for _, item := range bulkResponse.Items {
					logBulkUpdateError(item)
				}
			}
		}
	}

	return retry, nil
}