Exemplo n.º 1
0
// GetKeywordType is a crude way of determining if the template is using
// Logstash 5 keyword type, or Logstash 2 "raw" type.
func (es *ElasticSearch) GetKeywordType(index string) (string, error) {
	if index == "" {
		index = es.EventBaseIndex
	}
	template, err := es.GetTemplate(index)
	if err != nil {
		log.Warning("Failed to get template from Elastic Search, keyword resolution delayed.")
		return "", nil
	}

	version := template.GetMap(index).Get("version")
	log.Debug("Found template version %v", version)

	dynamicTemplates := template.GetMap(index).
		GetMap("mappings").
		GetMap("_default_").
		GetMapList("dynamic_templates")
	if dynamicTemplates == nil {
		log.Warning("Failed to parse template, keyword resolution delayed.")
		log.Warning("Template: %s", util.ToJson(template))
		return "", nil
	}
	for _, entry := range dynamicTemplates {
		if entry["string_fields"] != nil {
			mappingType := entry.GetMap("string_fields").
				GetMap("mapping").
				GetMap("fields").
				GetMap("keyword").
				Get("type")
			if mappingType == "keyword" {
				return "keyword", nil
			}

			if entry.GetMap("string_fields").GetMap("mapping").GetMap("fields").GetMap("raw") != nil {
				return "raw", nil
			}
		}
	}
	log.Warning("Failed to parse template, keyword resolution delayed.")
	log.Warning("Template: %s", util.ToJson(template))
	return "", nil
}
Exemplo n.º 2
0
func (es *ElasticSearch) Search(query interface{}) (*SearchResponse, error) {

	if es.keyword == "" {
		log.Warning("Search keyword not known, trying again.")
		es.InitKeyword()
	}

	path := fmt.Sprintf("%s/_search", es.EventSearchIndex)
	response, err := es.HttpClient.PostJson(path, query)
	if err != nil {
		return nil, errors.WithStack(&DatastoreError{
			Message: "Failed to connect to Elastic Search",
			Cause:   err,
		})
	}
	result := SearchResponse{}
	if err := es.Decode(response, &result); err != nil {
		log.Println("Failed to decode response...")
		return nil, err
	}
	return &result, nil
}
Exemplo n.º 3
0
// RemoveTagsFromAlertGroup removes the given tags from all alerts matching
// the provided parameters.
func (s *EventService) RemoveTagsFromAlertGroup(p core.AlertGroupQueryParams, tags []string) error {

	filter := []interface{}{
		ExistsQuery("event_type"),
		KeywordTermQuery("event_type", "alert", s.es.keyword),
		RangeQuery{
			Field: "timestamp",
			Gte:   p.MinTimestamp,
			Lte:   p.MaxTimestamp,
		},
		KeywordTermQuery("src_ip", p.SrcIP, s.es.keyword),
		KeywordTermQuery("dest_ip", p.DstIP, s.es.keyword),
		TermQuery("alert.signature_id", p.SignatureID),
	}

	for _, tag := range tags {
		filter = append(filter, TermQuery("tags", tag))
	}

	query := m{
		"query": m{
			"bool": m{
				"filter": filter,
			},
		},
		"_source": "tags",
		"sort": l{
			"_doc",
		},
		"size": 10000,
	}

	log.Println(util.ToJson(query))

	searchResponse, err := s.es.SearchScroll(query, "1m")
	if err != nil {
		log.Error("Failed to initialize scroll: %v", err)
		return err
	}

	scrollID := searchResponse.ScrollId

	for {

		log.Debug("Search response total: %d; hits: %d",
			searchResponse.Hits.Total, len(searchResponse.Hits.Hits))

		if len(searchResponse.Hits.Hits) == 0 {
			break
		}

		// We do this in a retry loop as some documents may fail to be
		// updated. Most likely rejected due to max thread count or
		// something.
		maxRetries := 5
		retries := 0
		for {
			retry, err := bulkUpdateTags(s.es, searchResponse.Hits.Hits,
				nil, tags)
			if err != nil {
				log.Error("BulkAddTags failed: %v", err)
				return err
			}
			if !retry {
				break
			}
			retries++
			if retries > maxRetries {
				log.Warning("Errors occurred archive events, not all events may have been archived.")
				break
			}
		}

		// Get next set of events to archive.
		searchResponse, err = s.es.Scroll(scrollID, "1m")
		if err != nil {
			log.Error("Failed to fetch from scroll: %v", err)
			return err
		}

	}

	response, err := s.es.DeleteScroll(scrollID)
	if err != nil {
		log.Error("Failed to delete scroll id: %v", err)
	}
	io.Copy(ioutil.Discard, response.Body)

	s.es.Refresh()

	return nil
}
Exemplo n.º 4
0
func (s *ReportService) ReportAggs(agg string, options core.ReportOptions) (interface{}, error) {

	size := int64(10)

	query := NewEventQuery()

	// Event type...
	if options.EventType != "" {
		query.EventType(options.EventType)
	}

	// Narrow the type even further...
	if options.DnsType != "" {
		query.AddFilter(TermQuery("dns.type", options.DnsType))
	}

	if options.QueryString != "" {
		query.AddFilter(QueryString(options.QueryString))
	}

	if options.AddressFilter != "" {
		query.ShouldHaveIp(options.AddressFilter, s.es.keyword)
	}

	if options.TimeRange != "" {
		err := query.AddTimeRangeFilter(options.TimeRange)
		if err != nil {
			return nil, err
		}
	}

	if options.Size > 0 {
		size = options.Size
	}

	aggregations := map[string]string{
		// Generic.
		"src_ip":    "keyword",
		"dest_ip":   "keyword",
		"src_port":  "term",
		"dest_port": "term",

		// Alert.
		"alert.category":  "keyword",
		"alert.signature": "keyword",

		// DNS.
		"dns.rrname": "keyword",
		"dns.rrtype": "keyword",
		"dns.rcode":  "keyword",
		"dns.rdata":  "keyword",

		// SSH.
		"ssh.client.software_version": "keyword",
		"ssh.server.software_version": "keyword",
	}

	aggType := aggregations[agg]
	if aggType == "" {
		log.Warning("Unknown aggregation type for %s, will use term.", agg)
		aggType = "term"
	}

	if aggType == "keyword" {
		query.Aggs[agg] = map[string]interface{}{
			"terms": map[string]interface{}{
				"field": fmt.Sprintf("%s.%s", agg, s.es.keyword),
				"size":  size,
			},
		}
	} else {
		query.Aggs[agg] = map[string]interface{}{
			"terms": map[string]interface{}{
				"field": agg,
				"size":  size,
			},
		}
	}

	response, err := s.es.Search(query)
	if err != nil {
		return nil, err
	}

	// Unwrap response.
	buckets := JsonMap(response.Aggregations[agg].(map[string]interface{})).GetMapList("buckets")
	data := []map[string]interface{}{}
	for _, bucket := range buckets {
		data = append(data, map[string]interface{}{
			"key":   bucket["key"],
			"count": bucket["doc_count"],
		})
	}

	return map[string]interface{}{
		"data": data,
	}, nil
}