Beispiel #1
0
func (f decodeJSONFields) Run(event common.MapStr) (common.MapStr, error) {
	var errs []string

	for _, field := range f.fields {
		data, err := event.GetValue(field)
		if err != nil && errors.Cause(err) != common.ErrKeyNotFound {
			debug("Error trying to GetValue for field : %s in event : %v", field, event)
			errs = append(errs, err.Error())
			continue
		}
		text, ok := data.(string)
		if ok {
			var output interface{}
			err := unmarshal(f.maxDepth, []byte(text), &output, f.processArray)
			if err != nil {
				debug("Error trying to unmarshal %s", event[field])
				errs = append(errs, err.Error())
				continue
			}

			_, err = event.Put(field, output)
			if err != nil {
				debug("Error trying to Put value %v for field : %s", output, field)
				errs = append(errs, err.Error())
				continue
			}
		}
	}

	if len(errs) > 0 {
		return event, fmt.Errorf(strings.Join(errs, ", "))
	}
	return event, nil
}
func TestValidJSONDepthTwo(t *testing.T) {
	input := common.MapStr{
		"msg":      "{\"log\":\"{\\\"level\\\":\\\"info\\\"}\",\"stream\":\"stderr\",\"count\":3}",
		"pipeline": "us1",
	}

	testConfig, _ = common.NewConfigFrom(map[string]interface{}{
		"fields":       fields,
		"processArray": false,
		"maxDepth":     2,
	})

	actual := getActualValue(t, testConfig, input)

	expected := common.MapStr{
		"msg": map[string]interface{}{
			"log": map[string]interface{}{
				"level": "info",
			},
			"stream": "stderr",
			"count":  3,
		},
		"pipeline": "us1",
	}

	assert.Equal(t, expected.String(), actual.String())

}
Beispiel #3
0
func (c *Condition) checkEquals(event common.MapStr) bool {

	for field, equalValue := range c.equals {

		value, err := event.GetValue(field)
		if err != nil {
			return false
		}

		intValue, err := extractInt(value)
		if err == nil {
			if intValue != equalValue.Int {
				return false
			}
		} else {
			sValue, err := extractString(value)
			if err != nil {
				logp.Warn("unexpected type %T in equals condition as it accepts only integers and strings. ", value)
				return false
			}
			if sValue != equalValue.Str {
				return false
			}
		}
	}

	return true

}
Beispiel #4
0
func (c *Condition) CheckRegexp(event common.MapStr) bool {

	for field, equalValue := range c.Regexp {

		value, err := event.GetValue(field)
		if err != nil {
			logp.Debug("filter", "unavailable field %s: %v", field, err)
			return false
		}

		switch value.(type) {
		case string:
			if !equalValue.MatchString(value.(string)) {
				return false
			}
		default:
			logp.Warn("unexpected type %T in regexp condition as it accepts only strings. ", value)
			return false
		}

	}

	return true

}
Beispiel #5
0
func (c *Condition) checkContains(event common.MapStr) bool {
outer:
	for field, equalValue := range c.contains {
		value, err := event.GetValue(field)
		if err != nil {
			return false
		}

		switch value.(type) {
		case string:
			if !strings.Contains(value.(string), equalValue) {
				return false
			}
		case *string:
			if !strings.Contains(*value.(*string), equalValue) {
				return false
			}
		case []string:
			for _, s := range value.([]string) {
				if strings.Contains(s, equalValue) {
					continue outer
				}
			}
			return false
		default:
			logp.Warn("unexpected type %T in contains condition as it accepts only strings.", value)
			return false
		}
	}

	return true
}
Beispiel #6
0
func MakePingAllIPPortFactory(
	fields common.MapStr,
	ports []uint16,
	f func(*net.IPAddr, uint16) (common.MapStr, error),
) func(*net.IPAddr) TaskRunner {
	if len(ports) == 1 {
		port := ports[0]
		fields := fields.Clone()
		fields["port"] = strconv.Itoa(int(port))
		return MakePingIPFactory(fields, func(ip *net.IPAddr) (common.MapStr, error) {
			return f(ip, port)
		})
	}

	return MakePingAllIPFactory(fields, func(ip *net.IPAddr) []func() (common.MapStr, error) {
		funcs := make([]func() (common.MapStr, error), len(ports))
		for i := range ports {
			port := ports[i]
			funcs[i] = func() (common.MapStr, error) {
				event, err := f(ip, port)
				if event == nil {
					event = common.MapStr{}
				}
				event["port"] = strconv.Itoa(int(port))
				return event, err
			}
		}
		return funcs
	})
}
Beispiel #7
0
func (c *Condition) CheckEquals(event common.MapStr) bool {

	for field, equalValue := range c.Equals {

		value, err := event.GetValue(field)
		if err != nil {
			logp.Debug("filter", "unavailable field %s: %v", field, err)
			return false
		}

		switch value.(type) {
		case uint8, uint16, uint32, uint64, int8, int16, int32, int64, int, uint:
			return value == equalValue.Int
		case string:
			return value == equalValue.Str
		default:
			logp.Warn("unexpected type %T in equals condition as it accepts only integers and strings. ", value)
			return false
		}

	}

	return true

}
Beispiel #8
0
func processGroups(groups []string, topic string, pids map[int32]int64) []common.MapStr {
	var events []common.MapStr
	for _, group := range groups {
		pid_offsets, err := getConsumerOffsets(group, topic, pids)
		if err == nil {
			for pid, offset := range pid_offsets {
				event := common.MapStr{
					"@timestamp": common.Time(time.Now()),
					"type":       "consumer",
					"partition":  pid,
					"topic":      topic,
					"group":      group,
					"offset":     offset,
				}
				size, ok := pids[pid]
				if ok {
					event.Update(common.MapStr{"lag": size - offset})
				}
				events = append(events, event)
			}
		} else {
			logp.Debug("kafkabeat", "No offsets for group %s on topic %s", group, topic)
		}
	}
	return events
}
Beispiel #9
0
// TODO: move to libbeat/common?
func fieldString(event common.MapStr, field string) (string, error) {
	type stringer interface {
		String() string
	}

	v, err := event.GetValue(field)
	if err != nil {
		return "", err
	}

	switch s := v.(type) {
	case string:
		return s, nil
	case []byte:
		return string(s), nil
	case stringer:
		return s.String(), nil
	case int8, int16, int32, int64, int:
		i := reflect.ValueOf(s).Int()
		return strconv.FormatInt(i, 10), nil
	case uint8, uint16, uint32, uint64, uint:
		u := reflect.ValueOf(s).Uint()
		return strconv.FormatUint(u, 10), nil
	case float32:
		return strconv.FormatFloat(float64(s), 'g', -1, 32), nil
	case float64:
		return strconv.FormatFloat(s, 'g', -1, 64), nil
	default:
		logp.Warn("Can not convert key '%v' value to string", v)
		return "", errConvertString
	}
}
Beispiel #10
0
func newTCPMonitorHostJob(
	scheme, host string, port uint16,
	tls *transport.TLSConfig,
	config *Config,
) (monitors.Job, error) {
	typ := config.Name
	timeout := config.Timeout
	jobName := jobName(typ, jobType(scheme), host, []uint16{port})
	validator := makeValidateConn(config)
	pingAddr := net.JoinHostPort(host, strconv.Itoa(int(port)))

	taskDialer, err := buildDialerChain(scheme, tls, config)
	if err != nil {
		return nil, err
	}

	return monitors.MakeSimpleJob(jobName, typ, func() (common.MapStr, error) {
		event := common.MapStr{
			"scheme": scheme,
			"port":   port,
			"host":   host,
		}
		dialer, err := taskDialer.BuildWithMeasures(event)
		if err != nil {
			return event, err
		}

		results, err := pingHost(dialer, pingAddr, timeout, validator)
		event.Update(results)
		return event, err
	}), nil
}
Beispiel #11
0
// filterEvent validates an event for common required fields with types.
// If event is to be filtered out the reason is returned as error.
func filterEvent(event common.MapStr) error {
	ts, ok := event["@timestamp"]
	if !ok {
		return errors.New("Missing '@timestamp' field from event")
	}

	_, ok = ts.(common.Time)
	if !ok {
		return errors.New("Invalid '@timestamp' field from event.")
	}

	err := event.EnsureCountField()
	if err != nil {
		return err
	}

	t, ok := event["type"]
	if !ok {
		return errors.New("Missing 'type' field from event.")
	}

	_, ok = t.(string)
	if !ok {
		return errors.New("Invalid 'type' field from event.")
	}

	return nil
}
Beispiel #12
0
func (f dropFields) Run(event common.MapStr) (common.MapStr, error) {
	for _, field := range f.Fields {
		err := event.Delete(field)
		if err != nil {
			return event, fmt.Errorf("Fail to delete key %s: %s", field, err)
		}

	}
	return event, nil
}
Beispiel #13
0
func (p addCloudMetadata) Run(event common.MapStr) (common.MapStr, error) {
	if len(p.metadata) == 0 {
		return event, nil
	}

	// This overwrites the meta.cloud if it exists. But the cloud key should be
	// reserved for this processor so this should happen.
	_, err := event.Put("meta.cloud", p.metadata)

	return event, err
}
Beispiel #14
0
// DeDotLabels returns a new common.MapStr containing a copy of the labels
// where the dots in each label name have been changed to an underscore.
func DeDotLabels(labels map[string]string) common.MapStr {
	outputLabels := common.MapStr{}
	for k, v := range labels {
		// This is necessary so that ES does not interpret '.' fields as new
		// nested JSON objects, and also makes this compatible with ES 2.x.
		label := strings.Replace(k, ".", "_", -1)
		outputLabels.Put(label, v)
	}

	return outputLabels
}
Beispiel #15
0
// TODO: move to libbeat/common?
func fieldString(event common.MapStr, field string) (string, error) {
	v, err := event.GetValue(field)
	if err != nil {
		return "", err
	}

	s, err := tryConvString(v)
	if err != nil {
		logp.Warn("Can not convert key '%v' value to string", v)
	}
	return s, err
}
Beispiel #16
0
func (f dropFields) Run(event common.MapStr) (common.MapStr, error) {
	errors := []string{}

	for _, field := range f.Fields {
		err := event.Delete(field)
		if err != nil {
			errors = append(errors, err.Error())
		}

	}
	return event, fmt.Errorf(strings.Join(errors, ", "))
}
Beispiel #17
0
func equalEvent(expectedEvent common.MapStr, event common.MapStr) bool {
	// Remove labels to test
	// expectedLabels := expectedEvent["containerLabels"]
	// labels := event["containerLabels"]
	expectedEvent["containerLabels"] = []common.MapStr{}
	event["containerLabels"] = []common.MapStr{}

	// test equality
	return expectedEvent.String() == event.String()

	// TODO test labels

}
Beispiel #18
0
func TestMissingKey(t *testing.T) {
	input := common.MapStr{
		"pipeline": "us1",
	}

	actual := getActualValue(t, testConfig, input)

	expected := common.MapStr{
		"pipeline": "us1",
	}

	assert.Equal(t, expected.String(), actual.String())
}
Beispiel #19
0
func (f includeFields) Run(event common.MapStr) (common.MapStr, error) {
	filtered := common.MapStr{}
	errs := []string{}

	for _, field := range f.Fields {
		err := event.CopyFieldsTo(filtered, field)
		// Ignore errors caused by a field not existing in the event.
		if err != nil && errors.Cause(err) != common.ErrKeyNotFound {
			errs = append(errs, err.Error())
		}
	}

	return filtered, fmt.Errorf(strings.Join(errs, ", "))
}
Beispiel #20
0
func (f *DropFields) Filter(event common.MapStr) (common.MapStr, error) {

	if f.Cond != nil && !f.Cond.Check(event) {
		return event, nil
	}

	for _, field := range f.Fields {
		err := event.Delete(field)
		if err != nil {
			return event, fmt.Errorf("Fail to delete key %s: %s", field, err)
		}

	}
	return event, nil
}
Beispiel #21
0
func TestInvalidJSON(t *testing.T) {
	input := common.MapStr{
		"msg":      "{\"log\":\"{\\\"level\\\":\\\"info\\\"}\",\"stream\":\"stderr\",\"count\":3",
		"pipeline": "us1",
	}

	actual := getActualValue(t, testConfig, input)

	expected := common.MapStr{
		"msg":      "{\"log\":\"{\\\"level\\\":\\\"info\\\"}\",\"stream\":\"stderr\",\"count\":3",
		"pipeline": "us1",
	}
	assert.Equal(t, expected.String(), actual.String())

}
Beispiel #22
0
func TestFieldNotString(t *testing.T) {
	input := common.MapStr{
		"msg":      123,
		"pipeline": "us1",
	}

	actual := getActualValue(t, testConfig, input)

	expected := common.MapStr{
		"msg":      123,
		"pipeline": "us1",
	}

	assert.Equal(t, expected.String(), actual.String())

}
Beispiel #23
0
func (f includeFields) Run(event common.MapStr) (common.MapStr, error) {
	filtered := common.MapStr{}

	for _, field := range f.Fields {
		hasKey, err := event.HasKey(field)
		if err != nil {
			return filtered, fmt.Errorf("Fail to check the key %s: %s", field, err)
		}

		if hasKey {
			errorOnCopy := event.CopyFieldsTo(filtered, field)
			if errorOnCopy != nil {
				return filtered, fmt.Errorf("Fail to copy key %s: %s", field, err)
			}
		}
	}

	return filtered, nil
}
Beispiel #24
0
func (f includeFields) Run(event common.MapStr) (common.MapStr, error) {
	filtered := common.MapStr{}
	errors := []string{}

	for _, field := range f.Fields {
		hasKey, err := event.HasKey(field)
		if err != nil {
			errors = append(errors, err.Error())
		}

		if hasKey {
			errorOnCopy := event.CopyFieldsTo(filtered, field)
			if errorOnCopy != nil {
				errors = append(errors, err.Error())
			}
		}
	}

	return filtered, fmt.Errorf(strings.Join(errors, ", "))
}
Beispiel #25
0
// Applies a sequence of filtering rules and returns the filtered event
func (filters *FilterList) Filter(event common.MapStr) common.MapStr {

	// Check if filters are set, just return event if not
	if len(filters.filters) == 0 {
		return event
	}

	// clone the event at first, before starting filtering
	filtered := event.Clone()
	var err error

	for _, filter := range filters.filters {
		filtered, err = filter.Filter(filtered)
		if err != nil {
			logp.Debug("filter", "fail to apply filtering rule %s: %s", filter, err)
		}
	}

	return filtered
}
Beispiel #26
0
func TestValidJSONDepthOne(t *testing.T) {
	input := common.MapStr{
		"msg":      "{\"log\":\"{\\\"level\\\":\\\"info\\\"}\",\"stream\":\"stderr\",\"count\":3}",
		"pipeline": "us1",
	}

	actual := getActualValue(t, testConfig, input)

	expected := common.MapStr{
		"msg": map[string]interface{}{
			"log":    "{\"level\":\"info\"}",
			"stream": "stderr",
			"count":  3,
		},
		"pipeline": "us1",
	}

	assert.Equal(t, expected.String(), actual.String())

}
Beispiel #27
0
func (c *client) filterEvent(event common.MapStr) *common.MapStr {

	if event = common.ConvertToGenericEvent(event); event == nil {
		logp.Err("fail to convert to a generic event")
		return nil

	}

	// process the event by applying the configured actions
	publishEvent := c.publisher.Processors.Run(event)
	if publishEvent == nil {
		// the event is dropped
		logp.Debug("publish", "Drop event %s", event.StringToPrint())
		return nil
	}
	if logp.IsDebug("publish") {
		logp.Debug("publish", "Publish: %s", publishEvent.StringToPrint())
	}
	return &publishEvent
}
Beispiel #28
0
func hashFieldValue(h hash.Hash32, event common.MapStr, field string) error {
	type stringer interface {
		String() string
	}

	type hashable interface {
		Hash32(h hash.Hash32) error
	}

	v, err := event.GetValue(field)
	if err != nil {
		return err
	}

	switch s := v.(type) {
	case hashable:
		err = s.Hash32(h)
	case string:
		_, err = h.Write([]byte(s))
	case []byte:
		_, err = h.Write(s)
	case stringer:
		_, err = h.Write([]byte(s.String()))
	case int8, int16, int32, int64, int,
		uint8, uint16, uint32, uint64, uint:
		err = binary.Write(h, binary.LittleEndian, v)
	case float32:
		tmp := strconv.FormatFloat(float64(s), 'g', -1, 32)
		_, err = h.Write([]byte(tmp))
	case float64:
		tmp := strconv.FormatFloat(s, 'g', -1, 32)
		_, err = h.Write([]byte(tmp))
	default:
		// try to hash using reflection:
		err = binary.Write(h, binary.LittleEndian, v)
		if err != nil {
			err = fmt.Errorf("can not hash key '%v' of unknown type", field)
		}
	}
	return err
}
Beispiel #29
0
func (c *Condition) checkRegexp(event common.MapStr) bool {

	for field, equalValue := range c.regexp {

		value, err := event.GetValue(field)
		if err != nil {
			return false
		}

		sValue, err := extractString(value)
		if err != nil {
			logp.Warn("unexpected type %T in regexp condition as it accepts only strings. ", value)
			return false
		}
		if !equalValue.MatchString(sValue) {
			return false
		}
	}

	return true

}
Beispiel #30
0
// Applies a sequence of processing rules and returns the filtered event
func (procs *Processors) Run(event common.MapStr) common.MapStr {

	// Check if processors are set, just return event if not
	if len(procs.list) == 0 {
		return event
	}

	// clone the event at first, before starting filtering
	filtered := event.Clone()
	var err error

	for _, p := range procs.list {
		filtered, err = p.Run(filtered)
		if err != nil {
			logp.Debug("filter", "fail to apply processor %s: %s", p, err)
		}
		if filtered == nil {
			// drop event
			return nil
		}
	}

	return filtered
}