func (f decodeJSONFields) Run(event common.MapStr) (common.MapStr, error) {
var errs []string
for _, field := range f.fields {
data, err := event.GetValue(field)
if err != nil && errors.Cause(err) != common.ErrKeyNotFound {
debug("Error trying to GetValue for field : %s in event : %v", field, event)
errs = append(errs, err.Error())
continue
}
text, ok := data.(string)
if ok {
var output interface{}
err := unmarshal(f.maxDepth, []byte(text), &output, f.processArray)
if err != nil {
debug("Error trying to unmarshal %s", event[field])
errs = append(errs, err.Error())
continue
}
_, err = event.Put(field, output)
if err != nil {
debug("Error trying to Put value %v for field : %s", output, field)
errs = append(errs, err.Error())
continue
}
}
}
if len(errs) > 0 {
return event, fmt.Errorf(strings.Join(errs, ", "))
}
return event, nil
}
func TestValidJSONDepthTwo(t *testing.T) {
input := common.MapStr{
"msg": "{\"log\":\"{\\\"level\\\":\\\"info\\\"}\",\"stream\":\"stderr\",\"count\":3}",
"pipeline": "us1",
}
testConfig, _ = common.NewConfigFrom(map[string]interface{}{
"fields": fields,
"processArray": false,
"maxDepth": 2,
})
actual := getActualValue(t, testConfig, input)
expected := common.MapStr{
"msg": map[string]interface{}{
"log": map[string]interface{}{
"level": "info",
},
"stream": "stderr",
"count": 3,
},
"pipeline": "us1",
}
assert.Equal(t, expected.String(), actual.String())
}
func (c *Condition) checkEquals(event common.MapStr) bool {
for field, equalValue := range c.equals {
value, err := event.GetValue(field)
if err != nil {
return false
}
intValue, err := extractInt(value)
if err == nil {
if intValue != equalValue.Int {
return false
}
} else {
sValue, err := extractString(value)
if err != nil {
logp.Warn("unexpected type %T in equals condition as it accepts only integers and strings. ", value)
return false
}
if sValue != equalValue.Str {
return false
}
}
}
return true
}
func (c *Condition) CheckRegexp(event common.MapStr) bool {
for field, equalValue := range c.Regexp {
value, err := event.GetValue(field)
if err != nil {
logp.Debug("filter", "unavailable field %s: %v", field, err)
return false
}
switch value.(type) {
case string:
if !equalValue.MatchString(value.(string)) {
return false
}
default:
logp.Warn("unexpected type %T in regexp condition as it accepts only strings. ", value)
return false
}
}
return true
}
func (c *Condition) checkContains(event common.MapStr) bool {
outer:
for field, equalValue := range c.contains {
value, err := event.GetValue(field)
if err != nil {
return false
}
switch value.(type) {
case string:
if !strings.Contains(value.(string), equalValue) {
return false
}
case *string:
if !strings.Contains(*value.(*string), equalValue) {
return false
}
case []string:
for _, s := range value.([]string) {
if strings.Contains(s, equalValue) {
continue outer
}
}
return false
default:
logp.Warn("unexpected type %T in contains condition as it accepts only strings.", value)
return false
}
}
return true
}
func MakePingAllIPPortFactory(
fields common.MapStr,
ports []uint16,
f func(*net.IPAddr, uint16) (common.MapStr, error),
) func(*net.IPAddr) TaskRunner {
if len(ports) == 1 {
port := ports[0]
fields := fields.Clone()
fields["port"] = strconv.Itoa(int(port))
return MakePingIPFactory(fields, func(ip *net.IPAddr) (common.MapStr, error) {
return f(ip, port)
})
}
return MakePingAllIPFactory(fields, func(ip *net.IPAddr) []func() (common.MapStr, error) {
funcs := make([]func() (common.MapStr, error), len(ports))
for i := range ports {
port := ports[i]
funcs[i] = func() (common.MapStr, error) {
event, err := f(ip, port)
if event == nil {
event = common.MapStr{}
}
event["port"] = strconv.Itoa(int(port))
return event, err
}
}
return funcs
})
}
func (c *Condition) CheckEquals(event common.MapStr) bool {
for field, equalValue := range c.Equals {
value, err := event.GetValue(field)
if err != nil {
logp.Debug("filter", "unavailable field %s: %v", field, err)
return false
}
switch value.(type) {
case uint8, uint16, uint32, uint64, int8, int16, int32, int64, int, uint:
return value == equalValue.Int
case string:
return value == equalValue.Str
default:
logp.Warn("unexpected type %T in equals condition as it accepts only integers and strings. ", value)
return false
}
}
return true
}
func processGroups(groups []string, topic string, pids map[int32]int64) []common.MapStr {
var events []common.MapStr
for _, group := range groups {
pid_offsets, err := getConsumerOffsets(group, topic, pids)
if err == nil {
for pid, offset := range pid_offsets {
event := common.MapStr{
"@timestamp": common.Time(time.Now()),
"type": "consumer",
"partition": pid,
"topic": topic,
"group": group,
"offset": offset,
}
size, ok := pids[pid]
if ok {
event.Update(common.MapStr{"lag": size - offset})
}
events = append(events, event)
}
} else {
logp.Debug("kafkabeat", "No offsets for group %s on topic %s", group, topic)
}
}
return events
}
// TODO: move to libbeat/common?
func fieldString(event common.MapStr, field string) (string, error) {
type stringer interface {
String() string
}
v, err := event.GetValue(field)
if err != nil {
return "", err
}
switch s := v.(type) {
case string:
return s, nil
case []byte:
return string(s), nil
case stringer:
return s.String(), nil
case int8, int16, int32, int64, int:
i := reflect.ValueOf(s).Int()
return strconv.FormatInt(i, 10), nil
case uint8, uint16, uint32, uint64, uint:
u := reflect.ValueOf(s).Uint()
return strconv.FormatUint(u, 10), nil
case float32:
return strconv.FormatFloat(float64(s), 'g', -1, 32), nil
case float64:
return strconv.FormatFloat(s, 'g', -1, 64), nil
default:
logp.Warn("Can not convert key '%v' value to string", v)
return "", errConvertString
}
}
func newTCPMonitorHostJob(
scheme, host string, port uint16,
tls *transport.TLSConfig,
config *Config,
) (monitors.Job, error) {
typ := config.Name
timeout := config.Timeout
jobName := jobName(typ, jobType(scheme), host, []uint16{port})
validator := makeValidateConn(config)
pingAddr := net.JoinHostPort(host, strconv.Itoa(int(port)))
taskDialer, err := buildDialerChain(scheme, tls, config)
if err != nil {
return nil, err
}
return monitors.MakeSimpleJob(jobName, typ, func() (common.MapStr, error) {
event := common.MapStr{
"scheme": scheme,
"port": port,
"host": host,
}
dialer, err := taskDialer.BuildWithMeasures(event)
if err != nil {
return event, err
}
results, err := pingHost(dialer, pingAddr, timeout, validator)
event.Update(results)
return event, err
}), nil
}
// filterEvent validates an event for common required fields with types.
// If event is to be filtered out the reason is returned as error.
func filterEvent(event common.MapStr) error {
ts, ok := event["@timestamp"]
if !ok {
return errors.New("Missing '@timestamp' field from event")
}
_, ok = ts.(common.Time)
if !ok {
return errors.New("Invalid '@timestamp' field from event.")
}
err := event.EnsureCountField()
if err != nil {
return err
}
t, ok := event["type"]
if !ok {
return errors.New("Missing 'type' field from event.")
}
_, ok = t.(string)
if !ok {
return errors.New("Invalid 'type' field from event.")
}
return nil
}
func (f dropFields) Run(event common.MapStr) (common.MapStr, error) {
for _, field := range f.Fields {
err := event.Delete(field)
if err != nil {
return event, fmt.Errorf("Fail to delete key %s: %s", field, err)
}
}
return event, nil
}
func (p addCloudMetadata) Run(event common.MapStr) (common.MapStr, error) {
if len(p.metadata) == 0 {
return event, nil
}
// This overwrites the meta.cloud if it exists. But the cloud key should be
// reserved for this processor so this should happen.
_, err := event.Put("meta.cloud", p.metadata)
return event, err
}
// DeDotLabels returns a new common.MapStr containing a copy of the labels
// where the dots in each label name have been changed to an underscore.
func DeDotLabels(labels map[string]string) common.MapStr {
outputLabels := common.MapStr{}
for k, v := range labels {
// This is necessary so that ES does not interpret '.' fields as new
// nested JSON objects, and also makes this compatible with ES 2.x.
label := strings.Replace(k, ".", "_", -1)
outputLabels.Put(label, v)
}
return outputLabels
}
// TODO: move to libbeat/common?
func fieldString(event common.MapStr, field string) (string, error) {
v, err := event.GetValue(field)
if err != nil {
return "", err
}
s, err := tryConvString(v)
if err != nil {
logp.Warn("Can not convert key '%v' value to string", v)
}
return s, err
}
func (f dropFields) Run(event common.MapStr) (common.MapStr, error) {
errors := []string{}
for _, field := range f.Fields {
err := event.Delete(field)
if err != nil {
errors = append(errors, err.Error())
}
}
return event, fmt.Errorf(strings.Join(errors, ", "))
}
func equalEvent(expectedEvent common.MapStr, event common.MapStr) bool {
// Remove labels to test
// expectedLabels := expectedEvent["containerLabels"]
// labels := event["containerLabels"]
expectedEvent["containerLabels"] = []common.MapStr{}
event["containerLabels"] = []common.MapStr{}
// test equality
return expectedEvent.String() == event.String()
// TODO test labels
}
func TestMissingKey(t *testing.T) {
input := common.MapStr{
"pipeline": "us1",
}
actual := getActualValue(t, testConfig, input)
expected := common.MapStr{
"pipeline": "us1",
}
assert.Equal(t, expected.String(), actual.String())
}
func (f includeFields) Run(event common.MapStr) (common.MapStr, error) {
filtered := common.MapStr{}
errs := []string{}
for _, field := range f.Fields {
err := event.CopyFieldsTo(filtered, field)
// Ignore errors caused by a field not existing in the event.
if err != nil && errors.Cause(err) != common.ErrKeyNotFound {
errs = append(errs, err.Error())
}
}
return filtered, fmt.Errorf(strings.Join(errs, ", "))
}
func (f *DropFields) Filter(event common.MapStr) (common.MapStr, error) {
if f.Cond != nil && !f.Cond.Check(event) {
return event, nil
}
for _, field := range f.Fields {
err := event.Delete(field)
if err != nil {
return event, fmt.Errorf("Fail to delete key %s: %s", field, err)
}
}
return event, nil
}
func TestInvalidJSON(t *testing.T) {
input := common.MapStr{
"msg": "{\"log\":\"{\\\"level\\\":\\\"info\\\"}\",\"stream\":\"stderr\",\"count\":3",
"pipeline": "us1",
}
actual := getActualValue(t, testConfig, input)
expected := common.MapStr{
"msg": "{\"log\":\"{\\\"level\\\":\\\"info\\\"}\",\"stream\":\"stderr\",\"count\":3",
"pipeline": "us1",
}
assert.Equal(t, expected.String(), actual.String())
}
func TestFieldNotString(t *testing.T) {
input := common.MapStr{
"msg": 123,
"pipeline": "us1",
}
actual := getActualValue(t, testConfig, input)
expected := common.MapStr{
"msg": 123,
"pipeline": "us1",
}
assert.Equal(t, expected.String(), actual.String())
}
func (f includeFields) Run(event common.MapStr) (common.MapStr, error) {
filtered := common.MapStr{}
for _, field := range f.Fields {
hasKey, err := event.HasKey(field)
if err != nil {
return filtered, fmt.Errorf("Fail to check the key %s: %s", field, err)
}
if hasKey {
errorOnCopy := event.CopyFieldsTo(filtered, field)
if errorOnCopy != nil {
return filtered, fmt.Errorf("Fail to copy key %s: %s", field, err)
}
}
}
return filtered, nil
}
func (f includeFields) Run(event common.MapStr) (common.MapStr, error) {
filtered := common.MapStr{}
errors := []string{}
for _, field := range f.Fields {
hasKey, err := event.HasKey(field)
if err != nil {
errors = append(errors, err.Error())
}
if hasKey {
errorOnCopy := event.CopyFieldsTo(filtered, field)
if errorOnCopy != nil {
errors = append(errors, err.Error())
}
}
}
return filtered, fmt.Errorf(strings.Join(errors, ", "))
}
// Applies a sequence of filtering rules and returns the filtered event
func (filters *FilterList) Filter(event common.MapStr) common.MapStr {
// Check if filters are set, just return event if not
if len(filters.filters) == 0 {
return event
}
// clone the event at first, before starting filtering
filtered := event.Clone()
var err error
for _, filter := range filters.filters {
filtered, err = filter.Filter(filtered)
if err != nil {
logp.Debug("filter", "fail to apply filtering rule %s: %s", filter, err)
}
}
return filtered
}
func TestValidJSONDepthOne(t *testing.T) {
input := common.MapStr{
"msg": "{\"log\":\"{\\\"level\\\":\\\"info\\\"}\",\"stream\":\"stderr\",\"count\":3}",
"pipeline": "us1",
}
actual := getActualValue(t, testConfig, input)
expected := common.MapStr{
"msg": map[string]interface{}{
"log": "{\"level\":\"info\"}",
"stream": "stderr",
"count": 3,
},
"pipeline": "us1",
}
assert.Equal(t, expected.String(), actual.String())
}
func (c *client) filterEvent(event common.MapStr) *common.MapStr {
if event = common.ConvertToGenericEvent(event); event == nil {
logp.Err("fail to convert to a generic event")
return nil
}
// process the event by applying the configured actions
publishEvent := c.publisher.Processors.Run(event)
if publishEvent == nil {
// the event is dropped
logp.Debug("publish", "Drop event %s", event.StringToPrint())
return nil
}
if logp.IsDebug("publish") {
logp.Debug("publish", "Publish: %s", publishEvent.StringToPrint())
}
return &publishEvent
}
func hashFieldValue(h hash.Hash32, event common.MapStr, field string) error {
type stringer interface {
String() string
}
type hashable interface {
Hash32(h hash.Hash32) error
}
v, err := event.GetValue(field)
if err != nil {
return err
}
switch s := v.(type) {
case hashable:
err = s.Hash32(h)
case string:
_, err = h.Write([]byte(s))
case []byte:
_, err = h.Write(s)
case stringer:
_, err = h.Write([]byte(s.String()))
case int8, int16, int32, int64, int,
uint8, uint16, uint32, uint64, uint:
err = binary.Write(h, binary.LittleEndian, v)
case float32:
tmp := strconv.FormatFloat(float64(s), 'g', -1, 32)
_, err = h.Write([]byte(tmp))
case float64:
tmp := strconv.FormatFloat(s, 'g', -1, 32)
_, err = h.Write([]byte(tmp))
default:
// try to hash using reflection:
err = binary.Write(h, binary.LittleEndian, v)
if err != nil {
err = fmt.Errorf("can not hash key '%v' of unknown type", field)
}
}
return err
}
func (c *Condition) checkRegexp(event common.MapStr) bool {
for field, equalValue := range c.regexp {
value, err := event.GetValue(field)
if err != nil {
return false
}
sValue, err := extractString(value)
if err != nil {
logp.Warn("unexpected type %T in regexp condition as it accepts only strings. ", value)
return false
}
if !equalValue.MatchString(sValue) {
return false
}
}
return true
}
// Applies a sequence of processing rules and returns the filtered event
func (procs *Processors) Run(event common.MapStr) common.MapStr {
// Check if processors are set, just return event if not
if len(procs.list) == 0 {
return event
}
// clone the event at first, before starting filtering
filtered := event.Clone()
var err error
for _, p := range procs.list {
filtered, err = p.Run(filtered)
if err != nil {
logp.Debug("filter", "fail to apply processor %s: %s", p, err)
}
if filtered == nil {
// drop event
return nil
}
}
return filtered
}