Пример #1
0
// New instantiates a new output plugin instance publishing to elasticsearch.
func New(cfg *common.Config, topologyExpire int) (outputs.Outputer, error) {
	if !cfg.HasField("bulk_max_size") {
		cfg.SetInt("bulk_max_size", -1, defaultBulkSize)
	}

	output := &elasticsearchOutput{}
	err := output.init(cfg, topologyExpire)
	if err != nil {
		return nil, err
	}
	return output, nil
}
Пример #2
0
func new(beatName string, cfg *common.Config, _ int) (outputs.Outputer, error) {

	if !cfg.HasField("index") {
		cfg.SetString("index", -1, beatName)
	}

	output := &logstash{}
	if err := output.init(cfg); err != nil {
		return nil, err
	}
	return output, nil
}
Пример #3
0
// New instantiates a new output plugin instance publishing to elasticsearch.
func New(beatName string, cfg *common.Config, topologyExpire int) (outputs.Outputer, error) {
	if !cfg.HasField("bulk_max_size") {
		cfg.SetInt("bulk_max_size", -1, defaultBulkSize)
	}

	if !cfg.HasField("index") {
		pattern := fmt.Sprintf("%v-%%{+yyyy.MM.dd}", beatName)
		cfg.SetString("index", -1, pattern)
	}

	output := &elasticsearchOutput{beatName: beatName}
	err := output.init(cfg, topologyExpire)
	if err != nil {
		return nil, err
	}
	return output, nil
}
Пример #4
0
func addCondition(
	cfg common.Config,
	p Processor,
) (Processor, error) {
	if !cfg.HasField("when") {
		return p, nil
	}
	sub, err := cfg.Child("when", -1)
	if err != nil {
		return nil, err
	}

	condConfig := ConditionConfig{}
	if err := sub.Unpack(&condConfig); err != nil {
		return nil, err
	}

	return NewConditionRule(condConfig, p)
}
Пример #5
0
func BuildSelectorFromConfig(
	cfg *common.Config,
	settings Settings,
) (Selector, error) {
	var sel []SelectorExpr

	key := settings.Key
	multiKey := settings.MultiKey
	found := false

	if cfg.HasField(multiKey) {
		found = true
		sub, err := cfg.Child(multiKey, -1)
		if err != nil {
			return Selector{}, err
		}

		var table []*common.Config
		if err := sub.Unpack(&table); err != nil {
			return Selector{}, err
		}

		for _, config := range table {
			action, err := buildSingle(config, key)
			if err != nil {
				return Selector{}, err
			}

			if action != nilSelector {
				sel = append(sel, action)
			}
		}
	}

	if settings.EnableSingleOnly && cfg.HasField(key) {
		found = true

		// expect event-format-string
		str, err := cfg.String(key, -1)
		if err != nil {
			return Selector{}, err
		}

		fmtstr, err := fmtstr.CompileEvent(str)
		if err != nil {
			return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key))
		}

		if fmtstr.IsConst() {
			str, err := fmtstr.Run(common.MapStr{})
			if err != nil {
				return Selector{}, err
			}

			if str != "" {
				sel = append(sel, ConstSelectorExpr(str))
			}
		} else {
			sel = append(sel, FmtSelectorExpr(fmtstr, ""))
		}
	}

	if settings.FailEmpty && !found {
		if settings.EnableSingleOnly {
			return Selector{}, fmt.Errorf("missing required '%v' or '%v' in %v",
				key, multiKey, cfg.Path())
		}

		return Selector{}, fmt.Errorf("missing required '%v' in %v",
			multiKey, cfg.Path())
	}

	return MakeSelector(sel...), nil
}
Пример #6
0
func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) {
	// TODO: check for unknown fields

	// 1. extract required key-word handler
	if !cfg.HasField(key) {
		return nil, fmt.Errorf("missing %v", cfg.PathOf(key))
	}

	str, err := cfg.String(key, -1)
	if err != nil {
		return nil, err
	}

	evtfmt, err := fmtstr.CompileEvent(str)
	if err != nil {
		return nil, fmt.Errorf("%v in %v", err, cfg.PathOf(key))
	}

	// 2. extract optional `default` value
	var otherwise string
	if cfg.HasField("default") {
		tmp, err := cfg.String("default", -1)
		if err != nil {
			return nil, err
		}
		otherwise = tmp
	}

	// 3. extract optional `mapping`
	mapping := struct {
		Table map[string]string `config:"mappings"`
	}{nil}
	if cfg.HasField("mappings") {
		if err := cfg.Unpack(&mapping); err != nil {
			return nil, err
		}
	}

	// 4. extract conditional
	var cond *processors.Condition
	if cfg.HasField("when") {
		sub, err := cfg.Child("when", -1)
		if err != nil {
			return nil, err
		}

		condConfig := processors.ConditionConfig{}
		if err := sub.Unpack(&condConfig); err != nil {
			return nil, err
		}

		tmp, err := processors.NewCondition(&condConfig)
		if err != nil {
			return nil, err
		}

		cond = tmp
	}

	// 5. build selector from available fields
	var sel SelectorExpr
	if len(mapping.Table) > 0 {
		if evtfmt.IsConst() {
			str, err := evtfmt.Run(common.MapStr{})
			if err != nil {
				return nil, err
			}

			str = mapping.Table[str]
			if str == "" {
				str = otherwise
			}

			if str == "" {
				sel = nilSelector
			} else {
				sel = ConstSelectorExpr(str)
			}
		} else {
			sel = &mapSelector{
				from:      FmtSelectorExpr(evtfmt, ""),
				to:        mapping.Table,
				otherwise: otherwise,
			}
		}
	} else {
		if evtfmt.IsConst() {
			str, err := evtfmt.Run(common.MapStr{})
			if err != nil {
				return nil, err
			}

			if str == "" {
				sel = nilSelector
			} else {
				sel = ConstSelectorExpr(str)
			}
		} else {
			sel = FmtSelectorExpr(evtfmt, otherwise)
		}
	}
	if cond != nil && sel != nilSelector {
		sel = ConditionalSelectorExpr(sel, cond)
	}

	return sel, nil
}
Пример #7
0
func (r *redisOut) init(cfg *common.Config, expireTopo int) error {
	config := defaultConfig
	if err := cfg.Unpack(&config); err != nil {
		return err
	}

	sendRetries := config.MaxRetries
	maxAttempts := config.MaxRetries + 1
	if sendRetries < 0 {
		maxAttempts = 0
	}

	var dataType redisDataType
	switch config.DataType {
	case "", "list":
		dataType = redisListType
	case "channel":
		dataType = redisChannelType
	default:
		return errors.New("Bad Redis data type")
	}

	if cfg.HasField("index") && !cfg.HasField("key") {
		s, err := cfg.String("index", -1)
		if err != nil {
			return err
		}
		if err := cfg.SetString("key", -1, s); err != nil {
			return err
		}
	}
	if !cfg.HasField("key") {
		cfg.SetString("key", -1, r.beatName)
	}

	key, err := outil.BuildSelectorFromConfig(cfg, outil.Settings{
		Key:              "key",
		MultiKey:         "keys",
		EnableSingleOnly: true,
		FailEmpty:        true,
	})
	if err != nil {
		return err
	}

	tls, err := outputs.LoadTLSConfig(config.TLS)
	if err != nil {
		return err
	}

	transp := &transport.Config{
		Timeout: config.Timeout,
		Proxy:   &config.Proxy,
		TLS:     tls,
		Stats: &transport.IOStats{
			Read:        statReadBytes,
			Write:       statWriteBytes,
			ReadErrors:  statReadErrors,
			WriteErrors: statWriteErrors,
		},
	}

	// configure topology support
	r.topology.init(transp, topoConfig{
		host:     config.HostTopology,
		password: config.PasswordTopology,
		db:       config.DbTopology,
		expire:   time.Duration(expireTopo) * time.Second,
	})

	// configure publisher clients
	clients, err := modeutil.MakeClients(cfg, func(host string) (mode.ProtocolClient, error) {
		t, err := transport.NewClient(transp, "tcp", host, config.Port)
		if err != nil {
			return nil, err
		}
		return newClient(t, config.Password, config.Db, key, dataType), nil
	})
	if err != nil {
		return err
	}

	logp.Info("Max Retries set to: %v", sendRetries)
	m, err := modeutil.NewConnectionMode(clients, modeutil.Settings{
		Failover:     !config.LoadBalance,
		MaxAttempts:  maxAttempts,
		Timeout:      config.Timeout,
		WaitRetry:    defaultWaitRetry,
		MaxWaitRetry: defaultMaxWaitRetry,
	})
	if err != nil {
		return err
	}

	r.mode = m
	return nil
}