// New instantiates a new output plugin instance publishing to elasticsearch.
func New(cfg *common.Config, topologyExpire int) (outputs.Outputer, error) {
if !cfg.HasField("bulk_max_size") {
cfg.SetInt("bulk_max_size", -1, defaultBulkSize)
}
output := &elasticsearchOutput{}
err := output.init(cfg, topologyExpire)
if err != nil {
return nil, err
}
return output, nil
}
func new(beatName string, cfg *common.Config, _ int) (outputs.Outputer, error) {
if !cfg.HasField("index") {
cfg.SetString("index", -1, beatName)
}
output := &logstash{}
if err := output.init(cfg); err != nil {
return nil, err
}
return output, nil
}
// New instantiates a new output plugin instance publishing to elasticsearch.
func New(beatName string, cfg *common.Config, topologyExpire int) (outputs.Outputer, error) {
if !cfg.HasField("bulk_max_size") {
cfg.SetInt("bulk_max_size", -1, defaultBulkSize)
}
if !cfg.HasField("index") {
pattern := fmt.Sprintf("%v-%%{+yyyy.MM.dd}", beatName)
cfg.SetString("index", -1, pattern)
}
output := &elasticsearchOutput{beatName: beatName}
err := output.init(cfg, topologyExpire)
if err != nil {
return nil, err
}
return output, nil
}
func addCondition(
cfg common.Config,
p Processor,
) (Processor, error) {
if !cfg.HasField("when") {
return p, nil
}
sub, err := cfg.Child("when", -1)
if err != nil {
return nil, err
}
condConfig := ConditionConfig{}
if err := sub.Unpack(&condConfig); err != nil {
return nil, err
}
return NewConditionRule(condConfig, p)
}
func BuildSelectorFromConfig(
cfg *common.Config,
settings Settings,
) (Selector, error) {
var sel []SelectorExpr
key := settings.Key
multiKey := settings.MultiKey
found := false
if cfg.HasField(multiKey) {
found = true
sub, err := cfg.Child(multiKey, -1)
if err != nil {
return Selector{}, err
}
var table []*common.Config
if err := sub.Unpack(&table); err != nil {
return Selector{}, err
}
for _, config := range table {
action, err := buildSingle(config, key)
if err != nil {
return Selector{}, err
}
if action != nilSelector {
sel = append(sel, action)
}
}
}
if settings.EnableSingleOnly && cfg.HasField(key) {
found = true
// expect event-format-string
str, err := cfg.String(key, -1)
if err != nil {
return Selector{}, err
}
fmtstr, err := fmtstr.CompileEvent(str)
if err != nil {
return Selector{}, fmt.Errorf("%v in %v", err, cfg.PathOf(key))
}
if fmtstr.IsConst() {
str, err := fmtstr.Run(common.MapStr{})
if err != nil {
return Selector{}, err
}
if str != "" {
sel = append(sel, ConstSelectorExpr(str))
}
} else {
sel = append(sel, FmtSelectorExpr(fmtstr, ""))
}
}
if settings.FailEmpty && !found {
if settings.EnableSingleOnly {
return Selector{}, fmt.Errorf("missing required '%v' or '%v' in %v",
key, multiKey, cfg.Path())
}
return Selector{}, fmt.Errorf("missing required '%v' in %v",
multiKey, cfg.Path())
}
return MakeSelector(sel...), nil
}
func buildSingle(cfg *common.Config, key string) (SelectorExpr, error) {
// TODO: check for unknown fields
// 1. extract required key-word handler
if !cfg.HasField(key) {
return nil, fmt.Errorf("missing %v", cfg.PathOf(key))
}
str, err := cfg.String(key, -1)
if err != nil {
return nil, err
}
evtfmt, err := fmtstr.CompileEvent(str)
if err != nil {
return nil, fmt.Errorf("%v in %v", err, cfg.PathOf(key))
}
// 2. extract optional `default` value
var otherwise string
if cfg.HasField("default") {
tmp, err := cfg.String("default", -1)
if err != nil {
return nil, err
}
otherwise = tmp
}
// 3. extract optional `mapping`
mapping := struct {
Table map[string]string `config:"mappings"`
}{nil}
if cfg.HasField("mappings") {
if err := cfg.Unpack(&mapping); err != nil {
return nil, err
}
}
// 4. extract conditional
var cond *processors.Condition
if cfg.HasField("when") {
sub, err := cfg.Child("when", -1)
if err != nil {
return nil, err
}
condConfig := processors.ConditionConfig{}
if err := sub.Unpack(&condConfig); err != nil {
return nil, err
}
tmp, err := processors.NewCondition(&condConfig)
if err != nil {
return nil, err
}
cond = tmp
}
// 5. build selector from available fields
var sel SelectorExpr
if len(mapping.Table) > 0 {
if evtfmt.IsConst() {
str, err := evtfmt.Run(common.MapStr{})
if err != nil {
return nil, err
}
str = mapping.Table[str]
if str == "" {
str = otherwise
}
if str == "" {
sel = nilSelector
} else {
sel = ConstSelectorExpr(str)
}
} else {
sel = &mapSelector{
from: FmtSelectorExpr(evtfmt, ""),
to: mapping.Table,
otherwise: otherwise,
}
}
} else {
if evtfmt.IsConst() {
str, err := evtfmt.Run(common.MapStr{})
if err != nil {
return nil, err
}
if str == "" {
sel = nilSelector
} else {
sel = ConstSelectorExpr(str)
}
} else {
sel = FmtSelectorExpr(evtfmt, otherwise)
}
}
if cond != nil && sel != nilSelector {
sel = ConditionalSelectorExpr(sel, cond)
}
return sel, nil
}
func (r *redisOut) init(cfg *common.Config, expireTopo int) error {
config := defaultConfig
if err := cfg.Unpack(&config); err != nil {
return err
}
sendRetries := config.MaxRetries
maxAttempts := config.MaxRetries + 1
if sendRetries < 0 {
maxAttempts = 0
}
var dataType redisDataType
switch config.DataType {
case "", "list":
dataType = redisListType
case "channel":
dataType = redisChannelType
default:
return errors.New("Bad Redis data type")
}
if cfg.HasField("index") && !cfg.HasField("key") {
s, err := cfg.String("index", -1)
if err != nil {
return err
}
if err := cfg.SetString("key", -1, s); err != nil {
return err
}
}
if !cfg.HasField("key") {
cfg.SetString("key", -1, r.beatName)
}
key, err := outil.BuildSelectorFromConfig(cfg, outil.Settings{
Key: "key",
MultiKey: "keys",
EnableSingleOnly: true,
FailEmpty: true,
})
if err != nil {
return err
}
tls, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
transp := &transport.Config{
Timeout: config.Timeout,
Proxy: &config.Proxy,
TLS: tls,
Stats: &transport.IOStats{
Read: statReadBytes,
Write: statWriteBytes,
ReadErrors: statReadErrors,
WriteErrors: statWriteErrors,
},
}
// configure topology support
r.topology.init(transp, topoConfig{
host: config.HostTopology,
password: config.PasswordTopology,
db: config.DbTopology,
expire: time.Duration(expireTopo) * time.Second,
})
// configure publisher clients
clients, err := modeutil.MakeClients(cfg, func(host string) (mode.ProtocolClient, error) {
t, err := transport.NewClient(transp, "tcp", host, config.Port)
if err != nil {
return nil, err
}
return newClient(t, config.Password, config.Db, key, dataType), nil
})
if err != nil {
return err
}
logp.Info("Max Retries set to: %v", sendRetries)
m, err := modeutil.NewConnectionMode(clients, modeutil.Settings{
Failover: !config.LoadBalance,
MaxAttempts: maxAttempts,
Timeout: config.Timeout,
WaitRetry: defaultWaitRetry,
MaxWaitRetry: defaultMaxWaitRetry,
})
if err != nil {
return err
}
r.mode = m
return nil
}