func New(config *ucfg.Config, _ int) (outputs.Outputer, error) {
c := &console{config: defaultConfig}
err := config.Unpack(&c.config)
if err != nil {
return nil, err
}
return c, nil
}
func (k *kafka) init(cfg *ucfg.Config) error {
debugf("initialize kafka output")
config := defaultConfig
if err := cfg.Unpack(&config); err != nil {
return err
}
libCfg, retries, err := newKafkaConfig(&config)
if err != nil {
return err
}
hosts := config.Hosts
if len(hosts) < 1 {
logp.Err("Kafka configuration failed with: %v", errNoHosts)
return errNoHosts
}
debugf("hosts: %v", hosts)
useType := config.UseType
topic := config.Topic
if topic == "" && !useType {
logp.Err("Kafka configuration failed with: %v", errNoTopicSet)
return errNoTopicSet
}
var clients []mode.AsyncProtocolClient
worker := 1
if config.Worker > 1 {
worker = config.Worker
}
for i := 0; i < worker; i++ {
client, err := newKafkaClient(hosts, topic, useType, libCfg)
if err != nil {
logp.Err("Failed to create kafka client: %v", err)
return err
}
clients = append(clients, client)
}
mode, err := mode.NewAsyncConnectionMode(
clients,
false,
retries, // retry implemented by kafka client
libCfg.Producer.Retry.Backoff,
libCfg.Net.WriteTimeout,
10*time.Second)
if err != nil {
logp.Err("Failed to configure kafka connection: %v", err)
return err
}
k.mode = mode
return nil
}
// NewOutput instantiates a new output plugin instance publishing to elasticsearch.
func New(cfg *ucfg.Config, topologyExpire int) (outputs.Outputer, error) {
if !cfg.HasField("bulk_max_size") {
cfg.SetInt("bulk_max_size", 0, defaultBulkSize)
}
output := &elasticsearchOutput{}
err := output.init(cfg, topologyExpire)
if err != nil {
return nil, err
}
return output, nil
}
func New(cfg *ucfg.Config, topologyExpire int) (outputs.Outputer, error) {
config := defaultConfig
if err := cfg.Unpack(&config); err != nil {
return nil, err
}
output := &redisOutput{}
if err := output.Init(&config, topologyExpire); err != nil {
return nil, err
}
return output, nil
}
func New(testMode bool, results publish.Transactions, cfg *ucfg.Config) (*Icmp, error) {
p := &Icmp{}
config := defaultConfig
if !testMode {
if err := cfg.Unpack(&config); err != nil {
return nil, err
}
}
if err := p.init(results, &config); err != nil {
return nil, err
}
return p, nil
}
func (lj *logstash) init(cfg *ucfg.Config) error {
config := defaultConfig
if err := cfg.Unpack(&config); err != nil {
return err
}
useTLS := (config.TLS != nil)
timeout := time.Duration(config.Timeout) * time.Second
sendRetries := config.MaxRetries
maxAttempts := sendRetries + 1
if sendRetries < 0 {
maxAttempts = 0
}
var clients []mode.ProtocolClient
var err error
if useTLS {
var tlsConfig *tls.Config
tlsConfig, err = outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
clients, err = mode.MakeClients(cfg,
makeClientFactory(&config, makeTLSClient(config.Port, tlsConfig)))
} else {
clients, err = mode.MakeClients(cfg,
makeClientFactory(&config, makeTCPClient(config.Port)))
}
if err != nil {
return err
}
logp.Info("Max Retries set to: %v", sendRetries)
m, err := mode.NewConnectionMode(clients, !config.LoadBalance,
maxAttempts, waitRetry, timeout, maxWaitRetry)
if err != nil {
return err
}
lj.mode = m
lj.index = config.Index
return nil
}
func New(
testMode bool,
results publish.Transactions,
cfg *ucfg.Config,
) (protos.Plugin, error) {
p := &Thrift{}
config := defaultConfig
if !testMode {
if err := cfg.Unpack(&config); err != nil {
return nil, err
}
}
if err := p.init(testMode, results, &config); err != nil {
return nil, err
}
return p, nil
}
func newOutputWorker(
cfg *ucfg.Config,
out outputs.Outputer,
ws *common.WorkerSignal,
hwm int,
bulkHWM int,
) *outputWorker {
config := defaultConfig
err := cfg.Unpack(&config)
if err != nil {
logp.Err("Failed to read output worker config: %v", err)
return nil
}
o := &outputWorker{
out: outputs.CastBulkOutputer(out),
config: config,
maxBulkSize: config.BulkMaxSize,
}
o.messageWorker.init(ws, hwm, bulkHWM, o)
return o
}
func ReadHostList(cfg *ucfg.Config) ([]string, error) {
config := struct {
Host string `config:"host"`
Hosts []string `config:"hosts"`
Worker int `config:"worker"`
}{
Worker: 1,
}
err := cfg.Unpack(&config)
if err != nil {
return nil, err
}
// TODO: remove config.Host
var lst []string
if len(config.Hosts) > 0 {
lst = config.Hosts
} else if config.Host != "" {
lst = []string{config.Host}
}
if len(lst) == 0 || config.Worker <= 1 {
return lst, nil
}
// duplicate entries config.Workers times
hosts := make([]string, 0, len(lst)*config.Worker)
for _, entry := range lst {
for i := 0; i < config.Worker; i++ {
hosts = append(hosts, entry)
}
}
return hosts, nil
}
func New(cfg *ucfg.Config, _ int) (outputs.Outputer, error) {
config := defaultConfig
if err := cfg.Unpack(&config); err != nil {
return nil, err
}
// disable bulk support in publisher pipeline
cfg.SetInt("flush_interval", 0, -1)
cfg.SetInt("bulk_max_size", 0, -1)
output := &fileOutput{}
if err := output.init(config); err != nil {
return nil, err
}
return output, nil
}
func (out *elasticsearchOutput) init(
cfg *ucfg.Config,
topologyExpire int,
) error {
config := defaultConfig
if err := cfg.Unpack(&config); err != nil {
return err
}
tlsConfig, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
clients, err := mode.MakeClients(cfg, makeClientFactory(tlsConfig, &config))
if err != nil {
return err
}
timeout := time.Duration(config.Timeout) * time.Second
maxRetries := config.MaxRetries
maxAttempts := maxRetries + 1 // maximum number of send attempts (-1 = infinite)
if maxRetries < 0 {
maxAttempts = 0
}
var waitRetry = time.Duration(1) * time.Second
var maxWaitRetry = time.Duration(60) * time.Second
out.clients = clients
loadBalance := config.LoadBalance
m, err := mode.NewConnectionMode(clients, !loadBalance,
maxAttempts, waitRetry, timeout, maxWaitRetry)
if err != nil {
return err
}
loadTemplate(config.Template, clients)
if config.SaveTopology {
err := out.EnableTTL()
if err != nil {
logp.Err("Fail to set _ttl mapping: %s", err)
// keep trying in the background
go func() {
for {
err := out.EnableTTL()
if err == nil {
break
}
logp.Err("Fail to set _ttl mapping: %s", err)
time.Sleep(5 * time.Second)
}
}()
}
}
out.TopologyExpire = 15000
if topologyExpire != 0 {
out.TopologyExpire = topologyExpire * 1000 // millisec
}
out.mode = m
out.index = config.Index
return nil
}