func (lj *logstash) init(cfg *common.Config) error {
config := defaultConfig
if err := cfg.Unpack(&config); err != nil {
return err
}
tls, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
transp := &transport.Config{
Timeout: config.Timeout,
Proxy: &config.Proxy,
TLS: tls,
Stats: &transport.IOStats{
Read: statReadBytes,
Write: statWriteBytes,
ReadErrors: statReadErrors,
WriteErrors: statWriteErrors,
},
}
logp.Info("Max Retries set to: %v", config.MaxRetries)
m, err := initConnectionMode(cfg, &config, transp)
if err != nil {
return err
}
lj.mode = m
lj.index = config.Index
return nil
}
func newKafkaConfig(config *kafkaConfig) (*sarama.Config, int, error) {
k := sarama.NewConfig()
modeRetries := 1
// configure network level properties
timeout := time.Duration(config.Timeout) * time.Second
k.Net.DialTimeout = timeout
k.Net.ReadTimeout = timeout
k.Net.WriteTimeout = timeout
k.Net.KeepAlive = config.KeepAlive
k.Producer.Timeout = config.BrokerTimeout
tls, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return nil, modeRetries, err
}
k.Net.TLS.Enable = tls != nil
k.Net.TLS.Config = tls
// TODO: configure metadata level properties
// use lib defaults
// configure producer API properties
if config.MaxMessageBytes != nil {
k.Producer.MaxMessageBytes = *config.MaxMessageBytes
}
if config.RequiredACKs != nil {
k.Producer.RequiredAcks = sarama.RequiredAcks(*config.RequiredACKs)
}
compressionMode, ok := compressionModes[strings.ToLower(config.Compression)]
if !ok {
return nil, modeRetries, fmt.Errorf("Unknown compression mode: %v", config.Compression)
}
k.Producer.Compression = compressionMode
k.Producer.Return.Successes = true // enable return channel for signaling
k.Producer.Return.Errors = true
if config.MaxRetries != nil {
retries := *config.MaxRetries
if retries < 0 {
retries = 10
modeRetries = -1
}
k.Producer.Retry.Max = retries
}
// configure client ID
k.ClientID = config.ClientID
if err := k.Validate(); err != nil {
logp.Err("Invalid kafka configuration: %v", err)
return nil, modeRetries, err
}
return k, modeRetries, nil
}
func newKafkaConfig(config *kafkaConfig) (*sarama.Config, error) {
k := sarama.NewConfig()
// configure network level properties
timeout := config.Timeout
k.Net.DialTimeout = timeout
k.Net.ReadTimeout = timeout
k.Net.WriteTimeout = timeout
k.Net.KeepAlive = config.KeepAlive
k.Producer.Timeout = config.BrokerTimeout
tls, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return nil, err
}
k.Net.TLS.Enable = tls != nil
k.Net.TLS.Config = tls
// TODO: configure metadata level properties
// use lib defaults
// configure producer API properties
if config.MaxMessageBytes != nil {
k.Producer.MaxMessageBytes = *config.MaxMessageBytes
}
if config.RequiredACKs != nil {
k.Producer.RequiredAcks = sarama.RequiredAcks(*config.RequiredACKs)
}
compressionMode, ok := compressionModes[strings.ToLower(config.Compression)]
if !ok {
return nil, fmt.Errorf("Unknown compression mode: %v", config.Compression)
}
k.Producer.Compression = compressionMode
k.Producer.Return.Successes = true // enable return channel for signaling
k.Producer.Return.Errors = true
// have retries being handled by libbeat, disable retries in sarama library
retryMax := config.MaxRetries
if retryMax < 0 {
retryMax = 1000
}
k.Producer.Retry.Max = retryMax
// configure per broker go channel buffering
k.ChannelBufferSize = config.ChanBufferSize
// configure client ID
k.ClientID = config.ClientID
if err := k.Validate(); err != nil {
logp.Err("Invalid kafka configuration: %v", err)
return nil, err
}
return k, nil
}
func (lj *logstash) init(cfg *common.Config) error {
config := defaultConfig
if err := cfg.Unpack(&config); err != nil {
return err
}
sendRetries := config.MaxRetries
maxAttempts := sendRetries + 1
if sendRetries < 0 {
maxAttempts = 0
}
tls, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
transp := &transport.Config{
Timeout: config.Timeout,
Proxy: &config.Proxy,
TLS: tls,
Stats: &transport.IOStats{
Read: statReadBytes,
Write: statWriteBytes,
ReadErrors: statReadErrors,
WriteErrors: statWriteErrors,
},
}
logp.Info("Max Retries set to: %v", sendRetries)
var m mode.ConnectionMode
if config.Pipelining == 0 {
clients, err := modeutil.MakeClients(cfg, makeClientFactory(&config, transp))
if err == nil {
m, err = modeutil.NewConnectionMode(clients, !config.LoadBalance,
maxAttempts, defaultWaitRetry, config.Timeout, defaultMaxWaitRetry)
}
} else {
clients, err := modeutil.MakeAsyncClients(cfg,
makeAsyncClientFactory(&config, transp))
if err == nil {
m, err = modeutil.NewAsyncConnectionMode(clients, !config.LoadBalance,
maxAttempts, defaultWaitRetry, config.Timeout, defaultMaxWaitRetry)
}
}
if err != nil {
return err
}
lj.mode = m
lj.index = config.Index
return nil
}
func connectTLS(certName string) transportFactory {
return func(addr string) (TransportClient, error) {
tlsConfig, err := outputs.LoadTLSConfig(&outputs.TLSConfig{
CAs: []string{certName + ".pem"},
})
if err != nil {
return nil, err
}
return newTLSClient(addr, 0, tlsConfig)
}
}
func create(
info monitors.Info,
cfg *common.Config,
) ([]monitors.Job, error) {
config := DefaultConfig
if err := cfg.Unpack(&config); err != nil {
return nil, err
}
tls, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return nil, err
}
defaultScheme := "tcp"
if tls != nil {
defaultScheme = "ssl"
}
addrs, err := collectHosts(&config, defaultScheme)
if err != nil {
return nil, err
}
if config.Socks5.URL != "" && !config.Socks5.LocalResolve {
var jobs []monitors.Job
for _, addr := range addrs {
scheme, host := addr.Scheme, addr.Host
for _, port := range addr.Ports {
job, err := newTCPMonitorHostJob(scheme, host, port, tls, &config)
if err != nil {
return nil, err
}
jobs = append(jobs, job)
}
}
return jobs, nil
}
jobs := make([]monitors.Job, len(addrs))
for i, addr := range addrs {
jobs[i], err = newTCPMonitorIPsJob(addr, tls, &config)
if err != nil {
return nil, err
}
}
return jobs, nil
}
func connectTLS(timeout time.Duration, certName string) TransportFactory {
return func(addr string, proxy *transport.ProxyConfig) (*transport.Client, error) {
tlsConfig, err := outputs.LoadTLSConfig(&outputs.TLSConfig{
CAs: []string{certName + ".pem"},
})
if err != nil {
return nil, err
}
cfg := transport.Config{
Proxy: proxy,
TLS: tlsConfig,
Timeout: timeout,
}
return transport.NewClient(&cfg, "tcp", addr, 0)
}
}
func (out *elasticsearchOutput) init(
cfg *common.Config,
topologyExpire int,
) error {
config := defaultConfig
if err := cfg.Unpack(&config); err != nil {
return err
}
tlsConfig, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
err = out.readTemplate(&config.Template)
if err != nil {
return err
}
clients, err := modeutil.MakeClients(cfg, makeClientFactory(tlsConfig, &config, out))
if err != nil {
return err
}
maxRetries := config.MaxRetries
maxAttempts := maxRetries + 1 // maximum number of send attempts (-1 = infinite)
if maxRetries < 0 {
maxAttempts = 0
}
var waitRetry = time.Duration(1) * time.Second
var maxWaitRetry = time.Duration(60) * time.Second
out.clients = clients
loadBalance := config.LoadBalance
m, err := modeutil.NewConnectionMode(clients, !loadBalance,
maxAttempts, waitRetry, config.Timeout, maxWaitRetry)
if err != nil {
return err
}
out.mode = m
out.index = config.Index
return nil
}
func (lj *logstash) init(cfg *ucfg.Config) error {
config := defaultConfig
if err := cfg.Unpack(&config); err != nil {
return err
}
useTLS := (config.TLS != nil)
timeout := time.Duration(config.Timeout) * time.Second
sendRetries := config.MaxRetries
maxAttempts := sendRetries + 1
if sendRetries < 0 {
maxAttempts = 0
}
var clients []mode.ProtocolClient
var err error
if useTLS {
var tlsConfig *tls.Config
tlsConfig, err = outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
clients, err = mode.MakeClients(cfg,
makeClientFactory(&config, makeTLSClient(config.Port, tlsConfig)))
} else {
clients, err = mode.MakeClients(cfg,
makeClientFactory(&config, makeTCPClient(config.Port)))
}
if err != nil {
return err
}
logp.Info("Max Retries set to: %v", sendRetries)
m, err := mode.NewConnectionMode(clients, !config.LoadBalance,
maxAttempts, waitRetry, timeout, maxWaitRetry)
if err != nil {
return err
}
lj.mode = m
lj.index = config.Index
return nil
}
func NewMockServerTLS(t *testing.T, to time.Duration, cert string, proxy *transport.ProxyConfig) *MockServer {
tcpListener, err := net.Listen("tcp", "localhost:0")
if err != nil {
t.Fatalf("failed to generate TCP listener")
}
tlsConfig, err := outputs.LoadTLSConfig(&outputs.TLSConfig{
Certificate: outputs.CertificateConfig{
Certificate: cert + ".pem",
Key: cert + ".key",
},
})
if err != nil {
t.Fatalf("failed to load certificate")
}
listener := tls.NewListener(tcpListener, tlsConfig.BuildModuleConfig(""))
server := &MockServer{Listener: listener, Timeout: to}
server.Handshake = func(client net.Conn) {
if server.Err != nil {
return
}
server.ClientDeadline(client, server.Timeout)
if server.Err != nil {
return
}
tlsConn, ok := client.(*tls.Conn)
if !ok {
server.Err = errors.New("no tls connection")
return
}
server.Err = tlsConn.Handshake()
}
server.Transp = func() (*transport.Client, error) {
return connectTLS(to, cert)(server.Addr(), proxy)
}
return server
}
func newMockServerTLS(t *testing.T, to time.Duration, cert string) *mockServer {
tcpListener, err := net.Listen("tcp", "localhost:0")
if err != nil {
t.Fatalf("failed to generate TCP listener")
}
tlsConfig, err := outputs.LoadTLSConfig(&outputs.TLSConfig{
Certificate: cert + ".pem",
CertificateKey: cert + ".key",
})
if err != nil {
t.Fatalf("failed to load certificate")
}
listener := tls.NewListener(tcpListener, tlsConfig)
server := &mockServer{Listener: listener, timeout: to}
server.handshake = func(client net.Conn) {
if server.err != nil {
return
}
server.clientDeadline(client, server.timeout)
if server.err != nil {
return
}
tlsConn, ok := client.(*tls.Conn)
if !ok {
server.err = errors.New("no tls connection")
return
}
server.err = tlsConn.Handshake()
}
server.transp = func() (TransportClient, error) {
return connectTLS(cert)(server.Addr())
}
return server
}
func (lj *logstash) init(cfg *common.Config) error {
config := defaultConfig
if err := cfg.Unpack(&config); err != nil {
return err
}
sendRetries := config.MaxRetries
maxAttempts := sendRetries + 1
if sendRetries < 0 {
maxAttempts = 0
}
tls, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
transp := &transport.Config{
Timeout: config.Timeout,
Proxy: &config.Proxy,
TLS: tls,
}
clients, err := mode.MakeClients(cfg, makeClientFactory(&config, transp))
if err != nil {
return err
}
logp.Info("Max Retries set to: %v", sendRetries)
m, err := mode.NewConnectionMode(clients, !config.LoadBalance,
maxAttempts, defaultWaitRetry, config.Timeout, defaultMaxWaitRetry)
if err != nil {
return err
}
lj.mode = m
lj.index = config.Index
return nil
}
// New create a new instance of the partition MetricSet
func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
config := defaultConfig
if err := base.Module().UnpackConfig(&config); err != nil {
return nil, err
}
tls, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return nil, err
}
cfg := sarama.NewConfig()
cfg.Net.DialTimeout = base.Module().Config().Timeout
cfg.Net.ReadTimeout = base.Module().Config().Timeout
cfg.ClientID = config.ClientID
cfg.Metadata.Retry.Max = config.Metadata.Retries
cfg.Metadata.Retry.Backoff = config.Metadata.Backoff
if tls != nil {
cfg.Net.TLS.Enable = true
cfg.Net.TLS.Config = tls.BuildModuleConfig("")
}
if config.Username != "" {
cfg.Net.SASL.Enable = true
cfg.Net.SASL.User = config.Username
cfg.Net.SASL.Password = config.Password
}
broker := sarama.NewBroker(base.Host())
return &MetricSet{
BaseMetricSet: base,
broker: broker,
cfg: cfg,
id: noID,
topics: config.Topics,
}, nil
}
func (p *Poller) runOneTime() error {
request := gorequest.New()
url := p.config.Url
method := p.config.Method
switch method {
case "get":
request.Get(url)
case "delete":
request.Delete(url)
case "head":
request.Head(url)
case "patch":
request.Patch(url)
case "post":
request.Post(url)
case "put":
request.Put(url)
default:
return fmt.Errorf("Unsupported HTTP method %g", method)
}
// set timeout
if p.config.Timeout != nil {
request.Timeout(time.Duration(*p.config.Timeout) * time.Second)
} else {
request.Timeout(DefaultTimeout)
}
// set authentication
if p.config.BasicAuth.Username != "" && p.config.BasicAuth.Password != "" {
request.BasicAuth.Username = p.config.BasicAuth.Username
request.BasicAuth.Password = p.config.BasicAuth.Password
}
// set tls config
useTLS := (p.config.TLS != nil)
if useTLS {
var err error
var tlsConfig *tls.Config
tlsConfig, err = outputs.LoadTLSConfig(p.config.TLS)
if err != nil {
return err
}
request.TLSClientConfig(tlsConfig)
}
// set body
if p.config.Body != "" {
switch method {
case "patch", "post", "put":
request.SendString(p.config.Body)
default:
}
}
// set headers
request.Header = p.config.Headers
// set proxy
if p.config.ProxyUrl != "" {
request.Proxy(p.config.ProxyUrl)
}
logp.Debug("Httpbeat", "Executing HTTP request: %v", request)
now := time.Now()
resp, body, errs := request.End()
if errs != nil {
logp.Err("An error occured while executing HTTP request: %v", errs)
return fmt.Errorf("An error occured while executing HTTP request: %v", errs)
}
requestEvent := Request{
Url: url,
Method: method,
Headers: p.config.Headers,
Body: p.config.Body,
}
responseEvent := Response{
StatusCode: resp.StatusCode,
Headers: p.GetResponseHeader(resp),
Body: body,
}
event := HttpEvent{
ReadTime: now,
DocumentType: p.config.DocumentType,
Fields: p.config.Fields,
Request: requestEvent,
Response: responseEvent,
}
p.httpbeat.events.PublishEvent(event.ToMapStr())
return nil
}
func (r *redisOut) init(cfg *common.Config, expireTopo int) error {
config := defaultConfig
if err := cfg.Unpack(&config); err != nil {
return err
}
sendRetries := config.MaxRetries
maxAttempts := config.MaxRetries + 1
if sendRetries < 0 {
maxAttempts = 0
}
var dataType redisDataType
switch config.DataType {
case "", "list":
dataType = redisListType
case "channel":
dataType = redisChannelType
default:
return errors.New("Bad Redis data type")
}
index := []byte(config.Index)
if len(index) == 0 {
return fmt.Errorf("missing %v", cfg.PathOf("index"))
}
tls, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
transp := &transport.Config{
Timeout: config.Timeout,
Proxy: &config.Proxy,
TLS: tls,
Stats: &transport.IOStats{
Read: statReadBytes,
Write: statWriteBytes,
ReadErrors: statReadErrors,
WriteErrors: statWriteErrors,
},
}
// configure topology support
r.topology.init(transp, topoConfig{
host: config.HostTopology,
password: config.PasswordTopology,
db: config.DbTopology,
expire: time.Duration(expireTopo) * time.Second,
})
// configure publisher clients
clients, err := modeutil.MakeClients(cfg, func(host string) (mode.ProtocolClient, error) {
t, err := transport.NewClient(transp, "tcp", host, config.Port)
if err != nil {
return nil, err
}
return newClient(t, config.Password, config.Db, index, dataType), nil
})
if err != nil {
return err
}
logp.Info("Max Retries set to: %v", sendRetries)
m, err := modeutil.NewConnectionMode(clients, !config.LoadBalance,
maxAttempts, defaultWaitRetry, config.Timeout, defaultMaxWaitRetry)
if err != nil {
return err
}
r.mode = m
return nil
}
func (out *elasticsearchOutput) init(
cfg *ucfg.Config,
topologyExpire int,
) error {
config := defaultConfig
if err := cfg.Unpack(&config); err != nil {
return err
}
tlsConfig, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
clients, err := mode.MakeClients(cfg, makeClientFactory(tlsConfig, &config))
if err != nil {
return err
}
timeout := time.Duration(config.Timeout) * time.Second
maxRetries := config.MaxRetries
maxAttempts := maxRetries + 1 // maximum number of send attempts (-1 = infinite)
if maxRetries < 0 {
maxAttempts = 0
}
var waitRetry = time.Duration(1) * time.Second
var maxWaitRetry = time.Duration(60) * time.Second
out.clients = clients
loadBalance := config.LoadBalance
m, err := mode.NewConnectionMode(clients, !loadBalance,
maxAttempts, waitRetry, timeout, maxWaitRetry)
if err != nil {
return err
}
loadTemplate(config.Template, clients)
if config.SaveTopology {
err := out.EnableTTL()
if err != nil {
logp.Err("Fail to set _ttl mapping: %s", err)
// keep trying in the background
go func() {
for {
err := out.EnableTTL()
if err == nil {
break
}
logp.Err("Fail to set _ttl mapping: %s", err)
time.Sleep(5 * time.Second)
}
}()
}
}
out.TopologyExpire = 15000
if topologyExpire != 0 {
out.TopologyExpire = topologyExpire * 1000 // millisec
}
out.mode = m
out.index = config.Index
return nil
}
func (r *redisOut) init(cfg *common.Config, expireTopo int) error {
config := defaultConfig
if err := cfg.Unpack(&config); err != nil {
return err
}
sendRetries := config.MaxRetries
maxAttempts := config.MaxRetries + 1
if sendRetries < 0 {
maxAttempts = 0
}
var dataType redisDataType
switch config.DataType {
case "", "list":
dataType = redisListType
case "channel":
dataType = redisChannelType
default:
return errors.New("Bad Redis data type")
}
if cfg.HasField("index") && !cfg.HasField("key") {
s, err := cfg.String("index", -1)
if err != nil {
return err
}
if err := cfg.SetString("key", -1, s); err != nil {
return err
}
}
if !cfg.HasField("key") {
cfg.SetString("key", -1, r.beatName)
}
key, err := outil.BuildSelectorFromConfig(cfg, outil.Settings{
Key: "key",
MultiKey: "keys",
EnableSingleOnly: true,
FailEmpty: true,
})
if err != nil {
return err
}
tls, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
transp := &transport.Config{
Timeout: config.Timeout,
Proxy: &config.Proxy,
TLS: tls,
Stats: &transport.IOStats{
Read: statReadBytes,
Write: statWriteBytes,
ReadErrors: statReadErrors,
WriteErrors: statWriteErrors,
},
}
// configure topology support
r.topology.init(transp, topoConfig{
host: config.HostTopology,
password: config.PasswordTopology,
db: config.DbTopology,
expire: time.Duration(expireTopo) * time.Second,
})
// configure publisher clients
clients, err := modeutil.MakeClients(cfg, func(host string) (mode.ProtocolClient, error) {
t, err := transport.NewClient(transp, "tcp", host, config.Port)
if err != nil {
return nil, err
}
return newClient(t, config.Password, config.Db, key, dataType), nil
})
if err != nil {
return err
}
logp.Info("Max Retries set to: %v", sendRetries)
m, err := modeutil.NewConnectionMode(clients, modeutil.Settings{
Failover: !config.LoadBalance,
MaxAttempts: maxAttempts,
Timeout: config.Timeout,
WaitRetry: defaultWaitRetry,
MaxWaitRetry: defaultMaxWaitRetry,
})
if err != nil {
return err
}
r.mode = m
return nil
}
func newKafkaConfig(config *kafkaConfig) (*sarama.Config, error) {
k := sarama.NewConfig()
// configure network level properties
timeout := config.Timeout
k.Net.DialTimeout = timeout
k.Net.ReadTimeout = timeout
k.Net.WriteTimeout = timeout
k.Net.KeepAlive = config.KeepAlive
k.Producer.Timeout = config.BrokerTimeout
tls, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return nil, err
}
if tls != nil {
k.Net.TLS.Enable = true
k.Net.TLS.Config = tls.BuildModuleConfig("")
}
if config.Username != "" {
k.Net.SASL.Enable = true
k.Net.SASL.User = config.Username
k.Net.SASL.Password = config.Password
}
// configure metadata update properties
k.Metadata.Retry.Max = config.Metadata.Retry.Max
k.Metadata.Retry.Backoff = config.Metadata.Retry.Backoff
k.Metadata.RefreshFrequency = config.Metadata.RefreshFreq
// configure producer API properties
if config.MaxMessageBytes != nil {
k.Producer.MaxMessageBytes = *config.MaxMessageBytes
}
if config.RequiredACKs != nil {
k.Producer.RequiredAcks = sarama.RequiredAcks(*config.RequiredACKs)
}
compressionMode, ok := compressionModes[strings.ToLower(config.Compression)]
if !ok {
return nil, fmt.Errorf("Unknown compression mode: '%v'", config.Compression)
}
k.Producer.Compression = compressionMode
k.Producer.Return.Successes = true // enable return channel for signaling
k.Producer.Return.Errors = true
// have retries being handled by libbeat, disable retries in sarama library
retryMax := config.MaxRetries
if retryMax < 0 {
retryMax = 1000
}
k.Producer.Retry.Max = retryMax
// TODO: k.Producer.Retry.Backoff = ?
// configure per broker go channel buffering
k.ChannelBufferSize = config.ChanBufferSize
// configure client ID
k.ClientID = config.ClientID
if err := k.Validate(); err != nil {
logp.Err("Invalid kafka configuration: %v", err)
return nil, err
}
version, ok := kafkaVersions[config.Version]
if !ok {
return nil, fmt.Errorf("Unknown/unsupported kafka version: %v", config.Version)
}
k.Version = version
return k, nil
}
func create(
info monitors.Info,
cfg *common.Config,
) ([]monitors.Job, error) {
config := defaultConfig
if err := cfg.Unpack(&config); err != nil {
return nil, err
}
tls, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return nil, err
}
var body []byte
var enc contentEncoder
if config.Check.SendBody != "" {
var err error
compression := config.Check.Compression
enc, err = getContentEncoder(compression.Type, compression.Level)
if err != nil {
return nil, err
}
buf := bytes.NewBuffer(nil)
err = enc.Encode(buf, bytes.NewBufferString(config.Check.SendBody))
if err != nil {
return nil, err
}
body = buf.Bytes()
}
validator := makeValidateResponse(&config.Check)
jobs := make([]monitors.Job, len(config.URLs))
if config.ProxyURL != "" {
transport, err := newRoundTripper(&config, tls)
if err != nil {
return nil, err
}
for i, url := range config.URLs {
jobs[i], err = newHTTPMonitorHostJob(url, &config, transport, enc, body, validator)
if err != nil {
return nil, err
}
}
} else {
for i, url := range config.URLs {
jobs[i], err = newHTTPMonitorIPsJob(&config, url, tls, enc, body, validator)
if err != nil {
return nil, err
}
}
}
return jobs, nil
}
func (lj *logstash) init(
config outputs.MothershipConfig,
topologyExpire int,
) error {
useTLS := (config.TLS != nil)
timeout := logstashDefaultTimeout
if config.Timeout != 0 {
timeout = time.Duration(config.Timeout) * time.Second
}
defaultPort := logstashDefaultPort
if config.Port != 0 {
defaultPort = config.Port
}
maxWindowSize := defaultMaxWindowSize
if config.BulkMaxSize != nil {
maxWindowSize = *config.BulkMaxSize
}
compressLevel := defaultCompressionLevel
if config.CompressionLevel != nil {
compressLevel = *config.CompressionLevel
}
var clients []mode.ProtocolClient
var err error
if useTLS {
var tlsConfig *tls.Config
tlsConfig, err = outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
clients, err = mode.MakeClients(config,
makeClientFactory(maxWindowSize, compressLevel, timeout,
makeTLSClient(defaultPort, tlsConfig)))
} else {
clients, err = mode.MakeClients(config,
makeClientFactory(maxWindowSize, compressLevel, timeout,
makeTCPClient(defaultPort)))
}
if err != nil {
return err
}
sendRetries := defaultSendRetries
if config.MaxRetries != nil {
sendRetries = *config.MaxRetries
}
logp.Info("Max Retries set to: %v", sendRetries)
maxAttempts := sendRetries + 1
if sendRetries < 0 {
maxAttempts = 0
}
loadBalance := config.LoadBalance != nil && *config.LoadBalance
m, err := mode.NewConnectionMode(clients, !loadBalance,
maxAttempts, waitRetry, timeout, maxWaitRetry)
if err != nil {
return err
}
lj.mode = m
lj.index = config.Index
return nil
}
func (p *Poller) runOneTime() error {
if p.request == nil {
p.request = gorequest.New()
}
url := p.config.Url
method := p.config.Method
switch method {
case "get":
p.request.Get(url)
case "delete":
p.request.Delete(url)
case "head":
p.request.Head(url)
case "patch":
p.request.Patch(url)
case "post":
p.request.Post(url)
case "put":
p.request.Put(url)
default:
return fmt.Errorf("Unsupported HTTP method %g", method)
}
outputFormat := p.config.OutputFormat
switch outputFormat {
case "":
outputFormat = config.DefaultOutputFormat
case "string":
case "json":
break
default:
return fmt.Errorf("Unsupported output format %g", outputFormat)
}
// set timeout
if p.config.Timeout != nil {
p.request.Timeout(time.Duration(*p.config.Timeout) * time.Second)
} else {
p.request.Timeout(config.DefaultTimeout)
}
// set authentication
if p.config.BasicAuth.Username != "" && p.config.BasicAuth.Password != "" {
p.request.BasicAuth.Username = p.config.BasicAuth.Username
p.request.BasicAuth.Password = p.config.BasicAuth.Password
}
// set tls config
useTLS := (p.config.SSL != nil)
if useTLS {
var err error
var tlsConfig *tls.Config
var tlsC *transport.TLSConfig
//tlsConfig, err = outputs.LoadTLSConfig(p.config.TLS)
tlsC, err = outputs.LoadTLSConfig(p.config.SSL)
tlsConfig = convertTLSConfig(tlsC)
if err != nil {
return err
}
p.request.TLSClientConfig(tlsConfig)
}
// set body
if p.config.Body != "" {
switch method {
case "patch", "post", "put":
p.request.SendString(p.config.Body)
default:
}
}
// set headers
p.request.Header = p.config.Headers
// set proxy
if p.config.ProxyUrl != "" {
p.request.Proxy(p.config.ProxyUrl)
}
logp.Debug("Httpbeat", "Executing HTTP request: %v", p.request)
now := time.Now()
resp, body, errs := p.request.End()
if errs != nil {
p.request = nil
logp.Err("An error occurred while executing HTTP request: %v", errs)
return fmt.Errorf("An error occurred while executing HTTP request: %v", errs)
}
requestEvent := Request{
Url: url,
Method: method,
Headers: p.config.Headers,
Body: p.config.Body,
}
var jsonBody map[string]interface{}
responseEvent := Response{
StatusCode: resp.StatusCode,
Headers: p.GetResponseHeader(resp),
}
if outputFormat == "string" {
responseEvent.Body = body
} else {
if outputFormat == "json" {
decoder := json.NewDecoder(strings.NewReader(body))
decoder.UseNumber()
errs := decoder.Decode(&jsonBody)
if errs != nil {
jsonBody = nil
logp.Err("An error occurred while marshalling response to JSON: %w", errs)
} else {
if p.config.JsonDotMode == "unflatten" {
jsonBody = unflat(jsonBody).(map[string]interface{})
} else if p.config.JsonDotMode == "replace" {
jsonBody = replaceDots(jsonBody).(map[string]interface{})
}
}
responseEvent.JsonBody = jsonBody
}
}
event := HttpEvent{
ReadTime: now,
DocumentType: p.config.DocumentType,
Fields: p.config.Fields,
Request: requestEvent,
Response: responseEvent,
}
p.httpbeat.client.PublishEvent(event.ToMapStr())
return nil
}
func (out *elasticsearchOutput) init(
config outputs.MothershipConfig,
topologyExpire int,
) error {
tlsConfig, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
clients, err := mode.MakeClients(config, makeClientFactory(tlsConfig, config))
if err != nil {
return err
}
timeout := elasticsearchDefaultTimeout
if config.Timeout != 0 {
timeout = time.Duration(config.Timeout) * time.Second
}
maxRetries := defaultMaxRetries
if config.MaxRetries != nil {
maxRetries = *config.MaxRetries
}
maxAttempts := maxRetries + 1 // maximum number of send attempts (-1 = infinite)
if maxRetries < 0 {
maxAttempts = 0
}
var waitRetry = time.Duration(1) * time.Second
var maxWaitRetry = time.Duration(60) * time.Second
var m mode.ConnectionMode
out.clients = clients
if len(clients) == 1 {
client := clients[0]
m, err = mode.NewSingleConnectionMode(client, maxAttempts,
waitRetry, timeout, maxWaitRetry)
} else {
loadBalance := config.LoadBalance == nil || *config.LoadBalance
if loadBalance {
m, err = mode.NewLoadBalancerMode(clients, maxAttempts,
waitRetry, timeout, maxWaitRetry)
} else {
m, err = mode.NewFailOverConnectionMode(clients, maxAttempts, waitRetry, timeout)
}
}
if err != nil {
return err
}
if config.Save_topology {
err := out.EnableTTL()
if err != nil {
logp.Err("Fail to set _ttl mapping: %s", err)
// keep trying in the background
go func() {
for {
err := out.EnableTTL()
if err == nil {
break
}
logp.Err("Fail to set _ttl mapping: %s", err)
time.Sleep(5 * time.Second)
}
}()
}
}
out.TopologyExpire = 15000
if topologyExpire != 0 {
out.TopologyExpire = topologyExpire * 1000 // millisec
}
out.mode = m
out.index = config.Index
return nil
}
func (out *elasticsearchOutput) init(
cfg *common.Config,
topologyExpire int,
) error {
config := defaultConfig
if err := cfg.Unpack(&config); err != nil {
return err
}
index, err := outil.BuildSelectorFromConfig(cfg, outil.Settings{
Key: "index",
MultiKey: "indices",
EnableSingleOnly: true,
FailEmpty: true,
})
if err != nil {
return err
}
tlsConfig, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
err = out.readTemplate(&config.Template)
if err != nil {
return err
}
out.index = index
pipeline, err := outil.BuildSelectorFromConfig(cfg, outil.Settings{
Key: "pipeline",
MultiKey: "pipelines",
EnableSingleOnly: true,
FailEmpty: false,
})
if err != nil {
return err
}
if !pipeline.IsEmpty() {
out.pipeline = &pipeline
}
clients, err := modeutil.MakeClients(cfg, makeClientFactory(tlsConfig, &config, out))
if err != nil {
return err
}
maxRetries := config.MaxRetries
maxAttempts := maxRetries + 1 // maximum number of send attempts (-1 = infinite)
if maxRetries < 0 {
maxAttempts = 0
}
var waitRetry = time.Duration(1) * time.Second
var maxWaitRetry = time.Duration(60) * time.Second
out.clients = clients
loadBalance := config.LoadBalance
m, err := modeutil.NewConnectionMode(clients, modeutil.Settings{
Failover: !loadBalance,
MaxAttempts: maxAttempts,
Timeout: config.Timeout,
WaitRetry: waitRetry,
MaxWaitRetry: maxWaitRetry,
})
if err != nil {
return err
}
out.mode = m
return nil
}
func New() (*Importer, error) {
importer := Importer{}
/* define the command line arguments */
cl, err := DefineCommandLine()
if err != nil {
cl.flagSet.Usage()
return nil, err
}
/* parse command line arguments */
err = cl.ParseCommandLine()
if err != nil {
return nil, err
}
importer.cl = cl
/* prepare the Elasticsearch index pattern */
fmtstr, err := fmtstr.CompileEvent(cl.opt.Index)
if err != nil {
return nil, fmt.Errorf("fail to build the Elasticsearch index pattern: %s", err)
}
indexSel := outil.MakeSelector(outil.FmtSelectorExpr(fmtstr, ""))
var tlsConfig outputs.TLSConfig
var tls *transport.TLSConfig
if cl.opt.Insecure {
tlsConfig.VerificationMode = transport.VerifyNone
}
if len(cl.opt.Certificate) > 0 && len(cl.opt.CertificateKey) > 0 {
tlsConfig.Certificate = outputs.CertificateConfig{
Certificate: cl.opt.Certificate,
Key: cl.opt.CertificateKey,
}
}
if len(cl.opt.CertificateAuthority) > 0 {
tlsConfig.CAs = []string{cl.opt.CertificateAuthority}
}
tls, err = outputs.LoadTLSConfig(&tlsConfig)
if err != nil {
return nil, fmt.Errorf("fail to load the SSL certificate: %s", err)
}
/* connect to Elasticsearch */
client, err := elasticsearch.NewClient(
elasticsearch.ClientSettings{
URL: cl.opt.ES,
Index: indexSel,
TLS: tls,
Username: cl.opt.User,
Password: cl.opt.Pass,
Timeout: 60 * time.Second,
},
nil,
)
if err != nil {
return nil, fmt.Errorf("fail to connect to Elasticsearch: %s", err)
}
importer.client = client
return &importer, nil
}
func (lj *logstash) init(
config outputs.MothershipConfig,
topologyExpire int,
) error {
useTLS := (config.TLS != nil)
timeout := logstashDefaultTimeout
if config.Timeout != 0 {
timeout = time.Duration(config.Timeout) * time.Second
}
defaultPort := logstashDefaultPort
if config.Port != 0 {
defaultPort = config.Port
}
maxWindowSize := defaultMaxWindowSize
if config.BulkMaxSize != nil {
maxWindowSize = *config.BulkMaxSize
}
var clients []mode.ProtocolClient
var err error
if useTLS {
var tlsConfig *tls.Config
tlsConfig, err = outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
clients, err = mode.MakeClients(config,
makeClientFactory(maxWindowSize, timeout,
func(host string) (TransportClient, error) {
return newTLSClient(host, defaultPort, tlsConfig)
}))
} else {
clients, err = mode.MakeClients(config,
makeClientFactory(maxWindowSize, timeout,
func(host string) (TransportClient, error) {
return newTCPClient(host, defaultPort)
}))
}
if err != nil {
return err
}
sendRetries := defaultSendRetries
if config.MaxRetries != nil {
sendRetries = *config.MaxRetries
}
logp.Info("Max Retries set to: %v", sendRetries)
maxAttempts := sendRetries + 1
if sendRetries < 0 {
maxAttempts = 0
}
var m mode.ConnectionMode
if len(clients) == 1 {
m, err = mode.NewSingleConnectionMode(clients[0],
maxAttempts, waitRetry, timeout, maxWaitRetry)
} else {
loadBalance := config.LoadBalance != nil && *config.LoadBalance
if loadBalance {
m, err = mode.NewLoadBalancerMode(clients, maxAttempts,
waitRetry, timeout, maxWaitRetry)
} else {
m, err = mode.NewFailOverConnectionMode(clients, maxAttempts, waitRetry, timeout)
}
}
if err != nil {
return err
}
lj.mode = m
lj.index = config.Index
return nil
}