func (lj *lumberjack) init(
beat string,
config outputs.MothershipConfig,
topologyExpire int,
) error {
useTLS := false
if config.TLS != nil {
useTLS = *config.TLS
}
timeout := lumberjackDefaultTimeout
if config.Timeout != 0 {
timeout = time.Duration(config.Timeout) * time.Second
}
var clients []ProtocolClient
var err error
if useTLS {
var tlsConfig *tls.Config
tlsConfig, err = outputs.LoadTLSConfig(config)
if err != nil {
return err
}
clients, err = makeClients(config, timeout,
func(host string) (TransportClient, error) {
return newTLSClient(host, tlsConfig)
})
} else {
clients, err = makeClients(config, timeout,
func(host string) (TransportClient, error) {
return newTCPClient(host)
})
}
if err != nil {
return err
}
var mode ConnectionMode
if len(clients) == 1 {
mode, err = newSingleConnectionMode(clients[0], waitRetry, timeout)
} else {
mode, err = newFailOverConnectionMode(clients, waitRetry, timeout)
}
if err != nil {
return err
}
lj.mode = mode
return nil
}
func newMockTLSServer(t *testing.T, to time.Duration, cert string) *mockLSServer {
tcpListener, err := net.Listen("tcp", "localhost:0")
if err != nil {
t.Fatalf("failed to generate TCP listener")
}
tlsConfig, err := outputs.LoadTLSConfig(&outputs.TLSConfig{
Certificate: cert + ".pem",
CertificateKey: cert + ".key",
})
if err != nil {
t.Fatalf("failed to load certificate")
}
listener := tls.NewListener(tcpListener, tlsConfig)
server := &mockLSServer{Listener: listener, timeout: to}
server.handshake = func(client net.Conn) {
if server.err != nil {
return
}
server.clientDeadline(client, server.timeout)
if server.err != nil {
return
}
tlsConn, ok := client.(*tls.Conn)
if !ok {
server.err = errors.New("no tls connection")
return
}
server.err = tlsConn.Handshake()
}
return server
}
func makeClients(
beat string,
config outputs.MothershipConfig,
) ([]mode.ProtocolClient, error) {
var urls []string
if len(config.Hosts) > 0 {
// use hosts setting
for _, host := range config.Hosts {
url, err := getURL(config.Protocol, config.Path, host)
if err != nil {
logp.Err("Invalid host param set: %s, Error: %v", host, err)
}
urls = append(urls, url)
}
} else {
// usage of host and port is deprecated as it is replaced by hosts
url := fmt.Sprintf("%s://%s:%d%s", config.Protocol, config.Host, config.Port, config.Path)
urls = append(urls, url)
}
index := beat
if config.Index != "" {
index = config.Index
}
tlsConfig, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return nil, err
}
var clients []mode.ProtocolClient
for _, url := range urls {
client := NewClient(url, index, tlsConfig, config.Username, config.Password)
clients = append(clients, client)
}
return clients, nil
}
func (lj *logstash) init(
beat string,
config outputs.MothershipConfig,
topologyExpire int,
) error {
useTLS := false
if config.TLS != nil {
useTLS = !config.TLS.Disabled
}
timeout := logstashDefaultTimeout
if config.Timeout != 0 {
timeout = time.Duration(config.Timeout) * time.Second
}
defaultPort := logstashDefaultPort
if config.Port != 0 {
defaultPort = config.Port
}
var clients []mode.ProtocolClient
var err error
if useTLS {
var tlsConfig *tls.Config
tlsConfig, err = outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
clients, err = mode.MakeClients(config, makeClientFactory(timeout,
func(host string) (TransportClient, error) {
return newTLSClient(host, defaultPort, tlsConfig)
}))
} else {
clients, err = mode.MakeClients(config, makeClientFactory(timeout,
func(host string) (TransportClient, error) {
return newTCPClient(host, defaultPort)
}))
}
if err != nil {
return err
}
sendRetries := defaultSendRetries
if config.Max_retries != nil {
sendRetries = *config.Max_retries
}
maxAttempts := sendRetries + 1
if sendRetries < 0 {
maxAttempts = 0
}
var m mode.ConnectionMode
if len(clients) == 1 {
m, err = mode.NewSingleConnectionMode(clients[0],
maxAttempts, waitRetry, timeout, maxWaitRetry)
} else {
loadBalance := config.LoadBalance != nil && *config.LoadBalance
if loadBalance {
m, err = mode.NewLoadBalancerMode(clients, maxAttempts,
waitRetry, timeout, maxWaitRetry)
} else {
m, err = mode.NewFailOverConnectionMode(clients, maxAttempts, waitRetry, timeout)
}
}
if err != nil {
return err
}
lj.mode = m
if config.Index != "" {
lj.index = config.Index
} else {
lj.index = beat
}
return nil
}
func (out *elasticsearchOutput) init(
beat string,
config outputs.MothershipConfig,
topologyExpire int,
) error {
tlsConfig, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
clients, err := mode.MakeClients(config, makeClientFactory(beat, tlsConfig, config))
if err != nil {
return err
}
timeout := elasticsearchDefaultTimeout
if config.Timeout != 0 {
timeout = time.Duration(config.Timeout) * time.Second
}
maxRetries := defaultMaxRetries
if config.MaxRetries != nil {
maxRetries = *config.MaxRetries
}
maxAttempts := maxRetries + 1 // maximum number of send attempts (-1 = infinite)
if maxRetries < 0 {
maxAttempts = 0
}
var waitRetry = time.Duration(1) * time.Second
var maxWaitRetry = time.Duration(60) * time.Second
var m mode.ConnectionMode
out.clients = clients
if len(clients) == 1 {
client := clients[0]
m, err = mode.NewSingleConnectionMode(client, maxAttempts,
waitRetry, timeout, maxWaitRetry)
} else {
loadBalance := config.LoadBalance == nil || *config.LoadBalance
if loadBalance {
m, err = mode.NewLoadBalancerMode(clients, maxAttempts,
waitRetry, timeout, maxWaitRetry)
} else {
m, err = mode.NewFailOverConnectionMode(clients, maxAttempts, waitRetry, timeout)
}
}
if err != nil {
return err
}
if config.Save_topology {
err := out.EnableTTL()
if err != nil {
logp.Err("Fail to set _ttl mapping: %s", err)
// keep trying in the background
go func() {
for {
err := out.EnableTTL()
if err == nil {
break
}
logp.Err("Fail to set _ttl mapping: %s", err)
time.Sleep(5 * time.Second)
}
}()
}
}
out.TopologyExpire = 15000
if topologyExpire != 0 {
out.TopologyExpire = topologyExpire * 1000 // millisec
}
out.mode = m
if config.Index != "" {
out.index = config.Index
} else {
out.index = beat
}
return nil
}
// Initialize Elasticsearch as output
func (out *elasticsearchOutput) Init(
beat string,
config outputs.MothershipConfig,
topologyExpire int,
) error {
if len(config.Protocol) == 0 {
config.Protocol = "http"
}
var urls []string
if len(config.Hosts) > 0 {
// use hosts setting
for _, host := range config.Hosts {
url, err := getURL(config.Protocol, config.Path, host)
if err != nil {
logp.Err("Invalid host param set: %s, Error: %v", host, err)
}
urls = append(urls, url)
}
} else {
// usage of host and port is deprecated as it is replaced by hosts
url := fmt.Sprintf("%s://%s:%d%s", config.Protocol, config.Host, config.Port, config.Path)
urls = append(urls, url)
}
tlsConfig, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
es := NewElasticsearch(urls, tlsConfig, config.Username, config.Password)
out.Conn = es
if config.Index != "" {
out.Index = config.Index
} else {
out.Index = beat
}
out.TopologyExpire = 15000
if topologyExpire != 0 {
out.TopologyExpire = topologyExpire /*sec*/ * 1000 // millisec
}
if config.Max_retries != nil {
out.Conn.SetMaxRetries(*config.Max_retries)
}
logp.Info("[ElasticsearchOutput] Using Elasticsearch %s", urls)
logp.Info("[ElasticsearchOutput] Using index pattern [%s-]YYYY.MM.DD", out.Index)
logp.Info("[ElasticsearchOutput] Topology expires after %ds", out.TopologyExpire/1000)
if config.Save_topology {
err := out.EnableTTL()
if err != nil {
logp.Err("Fail to set _ttl mapping: %s", err)
// keep trying in the background
go func() {
for {
err := out.EnableTTL()
if err == nil {
break
}
logp.Err("Fail to set _ttl mapping: %s", err)
time.Sleep(5 * time.Second)
}
}()
}
}
return nil
}
func TestLogstashTLS(t *testing.T) {
pem := "ca_test.pem"
key := "ca_test.key"
var serverErr error
var win, data *message
genCertsIfMIssing(t, pem, key)
tcpListener, err := net.Listen("tcp", "localhost:0")
if err != nil {
t.Fatalf("failed to generate TCP listener")
}
// create lumberjack output client
config := outputs.MothershipConfig{
TLS: &outputs.TLSConfig{
Certificate: pem,
CertificateKey: key,
CAs: []string{pem},
},
Timeout: 5,
Hosts: []string{tcpListener.Addr().String()},
}
tlsConfig, err := outputs.LoadTLSConfig(config.TLS)
if err != nil {
tcpListener.Close()
t.Fatalf("failed to load certificates")
}
listener := tls.NewListener(tcpListener, tlsConfig)
// start server
var wg sync.WaitGroup
var wgReady sync.WaitGroup
wg.Add(1)
wgReady.Add(1)
go func() {
defer wg.Done()
for i := 0; i < 3; i++ { // try up to 3 failed connection attempts
// server read timeout
timeout := 5 * time.Second
buf := streambuf.New(nil)
wgReady.Done()
client, err := listener.Accept()
if err != nil {
continue
}
tlsConn, ok := client.(*tls.Conn)
if !ok {
serverErr = errors.New("no tls connection")
return
}
if err := client.SetDeadline(time.Now().Add(timeout)); err != nil {
serverErr = err
return
}
err = tlsConn.Handshake()
if err != nil {
serverErr = err
return
}
if err := client.SetDeadline(time.Now().Add(timeout)); err != nil {
serverErr = err
return
}
win, err = sockReadMessage(buf, client)
if err != nil {
serverErr = err
return
}
if err := client.SetDeadline(time.Now().Add(timeout)); err != nil {
serverErr = err
return
}
data, err = sockReadMessage(buf, client)
if err != nil {
serverErr = err
return
}
serverErr = sockSendACK(client, 1)
return
}
}()
// send event to server
go func() {
wgReady.Wait()
output := newTestLumberjackOutput(t, "", &config)
event := testEvent()
output.PublishEvent(nil, time.Now(), event)
}()
wg.Wait()
listener.Close()
// validate output
assert.Nil(t, serverErr)
assert.NotNil(t, win)
assert.NotNil(t, data)
if data != nil {
assert.Equal(t, 1, len(data.events))
data = data.events[0]
assert.Equal(t, 10.0, data.doc["extra"])
assert.Equal(t, "message", data.doc["message"])
}
}
func (lj *lumberjack) init(
beat string,
config outputs.MothershipConfig,
topologyExpire int,
) error {
useTLS := false
if config.TLS != nil {
useTLS = !config.TLS.Disabled
}
timeout := lumberjackDefaultTimeout
if config.Timeout != 0 {
timeout = time.Duration(config.Timeout) * time.Second
}
var clients []mode.ProtocolClient
var err error
if useTLS {
var tlsConfig *tls.Config
tlsConfig, err = outputs.LoadTLSConfig(config.TLS)
if err != nil {
return err
}
clients, err = makeClients(config, timeout,
func(host string) (TransportClient, error) {
return newTLSClient(host, tlsConfig)
})
} else {
clients, err = makeClients(config, timeout,
func(host string) (TransportClient, error) {
return newTCPClient(host)
})
}
if err != nil {
return err
}
sendRetries := defaultSendRetries
if config.Max_retries != nil {
sendRetries = *config.Max_retries
}
var m mode.ConnectionMode
if len(clients) == 1 {
m, err = mode.NewSingleConnectionMode(clients[0],
sendRetries, waitRetry, timeout)
} else {
loadBalance := config.LoadBalance == nil || *config.LoadBalance
if loadBalance {
m, err = mode.NewLoadBalancerMode(clients, sendRetries, waitRetry, timeout)
} else {
m, err = mode.NewFailOverConnectionMode(clients, sendRetries, waitRetry, timeout)
}
}
if err != nil {
return err
}
lj.mode = m
return nil
}